diff --git a/docker/README.md b/docker/README.md index 6123e4c37e1478..0ba7dad70338e4 100644 --- a/docker/README.md +++ b/docker/README.md @@ -85,13 +85,13 @@ in the settings before use. Find settings in: - [Core services](envs/core-services/): API, workers, web, plugin daemon, and - sandboxes. [shared.defaults](envs/core-services/shared.defaults) contains the + sandboxes. [api-worker.defaults](envs/core-services/api-worker.defaults) contains the common API, worker, and Beat settings, grouped by function. API process options live in `api.defaults`; worker autoscaling lives in `worker.defaults`. - [Databases](envs/databases/): PostgreSQL, MySQL, and Redis. - [Vector stores](envs/vectorstores/): bundled and external vector databases, including their API/worker client settings. Providers without a dedicated - template are documented in `shared.defaults`. + template are documented in `api-worker.defaults`. - [Infrastructure](envs/infrastructure/): nginx, Certbot, proxies, and supporting services. ### Change a service setting @@ -226,11 +226,13 @@ Bundled service templates now use the `.defaults` extension. Existing service `.env` paths still work. Root `.env.example` remains the only required starting template, and the Compose generator creates only a missing root `.env`. -The shared application reference is now consolidated in -[shared.defaults](envs/core-services/shared.defaults). Existing `shared.env` -and other override paths still work; do not replace or rename your local files. -New customizations use the matching template listed above. Root `.env` retains -precedence, including for settings that have moved to a provider template. +Common API and worker application settings are documented in +[api-worker.defaults](envs/core-services/api-worker.defaults). New customizations +use `envs/core-services/api-worker.env`. Existing `shared.env` files and other +service override paths remain supported; keep local files and edit them in place. +If both `shared.env` and `api-worker.env` exist, `api-worker.env` takes precedence +for matching settings. Root `.env` retains precedence over both files, including +for settings documented in a provider template. Review accidental empty assignments left by older template copies. For example, `CELERY_WORKER_AMOUNT=` overrides the Docker default of `4` and lets the worker @@ -275,7 +277,7 @@ Weaviate. Update that value if you need different profiles. For HTTPS certificates, follow the [Certbot guide](certbot/README.md). -For OpenTelemetry, create `envs/core-services/shared.env` from `shared.defaults` +For OpenTelemetry, create `envs/core-services/api-worker.env` from `api-worker.defaults` if it does not exist. Uncomment `ENABLE_OTEL` and set it to `true`; uncomment `OTLP_BASE_ENDPOINT` and set it to your collector's endpoint, reachable from the API and worker containers. The same file documents optional `OTEL_*` @@ -298,7 +300,7 @@ provider's interaction configuration after changing the public origin. ### Ops tracing rolling upgrades New tracing tasks use `OPS_TRACE_QUEUE=ops_trace_v2` by default. Set it consistently on the API, -workers, and Beat in `docker/envs/core-services/shared.env` (or `docker/.env`). Default worker +workers, and Beat in `docker/envs/core-services/api-worker.env` (or `docker/.env`). Default worker subscriptions follow this value; explicit `CELERY_QUEUES` or `CELERY_WORKER_QUEUES` lists must include it and exclude legacy `ops_trace` while old workers are draining. @@ -445,10 +447,10 @@ For a directly exposed API, leave `RESPECT_XFORWARD_HEADERS_ENABLED=false`. The Python configuration and `api/.env.example` retain that default. To override the Compose default, set the variable in `docker/.env`, the shell used by Compose, or a Compose override. The API services' explicit `environment` entry takes -precedence over `envs/core-services/shared.env`. +precedence over `envs/core-services/api-worker.env` and legacy `shared.env` files. Existing `false` overrides in `docker/.env`, the invoking shell, or a Compose override remain in effect during upgrades. Move settings previously supplied only -by `shared.env` to one of those override locations if you need to preserve them. +by `api-worker.env` or `shared.env` to one of those override locations if you need to preserve them. After establishing the trusted ingress boundary, enable the setting to retain visitor IPs. diff --git a/docker/docker-compose-template.yaml b/docker/docker-compose-template.yaml index fd6f5e33799af8..c7bdd785021b81 100644 --- a/docker/docker-compose-template.yaml +++ b/docker/docker-compose-template.yaml @@ -3,6 +3,8 @@ x-shared-api-config: &shared-api-config env_file: - path: ./envs/core-services/shared.env required: false + - path: ./envs/core-services/api-worker.env + required: false - path: ./envs/security.env required: false - path: ./envs/databases/db-postgres.env @@ -60,7 +62,7 @@ x-shared-api-config: &shared-api-config - path: ./envs/core-services/api.env required: false - ./.env - - ./envs/core-services/shared.defaults + - ./envs/core-services/api-worker.defaults - ./envs/core-services/api.defaults networks: - ssrf_proxy_network @@ -71,6 +73,8 @@ x-shared-worker-config: &shared-worker-config env_file: - path: ./envs/core-services/shared.env required: false + - path: ./envs/core-services/api-worker.env + required: false - path: ./envs/security.env required: false - path: ./envs/databases/db-postgres.env @@ -128,7 +132,7 @@ x-shared-worker-config: &shared-worker-config - path: ./envs/core-services/worker.env required: false - ./.env - - ./envs/core-services/shared.defaults + - ./envs/core-services/api-worker.defaults - ./envs/core-services/worker.defaults networks: - ssrf_proxy_network @@ -147,6 +151,8 @@ x-knowledge-fs-celery-worker: &knowledge-fs-celery-worker env_file: - path: ./envs/core-services/shared.env required: false + - path: ./envs/core-services/api-worker.env + required: false - path: ./envs/security.env required: false - path: ./envs/databases/db-postgres.env @@ -162,7 +168,7 @@ x-knowledge-fs-celery-worker: &knowledge-fs-celery-worker - path: ./envs/core-services/worker.env required: false - ./.env - - ./envs/core-services/shared.defaults + - ./envs/core-services/api-worker.defaults - ./envs/core-services/worker.defaults environment: &knowledge-fs-celery-environment MODE: worker @@ -186,6 +192,8 @@ x-shared-worker-beat-config: &shared-worker-beat-config env_file: - path: ./envs/core-services/shared.env required: false + - path: ./envs/core-services/api-worker.env + required: false - path: ./envs/security.env required: false - path: ./envs/databases/db-postgres.env @@ -243,7 +251,7 @@ x-shared-worker-beat-config: &shared-worker-beat-config - path: ./envs/core-services/worker-beat.env required: false - ./.env - - ./envs/core-services/shared.defaults + - ./envs/core-services/api-worker.defaults - ./envs/core-services/worker-beat.defaults networks: - ssrf_proxy_network @@ -600,6 +608,8 @@ services: env_file: - path: ./envs/core-services/shared.env required: false + - path: ./envs/core-services/api-worker.env + required: false - path: ./envs/core-services/plugin-daemon.env required: false - path: ./envs/security.env @@ -611,7 +621,7 @@ services: - path: ./envs/databases/redis.env required: false - ./.env - - ./envs/core-services/shared.defaults + - ./envs/core-services/api-worker.defaults - ./envs/core-services/plugin-daemon.defaults networks: - ssrf_proxy_network diff --git a/docker/docker-compose.yaml b/docker/docker-compose.yaml index 347d6fd40171ce..1ccf4b38632956 100644 --- a/docker/docker-compose.yaml +++ b/docker/docker-compose.yaml @@ -9,6 +9,8 @@ x-shared-api-config: &shared-api-config env_file: - path: ./envs/core-services/shared.env required: false + - path: ./envs/core-services/api-worker.env + required: false - path: ./envs/security.env required: false - path: ./envs/databases/db-postgres.env @@ -66,7 +68,7 @@ x-shared-api-config: &shared-api-config - path: ./envs/core-services/api.env required: false - ./.env - - ./envs/core-services/shared.defaults + - ./envs/core-services/api-worker.defaults - ./envs/core-services/api.defaults networks: - ssrf_proxy_network @@ -77,6 +79,8 @@ x-shared-worker-config: &shared-worker-config env_file: - path: ./envs/core-services/shared.env required: false + - path: ./envs/core-services/api-worker.env + required: false - path: ./envs/security.env required: false - path: ./envs/databases/db-postgres.env @@ -134,7 +138,7 @@ x-shared-worker-config: &shared-worker-config - path: ./envs/core-services/worker.env required: false - ./.env - - ./envs/core-services/shared.defaults + - ./envs/core-services/api-worker.defaults - ./envs/core-services/worker.defaults networks: - ssrf_proxy_network @@ -153,6 +157,8 @@ x-knowledge-fs-celery-worker: &knowledge-fs-celery-worker env_file: - path: ./envs/core-services/shared.env required: false + - path: ./envs/core-services/api-worker.env + required: false - path: ./envs/security.env required: false - path: ./envs/databases/db-postgres.env @@ -168,7 +174,7 @@ x-knowledge-fs-celery-worker: &knowledge-fs-celery-worker - path: ./envs/core-services/worker.env required: false - ./.env - - ./envs/core-services/shared.defaults + - ./envs/core-services/api-worker.defaults - ./envs/core-services/worker.defaults environment: &knowledge-fs-celery-environment MODE: worker @@ -192,6 +198,8 @@ x-shared-worker-beat-config: &shared-worker-beat-config env_file: - path: ./envs/core-services/shared.env required: false + - path: ./envs/core-services/api-worker.env + required: false - path: ./envs/security.env required: false - path: ./envs/databases/db-postgres.env @@ -249,7 +257,7 @@ x-shared-worker-beat-config: &shared-worker-beat-config - path: ./envs/core-services/worker-beat.env required: false - ./.env - - ./envs/core-services/shared.defaults + - ./envs/core-services/api-worker.defaults - ./envs/core-services/worker-beat.defaults networks: - ssrf_proxy_network @@ -606,6 +614,8 @@ services: env_file: - path: ./envs/core-services/shared.env required: false + - path: ./envs/core-services/api-worker.env + required: false - path: ./envs/core-services/plugin-daemon.env required: false - path: ./envs/security.env @@ -617,7 +627,7 @@ services: - path: ./envs/databases/redis.env required: false - ./.env - - ./envs/core-services/shared.defaults + - ./envs/core-services/api-worker.defaults - ./envs/core-services/plugin-daemon.defaults networks: - ssrf_proxy_network diff --git a/docker/envs/core-services/shared.defaults b/docker/envs/core-services/api-worker.defaults similarity index 98% rename from docker/envs/core-services/shared.defaults rename to docker/envs/core-services/api-worker.defaults index d8d4dafa9e2ce3..b149783833ec7e 100644 --- a/docker/envs/core-services/shared.defaults +++ b/docker/envs/core-services/api-worker.defaults @@ -1,5 +1,6 @@ -# Shared API, worker and Beat configuration. Copy this file to shared.env to customize it. +# Common API and worker configuration, including Beat. Copy this file to api-worker.env to customize it. # Edit the copy; docker/.env takes precedence. Existing shared.env files remain supported. +# If both override files exist, api-worker.env takes precedence over shared.env. # Active assignments preserve Docker defaults and explicit overrides, including empty values. # Optional application/provider settings are commented examples: uncomment only what you need. # Do not uncomment empty values or placeholders unless you intend to set them. @@ -77,7 +78,7 @@ CELERY_WORKER_AMOUNT=${CELERY_WORKER_AMOUNT-4} # Celery broker and KnowledgeFS dispatch # ------------------------------ # Broker URL defaults/mappings live in api.defaults, worker.defaults and worker-beat.defaults. -# Set a common CELERY_BROKER_URL override in shared.env or docker/.env for every producer/consumer. +# Set a common CELERY_BROKER_URL override in api-worker.env or docker/.env for every producer/consumer. # Optional long-running KnowledgeFS Celery rollout: set identically in every API, worker and Beat # process sharing this Redis broker. It must exceed the KnowledgeFS hard task limit (7230s by # default). Keep unset to preserve Celery's existing default when this rollout is disabled. diff --git a/docker/tests/test_compose_environment.py b/docker/tests/test_compose_environment.py index 8de74992ea638f..672bf2566c8972 100644 --- a/docker/tests/test_compose_environment.py +++ b/docker/tests/test_compose_environment.py @@ -116,9 +116,17 @@ def test_copying_defaults_without_changes_preserves_service_values(self): def test_api_and_weaviate_defaults_and_isolated_overrides(self): cases = [ - ("core-services/shared", "worker", "CELERY_WORKER_AMOUNT", "CELERY_WORKER_AMOUNT", "4", "2", "3"), - ("core-services/shared", "api", "SERVER_WORKER_CLASS", "SERVER_WORKER_CLASS", "gevent", "sync", "gthread"), - ("core-services/shared", "api", "GUNICORN_TIMEOUT", "GUNICORN_TIMEOUT", "360", "120", "240"), + ("core-services/api-worker", "worker", "CELERY_WORKER_AMOUNT", "CELERY_WORKER_AMOUNT", "4", "2", "3"), + ( + "core-services/api-worker", + "api", + "SERVER_WORKER_CLASS", + "SERVER_WORKER_CLASS", + "gevent", + "sync", + "gthread", + ), + ("core-services/api-worker", "api", "GUNICORN_TIMEOUT", "GUNICORN_TIMEOUT", "360", "120", "240"), ( "vectorstores/weaviate", "weaviate", @@ -156,12 +164,54 @@ def test_api_and_weaviate_defaults_and_isolated_overrides(self): (self.project / target).unlink(missing_ok=True) root.write_text(initial_root) - def test_copying_shared_defaults_does_not_activate_optional_settings(self): + def test_copying_api_worker_defaults_does_not_activate_optional_settings(self): baseline = self.render() - self.copy_template("envs/core-services/shared.defaults", "envs/core-services/shared.env") + self.copy_template("envs/core-services/api-worker.defaults", "envs/core-services/api-worker.env") # Compare the entire configuration, including extra variables introduced by an unsafe copy. self.assertEqual(self.render(), baseline) + def test_api_worker_overrides_preserve_legacy_settings_and_precedence(self): + consumers = ( + "api", + "api_websocket", + "worker", + "worker_knowledge_fs_upgrade", + "worker_beat", + "knowledge_fs_document_worker", + "knowledge_fs_source_worker", + "knowledge_fs_research_worker", + "knowledge_fs_maintenance_worker", + "knowledge_fs_dispatch_worker", + "plugin_daemon", + ) + baseline = self.render() + legacy = self.project / "envs/core-services/shared.env" + legacy.write_text("DB_HOST=legacy-db\n") + + services = self.render() + for name in consumers: + self.assertEqual(services[name]["environment"]["DB_HOST"], "legacy-db", name) + + source = "envs/core-services/api-worker.defaults" + target = "envs/core-services/api-worker.env" + # An unchanged copy must not mask existing shared.env customizations. + self.copy_template(source, target) + self.assertEqual(self.render(), services) + + self.copy_template(source, target, {"DB_HOST": "api-worker-db"}) + services = self.render() + for name in consumers: + self.assertEqual(services[name]["environment"]["DB_HOST"], "api-worker-db", name) + for name in baseline.keys() - set(consumers): + self.assertEqual(services[name], baseline[name], name) + + # Change only the root file while retaining both local override files. + self.override_root({"DB_HOST": "root-db"}) + services = self.render() + for name in consumers: + self.assertEqual(services[name]["environment"]["DB_HOST"], "root-db", name) + self.assertEqual(legacy.read_text(), "DB_HOST=legacy-db\n") + def test_consolidated_settings_reach_consumers_and_respect_root_precedence(self): api_services = ("api", "api_websocket") worker_services = ( @@ -176,7 +226,7 @@ def test_consolidated_settings_reach_consumers_and_respect_root_precedence(self) vector_clients = (*api_services, "worker", "worker_knowledge_fs_upgrade", "worker_beat") cases = [ ( - "core-services/shared.defaults", + "core-services/api-worker.defaults", (*api_services, *worker_services, "worker_beat"), { "CODE_MAX_STRING_LENGTH": ("500000", "600000"), @@ -369,7 +419,7 @@ def test_service_overrides_and_root_precedence(self): def test_database_redis_overrides_reach_clients_and_commands(self): for name in ( - "shared", + "api-worker", "api", "worker", "worker-beat", @@ -451,7 +501,7 @@ def test_database_redis_overrides_reach_clients_and_commands(self): self.assertFalse((self.project / "should-not-exist").exists()) def test_root_plugin_credentials_reach_every_consumer(self): - for name in ("shared", "api", "worker", "worker-beat", "plugin-daemon", "dify-agent"): + for name in ("api-worker", "api", "worker", "worker-beat", "plugin-daemon", "dify-agent"): self.copy_template(f"envs/core-services/{name}.defaults", f"envs/core-services/{name}.env") api_services = ( "api",