From d3e7e86b15929d65a03eb911a234616eb6dd8d56 Mon Sep 17 00:00:00 2001 From: ai-ag2026 <261867348+ai-ag2026@users.noreply.github.com> Date: Fri, 31 Jul 2026 02:55:19 +0200 Subject: [PATCH] feat(worker): make the graceful-shutdown timeout configurable MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The worker poller's graceful-shutdown timeout is hardcoded to 30s in both shutdown paths (worker/main.py and the API-embedded poller in api/http.py). An in-flight retain is an LLM call that may legitimately run for minutes — HINDSIGHT_API_LLM_TIMEOUT defaults far above 30s — so every service stop cancels it mid-flight and the operation is lost. Add HINDSIGHT_API_SHUTDOWN_GRACE (default 30.0 = unchanged behaviour), wired through HindsightConfig like the other operational knobs, and use it at both call sites. Deployments size it together with their supervisor stop timeout (e.g. systemd TimeoutStopSec) so the supervisor never SIGKILLs mid-cleanup. Tests: config default + env parsing, in the pattern of the neighbouring config-wiring test files. --- .env.example | 1 + hindsight-api-slim/hindsight_api/api/http.py | 7 ++- hindsight-api-slim/hindsight_api/config.py | 6 +++ .../hindsight_api/worker/main.py | 7 ++- .../tests/test_shutdown_grace_config.py | 48 +++++++++++++++++++ 5 files changed, 67 insertions(+), 2 deletions(-) create mode 100644 hindsight-api-slim/tests/test_shutdown_grace_config.py diff --git a/.env.example b/.env.example index 9aebb9a5c9..fb7ac9f2b3 100644 --- a/.env.example +++ b/.env.example @@ -292,3 +292,4 @@ HINDSIGHT_API_LOG_LEVEL=info # When set, visitors see a login page and must enter the key before # accessing the dashboard or any /api/* routes (except /api/health). # HINDSIGHT_CP_ACCESS_KEY=your-shared-secret-key +# HINDSIGHT_API_SHUTDOWN_GRACE=30 # Seconds the worker poller waits for in-flight operations on shutdown; size together with your supervisor stop timeout (e.g. systemd TimeoutStopSec). diff --git a/hindsight-api-slim/hindsight_api/api/http.py b/hindsight-api-slim/hindsight_api/api/http.py index d66f641779..6ee70a0d11 100644 --- a/hindsight-api-slim/hindsight_api/api/http.py +++ b/hindsight-api-slim/hindsight_api/api/http.py @@ -3596,7 +3596,12 @@ async def lifespan(app: FastAPI): # Shutdown worker poller if running if poller is not None: - await poller.shutdown_graceful(timeout=30.0) + # Grace configurable (default 30s) so an in-flight retain — an LLM call + # that may legitimately run for minutes — is not cancelled mid-flight + # on service stop. Deployments size HINDSIGHT_API_SHUTDOWN_GRACE + # together with their supervisor stop timeout (e.g. systemd + # TimeoutStopSec) so the supervisor never SIGKILLs mid-cleanup. + await poller.shutdown_graceful(timeout=get_config().shutdown_grace) if poller_task is not None: poller_task.cancel() try: diff --git a/hindsight-api-slim/hindsight_api/config.py b/hindsight-api-slim/hindsight_api/config.py index 368d6d84f3..7c5afbcc4f 100644 --- a/hindsight-api-slim/hindsight_api/config.py +++ b/hindsight-api-slim/hindsight_api/config.py @@ -154,6 +154,7 @@ def normalize_config_dict(config: dict[str, Any]) -> dict[str, Any]: ENV_LLM_INITIAL_BACKOFF = "HINDSIGHT_API_LLM_INITIAL_BACKOFF" ENV_LLM_MAX_BACKOFF = "HINDSIGHT_API_LLM_MAX_BACKOFF" ENV_LLM_TIMEOUT = "HINDSIGHT_API_LLM_TIMEOUT" +ENV_SHUTDOWN_GRACE = "HINDSIGHT_API_SHUTDOWN_GRACE" ENV_LLM_REASONING_EFFORT = "HINDSIGHT_API_LLM_REASONING_EFFORT" ENV_LLM_GROQ_SERVICE_TIER = "HINDSIGHT_API_LLM_GROQ_SERVICE_TIER" ENV_LLM_OPENAI_SERVICE_TIER = "HINDSIGHT_API_LLM_OPENAI_SERVICE_TIER" @@ -863,6 +864,8 @@ def _parse_worker_slot_reservations() -> dict[str, int]: DEFAULT_LLM_INITIAL_BACKOFF = 1.0 # Initial backoff in seconds for retry exponential backoff DEFAULT_LLM_MAX_BACKOFF = 60.0 # Max backoff cap in seconds for retry exponential backoff DEFAULT_LLM_TIMEOUT = 120.0 # seconds +# Seconds the worker poller waits for in-flight operations on shutdown. +DEFAULT_SHUTDOWN_GRACE = 30.0 DEFAULT_LLM_REASONING_EFFORT = "low" DEFAULT_LLM_SEND_BANK_AS_USER = False # Opt-in: tag provider calls with user= @@ -2221,6 +2224,8 @@ class HindsightConfig: # Defaulted fields (source-compatible additions — existing direct constructor callers keep working). # Keep at the end of the dataclass; Python forbids non-default fields after default fields. embeddings_openai_batch_size: int = DEFAULT_EMBEDDINGS_OPENAI_BATCH_SIZE + # Seconds the worker poller waits for in-flight operations on shutdown. + shutdown_grace: float = DEFAULT_SHUTDOWN_GRACE embeddings_openai_dimensions: int | None = None embeddings_zeroentropy_api_key: str | None = None embeddings_zeroentropy_model: str = DEFAULT_EMBEDDINGS_ZEROENTROPY_MODEL @@ -2601,6 +2606,7 @@ def from_env(cls) -> "HindsightConfig": llm_initial_backoff=float(os.getenv(ENV_LLM_INITIAL_BACKOFF, str(DEFAULT_LLM_INITIAL_BACKOFF))), llm_max_backoff=float(os.getenv(ENV_LLM_MAX_BACKOFF, str(DEFAULT_LLM_MAX_BACKOFF))), llm_timeout=float(os.getenv(ENV_LLM_TIMEOUT, str(DEFAULT_LLM_TIMEOUT))), + shutdown_grace=float(os.getenv(ENV_SHUTDOWN_GRACE, str(DEFAULT_SHUTDOWN_GRACE))), llm_reasoning_effort=os.getenv(ENV_LLM_REASONING_EFFORT, DEFAULT_LLM_REASONING_EFFORT), llm_groq_service_tier=os.getenv(ENV_LLM_GROQ_SERVICE_TIER, DEFAULT_LLM_GROQ_SERVICE_TIER), llm_openai_service_tier=os.getenv(ENV_LLM_OPENAI_SERVICE_TIER, DEFAULT_LLM_OPENAI_SERVICE_TIER), diff --git a/hindsight-api-slim/hindsight_api/worker/main.py b/hindsight-api-slim/hindsight_api/worker/main.py index 3b314b88dd..10b2ebf479 100644 --- a/hindsight-api-slim/hindsight_api/worker/main.py +++ b/hindsight-api-slim/hindsight_api/worker/main.py @@ -349,7 +349,12 @@ def signal_handler(): server.should_exit = True print("Waiting for poller to finish...") - await poller.shutdown_graceful(timeout=30.0) + # Grace configurable (default 30s) so an in-flight retain — an LLM call + # that may legitimately run for minutes — is not cancelled mid-flight + # on service stop. Deployments size HINDSIGHT_API_SHUTDOWN_GRACE + # together with their supervisor stop timeout (e.g. systemd + # TimeoutStopSec) so the supervisor never SIGKILLs mid-cleanup. + await poller.shutdown_graceful(timeout=get_config().shutdown_grace) poller_task.cancel() try: await poller_task diff --git a/hindsight-api-slim/tests/test_shutdown_grace_config.py b/hindsight-api-slim/tests/test_shutdown_grace_config.py new file mode 100644 index 0000000000..0b04eaacae --- /dev/null +++ b/hindsight-api-slim/tests/test_shutdown_grace_config.py @@ -0,0 +1,48 @@ +"""HINDSIGHT_API_SHUTDOWN_GRACE config wiring. + +The worker poller's graceful-shutdown timeout was hardcoded to 30s. An +in-flight retain is an LLM call that may legitimately run for minutes +(HINDSIGHT_API_LLM_TIMEOUT defaults far above 30s); cancelling it mid-flight +on every service stop loses the operation. Deployments size this together +with their supervisor stop timeout (e.g. systemd TimeoutStopSec). +""" + +import os + +import pytest + + +@pytest.fixture(autouse=True) +def setup_test_env(): + from hindsight_api.config import clear_config_cache + + keys = ["HINDSIGHT_API_LLM_PROVIDER", "HINDSIGHT_API_SHUTDOWN_GRACE"] + original = {k: os.environ.get(k) for k in keys} + clear_config_cache() + yield + for k, v in original.items(): + if v is None: + os.environ.pop(k, None) + else: + os.environ[k] = v + clear_config_cache() + + +def test_default_shutdown_grace_is_30s(): + from hindsight_api.config import HindsightConfig + + os.environ["HINDSIGHT_API_LLM_PROVIDER"] = "mock" + os.environ.pop("HINDSIGHT_API_SHUTDOWN_GRACE", None) + + config = HindsightConfig.from_env() + assert config.shutdown_grace == 30.0 + + +def test_shutdown_grace_env_var_is_read(): + from hindsight_api.config import HindsightConfig + + os.environ["HINDSIGHT_API_LLM_PROVIDER"] = "mock" + os.environ["HINDSIGHT_API_SHUTDOWN_GRACE"] = "240" + + config = HindsightConfig.from_env() + assert config.shutdown_grace == 240.0