feat(worker): make the graceful-shutdown timeout configurable

.env.example

@@ -292,3 +292,4 @@ HINDSIGHT_API_LOG_LEVEL=info
 # When set, visitors see a login page and must enter the key before
 # accessing the dashboard or any /api/* routes (except /api/health).
 # HINDSIGHT_CP_ACCESS_KEY=your-shared-secret-key
+# HINDSIGHT_API_SHUTDOWN_GRACE=30  # Seconds the worker poller waits for in-flight operations on shutdown; size together with your supervisor stop timeout (e.g. systemd TimeoutStopSec).

hindsight-api-slim/hindsight_api/api/http.py

@@ -3596,7 +3596,12 @@ async def lifespan(app: FastAPI):
 
         # Shutdown worker poller if running
         if poller is not None:
-            await poller.shutdown_graceful(timeout=30.0)
+            # Grace configurable (default 30s) so an in-flight retain — an LLM call
+            # that may legitimately run for minutes — is not cancelled mid-flight
+            # on service stop. Deployments size HINDSIGHT_API_SHUTDOWN_GRACE
+            # together with their supervisor stop timeout (e.g. systemd
+            # TimeoutStopSec) so the supervisor never SIGKILLs mid-cleanup.
+            await poller.shutdown_graceful(timeout=get_config().shutdown_grace)
             if poller_task is not None:
                 poller_task.cancel()
                 try:

hindsight-api-slim/hindsight_api/config.py

@@ -154,6 +154,7 @@ def normalize_config_dict(config: dict[str, Any]) -> dict[str, Any]:
 ENV_LLM_INITIAL_BACKOFF = "HINDSIGHT_API_LLM_INITIAL_BACKOFF"
 ENV_LLM_MAX_BACKOFF = "HINDSIGHT_API_LLM_MAX_BACKOFF"
 ENV_LLM_TIMEOUT = "HINDSIGHT_API_LLM_TIMEOUT"
+ENV_SHUTDOWN_GRACE = "HINDSIGHT_API_SHUTDOWN_GRACE"
 ENV_LLM_REASONING_EFFORT = "HINDSIGHT_API_LLM_REASONING_EFFORT"
 ENV_LLM_GROQ_SERVICE_TIER = "HINDSIGHT_API_LLM_GROQ_SERVICE_TIER"
 ENV_LLM_OPENAI_SERVICE_TIER = "HINDSIGHT_API_LLM_OPENAI_SERVICE_TIER"
@@ -863,6 +864,8 @@ def _parse_worker_slot_reservations() -> dict[str, int]:
 DEFAULT_LLM_INITIAL_BACKOFF = 1.0  # Initial backoff in seconds for retry exponential backoff
 DEFAULT_LLM_MAX_BACKOFF = 60.0  # Max backoff cap in seconds for retry exponential backoff
 DEFAULT_LLM_TIMEOUT = 120.0  # seconds
+# Seconds the worker poller waits for in-flight operations on shutdown.
+DEFAULT_SHUTDOWN_GRACE = 30.0
 DEFAULT_LLM_REASONING_EFFORT = "low"
 DEFAULT_LLM_SEND_BANK_AS_USER = False  # Opt-in: tag provider calls with user=<bank_id>
 
@@ -2221,6 +2224,8 @@ class HindsightConfig:
     # Defaulted fields (source-compatible additions — existing direct constructor callers keep working).
     # Keep at the end of the dataclass; Python forbids non-default fields after default fields.
     embeddings_openai_batch_size: int = DEFAULT_EMBEDDINGS_OPENAI_BATCH_SIZE
+    # Seconds the worker poller waits for in-flight operations on shutdown.
+    shutdown_grace: float = DEFAULT_SHUTDOWN_GRACE
     embeddings_openai_dimensions: int | None = None
     embeddings_zeroentropy_api_key: str | None = None
     embeddings_zeroentropy_model: str = DEFAULT_EMBEDDINGS_ZEROENTROPY_MODEL
@@ -2601,6 +2606,7 @@ def from_env(cls) -> "HindsightConfig":
             llm_initial_backoff=float(os.getenv(ENV_LLM_INITIAL_BACKOFF, str(DEFAULT_LLM_INITIAL_BACKOFF))),
             llm_max_backoff=float(os.getenv(ENV_LLM_MAX_BACKOFF, str(DEFAULT_LLM_MAX_BACKOFF))),
             llm_timeout=float(os.getenv(ENV_LLM_TIMEOUT, str(DEFAULT_LLM_TIMEOUT))),
+            shutdown_grace=float(os.getenv(ENV_SHUTDOWN_GRACE, str(DEFAULT_SHUTDOWN_GRACE))),
             llm_reasoning_effort=os.getenv(ENV_LLM_REASONING_EFFORT, DEFAULT_LLM_REASONING_EFFORT),
             llm_groq_service_tier=os.getenv(ENV_LLM_GROQ_SERVICE_TIER, DEFAULT_LLM_GROQ_SERVICE_TIER),
             llm_openai_service_tier=os.getenv(ENV_LLM_OPENAI_SERVICE_TIER, DEFAULT_LLM_OPENAI_SERVICE_TIER),

hindsight-api-slim/hindsight_api/worker/main.py

@@ -349,7 +349,12 @@ def signal_handler():
         server.should_exit = True
 
         print("Waiting for poller to finish...")
-        await poller.shutdown_graceful(timeout=30.0)
+        # Grace configurable (default 30s) so an in-flight retain — an LLM call
+        # that may legitimately run for minutes — is not cancelled mid-flight
+        # on service stop. Deployments size HINDSIGHT_API_SHUTDOWN_GRACE
+        # together with their supervisor stop timeout (e.g. systemd
+        # TimeoutStopSec) so the supervisor never SIGKILLs mid-cleanup.
+        await poller.shutdown_graceful(timeout=get_config().shutdown_grace)
         poller_task.cancel()
         try:
             await poller_task

hindsight-api-slim/tests/test_shutdown_grace_config.py

@@ -0,0 +1,48 @@
+"""HINDSIGHT_API_SHUTDOWN_GRACE config wiring.
+
+The worker poller's graceful-shutdown timeout was hardcoded to 30s. An
+in-flight retain is an LLM call that may legitimately run for minutes
+(HINDSIGHT_API_LLM_TIMEOUT defaults far above 30s); cancelling it mid-flight
+on every service stop loses the operation. Deployments size this together
+with their supervisor stop timeout (e.g. systemd TimeoutStopSec).
+"""
+
+import os
+
+import pytest
+
+
+@pytest.fixture(autouse=True)
+def setup_test_env():
+    from hindsight_api.config import clear_config_cache
+
+    keys = ["HINDSIGHT_API_LLM_PROVIDER", "HINDSIGHT_API_SHUTDOWN_GRACE"]
+    original = {k: os.environ.get(k) for k in keys}
+    clear_config_cache()
+    yield
+    for k, v in original.items():
+        if v is None:
+            os.environ.pop(k, None)
+        else:
+            os.environ[k] = v
+    clear_config_cache()
+
+
+def test_default_shutdown_grace_is_30s():
+    from hindsight_api.config import HindsightConfig
+
+    os.environ["HINDSIGHT_API_LLM_PROVIDER"] = "mock"
+    os.environ.pop("HINDSIGHT_API_SHUTDOWN_GRACE", None)
+
+    config = HindsightConfig.from_env()
+    assert config.shutdown_grace == 30.0
+
+
+def test_shutdown_grace_env_var_is_read():
+    from hindsight_api.config import HindsightConfig
+
+    os.environ["HINDSIGHT_API_LLM_PROVIDER"] = "mock"
+    os.environ["HINDSIGHT_API_SHUTDOWN_GRACE"] = "240"
+
+    config = HindsightConfig.from_env()
+    assert config.shutdown_grace == 240.0