docker-compose.strfry.yml

# INCIDENT RATIONALE (2026-07-05 router split): the ingestion `strfry router` process used to
# run INSIDE the shared `strfry` container alongside the relay via dockur's strfry.sh entrypoint.
# On 2026-06-28 the router was SIGKILLed by the OOM killer, but strfry.sh only `wait`s on the
# relay pid, so the router's death went unnoticed and unrestarted — and the container stayed
# (healthy) because its healthcheck only probes the relay port (7777). Result: ~5h of silent
# zero-ingestion with an orphaned bloom plugin. The router is now isolated in its own
# `relay-fan-in` container (below) with restart:unless-stopped (auto-restart on ExitCode 137),
# an explicit mem_limit (deterministic cgroup OOM attribution), and a process-liveness
# healthcheck (unhealthy when ingestion is actually broken, not just when the relay port is up).
services:
  strfry:
    build:
      context: .
      dockerfile: Dockerfile.strfry
    # Explicit image name so relay-fan-in can reuse this exact tag deterministically, independent
    # of the compose project name (which is derived from the deploy dir and differs per host).
    image: deepfry-strfry
    container_name: strfry
    restart: unless-stopped
    # OOM protection (debug adapter-crash-heavy-query, 2026-06-28). On this 8 GiB host the
    # Docker VM (3.83 GiB) suffers VM-wide (global) OOM under heavy adapter load; the kernel
    # repeatedly killed strfry (3× on 2026-06-28) because it is the fattest unprotected
    # process (large LMDB mmap → ~126 MiB page tables). strfry holds the canonical event data
    # and must be the LAST sacrifice. -300 strongly deprioritises it as an OOM victim while
    # staying ABOVE dockerd (-500)/init (-999) so Docker's own infrastructure is never at
    # risk. Paired with oom_score_adj: 1000 on lmdb2graphql (the disposable read-lens), which
    # the kernel now kills first. NB: this steers WHICH process dies, not the underlying
    # capacity shortfall — the durable fix is more RAM / DB on local SSD.
    oom_score_adj: -300
    environment:
      STRFRY_PRIVATE_KEY: ${STRFRY_PRIVATE_KEY:-}
      # RELAY-ONLY: the `ROUTER` env var was intentionally REMOVED (2026-07-05 split).
      # dockur's strfry.sh only launches `./strfry router "$ROUTER"` when ROUTER is set;
      # with it unset this container runs ONLY `./strfry relay`. Ingestion now lives in the
      # dedicated `relay-fan-in` service below. Env vars are baked at container-create time,
      # so this container MUST be recreated (not just restarted) to stop launching the router
      # — see the Deployment & verification runbook in README.md.
    volumes:
      - ${STRFRY_DB_PATH:-./data/strfry-db}:/app/strfry-db
      - ./config/strfry/strfry.conf:/etc/strfry.conf:ro
      # NOTE: the router-only mounts (strfry-relay-fan-in.config + router.yaml) were moved to the
      # dedicated `relay-fan-in` service — the relay neither reads nor needs them.
      - ./config/whitelist/whitelist.yaml:/root/deepfry/whitelist.yaml
      # Bloom filter persistence (GATE-05): writable directory for the bloom plugin's
      # persisted filter so it survives container restarts. Override host path via
      # BLOOM_DATA_PATH env var. When using the bloom plugin, set bloom_path in
      # whitelist.yaml to a file under /root/deepfry/bloom-data/ (e.g.
      # bloom_path: /root/deepfry/bloom-data/bloom.dfbf).
      - ${BLOOM_DATA_PATH:-./data/bloom}:/root/deepfry/bloom-data
    ports:
      - "7777:7777" # Nostr WebSocket
    networks:
      - deepfry-net

  # Dedicated ingestion fan-in (2026-07-05 split — see INCIDENT RATIONALE at top of file).
  # Runs ONLY `./strfry router` — pulls events from ~174 upstream relays into the SAME
  # canonical LMDB the relay serves. Same stock dockur/strfry image (NOT forked); the bloom
  # plugin is baked in at /app/plugins/bloom.
  relay-fan-in:
    # Reuse the relay's already-built image — the router is the SAME strfry binary/image, differing
    # only by `command:`. Building a byte-identical second image (deepfry-relay-fan-in) is wasteful
    # and, worse, forces a redundant build at deploy time (which fails on hosts where the buildx
    # credential helper is unavailable, e.g. over SSH). Pin to the image the `strfry` service builds.
    image: deepfry-strfry
    depends_on:
      - strfry
    container_name: relay-fan-in
    # SPLIT-01: an OOM SIGKILL (ExitCode 137, OOMKilled=true) lands on THIS container and is
    # auto-restarted by Docker — closing the "silent unrestarted router death" gap.
    restart: unless-stopped
    # SPLIT-02: explicit memory ceiling so a fan-in runaway triggers a deterministic cgroup OOM
    # attributed to the fan-in, not VM-wide killer roulette against the canonical relay. Measured
    # 2026-07-06: the steady-state working set for ~174 upstream WS connections + per-conn buffers
    # + the LMDB mmap page tables sits at ~490 MiB, so the original 512 MiB left no headroom and the
    # container was OOM-killed (exit 137) → auto-restarted every ~2h (RestartCount climbed to 11/23h).
    # Raised to 1 GiB (~2× steady state) to stop the restart churn while still bounding a runaway and
    # keeping OOM attributed to the fan-in. NOTE (2026-07-08): slow creep past the limit is NOT a
    # router leak — it's page-table growth from the LMDB mmap (~8B of non-reclaimable PTE per touched
    # 4KiB page; ~620 MiB at full coverage of a 318 GB DB). Budget mem_limit ≈ anon steady state
    # (~500 MiB) + PTE full coverage + slack. Raise STRFRY_RELAY_FAN_IN_MEM_LIMIT to tune per host
    # (strfry host runs 2g since 2026-07-08; 1g OOM-cycled every ~18 h).
    mem_limit: ${STRFRY_RELAY_FAN_IN_MEM_LIMIT:-1g}
    # Do NOT set oom_score_adj here. The relay (-300) holds canonical data and must be the LAST
    # OOM sacrifice; the router is restartable/disposable, so its default score (0) makes it the
    # more acceptable victim. Its own mem_limit gives it a deterministic local ceiling regardless.
    working_dir: /app
    # Explicit command runs ONLY the router. We deliberately do NOT reuse strfry.sh here: that
    # entrypoint always launches `./strfry relay` first and `wait`s on it, which would start a
    # second relay binding :7777 in the same network — a conflict. Same override pattern the
    # quarantine service uses. Absolute /app/strfry because the binary is not on PATH.
    command: ["/app/strfry", "--config=/etc/strfry.conf", "router", "/app/strfry-relay-fan-in.config"]
    volumes:
      # IDENTICAL db + strfry.conf as the relay — required for safe multi-process LMDB. The
      # router WRITES down-events into the SAME LMDB the relay serves; if the two containers
      # disagree on the db dir or on dbParams.maxreaders=4096 (carried by strfry.conf, the prior
      # MDB_READERS_FULL fix) the reader table corrupts (MDB_BAD_RSLOT). Host bind-mount, never a
      # named volume (CLAUDE.md hard rule).
      - ${STRFRY_DB_PATH:-./data/strfry-db}:/app/strfry-db
      - ./config/strfry/strfry.conf:/etc/strfry.conf:ro
      # Router streams-only config (generated host-side by gen-relay-fan-in-config.sh, not committed).
      # :ro because strfry only reads it. Carries streams{} + pluginDown=/app/plugins/bloom.
      - ${STRFRY_RELAY_FAN_IN_CONFIG_PATH:-~/deepfry/strfry-relay-fan-in.config}:/app/strfry-relay-fan-in.config:ro
      # Router-owned plugin config, moved here from the relay service.
      - ./config/whitelist/router.yaml:/root/deepfry/router.yaml
      # The bloom plugin (pluginDown gate) reads bloom_path from whitelist.yaml.
      - ./config/whitelist/whitelist.yaml:/root/deepfry/whitelist.yaml
      # Writable bloom persistence — MANDATORY for pluginDown gating (T-260705-01). The router's
      # down-stream events are gated ONLY by pluginDown=/app/plugins/bloom (the global writePolicy
      # does not gate router-down events), so this dir must be mounted exactly as on the relay.
      - ${BLOOM_DATA_PATH:-./data/bloom}:/root/deepfry/bloom-data
      # SPLIT-03 ingestion-liveness healthcheck script (:ro, mirrors quarantine-db-guard.sh).
      - ./config/strfry/relay-fan-in-liveness.sh:/usr/local/bin/relay-fan-in-liveness.sh:ro
    # SPLIT-03: ingestion-liveness, NOT a port probe. The router exposes no server port; this
    # asserts the `strfry router` process is alive. Dead router → unhealthy → restart:unless-
    # stopped auto-recovery. (Freshness probes rejected: false-unhealthy during quiet bloom-gate
    # windows where zero events are legitimately written.)
    healthcheck:
      test: ["CMD-SHELL", "sh /usr/local/bin/relay-fan-in-liveness.sh"]
      interval: 1m
      timeout: 5s
      retries: 2
    # No `ports:` — the router is a WS client to upstreams + a local LMDB writer, no server port.
    networks:
      - deepfry-net

  # Internal quarantine relay for non-whitelisted events. See quarantine/SPEC.md.
  # Shares the Dockerfile with mainline strfry (both plugins ship in the image).
  # The db path MUST differ from mainline — enforced by quarantine-db-guard.sh.
  strfry-quarantine:
    build:
      context: .
      dockerfile: Dockerfile.strfry
    container_name: strfry-quarantine
    restart: unless-stopped
    entrypoint: ["/usr/local/bin/quarantine-db-guard.sh"]
    # Absolute path: the dockurr/strfry image keeps the binary at /app/strfry
    # and isn't in PATH, so the guard's `exec "$@"` would otherwise fail.
    command: ["/app/strfry", "--config=/etc/strfry.conf", "relay"]
    # Override the dockurr/strfry base image's baked-in HEALTHCHECK, which probes
    # http://localhost:7777/. This relay runs on 7778 (strfry-quarantine.conf), so the
    # inherited 7777 check always fails ("connection refused") and the container shows
    # (unhealthy) even though the relay is fine. Point the check at 7778.
    healthcheck:
      test: ["CMD-SHELL", "curl -ILfSs http://localhost:7778/ > /dev/null || exit 1"]
      interval: 1m
      timeout: 5s
      retries: 2
    working_dir: /app
    environment:
      QUARANTINE_EXPECTED_DB: "/app/strfry-quarantine-db/"
      MAINLINE_DB_PATH: "/app/strfry-db/"
    volumes:
      - ${STRFRY_QUARANTINE_DB_PATH:-./data/strfry-quarantine-db}:/app/strfry-quarantine-db
      - ./config/strfry/strfry-quarantine.conf:/etc/strfry.conf:ro
      - ./config/strfry/quarantine-db-guard.sh:/usr/local/bin/quarantine-db-guard.sh:ro
      # NOTE: the mainline ./data/strfry-db volume is intentionally NOT mounted here.
      # Adding it would be a destructive misconfiguration; the guard (exit 4) catches it.
    ports:
      - "7778:7778" # Nostr WebSocket (quarantine)
    networks:
      - deepfry-net

networks:
  deepfry-net:
    external: true
    # Network name is per-host: defaults to the shared deepfry-net, override via
    # DEEPFRY_NET_NAME in .env (e.g. strfry-net) on a split deploy. The named
    # network must already exist (`docker network create <name>`).
    name: ${DEEPFRY_NET_NAME:-deepfry-net}