docker-compose.monitoring.yml

# Observability stack: Prometheus (metrics) + Loki/Alloy (logs) + Grafana.
# Opt-in, layered onto whichever app stack you're running — it redeclares
# `web-net` (identical to docker-compose.yml/docker-compose.prod.yml) so
# Compose merges it into the same project rather than requiring a separate
# `external: true` network reference.
#
# Dev stack:
#   docker compose -f docker-compose.yml -f docker-compose.monitoring.yml up -d --build
# Prod stack:
#   docker compose -f docker-compose.prod.yml -f docker-compose.postgres.yml -f docker-compose.caddy.yml -f docker-compose.monitoring.yml up -d --build
#
# It ships every container's stdout/stderr (Caddy's JSON access logs,
# Postgres/Redis/backend console output, ...) into Loki via Alloy's Docker log
# discovery. Prometheus scrapes cAdvisor (per-container CPU/mem) and
# node-exporter (host CPU/mem/disk), which work with zero app
# instrumentation, plus a `backend` job in monitoring/prometheus/prometheus.yml
# that scrapes the Rust backend's own /metrics (HTTP request rate/latency/
# status via axum-prometheus). Note that job hardcodes port 5000 (the prod
# PORT) — it won't resolve against the dev stack's backend, which listens on
# 3000.
#
# Only Grafana is reachable from outside, through Caddy at
# /logging-grafana-main (see Caddyfile/Caddyfile.prod). Prometheus has no
# login of its own, so it's internal-only: no host port, no Caddy route —
# reachable solely as http://prometheus:9090 from other containers on
# monitoring-net (that's exactly how Grafana's own datasource reaches it).
services:

  # Internal-only: no host port, no Caddy route, and no longer on web-net.
  # Docker bridge networks are bidirectional, so sharing web-net with the three
  # internet-facing frontends meant any one of them could reach this
  # completely unauthenticated instance. Caddy joins monitoring-net instead so
  # its metrics listener stays scrapeable.
  prometheus:
    image: prom/prometheus:v3.12.0
    container_name: attendance-prometheus
    restart: unless-stopped
    security_opt:
      - no-new-privileges:true
    cap_drop:
      - ALL
    deploy:
      resources:
        limits:
          cpus: '1'
          memory: 1G
    volumes:
      - ./monitoring/prometheus/prometheus.yml:/etc/prometheus/prometheus.yml:ro
      - prometheus_data:/prometheus
    expose:
      - "9090"
    depends_on:
      - cadvisor
      - node-exporter
    networks:
      - monitoring-net

  # ponytail: on Docker Desktop (Mac/Windows) cAdvisor's container metrics
  # are limited/best-effort — it runs inside the Desktop VM, not directly on
  # the host. Fully accurate on a real Linux Docker host.
  cadvisor:
    image: ghcr.io/google/cadvisor:v0.60.5
    container_name: attendance-cadvisor
    restart: unless-stopped
    # `privileged: true` combined with a writable host /var/run — which
    # contains docker.sock — made any RCE in cAdvisor, or anything able to
    # reach cadvisor:8080 on monitoring-net, a path to host root. cAdvisor
    # needs read access to these paths, not write, and not full privilege.
    privileged: false
    security_opt:
      - no-new-privileges:true
    cap_drop:
      - ALL
    cap_add:
      # Required to stat cgroup and process paths owned by other users.
      - DAC_READ_SEARCH
    devices:
      - /dev/kmsg
    deploy:
      resources:
        limits:
          cpus: '0.5'
          memory: 256M
    volumes:
      - /:/rootfs:ro
      - /var/run:/var/run:ro
      - /sys:/sys:ro
    expose:
      - "8080"
    networks:
      - monitoring-net

  # ponytail: same Docker Desktop caveat as cAdvisor — reports the Desktop
  # VM's host stats, not literally your Mac's, on non-Linux dev machines.
  node-exporter:
    image: prom/node-exporter:v1.12.0
    container_name: attendance-node-exporter
    restart: unless-stopped
    pid: host
    # Every mount below is read-only and node-exporter needs no special
    # capability to read them — matches the cap_drop/security_opt pattern
    # applied to every other service in this stack.
    security_opt:
      - no-new-privileges:true
    cap_drop:
      - ALL
    deploy:
      resources:
        limits:
          cpus: '0.5'
          memory: 256M
    volumes:
      - /proc:/host/proc:ro
      - /sys:/host/sys:ro
      - /:/rootfs:ro
    command:
      - '--path.procfs=/host/proc'
      - '--path.sysfs=/host/sys'
      - '--path.rootfs=/rootfs'
      - '--collector.filesystem.mount-points-exclude=^/(sys|proc|dev|host|etc)($$|/)'
    expose:
      - "9100"
    networks:
      - monitoring-net

  loki:
    image: grafana/loki:3.7.0
    container_name: attendance-loki
    restart: unless-stopped
    security_opt:
      - no-new-privileges:true
    cap_drop:
      - ALL
    deploy:
      resources:
        limits:
          cpus: '1'
          memory: 512M
    command:
      - -config.file=/etc/loki/loki-config.yml
      - -config.expand-env=true
    environment:
      # AWS credentials for S3 backend storage. Loki's AWS SDK reads these
      # standard env vars automatically — no credentials in the config file.
      - AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID}
      - AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY}
      - AWS_DEFAULT_REGION=${AWS_REGION}
      - AWS_S3_BUCKET=${AWS_S3_BUCKET}
    volumes:
      - ./monitoring/loki/loki-config.yml:/etc/loki/loki-config.yml:ro
      - loki_data:/loki
    expose:
      - "3100"
    networks:
      - monitoring-net

  alloy:
    image: grafana/alloy:v1.17.0
    container_name: attendance-alloy
    restart: unless-stopped
    # Docker socket access (even read-only) lets this container enumerate
    # every container/image on the host — a meaningful privilege even
    # without write access, so it gets the same cap_drop/no-new-privileges
    # treatment as everything else rather than being left unhardened because
    # the mount itself is already read-only.
    security_opt:
      - no-new-privileges:true
    cap_drop:
      - ALL
    deploy:
      resources:
        limits:
          cpus: '0.5'
          memory: 512M
    command:
      - run
      - --server.http.listen-addr=0.0.0.0:12345
      - --storage.path=/var/lib/alloy/data
      - /etc/alloy/config.alloy
    volumes:
      - ./monitoring/alloy/config.alloy:/etc/alloy/config.alloy:ro
      - /var/run/docker.sock:/var/run/docker.sock:ro
      - alloy_data:/var/lib/alloy/data
    depends_on:
      - loki
    networks:
      - monitoring-net

  grafana:
    image: grafana/grafana:13.1.0
    container_name: attendance-grafana
    restart: unless-stopped
    # Reached only through Caddy at /logging-grafana-main — no host port
    # published. serve_from_sub_path=true + a matching root_url is required
    # (Grafana's own router expects the full prefixed path — Caddy passes
    # it through unstripped) or Grafana's emitted links/assets break.
    security_opt:
      - no-new-privileges:true
    cap_drop:
      - ALL
    deploy:
      resources:
        limits:
          cpus: '1'
          memory: 512M
    environment:
      - GF_SECURITY_ADMIN_USER=${GRAFANA_ADMIN_USER:-admin}
      # No weak fallback: this is reachable on the public internet in prod
      # (see Caddyfile.prod), protected only by this login.
      - GF_SECURITY_ADMIN_PASSWORD=${GRAFANA_ADMIN_PASSWORD:?GRAFANA_ADMIN_PASSWORD must be set in .env — run scripts/generate-secrets.sh}
      - GF_USERS_ALLOW_SIGN_UP=false
      - GF_SERVER_ROOT_URL=${GRAFANA_ROOT_URL:-http://localhost/logging-grafana-main/}
      - GF_SERVER_SERVE_FROM_SUB_PATH=true
    volumes:
      - grafana_data:/var/lib/grafana
      - ./monitoring/grafana/provisioning:/etc/grafana/provisioning:ro
      - ./monitoring/grafana/provisioning/dashboards:/etc/grafana/provisioning/dashboards:ro
    expose:
      - "3000"
    depends_on:
      - prometheus
      - loki
    networks:
      - monitoring-net
      - web-net

volumes:
  prometheus_data:
  loki_data:
  alloy_data:
  grafana_data:

networks:
  monitoring-net:
    driver: bridge
  web-net:
    driver: bridge