fix: alert on stalled rollout-checker CronJobs

cluster-objects/prometheusrule.yaml

@@ -1,15 +1,23 @@
-# Prometheus Operator alerting rules for the docs site's existing
-# `/api/metrics` endpoint (src/app/api/metrics/route.ts, src/lib/metrics.ts),
-# already scraped by cluster-objects/servicemonitor.yaml.
+# Prometheus Operator alerting rules for:
+#   1. the docs site's own `/api/metrics` endpoint
+#      (src/app/api/metrics/route.ts, src/lib/metrics.ts), already scraped
+#      by cluster-objects/servicemonitor.yaml; and
+#   2. the automated rollout-checker CronJobs (cluster-objects/job.yaml,
+#      cluster-objects/pr-job.yaml) that deploy new images, via
+#      kube-state-metrics' kube_cronjob_status_last_successful_time.
 #
 # This resource only takes effect if a Prometheus Operator with the
-# PrometheusRule CRD is already watching this namespace/selector, and only
-# if servicemonitor.yaml is applied so the referenced metrics are actually
-# being scraped. It does not install an operator, does not configure any
+# PrometheusRule CRD is already watching this namespace/selector — group 1
+# additionally needs servicemonitor.yaml applied so those metrics are
+# actually scraped, and group 2 additionally needs kube-state-metrics
+# running in-cluster (a near-universal companion to a Prometheus Operator
+# install). This file does not install an operator, does not configure any
 # exporter, and does not send data off-box — it only defines evaluation
-# rules over metrics already produced in-process by this application.
+# rules over metrics already produced by this application or by
+# kube-state-metrics, if present.
 #
-# Both alerts use metric/label names defined in src/lib/metrics.ts:
+# The first three alerts use metric/label names defined in
+# src/lib/metrics.ts:
 #   docs_api_requests_total{route, method, status_class}
 #   docs_api_request_duration_seconds{route, method}
 # `route` and `status_class` are fixed, bounded label sets (see the
@@ -74,3 +82,43 @@ spec:
               instrumented API routes has exceeded 1s over the last 5
               minutes (current value: {{ $value | humanizeDuration }}).
             runbook_url: "https://github.com/kubestellar/docs/blob/main/runbooks/api-error-rate-latency.md"
+        # The two alerts below cover the automated deploy pipeline itself
+        # (cluster-objects/job.yaml, cluster-objects/pr-job.yaml), not the
+        # application. kube_cronjob_status_last_successful_time is a
+        # kube-state-metrics metric (a near-universal companion to a
+        # Prometheus Operator install) — this does not add a new exporter,
+        # it only evaluates a metric that kube-state-metrics already
+        # exposes if it is present. Both CronJobs are concurrencyPolicy:
+        # Forbid with failedJobsHistoryLimit: 1, so a stalled or failing
+        # rollout-checker would otherwise go undetected: production/preview
+        # Deployments keep serving a stale image with no signal until
+        # someone happens to check `kubectl get cronjob -n docs`.
+        - alert: DocsRolloutCheckerStalled
+          expr: >-
+            time() - kube_cronjob_status_last_successful_time{namespace="docs", cronjob="nextra-rollout-checker"} > 900
+          for: 5m
+          labels:
+            severity: warning
+          annotations:
+            summary: "Production rollout-checker CronJob has not succeeded in 15m"
+            description: >-
+              `nextra-rollout-checker` (cluster-objects/job.yaml) runs every
+              2 minutes and auto-rolls out new `kubestellar-docs` images
+              from OCIR. It has not completed successfully in over 15
+              minutes, so new production image builds are not being
+              deployed.
+            runbook_url: "https://github.com/kubestellar/docs/blob/main/runbooks/rollout-checker-failure.md"
+        - alert: DocsPrRolloutCheckerStalled
+          expr: >-
+            time() - kube_cronjob_status_last_successful_time{namespace="docs", cronjob="nextra-pr-rollout-checker"} > 600
+          for: 5m
+          labels:
+            severity: warning
+          annotations:
+            summary: "PR-preview rollout-checker CronJob has not succeeded in 10m"
+            description: >-
+              `nextra-pr-rollout-checker` (cluster-objects/pr-job.yaml) runs
+              every 1 minute and reconciles PR preview deployments from
+              OCIR. It has not completed successfully in over 10 minutes,
+              so PR preview deploys/teardowns are not being reconciled.
+            runbook_url: "https://github.com/kubestellar/docs/blob/main/runbooks/rollout-checker-failure.md"

runbooks/README.md

@@ -9,6 +9,7 @@ looking at.
 | [`slo.md`](slo.md) | Establishing the readiness SLO/SLI for the production docs site and how the `healthz-monitor` alerting works. | Start here for any production readiness question. |
 | [`deploy-rollback.md`](deploy-rollback.md) | The live site is serving broken/stale content or a bad deploy went out (Netlify or container path) and needs to be rolled back. | `slo.md` |
 | [`api-error-rate-latency.md`](api-error-rate-latency.md) | The `DocsApiHighErrorRate` or `DocsApiHighRequestLatency` alert fires. | `cluster-objects/prometheusrule.yaml` `runbook_url` annotations |
+| [`rollout-checker-failure.md`](rollout-checker-failure.md) | The `DocsRolloutCheckerStalled` or `DocsPrRolloutCheckerStalled` alert fires — the `nextra-rollout-checker`/`nextra-pr-rollout-checker` CronJob (`cluster-objects/job.yaml`/`pr-job.yaml`) has stopped completing successfully, so new image rollouts are not happening. | `cluster-objects/prometheusrule.yaml` `runbook_url` annotations |
 | [`netlify-build-failure.md`](netlify-build-failure.md) | A Netlify build fails: a `[netlify-deploy-failure]` issue opens for a direct-to-main commit, or `netlify-error-reporter.yml` comments on a PR (the alert links this runbook and auto-closes on the next successful deploy, added in [#6783](https://github.com/kubestellar/docs/issues/6783)). | `netlify-error-reporter.yml` failure alert |
 | [`version-branch-rollback.md`](version-branch-rollback.md) | The version picker shows a wrong/missing version, `versions.ts`/`shared.json` changed unexpectedly, or `create-version-branch.yml`/`sync-console-release-versions.yml` needs investigating. | — |
 | [`scorecard-monitoring.md`](scorecard-monitoring.md) | The weekly OpenSSF Scorecard scan (`scorecard.yml`) fails — an automated `[scorecard-failure]` alert opens/comments on a `ci-failure` issue (added in [#6918](https://github.com/kubestellar/docs/pull/6918)); check for the known recurring `gcr.io` pull-failure root cause first. | `ci-failure`-labeled issues titled `[scorecard-failure] ...` |

runbooks/rollout-checker-failure.md

@@ -0,0 +1,84 @@
+# Runbook: Rollout-Checker CronJob Stalled
+
+## Scope
+
+Applies to the `DocsRolloutCheckerStalled` and `DocsPrRolloutCheckerStalled`
+alerts defined in `cluster-objects/prometheusrule.yaml`. Both fire from
+kube-state-metrics' `kube_cronjob_status_last_successful_time`, so they
+require a Prometheus Operator and kube-state-metrics already running in the
+cluster (no exporter is installed by this repo) — if neither alert has ever
+fired and you don't know whether that monitoring exists, the CronJobs
+themselves are still the thing to check by hand below.
+
+These two CronJobs (`cluster-objects/job.yaml`,
+`cluster-objects/pr-job.yaml`) are the only mechanism that deploys new
+`kubestellar-docs` images:
+
+- **`nextra-rollout-checker`** (schedule `*/2 * * * *`, namespace `docs`) —
+  polls OCIR for the latest `kubestellar-docs` image digest and
+  `kubectl set image`s the production Deployment
+  (`cluster-objects/deployment.yaml`) when a newer digest appears. A run
+  that finds no newer image still exits `0` ("skipping rollout"), so a
+  healthy CronJob does **not** mean a new image went out — only that the
+  checker itself is working.
+- **`nextra-pr-rollout-checker`** (schedule `*/1 * * * *`, namespace `docs`)
+  — the same mechanism for per-PR preview Deployments.
+
+Both scripts run with `set -e` (any failed step — missing env vars,
+`kubectl`/`oci` auth, a malformed OCI API response — exits the job
+non-zero), both CronJobs are `concurrencyPolicy: Forbid` with
+`backoffLimit: 0`, and `failedJobsHistoryLimit: 1` keeps at most one failed
+Job around. There is no retry beyond the next scheduled tick.
+
+- **`DocsRolloutCheckerStalled`** — fires when the production checker has
+  not completed a successful run in over 15 minutes (more than 7 missed
+  2-minute ticks).
+- **`DocsPrRolloutCheckerStalled`** — fires when the PR-preview checker has
+  not completed a successful run in over 10 minutes (more than 9 missed
+  1-minute ticks).
+
+## Diagnosis
+
+1. Confirm the CronJob is actually failing, not just slow to be scraped:
+   ```sh
+   kubectl get cronjob -n docs nextra-rollout-checker nextra-pr-rollout-checker
+   kubectl get jobs -n docs -l 'job-name' --sort-by=.metadata.creationTimestamp
+   ```
+   Look at `LAST SCHEDULE` and whether the most recent Job succeeded.
+2. Read the failing Job's logs — the script's `[DEBUG]`/`[ERROR]` lines
+   (`cluster-objects/job.yaml` / `cluster-objects/pr-job.yaml`
+   ConfigMap data) point at the failing step directly:
+   ```sh
+   kubectl logs -n docs job/<failing-job-name>
+   ```
+3. Common root causes, roughly in likelihood order:
+   - **Expired/rotated `oci-config-secret`** — `oci artifacts container
+     image list` fails auth. Re-provision the secret
+     (`cluster-objects/rbac.yaml` mounts it at `/home/appuser/.oci`).
+   - **RBAC regression** on `nextra-rollout-sa`
+     (`cluster-objects/rbac.yaml`) — `kubectl set image` / `kubectl
+     annotate` / `kubectl get deploy` denied. Diff the live Role against
+     the manifest.
+   - **OCI API outage/throttling** — transient; confirm via the OCI status
+     page, no action needed beyond waiting if the next scheduled run
+     recovers on its own.
+   - **Missing `COMPARTMENT_OCID`/`PROD_REPO_OCID`/`REPO_OCID`** env vars
+     after a Secret/ConfigMap edit — the script's `:?` guards fail fast
+     with an explicit `not set` error naming the variable.
+4. If the checker itself is healthy but a specific rollout is stuck (e.g.
+   repeated `kubectl set image` with no corresponding pod update), see
+   `runbooks/deploy-rollback.md` instead — this runbook is about the
+   *checker* not running, not about a bad image once rolled out.
+
+## Recovery
+
+- Fix the underlying cause (secret rotation, RBAC, OCI-side issue) — the
+  CronJob needs no manual retry; the next scheduled tick (within 1–2
+  minutes) resumes normal operation once the blocker is cleared.
+- If a production image needs to go out immediately and the checker can't
+  be fixed in time, you can run the same `kubectl set image` step by hand
+  using the digest from `oci artifacts container image list` — see the
+  script body in `cluster-objects/job.yaml` for the exact command.
+- Once the alert clears (next successful run updates
+  `kube_cronjob_status_last_successful_time`), Prometheus resolves it
+  automatically — no manual alert acknowledgment is needed.

src/__tests__/cluster-objects-metrics-consistency.test.ts

@@ -86,24 +86,32 @@ describe("cluster-objects manifests stay consistent with src/lib/metrics.ts", ()
   it("prometheusrule.yaml only references metric and label names that exist in src/lib/metrics.ts", () => {
     const doc = loadYaml("prometheusrule.yaml") as AnyRecord
     // Prometheus built-in metrics that any scrape target automatically
-    // exposes (see prometheus.io/docs/concepts/jobs_instances). These are
-    // legitimate to reference from alert rules without a corresponding
-    // definition in src/lib/metrics.ts.
+    // exposes (see prometheus.io/docs/concepts/jobs_instances), plus
+    // kube-state-metrics metrics that the DocsRolloutCheckerStalled /
+    // DocsPrRolloutCheckerStalled alerts rely on (a near-universal
+    // companion to a Prometheus Operator install, not something this repo
+    // provisions). These are legitimate to reference from alert rules
+    // without a corresponding definition in src/lib/metrics.ts.
     const prometheusBuiltinMetrics = ["up"]
+    const kubeStateMetrics = ["kube_cronjob_status_last_successful_time"]
     const knownMetricNames = [
       httpRequestsTotal.name,
       httpRequestDurationSeconds.name,
       ...prometheusBuiltinMetrics,
+      ...kubeStateMetrics,
     ]
     // `job` and `instance` are labels Prometheus attaches to every scraped
     // series (from scrape_configs.job_name / the target endpoint), so they
     // are always available even though src/lib/metrics.ts does not declare
-    // them.
+    // them. `namespace` and `cronjob` are labels kube-state-metrics attaches
+    // to kube_cronjob_status_last_successful_time.
     const prometheusBuiltinLabels = ["job", "instance"]
+    const kubeStateMetricsLabels = ["namespace", "cronjob"]
     const knownLabelNames = new Set([
       ...httpRequestsTotal.labelNames,
       ...httpRequestDurationSeconds.labelNames,
       ...prometheusBuiltinLabels,
+      ...kubeStateMetricsLabels,
     ])
 
     const groups = doc.spec?.groups ?? []