From 602902bd9706a79527b29298ea8478a0dfe3b478 Mon Sep 17 00:00:00 2001 From: operations Date: Sun, 4 Oct 2026 20:50:42 -0400 Subject: [PATCH] fix: alert on stalled rollout-checker CronJobs Add DocsRolloutCheckerStalled and DocsPrRolloutCheckerStalled alerts to cluster-objects/prometheusrule.yaml, backed by kube-state-metrics' kube_cronjob_status_last_successful_time. Neither nextra-rollout-checker nor nextra-pr-rollout-checker (cluster-objects/job.yaml, pr-job.yaml) had any failure alerting: both are concurrencyPolicy: Forbid with failedJobsHistoryLimit: 1 and no retry beyond the next scheduled tick, so a stuck/failing checker silently stops deploying new images with no signal. Adds runbooks/rollout-checker-failure.md and a runbooks/README.md index entry, and extends src/__tests__/cluster-objects-metrics-consistency.test.ts to recognize kube-state-metrics' metric/labels as a legitimate external source alongside the existing Prometheus built-ins. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Signed-off-by: operations --- cluster-objects/prometheusrule.yaml | 64 ++++++++++++-- runbooks/README.md | 1 + runbooks/rollout-checker-failure.md | 84 +++++++++++++++++++ ...luster-objects-metrics-consistency.test.ts | 16 +++- 4 files changed, 153 insertions(+), 12 deletions(-) create mode 100644 runbooks/rollout-checker-failure.md diff --git a/cluster-objects/prometheusrule.yaml b/cluster-objects/prometheusrule.yaml index a8dfa3f308..ff0278af4f 100644 --- a/cluster-objects/prometheusrule.yaml +++ b/cluster-objects/prometheusrule.yaml @@ -1,15 +1,23 @@ -# Prometheus Operator alerting rules for the docs site's existing -# `/api/metrics` endpoint (src/app/api/metrics/route.ts, src/lib/metrics.ts), -# already scraped by cluster-objects/servicemonitor.yaml. +# Prometheus Operator alerting rules for: +# 1. the docs site's own `/api/metrics` endpoint +# (src/app/api/metrics/route.ts, src/lib/metrics.ts), already scraped +# by cluster-objects/servicemonitor.yaml; and +# 2. the automated rollout-checker CronJobs (cluster-objects/job.yaml, +# cluster-objects/pr-job.yaml) that deploy new images, via +# kube-state-metrics' kube_cronjob_status_last_successful_time. # # This resource only takes effect if a Prometheus Operator with the -# PrometheusRule CRD is already watching this namespace/selector, and only -# if servicemonitor.yaml is applied so the referenced metrics are actually -# being scraped. It does not install an operator, does not configure any +# PrometheusRule CRD is already watching this namespace/selector — group 1 +# additionally needs servicemonitor.yaml applied so those metrics are +# actually scraped, and group 2 additionally needs kube-state-metrics +# running in-cluster (a near-universal companion to a Prometheus Operator +# install). This file does not install an operator, does not configure any # exporter, and does not send data off-box — it only defines evaluation -# rules over metrics already produced in-process by this application. +# rules over metrics already produced by this application or by +# kube-state-metrics, if present. # -# Both alerts use metric/label names defined in src/lib/metrics.ts: +# The first three alerts use metric/label names defined in +# src/lib/metrics.ts: # docs_api_requests_total{route, method, status_class} # docs_api_request_duration_seconds{route, method} # `route` and `status_class` are fixed, bounded label sets (see the @@ -74,3 +82,43 @@ spec: instrumented API routes has exceeded 1s over the last 5 minutes (current value: {{ $value | humanizeDuration }}). runbook_url: "https://github.com/kubestellar/docs/blob/main/runbooks/api-error-rate-latency.md" + # The two alerts below cover the automated deploy pipeline itself + # (cluster-objects/job.yaml, cluster-objects/pr-job.yaml), not the + # application. kube_cronjob_status_last_successful_time is a + # kube-state-metrics metric (a near-universal companion to a + # Prometheus Operator install) — this does not add a new exporter, + # it only evaluates a metric that kube-state-metrics already + # exposes if it is present. Both CronJobs are concurrencyPolicy: + # Forbid with failedJobsHistoryLimit: 1, so a stalled or failing + # rollout-checker would otherwise go undetected: production/preview + # Deployments keep serving a stale image with no signal until + # someone happens to check `kubectl get cronjob -n docs`. + - alert: DocsRolloutCheckerStalled + expr: >- + time() - kube_cronjob_status_last_successful_time{namespace="docs", cronjob="nextra-rollout-checker"} > 900 + for: 5m + labels: + severity: warning + annotations: + summary: "Production rollout-checker CronJob has not succeeded in 15m" + description: >- + `nextra-rollout-checker` (cluster-objects/job.yaml) runs every + 2 minutes and auto-rolls out new `kubestellar-docs` images + from OCIR. It has not completed successfully in over 15 + minutes, so new production image builds are not being + deployed. + runbook_url: "https://github.com/kubestellar/docs/blob/main/runbooks/rollout-checker-failure.md" + - alert: DocsPrRolloutCheckerStalled + expr: >- + time() - kube_cronjob_status_last_successful_time{namespace="docs", cronjob="nextra-pr-rollout-checker"} > 600 + for: 5m + labels: + severity: warning + annotations: + summary: "PR-preview rollout-checker CronJob has not succeeded in 10m" + description: >- + `nextra-pr-rollout-checker` (cluster-objects/pr-job.yaml) runs + every 1 minute and reconciles PR preview deployments from + OCIR. It has not completed successfully in over 10 minutes, + so PR preview deploys/teardowns are not being reconciled. + runbook_url: "https://github.com/kubestellar/docs/blob/main/runbooks/rollout-checker-failure.md" diff --git a/runbooks/README.md b/runbooks/README.md index 06a0b8e683..b1df09b814 100644 --- a/runbooks/README.md +++ b/runbooks/README.md @@ -9,6 +9,7 @@ looking at. | [`slo.md`](slo.md) | Establishing the readiness SLO/SLI for the production docs site and how the `healthz-monitor` alerting works. | Start here for any production readiness question. | | [`deploy-rollback.md`](deploy-rollback.md) | The live site is serving broken/stale content or a bad deploy went out (Netlify or container path) and needs to be rolled back. | `slo.md` | | [`api-error-rate-latency.md`](api-error-rate-latency.md) | The `DocsApiHighErrorRate` or `DocsApiHighRequestLatency` alert fires. | `cluster-objects/prometheusrule.yaml` `runbook_url` annotations | +| [`rollout-checker-failure.md`](rollout-checker-failure.md) | The `DocsRolloutCheckerStalled` or `DocsPrRolloutCheckerStalled` alert fires — the `nextra-rollout-checker`/`nextra-pr-rollout-checker` CronJob (`cluster-objects/job.yaml`/`pr-job.yaml`) has stopped completing successfully, so new image rollouts are not happening. | `cluster-objects/prometheusrule.yaml` `runbook_url` annotations | | [`netlify-build-failure.md`](netlify-build-failure.md) | A Netlify build fails: a `[netlify-deploy-failure]` issue opens for a direct-to-main commit, or `netlify-error-reporter.yml` comments on a PR (the alert links this runbook and auto-closes on the next successful deploy, added in [#6783](https://github.com/kubestellar/docs/issues/6783)). | `netlify-error-reporter.yml` failure alert | | [`version-branch-rollback.md`](version-branch-rollback.md) | The version picker shows a wrong/missing version, `versions.ts`/`shared.json` changed unexpectedly, or `create-version-branch.yml`/`sync-console-release-versions.yml` needs investigating. | — | | [`scorecard-monitoring.md`](scorecard-monitoring.md) | The weekly OpenSSF Scorecard scan (`scorecard.yml`) fails — an automated `[scorecard-failure]` alert opens/comments on a `ci-failure` issue (added in [#6918](https://github.com/kubestellar/docs/pull/6918)); check for the known recurring `gcr.io` pull-failure root cause first. | `ci-failure`-labeled issues titled `[scorecard-failure] ...` | diff --git a/runbooks/rollout-checker-failure.md b/runbooks/rollout-checker-failure.md new file mode 100644 index 0000000000..5f19d213f6 --- /dev/null +++ b/runbooks/rollout-checker-failure.md @@ -0,0 +1,84 @@ +# Runbook: Rollout-Checker CronJob Stalled + +## Scope + +Applies to the `DocsRolloutCheckerStalled` and `DocsPrRolloutCheckerStalled` +alerts defined in `cluster-objects/prometheusrule.yaml`. Both fire from +kube-state-metrics' `kube_cronjob_status_last_successful_time`, so they +require a Prometheus Operator and kube-state-metrics already running in the +cluster (no exporter is installed by this repo) — if neither alert has ever +fired and you don't know whether that monitoring exists, the CronJobs +themselves are still the thing to check by hand below. + +These two CronJobs (`cluster-objects/job.yaml`, +`cluster-objects/pr-job.yaml`) are the only mechanism that deploys new +`kubestellar-docs` images: + +- **`nextra-rollout-checker`** (schedule `*/2 * * * *`, namespace `docs`) — + polls OCIR for the latest `kubestellar-docs` image digest and + `kubectl set image`s the production Deployment + (`cluster-objects/deployment.yaml`) when a newer digest appears. A run + that finds no newer image still exits `0` ("skipping rollout"), so a + healthy CronJob does **not** mean a new image went out — only that the + checker itself is working. +- **`nextra-pr-rollout-checker`** (schedule `*/1 * * * *`, namespace `docs`) + — the same mechanism for per-PR preview Deployments. + +Both scripts run with `set -e` (any failed step — missing env vars, +`kubectl`/`oci` auth, a malformed OCI API response — exits the job +non-zero), both CronJobs are `concurrencyPolicy: Forbid` with +`backoffLimit: 0`, and `failedJobsHistoryLimit: 1` keeps at most one failed +Job around. There is no retry beyond the next scheduled tick. + +- **`DocsRolloutCheckerStalled`** — fires when the production checker has + not completed a successful run in over 15 minutes (more than 7 missed + 2-minute ticks). +- **`DocsPrRolloutCheckerStalled`** — fires when the PR-preview checker has + not completed a successful run in over 10 minutes (more than 9 missed + 1-minute ticks). + +## Diagnosis + +1. Confirm the CronJob is actually failing, not just slow to be scraped: + ```sh + kubectl get cronjob -n docs nextra-rollout-checker nextra-pr-rollout-checker + kubectl get jobs -n docs -l 'job-name' --sort-by=.metadata.creationTimestamp + ``` + Look at `LAST SCHEDULE` and whether the most recent Job succeeded. +2. Read the failing Job's logs — the script's `[DEBUG]`/`[ERROR]` lines + (`cluster-objects/job.yaml` / `cluster-objects/pr-job.yaml` + ConfigMap data) point at the failing step directly: + ```sh + kubectl logs -n docs job/ + ``` +3. Common root causes, roughly in likelihood order: + - **Expired/rotated `oci-config-secret`** — `oci artifacts container + image list` fails auth. Re-provision the secret + (`cluster-objects/rbac.yaml` mounts it at `/home/appuser/.oci`). + - **RBAC regression** on `nextra-rollout-sa` + (`cluster-objects/rbac.yaml`) — `kubectl set image` / `kubectl + annotate` / `kubectl get deploy` denied. Diff the live Role against + the manifest. + - **OCI API outage/throttling** — transient; confirm via the OCI status + page, no action needed beyond waiting if the next scheduled run + recovers on its own. + - **Missing `COMPARTMENT_OCID`/`PROD_REPO_OCID`/`REPO_OCID`** env vars + after a Secret/ConfigMap edit — the script's `:?` guards fail fast + with an explicit `not set` error naming the variable. +4. If the checker itself is healthy but a specific rollout is stuck (e.g. + repeated `kubectl set image` with no corresponding pod update), see + `runbooks/deploy-rollback.md` instead — this runbook is about the + *checker* not running, not about a bad image once rolled out. + +## Recovery + +- Fix the underlying cause (secret rotation, RBAC, OCI-side issue) — the + CronJob needs no manual retry; the next scheduled tick (within 1–2 + minutes) resumes normal operation once the blocker is cleared. +- If a production image needs to go out immediately and the checker can't + be fixed in time, you can run the same `kubectl set image` step by hand + using the digest from `oci artifacts container image list` — see the + script body in `cluster-objects/job.yaml` for the exact command. +- Once the alert clears (next successful run updates + `kube_cronjob_status_last_successful_time`), Prometheus resolves it + automatically — no manual alert acknowledgment is needed. diff --git a/src/__tests__/cluster-objects-metrics-consistency.test.ts b/src/__tests__/cluster-objects-metrics-consistency.test.ts index 1b8b166ac3..5288acbb13 100644 --- a/src/__tests__/cluster-objects-metrics-consistency.test.ts +++ b/src/__tests__/cluster-objects-metrics-consistency.test.ts @@ -86,24 +86,32 @@ describe("cluster-objects manifests stay consistent with src/lib/metrics.ts", () it("prometheusrule.yaml only references metric and label names that exist in src/lib/metrics.ts", () => { const doc = loadYaml("prometheusrule.yaml") as AnyRecord // Prometheus built-in metrics that any scrape target automatically - // exposes (see prometheus.io/docs/concepts/jobs_instances). These are - // legitimate to reference from alert rules without a corresponding - // definition in src/lib/metrics.ts. + // exposes (see prometheus.io/docs/concepts/jobs_instances), plus + // kube-state-metrics metrics that the DocsRolloutCheckerStalled / + // DocsPrRolloutCheckerStalled alerts rely on (a near-universal + // companion to a Prometheus Operator install, not something this repo + // provisions). These are legitimate to reference from alert rules + // without a corresponding definition in src/lib/metrics.ts. const prometheusBuiltinMetrics = ["up"] + const kubeStateMetrics = ["kube_cronjob_status_last_successful_time"] const knownMetricNames = [ httpRequestsTotal.name, httpRequestDurationSeconds.name, ...prometheusBuiltinMetrics, + ...kubeStateMetrics, ] // `job` and `instance` are labels Prometheus attaches to every scraped // series (from scrape_configs.job_name / the target endpoint), so they // are always available even though src/lib/metrics.ts does not declare - // them. + // them. `namespace` and `cronjob` are labels kube-state-metrics attaches + // to kube_cronjob_status_last_successful_time. const prometheusBuiltinLabels = ["job", "instance"] + const kubeStateMetricsLabels = ["namespace", "cronjob"] const knownLabelNames = new Set([ ...httpRequestsTotal.labelNames, ...httpRequestDurationSeconds.labelNames, ...prometheusBuiltinLabels, + ...kubeStateMetricsLabels, ]) const groups = doc.spec?.groups ?? []