fix(ci): recover stuck dev Elasticsearch rollout

.github/workflows/cd-k8s.yml

@@ -147,6 +147,7 @@ jobs:
           kubectl kustomize "$overlay_dir" >/tmp/matchid-k8s-rendered.yaml
           kubectl -n "$NAMESPACE" delete deploy/redis --ignore-not-found=true
           kubectl apply -f /tmp/matchid-k8s-rendered.yaml
+          bash deploy/k8s/scripts/recover-stuck-statefulset.sh "$NAMESPACE" elasticsearch
 
       - name: Wait for runtime
         run: |

.github/workflows/cd.yml

@@ -678,6 +678,7 @@ jobs:
           kubectl kustomize "$overlay_dir" >/tmp/matchid-k8s-rendered.yaml
           kubectl -n "$NAMESPACE" delete deploy/redis --ignore-not-found=true
           kubectl apply -f /tmp/matchid-k8s-rendered.yaml
+          bash deploy/k8s/scripts/recover-stuck-statefulset.sh "$NAMESPACE" elasticsearch
 
       - name: Wait for runtime
         run: |

deploy/k8s/scripts/recover-stuck-statefulset.sh

@@ -0,0 +1,44 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+namespace="${1:?namespace is required}"
+statefulset="${2:?statefulset name is required}"
+pod="${3:-${statefulset}-0}"
+
+if ! kubectl -n "$namespace" get pod "$pod" >/dev/null 2>&1; then
+  exit 0
+fi
+
+# An OrderedReady StatefulSet can remain blocked on an unhealthy pod from the
+# previous revision. Wait for the controller to publish the revision produced
+# by the manifest that was just applied before deciding whether recovery is
+# needed.
+for attempt in $(seq 1 30); do
+  revision_status="$(
+    kubectl -n "$namespace" get statefulset "$statefulset" \
+      -o jsonpath='{.metadata.generation}{"\t"}{.status.observedGeneration}{"\t"}{.status.updateRevision}'
+  )"
+  IFS=$'\t' read -r generation observed_generation desired_revision <<< "$revision_status"
+
+  if [ -n "${desired_revision:-}" ] && [ "$generation" = "${observed_generation:-}" ]; then
+    break
+  fi
+
+  if [ "$attempt" -eq 30 ]; then
+    echo "StatefulSet $statefulset did not publish its desired revision" >&2
+    exit 1
+  fi
+  sleep 1
+done
+
+pod_status="$(
+  kubectl -n "$namespace" get pod "$pod" \
+    -o jsonpath='{.metadata.labels.controller-revision-hash}{"\t"}{.status.conditions[?(@.type=="Ready")].status}'
+)"
+IFS=$'\t' read -r pod_revision pod_ready <<< "$pod_status"
+
+if [ "${pod_revision:-}" != "$desired_revision" ] && [ "${pod_ready:-}" != "True" ]; then
+  echo "Recreating non-ready $pod from revision ${pod_revision:-unknown} at desired revision $desired_revision"
+  kubectl -n "$namespace" delete pod "$pod"
+fi

deploy/k8s/tests/recover-stuck-statefulset.sh

@@ -0,0 +1,61 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+root_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd)"
+subject="$root_dir/deploy/k8s/scripts/recover-stuck-statefulset.sh"
+tmp_dir="$(mktemp -d)"
+trap 'rm -rf "$tmp_dir"' EXIT
+
+cat > "$tmp_dir/kubectl" <<'EOF'
+#!/usr/bin/env bash
+set -euo pipefail
+
+case "$*" in
+  *" get pod elasticsearch-0 -o jsonpath="*)
+    printf '%s\t%s' "$POD_REVISION" "$POD_READY"
+    ;;
+  *" get pod elasticsearch-0")
+    ;;
+  *" get statefulset elasticsearch -o jsonpath="*)
+    printf '5\t5\t%s' "$DESIRED_REVISION"
+    ;;
+  *" delete pod elasticsearch-0")
+    printf 'delete\n' >> "$KUBECTL_TEST_LOG"
+    ;;
+  *)
+    echo "unexpected kubectl call: $*" >&2
+    exit 1
+    ;;
+esac
+EOF
+chmod +x "$tmp_dir/kubectl"
+
+export PATH="$tmp_dir:$PATH"
+export KUBECTL_TEST_LOG="$tmp_dir/calls"
+
+assert_deleted() {
+  local pod_revision="$1"
+  local pod_ready="$2"
+
+  : > "$KUBECTL_TEST_LOG"
+  DESIRED_REVISION=new POD_REVISION="$pod_revision" POD_READY="$pod_ready" \
+    bash "$subject" matchid-dev elasticsearch >/dev/null
+  grep -qx delete "$KUBECTL_TEST_LOG"
+}
+
+assert_not_deleted() {
+  local pod_revision="$1"
+  local pod_ready="$2"
+
+  : > "$KUBECTL_TEST_LOG"
+  DESIRED_REVISION=new POD_REVISION="$pod_revision" POD_READY="$pod_ready" \
+    bash "$subject" matchid-dev elasticsearch >/dev/null
+  test ! -s "$KUBECTL_TEST_LOG"
+}
+
+assert_deleted old False
+assert_not_deleted new False
+assert_not_deleted old True
+
+echo "recover-stuck-statefulset tests passed"