diff --git a/.github/workflows/cd-k8s.yml b/.github/workflows/cd-k8s.yml index de50da15..69b4f29b 100644 --- a/.github/workflows/cd-k8s.yml +++ b/.github/workflows/cd-k8s.yml @@ -147,6 +147,7 @@ jobs: kubectl kustomize "$overlay_dir" >/tmp/matchid-k8s-rendered.yaml kubectl -n "$NAMESPACE" delete deploy/redis --ignore-not-found=true kubectl apply -f /tmp/matchid-k8s-rendered.yaml + bash deploy/k8s/scripts/recover-stuck-statefulset.sh "$NAMESPACE" elasticsearch - name: Wait for runtime run: | diff --git a/.github/workflows/cd.yml b/.github/workflows/cd.yml index 9e3e4fad..5b7c4249 100644 --- a/.github/workflows/cd.yml +++ b/.github/workflows/cd.yml @@ -678,6 +678,7 @@ jobs: kubectl kustomize "$overlay_dir" >/tmp/matchid-k8s-rendered.yaml kubectl -n "$NAMESPACE" delete deploy/redis --ignore-not-found=true kubectl apply -f /tmp/matchid-k8s-rendered.yaml + bash deploy/k8s/scripts/recover-stuck-statefulset.sh "$NAMESPACE" elasticsearch - name: Wait for runtime run: | diff --git a/deploy/k8s/scripts/recover-stuck-statefulset.sh b/deploy/k8s/scripts/recover-stuck-statefulset.sh new file mode 100644 index 00000000..a961aa4a --- /dev/null +++ b/deploy/k8s/scripts/recover-stuck-statefulset.sh @@ -0,0 +1,44 @@ +#!/usr/bin/env bash + +set -euo pipefail + +namespace="${1:?namespace is required}" +statefulset="${2:?statefulset name is required}" +pod="${3:-${statefulset}-0}" + +if ! kubectl -n "$namespace" get pod "$pod" >/dev/null 2>&1; then + exit 0 +fi + +# An OrderedReady StatefulSet can remain blocked on an unhealthy pod from the +# previous revision. Wait for the controller to publish the revision produced +# by the manifest that was just applied before deciding whether recovery is +# needed. +for attempt in $(seq 1 30); do + revision_status="$( + kubectl -n "$namespace" get statefulset "$statefulset" \ + -o jsonpath='{.metadata.generation}{"\t"}{.status.observedGeneration}{"\t"}{.status.updateRevision}' + )" + IFS=$'\t' read -r generation observed_generation desired_revision <<< "$revision_status" + + if [ -n "${desired_revision:-}" ] && [ "$generation" = "${observed_generation:-}" ]; then + break + fi + + if [ "$attempt" -eq 30 ]; then + echo "StatefulSet $statefulset did not publish its desired revision" >&2 + exit 1 + fi + sleep 1 +done + +pod_status="$( + kubectl -n "$namespace" get pod "$pod" \ + -o jsonpath='{.metadata.labels.controller-revision-hash}{"\t"}{.status.conditions[?(@.type=="Ready")].status}' +)" +IFS=$'\t' read -r pod_revision pod_ready <<< "$pod_status" + +if [ "${pod_revision:-}" != "$desired_revision" ] && [ "${pod_ready:-}" != "True" ]; then + echo "Recreating non-ready $pod from revision ${pod_revision:-unknown} at desired revision $desired_revision" + kubectl -n "$namespace" delete pod "$pod" +fi diff --git a/deploy/k8s/tests/recover-stuck-statefulset.sh b/deploy/k8s/tests/recover-stuck-statefulset.sh new file mode 100644 index 00000000..0c78e98b --- /dev/null +++ b/deploy/k8s/tests/recover-stuck-statefulset.sh @@ -0,0 +1,61 @@ +#!/usr/bin/env bash + +set -euo pipefail + +root_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd)" +subject="$root_dir/deploy/k8s/scripts/recover-stuck-statefulset.sh" +tmp_dir="$(mktemp -d)" +trap 'rm -rf "$tmp_dir"' EXIT + +cat > "$tmp_dir/kubectl" <<'EOF' +#!/usr/bin/env bash +set -euo pipefail + +case "$*" in + *" get pod elasticsearch-0 -o jsonpath="*) + printf '%s\t%s' "$POD_REVISION" "$POD_READY" + ;; + *" get pod elasticsearch-0") + ;; + *" get statefulset elasticsearch -o jsonpath="*) + printf '5\t5\t%s' "$DESIRED_REVISION" + ;; + *" delete pod elasticsearch-0") + printf 'delete\n' >> "$KUBECTL_TEST_LOG" + ;; + *) + echo "unexpected kubectl call: $*" >&2 + exit 1 + ;; +esac +EOF +chmod +x "$tmp_dir/kubectl" + +export PATH="$tmp_dir:$PATH" +export KUBECTL_TEST_LOG="$tmp_dir/calls" + +assert_deleted() { + local pod_revision="$1" + local pod_ready="$2" + + : > "$KUBECTL_TEST_LOG" + DESIRED_REVISION=new POD_REVISION="$pod_revision" POD_READY="$pod_ready" \ + bash "$subject" matchid-dev elasticsearch >/dev/null + grep -qx delete "$KUBECTL_TEST_LOG" +} + +assert_not_deleted() { + local pod_revision="$1" + local pod_ready="$2" + + : > "$KUBECTL_TEST_LOG" + DESIRED_REVISION=new POD_REVISION="$pod_revision" POD_READY="$pod_ready" \ + bash "$subject" matchid-dev elasticsearch >/dev/null + test ! -s "$KUBECTL_TEST_LOG" +} + +assert_deleted old False +assert_not_deleted new False +assert_not_deleted old True + +echo "recover-stuck-statefulset tests passed"