Prevent stale release approvals from blocking newer runtimes

.github/scripts/cancel-superseded-release-watchers.sh

@@ -0,0 +1,63 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+: "${GH_TOKEN:?GH_TOKEN is required}"
+: "${GITHUB_REPOSITORY:?GITHUB_REPOSITORY is required}"
+: "${GITHUB_RUN_ID:?GITHUB_RUN_ID is required}"
+: "${GITHUB_OUTPUT:?GITHUB_OUTPUT is required}"
+
+current_spec=${1:?usage: cancel-superseded-release-watchers.sh <current-spec>}
+[[ "$current_spec" =~ ^[0-9]+$ ]] \
+  || { echo "current spec must be an integer, got: $current_spec"; exit 2; }
+
+workflow=watch-mainnet-release.yml
+release_slot_available=true
+run_ids=$(
+  for status in waiting in_progress queued pending; do
+    gh api --paginate \
+      "repos/$GITHUB_REPOSITORY/actions/workflows/$workflow/runs?status=$status&per_page=100" \
+      --jq '.workflow_runs[].id'
+  done | sort -u
+)
+
+while read -r run_id; do
+  [ -n "$run_id" ] || continue
+  [ "$run_id" != "$GITHUB_RUN_ID" ] || continue
+
+  jobs=$(gh api --paginate \
+    "repos/$GITHUB_REPOSITORY/actions/runs/$run_id/jobs?filter=all&per_page=100" \
+    --jq '.jobs[] | [.name, .status] | @tsv')
+  run_spec=$(sed -nE 's/^Cut GitHub release v([0-9]+)[[:space:]].*$/\1/p' \
+    <<<"$jobs" | sort -n | tail -n 1)
+  [ -n "$run_spec" ] || continue
+
+  if [ "$run_spec" -eq "$current_spec" ] && \
+      grep -Eq $'\t(in_progress|waiting|queued|pending|requested)$' <<<"$jobs"; then
+    echo "Watcher run $run_id already owns the runtime v$current_spec release slot"
+    release_slot_available=false
+    continue
+  fi
+
+  # Cancel only while protected publication work is still waiting. Once any
+  # job has begun executing, leave the run alone rather than interrupting a
+  # package upload or branch update halfway through.
+  if grep -Eq $'\t(in_progress)$' <<<"$jobs"; then
+    continue
+  fi
+  if ! grep -Eq \
+      $'^(Cut GitHub release v[0-9]+|Publish Python packages to PyPI|Publish Rust crates to crates.io|Deploy production website and docs)\t(waiting|queued|pending|requested)$' \
+      <<<"$jobs"; then
+    continue
+  fi
+
+  if [ "$run_spec" -lt "$current_spec" ]; then
+    echo "Canceling watcher run $run_id for superseded runtime v$run_spec"
+    # A run can finish between the list and cancel requests. Treat that race
+    # as cleanup already accomplished rather than blocking the current release.
+    gh api --method POST \
+      "repos/$GITHUB_REPOSITORY/actions/runs/$run_id/cancel" \
+      >/dev/null || echo "Watcher run $run_id was no longer cancelable"
+  fi
+done <<<"$run_ids"
+
+echo "release_slot_available=$release_slot_available" >> "$GITHUB_OUTPUT"

.github/scripts/test-cancel-superseded-release-watchers.sh

@@ -0,0 +1,73 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+script_dir=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
+helper="$script_dir/cancel-superseded-release-watchers.sh"
+tmp=$(mktemp -d)
+trap 'rm -rf "$tmp"' EXIT
+mkdir -p "$tmp/bin"
+
+cat > "$tmp/bin/gh" <<'EOF'
+#!/usr/bin/env bash
+set -euo pipefail
+
+endpoint=''
+for argument in "$@"; do
+  case "$argument" in
+    repos/*) endpoint="$argument" ;;
+  esac
+done
+
+case "$endpoint" in
+  *'/runs?status=waiting&'*) printf '%s\n' 100 200 300 400 500 ;;
+  *'/runs?status='*) ;;
+  */runs/100/jobs*)
+    printf '%s\t%s\n' 'Cut GitHub release v441' completed
+    printf '%s\t%s\n' 'Publish Python packages to PyPI' in_progress
+    ;;
+  */runs/200/jobs*)
+    printf '%s\t%s\n' 'Cut GitHub release v442' waiting
+    ;;
+  */runs/300/jobs*)
+    printf '%s\t%s\n' 'Cut GitHub release v443' waiting
+    ;;
+  */runs/400/jobs*)
+    printf '%s\t%s\n' 'Cut GitHub release v444' waiting
+    ;;
+  */runs/500/jobs*)
+    printf '%s\t%s\n' 'Cut GitHub release' completed
+    printf '%s\t%s\n' 'Publish Python packages to PyPI' waiting
+    ;;
+  */runs/200/cancel) printf '%s\n' 200 >> "$MOCK_CANCELS" ;;
+  *) echo "unexpected mock gh endpoint: $endpoint" >&2; exit 2 ;;
+esac
+EOF
+chmod +x "$tmp/bin/gh"
+
+export PATH="$tmp/bin:$PATH"
+export GH_TOKEN=test-job-token
+export GITHUB_REPOSITORY=RaoFoundation/subtensor
+export GITHUB_RUN_ID=100
+export MOCK_CANCELS="$tmp/cancels"
+export GITHUB_OUTPUT="$tmp/output"
+: > "$MOCK_CANCELS"
+: > "$GITHUB_OUTPUT"
+
+output=$("$helper" 443)
+grep -qx '200' "$MOCK_CANCELS"
+grep -q 'Canceling watcher run 200 for superseded runtime v442' <<<"$output"
+grep -q 'Watcher run 300 already owns the runtime v443 release slot' <<<"$output"
+grep -qx 'release_slot_available=false' "$GITHUB_OUTPUT"
+
+: > "$MOCK_CANCELS"
+: > "$GITHUB_OUTPUT"
+"$helper" 441 >/dev/null
+[ ! -s "$MOCK_CANCELS" ]
+grep -qx 'release_slot_available=true' "$GITHUB_OUTPUT"
+
+if "$helper" not-a-spec >/dev/null 2>&1; then
+  echo "expected a malformed spec to fail" >&2
+  exit 1
+fi
+
+echo "superseded release watcher cancellation tests passed"

.github/workflows/runtime-checks.yml

@@ -138,6 +138,7 @@ jobs:
       - run: .github/scripts/test-snapshot-artifact.sh
       - run: .github/scripts/test-download-artifact.sh
       - run: .github/scripts/test-select-shared-release-artifact.sh
+      - run: .github/scripts/test-cancel-superseded-release-watchers.sh
       - run: .github/scripts/test-runtime-change-filter.sh
       - run: .github/scripts/test-clone-regression-phase.sh
 

.github/workflows/watch-mainnet-release.yml

@@ -26,10 +26,6 @@ on:
     - cron: "*/10 * * * *"
   workflow_dispatch:
 
-concurrency:
-  group: watch-mainnet-release
-  cancel-in-progress: false
-
 env:
   MAINNET_HTTP: https://entrypoint-finney.opentensor.ai:443
 
@@ -42,9 +38,10 @@ jobs:
     runs-on: [self-hosted, fireactions-turbo-8]
     permissions:
       contents: read
-      actions: read # list/download the release-train artifact
+      actions: write # download artifacts and cancel superseded watcher runs
     outputs:
       release_needed: ${{ steps.compare.outputs.release_needed }}
+      release_slot_available: ${{ steps.queue.outputs.release_slot_available }}
       python_needed: ${{ steps.compare.outputs.python_needed }}
       spec_version: ${{ steps.compare.outputs.spec_version }}
       release_tag: ${{ steps.compare.outputs.release_tag }}
@@ -214,11 +211,29 @@ jobs:
           echo "release_needed=$release_needed" >> "$GITHUB_OUTPUT"
           echo "python_needed=$python_needed" >> "$GITHUB_OUTPUT"
 
+      # A protected-environment approval can wait indefinitely. Once the chain
+      # has moved to a newer runtime, older approval jobs are no longer safe or
+      # useful, so retire them without disturbing another watcher for this spec.
+      - name: Cancel superseded release watchers
+        id: queue
+        continue-on-error: true
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          CURRENT_SPEC: ${{ steps.compare.outputs.spec_version }}
+        run: .github/scripts/cancel-superseded-release-watchers.sh "$CURRENT_SPEC"
+
   release:
-    name: Cut GitHub release
+    name: Cut GitHub release v${{ needs.check.outputs.spec_version }}
     needs: check
-    if: needs.check.outputs.release_needed == 'true'
+    if: >-
+      needs.check.outputs.release_needed == 'true' &&
+      needs.check.outputs.release_slot_available == 'true'
     runs-on: [self-hosted, fireactions-turbo-8]
+    # Serialize retries for one runtime without allowing an abandoned approval
+    # for an older runtime to block finalization of the runtime now on chain.
+    concurrency:
+      group: watch-mainnet-release-${{ needs.check.outputs.release_tag }}
+      cancel-in-progress: false
     # MIRROR_DEPLOY_KEY lives in the mainnet environment secrets; one
     # approval of the run covers this and the publish jobs below.
     environment: mainnet
@@ -234,6 +249,37 @@ jobs:
           ref: ${{ needs.check.outputs.sha }}
           ssh-key: ${{ secrets.MIRROR_DEPLOY_KEY }}
 
+      # Approval can arrive long after the check job captured chain state. Fail
+      # closed before touching the mirror or release if a newer runtime won in
+      # the meantime.
+      - name: Revalidate finalized runtime after approval
+        env:
+          EXPECTED_SPEC: ${{ needs.check.outputs.spec_version }}
+          EXPECTED_CODE_HASH: ${{ needs.check.outputs.code_hash }}
+        # Keep this check inline: the released commit checked out above may
+        # predate this workflow and cannot be expected to contain a new helper.
+        run: |
+          set -euo pipefail
+          finalized_head=$(curl -sf -H "Content-Type: application/json" \
+            -d '{"id":1,"jsonrpc":"2.0","method":"chain_getFinalizedHead","params":[]}' \
+            "$MAINNET_HTTP" | jq -er \
+              '.result | strings | select(test("^0x[0-9a-f]{64}$"))')
+          runtime_request=$(jq -cn --arg block "$finalized_head" \
+            '{id: 1, jsonrpc: "2.0", method: "state_getRuntimeVersion", params: [$block]}')
+          current_spec=$(curl -sf -H "Content-Type: application/json" \
+            -d "$runtime_request" "$MAINNET_HTTP" | jq -er '.result.specVersion')
+          code_hash_request=$(jq -cn --arg block "$finalized_head" \
+            '{id: 1, jsonrpc: "2.0", method: "state_getStorageHash",
+              params: ["0x3a636f6465", $block]}')
+          current_code_hash=$(curl -sf -H "Content-Type: application/json" \
+            -d "$code_hash_request" "$MAINNET_HTTP" | jq -er \
+              '.result | strings | select(test("^0x[0-9a-f]{64}$"))')
+
+          [ "$current_spec" = "$EXPECTED_SPEC" ] \
+            || { echo "mainnet advanced to spec $current_spec; expected $EXPECTED_SPEC"; exit 1; }
+          [ "$current_code_hash" = "$EXPECTED_CODE_HASH" ] \
+            || { echo "mainnet code hash changed to $current_code_hash; expected $EXPECTED_CODE_HASH"; exit 1; }
+
       # The release train uploaded the deterministic srtool build and the
       # multisig call data as the mainnet-upgrade-<spec> workflow artifact
       # (90-day retention). Attach it to the release so the deterministic
@@ -353,7 +399,9 @@ jobs:
   build-core:
     name: Build bittensor-core wheels
     needs: check
-    if: needs.check.outputs.python_needed == 'true'
+    if: >-
+      needs.check.outputs.release_slot_available == 'true' &&
+      needs.check.outputs.python_needed == 'true'
     uses: ./.github/workflows/build-core-wheels.yml
     with:
       ref: ${{ needs.check.outputs.sha }}