From 07bc4e2162bb3475bc1bdef2993e4ef8beac7601 Mon Sep 17 00:00:00 2001 From: UnArbosFive Date: Thu, 6 Aug 2026 18:38:01 +0200 Subject: [PATCH] fix: make release watcher supersession-safe --- .../cancel-superseded-release-watchers.sh | 63 ++++++++++++++++ ...test-cancel-superseded-release-watchers.sh | 73 +++++++++++++++++++ .github/workflows/runtime-checks.yml | 1 + .github/workflows/watch-mainnet-release.yml | 64 ++++++++++++++-- 4 files changed, 193 insertions(+), 8 deletions(-) create mode 100755 .github/scripts/cancel-superseded-release-watchers.sh create mode 100755 .github/scripts/test-cancel-superseded-release-watchers.sh diff --git a/.github/scripts/cancel-superseded-release-watchers.sh b/.github/scripts/cancel-superseded-release-watchers.sh new file mode 100755 index 0000000000..1d3fd96e58 --- /dev/null +++ b/.github/scripts/cancel-superseded-release-watchers.sh @@ -0,0 +1,63 @@ +#!/usr/bin/env bash +set -euo pipefail + +: "${GH_TOKEN:?GH_TOKEN is required}" +: "${GITHUB_REPOSITORY:?GITHUB_REPOSITORY is required}" +: "${GITHUB_RUN_ID:?GITHUB_RUN_ID is required}" +: "${GITHUB_OUTPUT:?GITHUB_OUTPUT is required}" + +current_spec=${1:?usage: cancel-superseded-release-watchers.sh } +[[ "$current_spec" =~ ^[0-9]+$ ]] \ + || { echo "current spec must be an integer, got: $current_spec"; exit 2; } + +workflow=watch-mainnet-release.yml +release_slot_available=true +run_ids=$( + for status in waiting in_progress queued pending; do + gh api --paginate \ + "repos/$GITHUB_REPOSITORY/actions/workflows/$workflow/runs?status=$status&per_page=100" \ + --jq '.workflow_runs[].id' + done | sort -u +) + +while read -r run_id; do + [ -n "$run_id" ] || continue + [ "$run_id" != "$GITHUB_RUN_ID" ] || continue + + jobs=$(gh api --paginate \ + "repos/$GITHUB_REPOSITORY/actions/runs/$run_id/jobs?filter=all&per_page=100" \ + --jq '.jobs[] | [.name, .status] | @tsv') + run_spec=$(sed -nE 's/^Cut GitHub release v([0-9]+)[[:space:]].*$/\1/p' \ + <<<"$jobs" | sort -n | tail -n 1) + [ -n "$run_spec" ] || continue + + if [ "$run_spec" -eq "$current_spec" ] && \ + grep -Eq $'\t(in_progress|waiting|queued|pending|requested)$' <<<"$jobs"; then + echo "Watcher run $run_id already owns the runtime v$current_spec release slot" + release_slot_available=false + continue + fi + + # Cancel only while protected publication work is still waiting. Once any + # job has begun executing, leave the run alone rather than interrupting a + # package upload or branch update halfway through. + if grep -Eq $'\t(in_progress)$' <<<"$jobs"; then + continue + fi + if ! grep -Eq \ + $'^(Cut GitHub release v[0-9]+|Publish Python packages to PyPI|Publish Rust crates to crates.io|Deploy production website and docs)\t(waiting|queued|pending|requested)$' \ + <<<"$jobs"; then + continue + fi + + if [ "$run_spec" -lt "$current_spec" ]; then + echo "Canceling watcher run $run_id for superseded runtime v$run_spec" + # A run can finish between the list and cancel requests. Treat that race + # as cleanup already accomplished rather than blocking the current release. + gh api --method POST \ + "repos/$GITHUB_REPOSITORY/actions/runs/$run_id/cancel" \ + >/dev/null || echo "Watcher run $run_id was no longer cancelable" + fi +done <<<"$run_ids" + +echo "release_slot_available=$release_slot_available" >> "$GITHUB_OUTPUT" diff --git a/.github/scripts/test-cancel-superseded-release-watchers.sh b/.github/scripts/test-cancel-superseded-release-watchers.sh new file mode 100755 index 0000000000..c6e70c3eba --- /dev/null +++ b/.github/scripts/test-cancel-superseded-release-watchers.sh @@ -0,0 +1,73 @@ +#!/usr/bin/env bash +set -euo pipefail + +script_dir=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +helper="$script_dir/cancel-superseded-release-watchers.sh" +tmp=$(mktemp -d) +trap 'rm -rf "$tmp"' EXIT +mkdir -p "$tmp/bin" + +cat > "$tmp/bin/gh" <<'EOF' +#!/usr/bin/env bash +set -euo pipefail + +endpoint='' +for argument in "$@"; do + case "$argument" in + repos/*) endpoint="$argument" ;; + esac +done + +case "$endpoint" in + *'/runs?status=waiting&'*) printf '%s\n' 100 200 300 400 500 ;; + *'/runs?status='*) ;; + */runs/100/jobs*) + printf '%s\t%s\n' 'Cut GitHub release v441' completed + printf '%s\t%s\n' 'Publish Python packages to PyPI' in_progress + ;; + */runs/200/jobs*) + printf '%s\t%s\n' 'Cut GitHub release v442' waiting + ;; + */runs/300/jobs*) + printf '%s\t%s\n' 'Cut GitHub release v443' waiting + ;; + */runs/400/jobs*) + printf '%s\t%s\n' 'Cut GitHub release v444' waiting + ;; + */runs/500/jobs*) + printf '%s\t%s\n' 'Cut GitHub release' completed + printf '%s\t%s\n' 'Publish Python packages to PyPI' waiting + ;; + */runs/200/cancel) printf '%s\n' 200 >> "$MOCK_CANCELS" ;; + *) echo "unexpected mock gh endpoint: $endpoint" >&2; exit 2 ;; +esac +EOF +chmod +x "$tmp/bin/gh" + +export PATH="$tmp/bin:$PATH" +export GH_TOKEN=test-job-token +export GITHUB_REPOSITORY=RaoFoundation/subtensor +export GITHUB_RUN_ID=100 +export MOCK_CANCELS="$tmp/cancels" +export GITHUB_OUTPUT="$tmp/output" +: > "$MOCK_CANCELS" +: > "$GITHUB_OUTPUT" + +output=$("$helper" 443) +grep -qx '200' "$MOCK_CANCELS" +grep -q 'Canceling watcher run 200 for superseded runtime v442' <<<"$output" +grep -q 'Watcher run 300 already owns the runtime v443 release slot' <<<"$output" +grep -qx 'release_slot_available=false' "$GITHUB_OUTPUT" + +: > "$MOCK_CANCELS" +: > "$GITHUB_OUTPUT" +"$helper" 441 >/dev/null +[ ! -s "$MOCK_CANCELS" ] +grep -qx 'release_slot_available=true' "$GITHUB_OUTPUT" + +if "$helper" not-a-spec >/dev/null 2>&1; then + echo "expected a malformed spec to fail" >&2 + exit 1 +fi + +echo "superseded release watcher cancellation tests passed" diff --git a/.github/workflows/runtime-checks.yml b/.github/workflows/runtime-checks.yml index 3e2bd43dec..bd1102282b 100644 --- a/.github/workflows/runtime-checks.yml +++ b/.github/workflows/runtime-checks.yml @@ -138,6 +138,7 @@ jobs: - run: .github/scripts/test-snapshot-artifact.sh - run: .github/scripts/test-download-artifact.sh - run: .github/scripts/test-select-shared-release-artifact.sh + - run: .github/scripts/test-cancel-superseded-release-watchers.sh - run: .github/scripts/test-runtime-change-filter.sh - run: .github/scripts/test-clone-regression-phase.sh diff --git a/.github/workflows/watch-mainnet-release.yml b/.github/workflows/watch-mainnet-release.yml index 2ac010518c..a2825b5890 100644 --- a/.github/workflows/watch-mainnet-release.yml +++ b/.github/workflows/watch-mainnet-release.yml @@ -26,10 +26,6 @@ on: - cron: "*/10 * * * *" workflow_dispatch: -concurrency: - group: watch-mainnet-release - cancel-in-progress: false - env: MAINNET_HTTP: https://entrypoint-finney.opentensor.ai:443 @@ -42,9 +38,10 @@ jobs: runs-on: [self-hosted, fireactions-turbo-8] permissions: contents: read - actions: read # list/download the release-train artifact + actions: write # download artifacts and cancel superseded watcher runs outputs: release_needed: ${{ steps.compare.outputs.release_needed }} + release_slot_available: ${{ steps.queue.outputs.release_slot_available }} python_needed: ${{ steps.compare.outputs.python_needed }} spec_version: ${{ steps.compare.outputs.spec_version }} release_tag: ${{ steps.compare.outputs.release_tag }} @@ -214,11 +211,29 @@ jobs: echo "release_needed=$release_needed" >> "$GITHUB_OUTPUT" echo "python_needed=$python_needed" >> "$GITHUB_OUTPUT" + # A protected-environment approval can wait indefinitely. Once the chain + # has moved to a newer runtime, older approval jobs are no longer safe or + # useful, so retire them without disturbing another watcher for this spec. + - name: Cancel superseded release watchers + id: queue + continue-on-error: true + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + CURRENT_SPEC: ${{ steps.compare.outputs.spec_version }} + run: .github/scripts/cancel-superseded-release-watchers.sh "$CURRENT_SPEC" + release: - name: Cut GitHub release + name: Cut GitHub release v${{ needs.check.outputs.spec_version }} needs: check - if: needs.check.outputs.release_needed == 'true' + if: >- + needs.check.outputs.release_needed == 'true' && + needs.check.outputs.release_slot_available == 'true' runs-on: [self-hosted, fireactions-turbo-8] + # Serialize retries for one runtime without allowing an abandoned approval + # for an older runtime to block finalization of the runtime now on chain. + concurrency: + group: watch-mainnet-release-${{ needs.check.outputs.release_tag }} + cancel-in-progress: false # MIRROR_DEPLOY_KEY lives in the mainnet environment secrets; one # approval of the run covers this and the publish jobs below. environment: mainnet @@ -234,6 +249,37 @@ jobs: ref: ${{ needs.check.outputs.sha }} ssh-key: ${{ secrets.MIRROR_DEPLOY_KEY }} + # Approval can arrive long after the check job captured chain state. Fail + # closed before touching the mirror or release if a newer runtime won in + # the meantime. + - name: Revalidate finalized runtime after approval + env: + EXPECTED_SPEC: ${{ needs.check.outputs.spec_version }} + EXPECTED_CODE_HASH: ${{ needs.check.outputs.code_hash }} + # Keep this check inline: the released commit checked out above may + # predate this workflow and cannot be expected to contain a new helper. + run: | + set -euo pipefail + finalized_head=$(curl -sf -H "Content-Type: application/json" \ + -d '{"id":1,"jsonrpc":"2.0","method":"chain_getFinalizedHead","params":[]}' \ + "$MAINNET_HTTP" | jq -er \ + '.result | strings | select(test("^0x[0-9a-f]{64}$"))') + runtime_request=$(jq -cn --arg block "$finalized_head" \ + '{id: 1, jsonrpc: "2.0", method: "state_getRuntimeVersion", params: [$block]}') + current_spec=$(curl -sf -H "Content-Type: application/json" \ + -d "$runtime_request" "$MAINNET_HTTP" | jq -er '.result.specVersion') + code_hash_request=$(jq -cn --arg block "$finalized_head" \ + '{id: 1, jsonrpc: "2.0", method: "state_getStorageHash", + params: ["0x3a636f6465", $block]}') + current_code_hash=$(curl -sf -H "Content-Type: application/json" \ + -d "$code_hash_request" "$MAINNET_HTTP" | jq -er \ + '.result | strings | select(test("^0x[0-9a-f]{64}$"))') + + [ "$current_spec" = "$EXPECTED_SPEC" ] \ + || { echo "mainnet advanced to spec $current_spec; expected $EXPECTED_SPEC"; exit 1; } + [ "$current_code_hash" = "$EXPECTED_CODE_HASH" ] \ + || { echo "mainnet code hash changed to $current_code_hash; expected $EXPECTED_CODE_HASH"; exit 1; } + # The release train uploaded the deterministic srtool build and the # multisig call data as the mainnet-upgrade- workflow artifact # (90-day retention). Attach it to the release so the deterministic @@ -353,7 +399,9 @@ jobs: build-core: name: Build bittensor-core wheels needs: check - if: needs.check.outputs.python_needed == 'true' + if: >- + needs.check.outputs.release_slot_available == 'true' && + needs.check.outputs.python_needed == 'true' uses: ./.github/workflows/build-core-wheels.yml with: ref: ${{ needs.check.outputs.sha }}