diff --git a/AGENTS.md b/AGENTS.md index d3bb732..0175ef1 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -752,6 +752,8 @@ npm run push -- --strict # Abort push if any validator npm run apply -- # Pull then push (full sync) npm run validate -- # Lint resources locally (fails fast on schema drift) npm run sim -- --suite --target # Run a simulation suite against an assistant/squad +npm run rollback -- --to # Re-apply a snapshot taken before a push +npm run rollback -- --list # List available snapshots # Testing npm run call -- -a # Call an assistant via WebSocket diff --git a/improvements.md b/improvements.md index 2df5c8e..6d9590f 100644 --- a/improvements.md +++ b/improvements.md @@ -54,7 +54,7 @@ you which stack PR closes the row.** | --- | -------------------------------------------------------- | -------------------------------------------------- | ---------- | --------------------------------- | | 1 | `push` drift detection | Prevent silent overwrites of dashboard edits | #4 | RESOLVED 2026-04-30 (Stack G) | | 2 | `apply` same-file conflict | `apply` drops concurrent same-file dashboard edits | #4 | Partial — Stack G GET on push | -| 3 | Rollback | Current undo can clobber newer live changes | #4, #5 | Open (Stack H planned) | +| 3 | Rollback | Current undo can clobber newer live changes | #4, #5 | RESOLVED 2026-04-30 (Stack H) | | 4 | State schema content hashes | Architectural unlock for #1, #2, #3, #6, #7 | None | RESOLVED 2026-04-30 (Stack F) | | 5 | `push --dry-run` | Cheapest operator-safety win | None | RESOLVED 2026-04-30 (Stack C) | | 6 | API-level optimistic concurrency | Server-side conflict rejection | Platform | Deferred (Stack I, gated) | diff --git a/package.json b/package.json index ef33816..2b124d5 100644 --- a/package.json +++ b/package.json @@ -15,6 +15,7 @@ "eval": "tsx src/eval.ts", "validate": "tsx src/validate-cmd.ts", "sim": "tsx src/sim-cmd.ts", + "rollback": "tsx src/rollback-cmd.ts", "build": "tsc --noEmit", "test": "node --import tsx --test tests/*.test.ts" }, diff --git a/src/push.ts b/src/push.ts index 5269345..e086a30 100644 --- a/src/push.ts +++ b/src/push.ts @@ -14,6 +14,20 @@ import { } from "./config.ts"; import { summarizeFindings, validateResources } from "./validate.ts"; import { checkDriftForUpdate } from "./drift.ts"; +import { writeSnapshot } from "./snapshot.ts"; + +// Map a resource label to its state-file key. Used for snapshotting (Stack H) +// — snapshot directories are keyed by the same names the state file uses. +const RESOURCE_LABEL_TO_TYPE: Record = { + tool: "tools", + "structured output": "structuredOutputs", + assistant: "assistants", + squad: "squads", + personality: "personalities", + scenario: "scenarios", + simulation: "simulations", + "simulation suite": "simulationSuites", +}; import { hashPayload, loadState, @@ -88,6 +102,8 @@ async function upsertResourceWithStateRecovery(options: { // payload, hash it, and compare to lastPulledHash. Refuse to overwrite // without --overwrite. Skipped in dry-run because the operator just // wants to see what would happen, and skipped if no baseline hash. + // Stack H — when we successfully fetch the platform payload, snapshot + // it (and our outgoing payload) so `npm run rollback` has a target. if (!DRY_RUN) { const stateEntry = stateSection[resourceId]; if (stateEntry) { @@ -113,6 +129,47 @@ async function upsertResourceWithStateRecovery(options: { ". Continuing.", ); } + + // Snapshot the current platform payload + our outgoing payload to a + // per-push directory so rollback can revert. Costs one extra GET per + // resource — acceptable for the safety guarantee. (Follow-up: plumb + // drift's GET result through to avoid the duplicate fetch.) + try { + const resourceType = RESOURCE_LABEL_TO_TYPE[resourceLabel]; + if (resourceType) { + const platformResponse = await fetch( + `${VAPI_BASE_URL}${updateEndpoint}`, + { + method: "GET", + headers: { + Authorization: `Bearer ${process.env.VAPI_TOKEN}`, + }, + }, + ); + if (platformResponse.ok) { + const platformPayloadForSnapshot = await platformResponse.json(); + await writeSnapshot({ + baseDir: BASE_DIR, + env: VAPI_ENV, + resourceType, + resourceId, + payload: { + outgoing: updatePayload, + platform: platformPayloadForSnapshot, + }, + }); + } + } + } catch (snapshotErr) { + // Snapshot failures should NOT block the push — the snapshot is a + // safety net, not a precondition. Log and move on. + console.warn( + ` ⚠️ snapshot failed for ${resourceLabel} ${resourceId}: ` + + (snapshotErr instanceof Error + ? snapshotErr.message + : String(snapshotErr)), + ); + } } } diff --git a/src/rollback-cmd.ts b/src/rollback-cmd.ts new file mode 100644 index 0000000..b931968 --- /dev/null +++ b/src/rollback-cmd.ts @@ -0,0 +1,212 @@ +// CLI entry: `npm run rollback -- --to ` | +// `npm run rollback -- --list` +// +// Reads .vapi-state..snapshots///.json +// and re-applies the captured *platform* payload via PATCH, restoring the +// dashboard to its state at that snapshot moment. +// +// Self-contained (does not import config.ts) so it can run in isolation +// without triggering the global CLI parser. + +import { existsSync, readFileSync } from "fs"; +import { dirname, join } from "path"; +import { fileURLToPath } from "url"; +import { + listSnapshotTimestamps, + loadSnapshot, +} from "./snapshot.ts"; + +const __dirname = dirname(fileURLToPath(import.meta.url)); +const BASE_DIR = join(__dirname, ".."); + +interface RollbackEnv { + env: string; + token: string; + baseUrl: string; +} + +function loadEnvFile(env: string): RollbackEnv { + const envFiles = [ + join(BASE_DIR, `.env.${env}`), + join(BASE_DIR, `.env.${env}.local`), + join(BASE_DIR, ".env.local"), + ]; + const envVars: Record = {}; + for (const envFile of envFiles) { + if (!existsSync(envFile)) continue; + for (const line of readFileSync(envFile, "utf-8").split("\n")) { + const trimmed = line.trim(); + if (!trimmed || trimmed.startsWith("#")) continue; + const eq = trimmed.indexOf("="); + if (eq === -1) continue; + const key = trimmed.slice(0, eq).trim(); + let value = trimmed.slice(eq + 1).trim(); + if ( + (value.startsWith('"') && value.endsWith('"')) || + (value.startsWith("'") && value.endsWith("'")) + ) { + value = value.slice(1, -1); + } + if (envVars[key] === undefined) envVars[key] = value; + } + } + const token = process.env.VAPI_TOKEN || envVars.VAPI_TOKEN; + const baseUrl = + process.env.VAPI_BASE_URL || + envVars.VAPI_BASE_URL || + "https://api.vapi.ai"; + if (!token) { + console.error(`❌ VAPI_TOKEN not found. Create .env.${env} with VAPI_TOKEN=your-token`); + process.exit(1); + } + return { env, token, baseUrl }; +} + +function printUsage(): void { + console.error( + [ + "Usage:", + " npm run rollback -- --list", + " npm run rollback -- --to ", + "", + "Snapshots are written automatically before each `npm run push` operation", + "to .vapi-state..snapshots//. Use --list to inspect available", + "timestamps; use --to to re-apply the platform payloads from that snapshot.", + ].join("\n"), + ); +} + +const ENDPOINT_MAP: Record = { + tools: "/tool", + structuredOutputs: "/structured-output", + assistants: "/assistant", + squads: "/squad", + personalities: "/eval/simulation/personality", + scenarios: "/eval/simulation/scenario", + simulations: "/eval/simulation", + simulationSuites: "/eval/simulation/suite", + evals: "/eval", +}; + +interface ParsedArgs { + env: string; + list: boolean; + to?: string; +} + +function parseArgs(): ParsedArgs { + const args = process.argv.slice(2); + const env = args[0]; + if (!env) { + printUsage(); + process.exit(1); + } + const SLUG_RE = /^[a-z0-9]([a-z0-9-]*[a-z0-9])?$/; + if (!SLUG_RE.test(env)) { + console.error(`❌ Invalid org name: ${env}`); + process.exit(1); + } + const parsed: ParsedArgs = { env, list: false }; + for (let i = 1; i < args.length; i++) { + const a = args[i]; + if (a === "--list") parsed.list = true; + else if (a === "--to") parsed.to = args[++i]; + else if (a === "--help" || a === "-h") { + printUsage(); + process.exit(0); + } + } + if (!parsed.list && !parsed.to) { + console.error("❌ Specify --list or --to "); + printUsage(); + process.exit(1); + } + return parsed; +} + +async function main(): Promise { + const args = parseArgs(); + + if (args.list) { + const timestamps = await listSnapshotTimestamps(BASE_DIR, args.env); + if (timestamps.length === 0) { + console.log(`No snapshots found for ${args.env}.`); + return; + } + console.log(`Snapshots for ${args.env}:`); + for (const t of timestamps) console.log(` ${t}`); + return; + } + + const cfg = loadEnvFile(args.env); + const entries = await loadSnapshot(BASE_DIR, args.env, args.to!); + if (entries.length === 0) { + console.log("Snapshot directory exists but contains no resources."); + return; + } + + // We need state so we can resolve resourceId → UUID for the PATCH path. + // Snapshot files don't store the UUID directly because the snapshot is + // keyed by resourceId; the same resourceId points at the same UUID across + // pushes (unless renamed, in which case the snapshot is stale anyway). + const stateFile = join(BASE_DIR, `.vapi-state.${args.env}.json`); + if (!existsSync(stateFile)) { + console.error(`❌ State file not found: ${stateFile}`); + process.exit(1); + } + const state = JSON.parse(readFileSync(stateFile, "utf-8")) as Record< + string, + Record + >; + + console.log(`🔁 Rollback ${args.env} → snapshot ${args.to}`); + console.log(` ${entries.length} resource(s) to restore\n`); + + let restored = 0; + let skipped = 0; + for (const entry of entries) { + const endpoint = ENDPOINT_MAP[entry.resourceType]; + if (!endpoint) { + console.warn(` ⚠️ Unknown resource type: ${entry.resourceType}, skipping`); + skipped++; + continue; + } + const section = state[entry.resourceType]; + const uuid = section?.[entry.resourceId]?.uuid; + if (!uuid) { + console.warn( + ` ⚠️ No UUID in state for ${entry.resourceType}/${entry.resourceId} — skipping`, + ); + skipped++; + continue; + } + process.stdout.write(` 🔁 ${entry.resourceType}/${entry.resourceId} ... `); + const response = await fetch(`${cfg.baseUrl}${endpoint}/${uuid}`, { + method: "PATCH", + headers: { + Authorization: `Bearer ${cfg.token}`, + "Content-Type": "application/json", + }, + body: JSON.stringify(entry.payload.platform), + }); + if (!response.ok) { + const text = await response.text(); + console.log(`❌ ${response.status}`); + console.error(` ${text}`); + skipped++; + continue; + } + console.log("✅"); + restored++; + } + + console.log( + `\n📊 Rollback summary: ${restored} restored, ${skipped} skipped`, + ); + if (skipped > 0) process.exit(1); +} + +main().catch((error) => { + console.error("\n❌ Rollback failed:", error instanceof Error ? error.message : error); + process.exit(1); +}); diff --git a/src/snapshot.ts b/src/snapshot.ts new file mode 100644 index 0000000..fff2b9b --- /dev/null +++ b/src/snapshot.ts @@ -0,0 +1,122 @@ +// ───────────────────────────────────────────────────────────────────────────── +// Snapshot-on-push — Stack H +// +// Before each PATCH, write the *outgoing* (local) payload AND the *current +// platform* payload to a per-push directory: +// +// .vapi-state..snapshots///.json +// +// `npm run rollback -- --to ` re-applies each +// `platform` payload as a PATCH, restoring the dashboard to its state at +// the moment of the snapshot. +// +// Reuses Stack G's drift-fetch path: when drift detection ran for this +// PATCH, the GET'd platform payload is passed in here so we don't pay a +// second GET. Snapshots are local-operator state and are gitignored. +// ───────────────────────────────────────────────────────────────────────────── + +import { existsSync } from "fs"; +import { mkdir, readFile, readdir, writeFile } from "fs/promises"; +import { join } from "path"; +import { sortedKeysReplacer } from "./state-serialize.ts"; + +export function snapshotsRoot(baseDir: string, env: string): string { + return join(baseDir, `.vapi-state.${env}.snapshots`); +} + +let activeRunDir: string | null = null; + +// Pin a single timestamp per push run. All resources written during the run +// share one directory so rollback can target an entire push, not individual +// PATCHes. +export function getRunSnapshotDir(baseDir: string, env: string): string { + if (!activeRunDir) { + const timestamp = new Date().toISOString().replace(/[:.]/g, "-"); + activeRunDir = join(snapshotsRoot(baseDir, env), timestamp); + } + return activeRunDir; +} + +// Test-only: reset the in-process timestamp so successive pushes within a +// single test produce distinct snapshot dirs. +export function _resetRunSnapshotDir(): void { + activeRunDir = null; +} + +export interface SnapshotPayload { + // What WE were about to push. + outgoing: unknown; + // What was on the dashboard right before our push (drift baseline). + platform: unknown; +} + +export async function writeSnapshot(options: { + baseDir: string; + env: string; + resourceType: string; + resourceId: string; + payload: SnapshotPayload; +}): Promise { + const dir = join(getRunSnapshotDir(options.baseDir, options.env), options.resourceType); + await mkdir(dir, { recursive: true }); + const fileName = `${options.resourceId.replace(/\//g, "__")}.json`; + const filePath = join(dir, fileName); + await writeFile( + filePath, + JSON.stringify(options.payload, sortedKeysReplacer, 2) + "\n", + ); + return filePath; +} + +// ───────────────────────────────────────────────────────────────────────────── +// Rollback support +// ───────────────────────────────────────────────────────────────────────────── + +export async function listSnapshotTimestamps( + baseDir: string, + env: string, +): Promise { + const root = snapshotsRoot(baseDir, env); + if (!existsSync(root)) return []; + const entries = await readdir(root, { withFileTypes: true }); + return entries + .filter((e) => e.isDirectory()) + .map((e) => e.name) + .sort(); +} + +export interface SnapshotEntry { + resourceType: string; + resourceId: string; + payload: SnapshotPayload; +} + +export async function loadSnapshot( + baseDir: string, + env: string, + timestamp: string, +): Promise { + const dir = join(snapshotsRoot(baseDir, env), timestamp); + if (!existsSync(dir)) { + throw new Error(`Snapshot not found: ${dir}`); + } + const types = await readdir(dir, { withFileTypes: true }); + const entries: SnapshotEntry[] = []; + for (const t of types) { + if (!t.isDirectory()) continue; + const typeDir = join(dir, t.name); + const files = await readdir(typeDir); + for (const f of files) { + if (!f.endsWith(".json")) continue; + const content = await readFile(join(typeDir, f), "utf-8"); + const payload = JSON.parse(content) as SnapshotPayload; + const resourceId = f.replace(/\.json$/, "").replace(/__/g, "/"); + entries.push({ + resourceType: t.name, + resourceId, + payload, + }); + } + } + return entries; +} diff --git a/tests/snapshot.test.ts b/tests/snapshot.test.ts new file mode 100644 index 0000000..8aef921 --- /dev/null +++ b/tests/snapshot.test.ts @@ -0,0 +1,142 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import { mkdtempSync, rmSync, readFileSync } from "node:fs"; +import { join } from "node:path"; +import { tmpdir } from "node:os"; +import { + _resetRunSnapshotDir, + listSnapshotTimestamps, + loadSnapshot, + snapshotsRoot, + writeSnapshot, +} from "../src/snapshot.ts"; + +// Stack H — snapshot writer / reader unit tests. +// `rollback-cmd.ts` itself is a thin CLI shell over PATCH; covered manually. + +test("writeSnapshot writes outgoing+platform pair to per-run directory", async () => { + _resetRunSnapshotDir(); + const tempDir = mkdtempSync(join(tmpdir(), "vapi-snapshot-")); + try { + const filePath = await writeSnapshot({ + baseDir: tempDir, + env: "test-env", + resourceType: "assistants", + resourceId: "agent-a", + payload: { + outgoing: { name: "agent-a", version: 2 }, + platform: { name: "agent-a", version: 1 }, + }, + }); + const content = JSON.parse(readFileSync(filePath, "utf-8")); + assert.equal(content.outgoing.version, 2); + assert.equal(content.platform.version, 1); + } finally { + rmSync(tempDir, { recursive: true, force: true }); + _resetRunSnapshotDir(); + } +}); + +test("writeSnapshot stamps under snapshotsRoot///.json", async () => { + _resetRunSnapshotDir(); + const tempDir = mkdtempSync(join(tmpdir(), "vapi-snapshot-")); + try { + const filePath = await writeSnapshot({ + baseDir: tempDir, + env: "test-env", + resourceType: "tools", + resourceId: "end-call", + payload: { outgoing: {}, platform: {} }, + }); + assert.ok( + filePath.startsWith(snapshotsRoot(tempDir, "test-env")), + `expected ${filePath} to live under ${snapshotsRoot(tempDir, "test-env")}`, + ); + assert.ok(filePath.endsWith("/tools/end-call.json")); + } finally { + rmSync(tempDir, { recursive: true, force: true }); + _resetRunSnapshotDir(); + } +}); + +test("writeSnapshot escapes nested resourceIds (e.g. support/intake)", async () => { + _resetRunSnapshotDir(); + const tempDir = mkdtempSync(join(tmpdir(), "vapi-snapshot-")); + try { + const filePath = await writeSnapshot({ + baseDir: tempDir, + env: "test-env", + resourceType: "assistants", + resourceId: "support/intake", + payload: { outgoing: {}, platform: {} }, + }); + // Nested IDs use `__` as path separator escape so the snapshot file + // doesn't create accidental subdirs that confuse the loader. + assert.ok(filePath.endsWith("support__intake.json")); + } finally { + rmSync(tempDir, { recursive: true, force: true }); + _resetRunSnapshotDir(); + } +}); + +test("loadSnapshot round-trips written entries", async () => { + _resetRunSnapshotDir(); + const tempDir = mkdtempSync(join(tmpdir(), "vapi-snapshot-")); + try { + await writeSnapshot({ + baseDir: tempDir, + env: "test-env", + resourceType: "assistants", + resourceId: "agent-a", + payload: { + outgoing: { name: "Agent A", n: 2 }, + platform: { name: "Agent A", n: 1 }, + }, + }); + await writeSnapshot({ + baseDir: tempDir, + env: "test-env", + resourceType: "tools", + resourceId: "end-call", + payload: { outgoing: {}, platform: { fn: "x" } }, + }); + + const stamps = await listSnapshotTimestamps(tempDir, "test-env"); + assert.equal(stamps.length, 1); + const entries = await loadSnapshot(tempDir, "test-env", stamps[0]!); + assert.equal(entries.length, 2); + + const agent = entries.find((e) => e.resourceId === "agent-a"); + assert.ok(agent); + assert.equal((agent!.payload.platform as { n: number }).n, 1); + + const tool = entries.find((e) => e.resourceId === "end-call"); + assert.ok(tool); + assert.equal((tool!.payload.platform as { fn: string }).fn, "x"); + } finally { + rmSync(tempDir, { recursive: true, force: true }); + _resetRunSnapshotDir(); + } +}); + +test("listSnapshotTimestamps returns empty array when no snapshots", async () => { + const tempDir = mkdtempSync(join(tmpdir(), "vapi-snapshot-")); + try { + const stamps = await listSnapshotTimestamps(tempDir, "test-env"); + assert.deepEqual(stamps, []); + } finally { + rmSync(tempDir, { recursive: true, force: true }); + } +}); + +test("loadSnapshot throws when timestamp directory is missing", async () => { + const tempDir = mkdtempSync(join(tmpdir(), "vapi-snapshot-")); + try { + await assert.rejects( + loadSnapshot(tempDir, "test-env", "missing-timestamp"), + /Snapshot not found/, + ); + } finally { + rmSync(tempDir, { recursive: true, force: true }); + } +});