From 3897ac049c07fbc101bf6d2b87e21efe9444dd29 Mon Sep 17 00:00:00 2001 From: nova28 <17953305+nova28@users.noreply.github.com> Date: Wed, 9 Sep 2026 00:56:12 +0800 Subject: [PATCH 01/15] feat(office): key run dedup on assignment generation identity Fix silent dedup collisions where task_assigned:: keys were permanently unique per pair, causing legitimate re-assignments after 24h to be swallowed at Debug level with no trace. - Add tasks.assignment_generation, bumped on create and reassignment, carried (never re-read) through task events into dedup keys. - Route task_assigned, blocker-resolution, agent_error, routine dispatch, and agent-supplied keys through shared builders in internal/runs/dedupkeys so convergent producers derive identical keys for the same occurrence. - Producers that cannot resolve a generation enqueue keyless instead of minting a permanently-unique fallback key. - Report every dedup decision (windowed/durable hit, keyless) through shared reporters in internal/runs/service with expvar counters, replacing the previously discarded QueueOutcome. Implements docs/specs/office/requirements/run-dedup-generation.md. --- .../internal/backendapp/adapters_office.go | 2 + .../backendapp/adapters_office_wakeup_test.go | 95 +++ .../office/approvals/handler_security_test.go | 5 +- .../internal/office/approvals/service.go | 6 +- .../internal/office/approvals/service_test.go | 5 +- .../internal/office/channels/service.go | 2 +- .../internal/office/dashboard/handler_test.go | 1 + .../internal/office/dashboard/service.go | 10 +- .../office/dashboard/service_tasks.go | 18 +- .../dashboard/session_termination_test.go | 2 +- .../internal/office/onboarding/service.go | 7 +- .../repository/sqlite/base_migrations.go | 8 +- .../office/repository/sqlite/tasks.go | 44 +- .../repository/sqlite/tasks_ops_test.go | 62 +- .../office/repository/sqlite/tasks_test.go | 1 + .../run_dedup_generation_dispatch_test.go | 90 +++ .../run_dedup_generation_keys_test.go | 84 +++ .../internal/office/routines/service.go | 70 ++- .../internal/office/runtime/actions.go | 24 +- .../internal/office/runtime/actions_test.go | 5 +- .../office/scheduler/approval_adapter.go | 2 +- .../office/scheduler/dashboard_adapter.go | 13 +- .../internal/office/scheduler/reactivity.go | 105 +++- .../reactivity_children_completed_test.go | 2 +- apps/backend/internal/office/scheduler/run.go | 64 +- .../service/agent_working_status_test.go | 2 +- .../internal/office/service/base_test.go | 1 + .../internal/office/service/channels.go | 2 +- .../continuation_summary_reader_test.go | 4 +- .../office/service/event_subscribers.go | 50 +- .../event_subscribers_decision_test.go | 16 +- .../service/event_subscribers_engine_test.go | 4 +- .../event_subscribers_run_output_test.go | 18 +- .../office/service/event_subscribers_test.go | 12 +- .../internal/office/service/failure.go | 6 +- .../internal/office/service/failure_test.go | 2 +- apps/backend/internal/office/service/retry.go | 7 +- .../service/retry_ceo_self_escalation_test.go | 2 +- .../office/service/retry_ratelimit_test.go | 6 +- apps/backend/internal/office/service/run.go | 30 +- .../service/run_lifecycle_events_test.go | 14 +- .../internal/office/service/run_test.go | 14 +- .../scheduler_checkout_contention_test.go | 2 +- .../service/scheduler_checkout_error_test.go | 2 +- .../scheduler_checkout_inactive_agent_test.go | 4 +- .../scheduler_checkout_release_test.go | 6 +- .../office/service/scheduler_features_test.go | 18 +- .../scheduler_integration_routing_test.go | 8 +- .../service/scheduler_integration_test.go | 14 +- .../office/service/scheduler_recovery.go | 4 +- .../service/scheduler_run_outcome_test.go | 16 +- .../office/service/scheduler_runs_test.go | 8 +- .../service/scheduler_taskless_launch_test.go | 16 +- .../internal/office/service/task_assignee.go | 6 +- .../office/service/task_starter_test.go | 16 +- .../wo46_idle_skip_routine_dispatch_test.go | 4 +- .../internal/office/shared/interfaces.go | 8 +- .../orchestrator/event_handlers_workflow.go | 12 +- ...handlers_workflow_office_autostart_test.go | 14 + .../internal/runs/dedupkeys/dedupkeys.go | 38 ++ .../internal/runs/dedupkeys/dedupkeys_test.go | 43 ++ apps/backend/internal/runs/service/dedup.go | 116 ++++ .../internal/runs/service/dedup_test.go | 127 ++++ .../internal/runs/service/metrics_vars.go | 34 ++ .../runs/service/queue_outcome_none_test.go | 22 + apps/backend/internal/runs/service/service.go | 58 +- .../task/repository/sqlite/base_migrations.go | 6 + .../internal/task/repository/sqlite/task.go | 8 + .../internal/task/service/service_tasks.go | 14 +- ...ervice_tasks_assignment_generation_test.go | 119 ++++ .../internal/workflow/engine/adapters.go | 6 + .../requirements/run-dedup-generation.md | 337 +++++++++++ .../system-design/run-dedup-generation-01.md | 568 ++++++++++++++++++ .../system-design/run-dedup-generation-02.md | 393 ++++++++++++ .../system-design/run-dedup-generation-03.md | 505 ++++++++++++++++ 75 files changed, 3165 insertions(+), 304 deletions(-) create mode 100644 apps/backend/internal/backendapp/adapters_office_wakeup_test.go create mode 100644 apps/backend/internal/office/routines/run_dedup_generation_dispatch_test.go create mode 100644 apps/backend/internal/office/routines/run_dedup_generation_keys_test.go create mode 100644 apps/backend/internal/runs/dedupkeys/dedupkeys.go create mode 100644 apps/backend/internal/runs/dedupkeys/dedupkeys_test.go create mode 100644 apps/backend/internal/runs/service/dedup.go create mode 100644 apps/backend/internal/runs/service/dedup_test.go create mode 100644 apps/backend/internal/runs/service/metrics_vars.go create mode 100644 apps/backend/internal/runs/service/queue_outcome_none_test.go create mode 100644 apps/backend/internal/task/service/service_tasks_assignment_generation_test.go create mode 100644 docs/specs/office/requirements/run-dedup-generation.md create mode 100644 docs/specs/office/system-design/run-dedup-generation-01.md create mode 100644 docs/specs/office/system-design/run-dedup-generation-02.md create mode 100644 docs/specs/office/system-design/run-dedup-generation-03.md diff --git a/apps/backend/internal/backendapp/adapters_office.go b/apps/backend/internal/backendapp/adapters_office.go index b4e1669a456..378f5fd3cb1 100644 --- a/apps/backend/internal/backendapp/adapters_office.go +++ b/apps/backend/internal/backendapp/adapters_office.go @@ -17,6 +17,7 @@ import ( officeroutines "github.com/kandev/kandev/internal/office/routines" officeservice "github.com/kandev/kandev/internal/office/service" officewakeup "github.com/kandev/kandev/internal/office/wakeup" + runsservice "github.com/kandev/kandev/internal/runs/service" "github.com/kandev/kandev/internal/task/models" tasksqlite "github.com/kandev/kandev/internal/task/repository/sqlite" taskservice "github.com/kandev/kandev/internal/task/service" @@ -266,6 +267,7 @@ func (a *routineWakeupAdapter) CreateWakeupRequest( } if err := a.repo.CreateWakeupRequest(ctx, row); err != nil { if errors.Is(err, officesqlite.ErrWakeupIdempotencyConflict) { + runsservice.ReportDurableDedup(runsservice.QueueSourceWakeup, req.Reason, req.IdempotencyKey, req.AgentProfileID) return officeroutines.ErrWakeupAlreadyRequested } return err diff --git a/apps/backend/internal/backendapp/adapters_office_wakeup_test.go b/apps/backend/internal/backendapp/adapters_office_wakeup_test.go new file mode 100644 index 00000000000..f627b8e9157 --- /dev/null +++ b/apps/backend/internal/backendapp/adapters_office_wakeup_test.go @@ -0,0 +1,95 @@ +package backendapp + +import ( + "context" + "errors" + "expvar" + "strings" + "testing" + + "github.com/jmoiron/sqlx" + _ "github.com/mattn/go-sqlite3" + + settingsstore "github.com/kandev/kandev/internal/agent/settings/store" + officesqlite "github.com/kandev/kandev/internal/office/repository/sqlite" + officeroutines "github.com/kandev/kandev/internal/office/routines" +) + +func newTestRoutineWakeupAdapter(t *testing.T) *routineWakeupAdapter { + t.Helper() + db, err := sqlx.Open("sqlite3", ":memory:") + if err != nil { + t.Fatalf("open sqlite: %v", err) + } + t.Cleanup(func() { _ = db.Close() }) + if _, _, err := settingsstore.Provide(db, db, nil); err != nil { + t.Fatalf("settings store init: %v", err) + } + repo, err := officesqlite.NewWithDB(db, db, nil) + if err != nil { + t.Fatalf("new office repo: %v", err) + } + return &routineWakeupAdapter{repo: repo} +} + +// AC-OFFICE-RUN-DEDUP-004.6: a durable idempotency conflict on the wakeup +// queue moves office_run_dedup_total{queue="wakeup",kind="durable"} through +// ReportDurableDedup, not ReportInsertResult - the adapter does its own +// errors.Is on the repository's sentinel and reports the conflict itself. +func TestRoutineWakeupAdapter_CreateWakeupRequest_DurableConflictReportsCounter(t *testing.T) { + adapter := newTestRoutineWakeupAdapter(t) + ctx := context.Background() + reason := "test_routine_wakeup_conflict_" + t.Name() + key := "routine:r1:t1:tick:1234567890" + + first := &officeroutines.WakeupRequest{ + ID: "wakeup-1", AgentProfileID: "agent-1", Source: "cron", + Reason: reason, IdempotencyKey: key, + } + if err := adapter.CreateWakeupRequest(ctx, first); err != nil { + t.Fatalf("first create: %v", err) + } + + second := &officeroutines.WakeupRequest{ + ID: "wakeup-2", AgentProfileID: "agent-1", Source: "cron", + Reason: reason, IdempotencyKey: key, + } + err := adapter.CreateWakeupRequest(ctx, second) + if !errors.Is(err, officesqlite.ErrWakeupIdempotencyConflict) { + t.Fatalf("second create error = %v, want ErrWakeupIdempotencyConflict", err) + } + + if !counterHasLabel(t, "office_run_dedup_total", "reason="+reason, "kind=durable", "queue=wakeup") { + t.Fatal("expected office_run_dedup_total to carry a durable/wakeup entry for this reason") + } +} + +// counterHasLabel reports whether the named expvar.Map has an entry whose +// key contains every given substring. Counters are process-global expvar +// state (no reset hook), so tests assert presence/growth rather than exact +// values that could collide with other tests in the same binary run. +func counterHasLabel(t *testing.T, mapName string, substrs ...string) bool { + t.Helper() + v := expvar.Get(mapName) + if v == nil { + t.Fatalf("expvar map %q not registered", mapName) + } + m, ok := v.(*expvar.Map) + if !ok { + t.Fatalf("expvar %q is not a *expvar.Map", mapName) + } + found := false + m.Do(func(kv expvar.KeyValue) { + matches := true + for _, s := range substrs { + if !strings.Contains(kv.Key, s) { + matches = false + break + } + } + if matches { + found = true + } + }) + return found +} diff --git a/apps/backend/internal/office/approvals/handler_security_test.go b/apps/backend/internal/office/approvals/handler_security_test.go index 76d7bac69a3..7cb8c50de43 100644 --- a/apps/backend/internal/office/approvals/handler_security_test.go +++ b/apps/backend/internal/office/approvals/handler_security_test.go @@ -18,6 +18,7 @@ import ( "github.com/kandev/kandev/internal/office/models" "github.com/kandev/kandev/internal/office/repository/sqlite" "github.com/kandev/kandev/internal/office/shared" + runsservice "github.com/kandev/kandev/internal/runs/service" ) // approvalHandlerFixture wires the minimal stack needed to exercise the @@ -72,7 +73,9 @@ func (s *silentActivityLogger) LogActivityWithRun(_ context.Context, _, _, _, _, type silentRunQueuer struct{} -func (s *silentRunQueuer) QueueRun(_ context.Context, _, _, _, _ string) error { return nil } +func (s *silentRunQueuer) QueueRun(_ context.Context, _, _, _, _ string) (runsservice.QueueOutcome, error) { + return runsservice.QueueOutcomeQueued, nil +} // seedApprovalAgent creates an agent_profiles row with the given role // and permissions, returning the persisted instance. diff --git a/apps/backend/internal/office/approvals/service.go b/apps/backend/internal/office/approvals/service.go index aaef3d2a01b..5408f7645a2 100644 --- a/apps/backend/internal/office/approvals/service.go +++ b/apps/backend/internal/office/approvals/service.go @@ -9,6 +9,7 @@ import ( "github.com/kandev/kandev/internal/common/logger" "github.com/kandev/kandev/internal/office/models" "github.com/kandev/kandev/internal/office/shared" + runsservice "github.com/kandev/kandev/internal/runs/service" "go.uber.org/zap" ) @@ -29,7 +30,7 @@ type AgentWriter interface { // RunQueuer enqueues run requests for agent instances. type RunQueuer interface { - QueueRun(ctx context.Context, agentInstanceID, reason, payload, idempotencyKey string) error + QueueRun(ctx context.Context, agentInstanceID, reason, payload, idempotencyKey string) (runsservice.QueueOutcome, error) } // ApprovalService handles approval CRUD and decide logic. @@ -215,6 +216,7 @@ func (s *ApprovalService) queueApprovalRun(ctx context.Context, approval *Approv approval.ID, approval.Type, approval.Status, approval.DecisionNote, ) idempotencyKey := "approval:" + approval.ID - return s.runs.QueueRun(ctx, approval.RequestedByAgentProfileID, + _, err := s.runs.QueueRun(ctx, approval.RequestedByAgentProfileID, "approval_resolved", payload, idempotencyKey) + return err } diff --git a/apps/backend/internal/office/approvals/service_test.go b/apps/backend/internal/office/approvals/service_test.go index 1186011164c..33206ea7b7f 100644 --- a/apps/backend/internal/office/approvals/service_test.go +++ b/apps/backend/internal/office/approvals/service_test.go @@ -11,6 +11,7 @@ import ( "github.com/kandev/kandev/internal/office/approvals" "github.com/kandev/kandev/internal/office/models" "github.com/kandev/kandev/internal/office/repository/sqlite" + runsservice "github.com/kandev/kandev/internal/runs/service" ) // noopActivityLogger implements shared.ActivityLogger without importing shared. @@ -23,7 +24,9 @@ func (n *noopActivityLogger) LogActivityWithRun(_ context.Context, _, _, _, _, _ // noopRunQueuer implements approvals.RunQueuer as a no-op. type noopRunQueuer struct{} -func (n *noopRunQueuer) QueueRun(_ context.Context, _, _, _, _ string) error { return nil } +func (n *noopRunQueuer) QueueRun(_ context.Context, _, _, _, _ string) (runsservice.QueueOutcome, error) { + return runsservice.QueueOutcomeQueued, nil +} type fakeAgentWriter struct { statuses map[string]string diff --git a/apps/backend/internal/office/channels/service.go b/apps/backend/internal/office/channels/service.go index 5311fa31d65..7971ffdf6da 100644 --- a/apps/backend/internal/office/channels/service.go +++ b/apps/backend/internal/office/channels/service.go @@ -103,7 +103,7 @@ func (s *ChannelService) SetupChannel(ctx context.Context, channel *models.Chann // workflow_step_participants. Channel tasks have no workflow_step_id; // the participant row is keyed at (step_id="", task_id) and the // runner projection still resolves it. - if err := s.repo.UpdateTaskAssignee(ctx, taskID, channel.AgentProfileID); err != nil { + if _, err := s.repo.UpdateTaskAssignee(ctx, taskID, channel.AgentProfileID); err != nil { _ = s.repo.DeleteChannel(ctx, channel.ID) return fmt.Errorf("set channel task assignee: %w", err) } diff --git a/apps/backend/internal/office/dashboard/handler_test.go b/apps/backend/internal/office/dashboard/handler_test.go index 4c79e9066f4..940bc7ef527 100644 --- a/apps/backend/internal/office/dashboard/handler_test.go +++ b/apps/backend/internal/office/dashboard/handler_test.go @@ -141,6 +141,7 @@ func newTestDeps(t *testing.T) *testDeps { project_id TEXT DEFAULT '', assignee_agent_profile_id TEXT DEFAULT '', assignee_user_id TEXT NOT NULL DEFAULT '', + assignment_generation INTEGER NOT NULL DEFAULT 0, labels TEXT DEFAULT '[]', metadata TEXT DEFAULT '{}', identifier TEXT DEFAULT '', diff --git a/apps/backend/internal/office/dashboard/service.go b/apps/backend/internal/office/dashboard/service.go index 53aafc317ed..1bdacb2d5bf 100644 --- a/apps/backend/internal/office/dashboard/service.go +++ b/apps/backend/internal/office/dashboard/service.go @@ -73,7 +73,10 @@ type Repository interface { GetRunsByCommentIDs(ctx context.Context, commentIDs []string) (map[string]sqlite.CommentRunStatus, error) UpdateTaskState(ctx context.Context, taskID, state string) error GetTaskExecutionFields(ctx context.Context, taskID string) (*sqlite.TaskExecutionFields, error) - UpdateTaskAssignee(ctx context.Context, taskID, assigneeID string) error + // UpdateTaskAssignee returns the task's assignment_generation after the + // bump, read back inside the same transaction that wrote the runner + // seat (see the sqlite implementation's doc comment). + UpdateTaskAssignee(ctx context.Context, taskID, assigneeID string) (int64, error) UpdateTaskPriority(ctx context.Context, taskID, priority string) error UpdateTaskProjectID(ctx context.Context, taskID, projectID string) error GetTaskProjectID(ctx context.Context, taskID string) (string, error) @@ -252,6 +255,11 @@ type MarkFixedHandler interface { type TaskReactivityChange struct { NewStatus *string NewAssigneeID *string + // AssignmentGeneration is the value UpdateTaskAssignee's transaction + // committed and read back, carried here rather than re-read. Nil means + // the caller could not supply one (e.g. the read-back itself failed); + // the pipeline then enqueues the task_assigned wake keyless. + AssignmentGeneration *int64 // PrevAssigneeID is the assignee BEFORE the mutation. Required when // NewAssigneeID is set so the pipeline can detect a real change and // hand off the previous assignee's session. diff --git a/apps/backend/internal/office/dashboard/service_tasks.go b/apps/backend/internal/office/dashboard/service_tasks.go index b83b974bda6..775e7fe1ddf 100644 --- a/apps/backend/internal/office/dashboard/service_tasks.go +++ b/apps/backend/internal/office/dashboard/service_tasks.go @@ -868,7 +868,8 @@ func (s *DashboardService) SetTaskAssigneeAsAgent(ctx context.Context, callerAge zap.String("task_id", taskID), zap.Error(err)) } } - if err := s.repo.UpdateTaskAssignee(ctx, taskID, assigneeID); err != nil { + generation, err := s.repo.UpdateTaskAssignee(ctx, taskID, assigneeID) + if err != nil { return err } @@ -876,14 +877,16 @@ func (s *DashboardService) SetTaskAssigneeAsAgent(ctx context.Context, callerAge // Reactivity pipeline — wakes the new assignee with task_assigned // and hard-cancels the previous assignee's active session. - s.runReactivityForAssigneeChange(ctx, taskID, prevAssignee, assigneeID, callerAgentID) + s.runReactivityForAssigneeChange(ctx, taskID, prevAssignee, assigneeID, callerAgentID, generation) return nil } // runReactivityForAssigneeChange invokes the reactivity pipeline for an // assignee change. Best-effort — failures are logged, never propagated. +// generation is the value UpdateTaskAssignee's transaction just committed +// and read back; it is carried onto the mutation rather than re-read. func (s *DashboardService) runReactivityForAssigneeChange( - ctx context.Context, taskID, prevAssigneeID, newAssigneeID, callerAgentID string, + ctx context.Context, taskID, prevAssigneeID, newAssigneeID, callerAgentID string, generation int64, ) { if s.reactivity == nil { return @@ -893,10 +896,11 @@ func (s *DashboardService) runReactivityForAssigneeChange( actorType = "agent" } change := TaskReactivityChange{ - NewAssigneeID: &newAssigneeID, - PrevAssigneeID: prevAssigneeID, - ActorID: callerAgentID, - ActorType: actorType, + NewAssigneeID: &newAssigneeID, + AssignmentGeneration: &generation, + PrevAssigneeID: prevAssigneeID, + ActorID: callerAgentID, + ActorType: actorType, } // preStatus="" — assignee changes don't depend on the prev status. result, err := s.reactivity.ApplyTaskMutation(ctx, taskID, "", change) diff --git a/apps/backend/internal/office/dashboard/session_termination_test.go b/apps/backend/internal/office/dashboard/session_termination_test.go index 0ee901e7f27..df96ce381fa 100644 --- a/apps/backend/internal/office/dashboard/session_termination_test.go +++ b/apps/backend/internal/office/dashboard/session_termination_test.go @@ -90,7 +90,7 @@ func TestSetTaskAssignee_TerminatesPrevSession(t *testing.T) { insertTestTask(t, deps.db, "task-r", "ws-r", "Reassign", "todo", 2) // Seed prev assignee directly via the underlying repo update. - if err := deps.repo.UpdateTaskAssignee(context.Background(), "task-r", "agent-prev"); err != nil { + if _, err := deps.repo.UpdateTaskAssignee(context.Background(), "task-r", "agent-prev"); err != nil { t.Fatalf("seed prev assignee: %v", err) } diff --git a/apps/backend/internal/office/onboarding/service.go b/apps/backend/internal/office/onboarding/service.go index 88ca65afbf6..aff0ca4eefe 100644 --- a/apps/backend/internal/office/onboarding/service.go +++ b/apps/backend/internal/office/onboarding/service.go @@ -17,6 +17,7 @@ import ( "github.com/kandev/kandev/internal/office/repository/sqlite" "github.com/kandev/kandev/internal/office/routing" "github.com/kandev/kandev/internal/office/shared" + runsservice "github.com/kandev/kandev/internal/runs/service" taskservice "github.com/kandev/kandev/internal/task/service" "go.uber.org/zap" @@ -349,7 +350,11 @@ func (s *OnboardingService) maybeCreateOnboardingTask( return "" } if s.runQueuer != nil { - if wakeErr := s.runQueuer.QueueRun(ctx, agentID, runReasonTaskAssigned, + // A third task_assigned producer alongside queueTaskAssignedRun; it is + // never handed the assigning transaction's generation, so it enqueues + // keyless rather than deriving a divergent key. + runsservice.ReportKeylessEnqueue(runReasonTaskAssigned, runsservice.KeylessCauseUnresolved, "onboarding_no_generation") + if _, wakeErr := s.runQueuer.QueueRun(ctx, agentID, runReasonTaskAssigned, fmt.Sprintf(`{"task_id":%q}`, taskID), ""); wakeErr != nil { s.logger.Warn("enqueue onboarding run failed", zap.Error(wakeErr)) } diff --git a/apps/backend/internal/office/repository/sqlite/base_migrations.go b/apps/backend/internal/office/repository/sqlite/base_migrations.go index 2d950c25eb6..7db2fbd23fd 100644 --- a/apps/backend/internal/office/repository/sqlite/base_migrations.go +++ b/apps/backend/internal/office/repository/sqlite/base_migrations.go @@ -567,6 +567,7 @@ func (r *Repository) runTaskPriorityRecreate() error { {"external_id", `ALTER TABLE tasks ADD COLUMN external_id TEXT COLLATE BINARY`}, {"external_id_settled_at", `ALTER TABLE tasks ADD COLUMN external_id_settled_at TIMESTAMP`}, {"assignee_user_id", `ALTER TABLE tasks ADD COLUMN assignee_user_id TEXT NOT NULL DEFAULT ''`}, + {"assignment_generation", `ALTER TABLE tasks ADD COLUMN assignment_generation INTEGER NOT NULL DEFAULT 0`}, } for _, column := range legacyColumns { if _, err := conn.ExecContext(ctx, column.stmt); err != nil && !db.IsDuplicateColumnError(err) { @@ -629,7 +630,8 @@ func taskPriorityMigrationStatements() []string { checkout_run_id TEXT, external_id TEXT COLLATE BINARY, external_id_settled_at TIMESTAMP, - assignee_user_id TEXT NOT NULL DEFAULT '' + assignee_user_id TEXT NOT NULL DEFAULT '', + assignment_generation INTEGER NOT NULL DEFAULT 0 )`, // archived_by_cascade_id and external_id/external_id_settled_at are // added to the task schema by task/repository/sqlite/base.go @@ -649,7 +651,7 @@ func taskPriorityMigrationStatements() []string { origin, project_id, labels, identifier, checkout_agent_id, checkout_at, checkout_run_id, - external_id, external_id_settled_at, assignee_user_id + external_id, external_id_settled_at, assignee_user_id, assignment_generation ) SELECT id, COALESCE(workspace_id,''), COALESCE(workflow_id,''), COALESCE(workflow_step_id,''), title, COALESCE(description,''), @@ -664,7 +666,7 @@ func taskPriorityMigrationStatements() []string { COALESCE(labels,'[]'), identifier, checkout_agent_id, checkout_at, checkout_run_id, external_id, external_id_settled_at, - COALESCE(assignee_user_id,'') + COALESCE(assignee_user_id,''), COALESCE(assignment_generation,0) FROM tasks`, `DROP TABLE tasks`, `ALTER TABLE tasks_priority_new RENAME TO tasks`, diff --git a/apps/backend/internal/office/repository/sqlite/tasks.go b/apps/backend/internal/office/repository/sqlite/tasks.go index 0e9c6ba0ab7..636518edb88 100644 --- a/apps/backend/internal/office/repository/sqlite/tasks.go +++ b/apps/backend/internal/office/repository/sqlite/tasks.go @@ -134,21 +134,32 @@ func (r *Repository) UpdateTaskState(ctx context.Context, taskID, state string) // task_id) so the projection's per-task runner clause still resolves it // (the (step_id="" / step_id="") match holds because the SELECT joins // step_id = task.workflow_step_id which is also ""). -func (r *Repository) UpdateTaskAssignee(ctx context.Context, taskID, assigneeID string) error { +// +// This is one of the two assignment_generation bump sites (the other is +// insertTaskTx -> upsertRunnerInTx on create). It increments +// tasks.assignment_generation unconditionally on every committed call — +// including a repeat assignment to the agent that already holds the seat, +// which is a real occurrence, not a no-op — and reads the new value back +// inside this same transaction before Commit, returning it so callers carry +// it forward instead of re-reading it later (a later re-read could observe a +// different, more recent occurrence's value). A read-back failure rolls the +// whole assignment back rather than commit a write whose generation could +// not be reported. +func (r *Repository) UpdateTaskAssignee(ctx context.Context, taskID, assigneeID string) (int64, error) { var stepID string err := r.ro.QueryRowxContext(ctx, r.ro.Rebind( `SELECT COALESCE(workflow_step_id, '') FROM tasks WHERE id = ?`), taskID).Scan(&stepID) if err != nil { if err == sql.ErrNoRows { - return fmt.Errorf("task not found: %s", taskID) + return 0, fmt.Errorf("task not found: %s", taskID) } - return err + return 0, err } tx, err := r.db.BeginTxx(ctx, nil) if err != nil { - return err + return 0, err } defer func() { _ = tx.Rollback() }() @@ -157,7 +168,7 @@ func (r *Repository) UpdateTaskAssignee(ctx context.Context, taskID, assigneeID DELETE FROM workflow_step_participants WHERE step_id = ? AND task_id = ? AND role = 'runner' `), stepID, taskID); err != nil { - return err + return 0, err } } else { var existing string @@ -170,7 +181,7 @@ func (r *Repository) UpdateTaskAssignee(ctx context.Context, taskID, assigneeID if _, err := tx.ExecContext(ctx, tx.Rebind( `UPDATE workflow_step_participants SET agent_profile_id = ? WHERE id = ?`), assigneeID, existing); err != nil { - return err + return 0, err } case sql.ErrNoRows: if _, err := tx.ExecContext(ctx, tx.Rebind(` @@ -178,19 +189,30 @@ func (r *Repository) UpdateTaskAssignee(ctx context.Context, taskID, assigneeID (id, step_id, task_id, role, agent_profile_id, decision_required, position, created_at) VALUES (?, ?, ?, 'runner', ?, 0, 0, ?) `), newParticipantUUID(), stepID, taskID, assigneeID, time.Now().UTC()); err != nil { - return err + return 0, err } default: - return probeErr + return 0, probeErr } } if _, err := tx.ExecContext(ctx, tx.Rebind(` - UPDATE tasks SET updated_at = CURRENT_TIMESTAMP WHERE id = ? + UPDATE tasks SET assignment_generation = assignment_generation + 1, updated_at = CURRENT_TIMESTAMP WHERE id = ? `), taskID); err != nil { - return err + return 0, err + } + + var generation int64 + if err := tx.QueryRowxContext(ctx, tx.Rebind( + `SELECT assignment_generation FROM tasks WHERE id = ?`), + taskID).Scan(&generation); err != nil { + return 0, err + } + + if err := tx.Commit(); err != nil { + return 0, err } - return tx.Commit() + return generation, nil } // TaskBasicInfo contains the minimal task fields needed for prompt building. diff --git a/apps/backend/internal/office/repository/sqlite/tasks_ops_test.go b/apps/backend/internal/office/repository/sqlite/tasks_ops_test.go index 18ff7cce145..d0012a48aa6 100644 --- a/apps/backend/internal/office/repository/sqlite/tasks_ops_test.go +++ b/apps/backend/internal/office/repository/sqlite/tasks_ops_test.go @@ -330,9 +330,13 @@ func TestUpdateTaskAssignee_WritesUpdatesAndClearsRunnerRow(t *testing.T) { t.Fatalf("seed task: %v", err) } - if err := repo.UpdateTaskAssignee(ctx, "as-1", "agent-one"); err != nil { + gen1, err := repo.UpdateTaskAssignee(ctx, "as-1", "agent-one") + if err != nil { t.Fatalf("UpdateTaskAssignee (insert): %v", err) } + if gen1 != 1 { + t.Fatalf("generation = %d, want 1 for the first assignment", gen1) + } if got := taskRunner(t, repo, "as-1"); got != "agent-one" { t.Fatalf("runner = %q, want agent-one", got) } @@ -340,10 +344,15 @@ func TestUpdateTaskAssignee_WritesUpdatesAndClearsRunnerRow(t *testing.T) { t.Fatalf("runner rows = %d, want 1", n) } - // A second assignment updates the existing row instead of adding one. - if err := repo.UpdateTaskAssignee(ctx, "as-1", "agent-two"); err != nil { + // A second assignment updates the existing row instead of adding one, + // and bumps the generation again. + gen2, err := repo.UpdateTaskAssignee(ctx, "as-1", "agent-two") + if err != nil { t.Fatalf("UpdateTaskAssignee (update): %v", err) } + if gen2 != 2 { + t.Fatalf("generation = %d, want 2 for the second assignment", gen2) + } if got := taskRunner(t, repo, "as-1"); got != "agent-two" { t.Errorf("runner = %q, want agent-two", got) } @@ -351,10 +360,15 @@ func TestUpdateTaskAssignee_WritesUpdatesAndClearsRunnerRow(t *testing.T) { t.Errorf("runner rows = %d, want the row updated in place", n) } - // An empty assignee deletes the runner row. - if err := repo.UpdateTaskAssignee(ctx, "as-1", ""); err != nil { + // An empty assignee deletes the runner row and still bumps the + // generation: an unassignment is itself an assignment write. + gen3, err := repo.UpdateTaskAssignee(ctx, "as-1", "") + if err != nil { t.Fatalf("UpdateTaskAssignee (clear): %v", err) } + if gen3 != 3 { + t.Fatalf("generation = %d, want 3 for the clear", gen3) + } if n := runnerRowCount(t, repo, "as-1"); n != 0 { t.Errorf("runner rows = %d, want 0 after clearing", n) } @@ -362,7 +376,7 @@ func TestUpdateTaskAssignee_WritesUpdatesAndClearsRunnerRow(t *testing.T) { t.Errorf("runner = %q, want empty after clearing", got) } - err := repo.UpdateTaskAssignee(ctx, "missing", "agent-one") + _, err = repo.UpdateTaskAssignee(ctx, "missing", "agent-one") if err == nil { t.Fatal("UpdateTaskAssignee(missing) = nil error, want not-found") } @@ -371,6 +385,34 @@ func TestUpdateTaskAssignee_WritesUpdatesAndClearsRunnerRow(t *testing.T) { } } +// TestUpdateTaskAssignee_RepeatAssignmentBumpsGeneration covers AC-001.3: a +// repeat assignment to the agent that already holds the seat is a real +// occurrence (the operator is asking for the work again), not a no-op, so it +// must mint a new generation rather than leave the counter unchanged. +func TestUpdateTaskAssignee_RepeatAssignmentBumpsGeneration(t *testing.T) { + repo := newSearchTestRepo(t) + ctx := context.Background() + + if _, err := repo.ExecRaw(ctx, ` + INSERT INTO tasks (id, workspace_id, workflow_step_id, title, created_at, updated_at) + VALUES ('as-repeat', 'ws-1', 'step-1', 'Repeat assign', datetime('now'), datetime('now')) + `); err != nil { + t.Fatalf("seed task: %v", err) + } + + gen1, err := repo.UpdateTaskAssignee(ctx, "as-repeat", "agent-a") + if err != nil { + t.Fatalf("first assignment: %v", err) + } + gen2, err := repo.UpdateTaskAssignee(ctx, "as-repeat", "agent-a") + if err != nil { + t.Fatalf("repeat assignment: %v", err) + } + if gen2 != gen1+1 { + t.Fatalf("repeat assignment generation = %d, want %d (one more than %d)", gen2, gen1+1, gen1) + } +} + // Tasks created outside a workflow have no workflow_step_id; the runner row // is keyed on the empty step so the projection still resolves it. func TestUpdateTaskAssignee_WorksWithoutAWorkflowStep(t *testing.T) { @@ -384,7 +426,7 @@ func TestUpdateTaskAssignee_WorksWithoutAWorkflowStep(t *testing.T) { t.Fatalf("seed task: %v", err) } - if err := repo.UpdateTaskAssignee(ctx, "as-nostep", "agent-channel"); err != nil { + if _, err := repo.UpdateTaskAssignee(ctx, "as-nostep", "agent-channel"); err != nil { t.Fatalf("UpdateTaskAssignee: %v", err) } if got := taskRunner(t, repo, "as-nostep"); got != "agent-channel" { @@ -405,13 +447,13 @@ func TestUpdateTaskAssignee_LeavesOtherTasksAlone(t *testing.T) { t.Fatalf("seed tasks: %v", err) } - if err := repo.UpdateTaskAssignee(ctx, "as-a", "agent-a"); err != nil { + if _, err := repo.UpdateTaskAssignee(ctx, "as-a", "agent-a"); err != nil { t.Fatalf("assign as-a: %v", err) } - if err := repo.UpdateTaskAssignee(ctx, "as-b", "agent-b"); err != nil { + if _, err := repo.UpdateTaskAssignee(ctx, "as-b", "agent-b"); err != nil { t.Fatalf("assign as-b: %v", err) } - if err := repo.UpdateTaskAssignee(ctx, "as-b", ""); err != nil { + if _, err := repo.UpdateTaskAssignee(ctx, "as-b", ""); err != nil { t.Fatalf("clear as-b: %v", err) } diff --git a/apps/backend/internal/office/repository/sqlite/tasks_test.go b/apps/backend/internal/office/repository/sqlite/tasks_test.go index 02145ba51e7..a6fadfa6a1d 100644 --- a/apps/backend/internal/office/repository/sqlite/tasks_test.go +++ b/apps/backend/internal/office/repository/sqlite/tasks_test.go @@ -36,6 +36,7 @@ func newSearchTestRepo(t *testing.T) *sqlite.Repository { workflow_step_id TEXT NOT NULL DEFAULT '', title TEXT NOT NULL DEFAULT '', assignee_user_id TEXT NOT NULL DEFAULT '', + assignment_generation INTEGER NOT NULL DEFAULT 0, description TEXT DEFAULT '', state TEXT DEFAULT 'TODO', priority TEXT NOT NULL DEFAULT 'medium' CHECK (priority IN ('critical','high','medium','low')), diff --git a/apps/backend/internal/office/routines/run_dedup_generation_dispatch_test.go b/apps/backend/internal/office/routines/run_dedup_generation_dispatch_test.go new file mode 100644 index 00000000000..b17ba118fe7 --- /dev/null +++ b/apps/backend/internal/office/routines/run_dedup_generation_dispatch_test.go @@ -0,0 +1,90 @@ +package routines_test + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/kandev/kandev/internal/office/models" +) + +// AC-OFFICE-RUN-DEDUP-001.9: the cron dispatch path threads the trigger's +// claimed tick into the wakeup-request's idempotency key rather than +// re-reading trigger.next_run_at after TickScheduledTriggers has already +// advanced it to the next slot. Driving this through TickScheduledTriggers +// (not the private key builder) is what actually pins the wiring: a future +// change that re-reads the trigger row after the advance would still pass +// the unit-level key-format tests but fail here. +func TestTickScheduledTriggers_WakeupKeyNamesClaimedTick_NotAdvancedRow(t *testing.T) { + svc := newTestRoutineService(t) + ctx := context.Background() + + routine := newLightweightTestRoutine(t, svc) + trigger := &models.RoutineTrigger{ + RoutineID: routine.ID, + Kind: "cron", + CronExpression: "* * * * *", + Timezone: "UTC", + Enabled: true, + } + if err := svc.CreateRoutineTrigger(ctx, trigger); err != nil { + t.Fatalf("create trigger: %v", err) + } + triggers, err := svc.ListRoutineTriggers(ctx, routine.ID) + if err != nil || len(triggers) != 1 { + t.Fatalf("list triggers: %v (n=%d)", err, len(triggers)) + } + claimedTick := *triggers[0].NextRunAt + + enq := &fakeWakeupEnqueuer{} + svc.SetWakeupEnqueuer(enq) + + if err := svc.TickScheduledTriggers(ctx, claimedTick.Add(2*time.Minute)); err != nil { + t.Fatalf("tick scheduled triggers: %v", err) + } + if len(enq.created) != 1 { + t.Fatalf("expected 1 wakeup-request created, got %d", len(enq.created)) + } + + want := fmt.Sprintf("routine:%s:%s:tick:%d", routine.ID, triggers[0].ID, claimedTick.Unix()) + got := enq.created[0].IdempotencyKey + if got != want { + t.Fatalf("idempotency key = %q, want %q (the claimed tick, not the row's advanced next_run_at)", got, want) + } +} + +// AC-OFFICE-RUN-DEDUP-001.2: two manual "Fire now" calls inside the same +// minute are two distinct occurrences (an operator asking twice) and must +// mint two distinct keys - the exact collision the old unix-minute key +// format reached (both fires share triggerID == "" and can share a minute). +func TestFireManual_TwoFiresInSameMinute_MintDistinctKeys(t *testing.T) { + svc := newTestRoutineService(t) + ctx := context.Background() + + routine := newLightweightTestRoutine(t, svc) + routine.ConcurrencyPolicy = "always_create" + if err := svc.UpdateRoutine(ctx, routine); err != nil { + t.Fatalf("update routine: %v", err) + } + + enq := &fakeWakeupEnqueuer{} + svc.SetWakeupEnqueuer(enq) + + if _, err := svc.FireManual(ctx, routine.ID, map[string]string{"name": "one"}); err != nil { + t.Fatalf("first fire: %v", err) + } + if _, err := svc.FireManual(ctx, routine.ID, map[string]string{"name": "two"}); err != nil { + t.Fatalf("second fire: %v", err) + } + if len(enq.created) != 2 { + t.Fatalf("expected 2 wakeup-requests created, got %d", len(enq.created)) + } + first, second := enq.created[0].IdempotencyKey, enq.created[1].IdempotencyKey + if first == "" || second == "" { + t.Fatalf("expected non-empty keys, got %q and %q", first, second) + } + if first == second { + t.Fatalf("two distinct manual fires must mint distinct keys, both got %q", first) + } +} diff --git a/apps/backend/internal/office/routines/run_dedup_generation_keys_test.go b/apps/backend/internal/office/routines/run_dedup_generation_keys_test.go new file mode 100644 index 00000000000..567c6fbf9bc --- /dev/null +++ b/apps/backend/internal/office/routines/run_dedup_generation_keys_test.go @@ -0,0 +1,84 @@ +package routines + +import ( + "fmt" + "testing" + "time" + + "github.com/kandev/kandev/internal/office/shared" +) + +// AC-OFFICE-RUN-DEDUP-001.1 / .4: a cron fire's key names the claimed +// scheduled slot, and redelivering the same slot (the same claimedTick) +// reproduces the same key byte for byte. +func TestBuildRoutineIdempotencyKey_Cron_UsesClaimedTick(t *testing.T) { + tick := time.Date(2026, 5, 10, 12, 0, 0, 0, time.UTC) + + key := buildRoutineIdempotencyKey(shared.RoutineSourceCron, "routine-1", "trigger-1", &tick, "run-ignored") + + want := fmt.Sprintf("routine:%s:%s:tick:%d", "routine-1", "trigger-1", tick.Unix()) + if key != want { + t.Fatalf("key = %q, want %q", key, want) + } + + // A second claim of the identical slot (redelivery) mints the same key. + repeat := buildRoutineIdempotencyKey(shared.RoutineSourceCron, "routine-1", "trigger-1", &tick, "run-different") + if repeat != key { + t.Fatalf("redelivery of the same slot produced %q, want %q (identical to the first)", repeat, key) + } +} + +// AC-OFFICE-RUN-DEDUP-001.2: two distinct cron slots (a genuinely later +// tick) mint two distinct keys. +func TestBuildRoutineIdempotencyKey_Cron_DistinctTicksDiffer(t *testing.T) { + tickOne := time.Date(2026, 5, 10, 12, 0, 0, 0, time.UTC) + tickTwo := tickOne.Add(time.Minute) + + keyOne := buildRoutineIdempotencyKey(shared.RoutineSourceCron, "routine-1", "trigger-1", &tickOne, "run-a") + keyTwo := buildRoutineIdempotencyKey(shared.RoutineSourceCron, "routine-1", "trigger-1", &tickTwo, "run-b") + + if keyOne == keyTwo { + t.Fatalf("distinct cron slots must mint distinct keys, both got %q", keyOne) + } +} + +// AC-OFFICE-RUN-DEDUP-003.3: a cron fire with no claimed tick (the +// exported entry point called directly with a nil tick, bypassing the +// live processCronTrigger guard) has no occurrence identity and goes +// keyless with cause=unresolved. +func TestBuildRoutineIdempotencyKey_Cron_NoClaimedTickGoesKeyless(t *testing.T) { + key := buildRoutineIdempotencyKey(shared.RoutineSourceCron, "routine-1", "trigger-1", nil, "run-1") + if key != "" { + t.Fatalf("key = %q, want empty (keyless) for a cron fire with no claimed tick", key) + } +} + +// Manual and webhook fires claim no slot, so two fires of one routine are +// two distinct occurrences by design: each RoutineRun.ID mints its own key. +func TestBuildRoutineIdempotencyKey_ManualAndWebhook_UseRoutineRunID(t *testing.T) { + for _, source := range []string{"manual", "webhook"} { + key := buildRoutineIdempotencyKey(source, "routine-1", "", nil, "run-1") + want := "routine:routine-1:run:run-1" + if key != want { + t.Errorf("source=%q key = %q, want %q", source, key, want) + } + } + + // Two distinct manual fires -> two distinct keys (the collision the + // old unix-minute key format could reach). + first := buildRoutineIdempotencyKey("manual", "routine-1", "", nil, "run-a") + second := buildRoutineIdempotencyKey("manual", "routine-1", "", nil, "run-b") + if first == second { + t.Fatalf("two distinct manual fires must mint distinct keys, both got %q", first) + } +} + +// An unrecognised RoutineRun.Source names no occurrence this table covers +// and goes keyless with cause=unresolved, the same direction as a cron +// fire with no claimed tick. +func TestBuildRoutineIdempotencyKey_UnrecognisedSourceGoesKeyless(t *testing.T) { + key := buildRoutineIdempotencyKey("some_future_source", "routine-1", "", nil, "run-1") + if key != "" { + t.Fatalf("key = %q, want empty (keyless) for an unrecognised source", key) + } +} diff --git a/apps/backend/internal/office/routines/service.go b/apps/backend/internal/office/routines/service.go index 3879a42d4bf..d1af4118ae9 100644 --- a/apps/backend/internal/office/routines/service.go +++ b/apps/backend/internal/office/routines/service.go @@ -14,6 +14,7 @@ import ( "github.com/kandev/kandev/internal/common/logger" "github.com/kandev/kandev/internal/office/models" "github.com/kandev/kandev/internal/office/shared" + runsservice "github.com/kandev/kandev/internal/runs/service" taskservice "github.com/kandev/kandev/internal/task/service" ) @@ -408,6 +409,11 @@ func (s *RoutineService) processCronTrigger(ctx context.Context, trigger *Routin runCount = 1 advanceTo = now } + // The claimed tick: UpdateTriggerNextRun below advances the trigger + // row's next_run_at to the next slot, so trigger.NextRunAt is captured + // here, before that write, and carried through to the key builder + // rather than re-read afterward. + claimedTick := trigger.NextRunAt if err := s.repo.UpdateTriggerNextRun(ctx, trigger.ID, &advanceTo); err != nil { s.logger.Warn("update trigger next_run_at failed", zap.String("trigger_id", trigger.ID), zap.Error(err)) @@ -421,7 +427,7 @@ func (s *RoutineService) processCronTrigger(ctx context.Context, trigger *Routin // runCount-1 as missed_ticks ("you missed N since the last fire"). missedForPayload = runCount - 1 } - _, err = s.DispatchRoutineRunWithMissed(ctx, routine, trigger, shared.RoutineSourceCron, nil, missedForPayload) + _, err = s.DispatchRoutineRunWithMissed(ctx, routine, trigger, shared.RoutineSourceCron, nil, missedForPayload, claimedTick) return err } @@ -493,7 +499,7 @@ func (s *RoutineService) DispatchRoutineRun( source string, provided map[string]string, ) (*RoutineRun, error) { - return s.dispatchRoutineRun(ctx, routine, trigger, source, provided, "", 0) + return s.dispatchRoutineRun(ctx, routine, trigger, source, provided, "", 0, nil) } // DispatchRoutineRunWithIdempotencyKey dispatches a fire with an explicit @@ -507,7 +513,7 @@ func (s *RoutineService) DispatchRoutineRunWithIdempotencyKey( provided map[string]string, idempotencyKey string, ) (*RoutineRun, error) { - return s.dispatchRoutineRun(ctx, routine, trigger, source, provided, idempotencyKey, 0) + return s.dispatchRoutineRun(ctx, routine, trigger, source, provided, idempotencyKey, 0, nil) } // DispatchRoutineRunWithMissed is the cron-tick entry point that @@ -517,6 +523,12 @@ func (s *RoutineService) DispatchRoutineRunWithIdempotencyKey( // learns "you missed N-1 ticks" via wakeup.RoutinePayload.MissedTicks. // Manual fires (UI / API) call DispatchRoutineRun directly with no // missed-tick attribution. +// +// claimedTick is the scheduled tick processCronTrigger claimed off the +// trigger row before advancing it — the dedup key's occurrence identity +// for a cron fire. It travels as a parameter rather than being re-read +// from the trigger row, which by dispatch time already names the next +// slot. func (s *RoutineService) DispatchRoutineRunWithMissed( ctx context.Context, routine *Routine, @@ -524,8 +536,9 @@ func (s *RoutineService) DispatchRoutineRunWithMissed( source string, provided map[string]string, missedTicks int, + claimedTick *time.Time, ) (*RoutineRun, error) { - return s.dispatchRoutineRun(ctx, routine, trigger, source, provided, "", missedTicks) + return s.dispatchRoutineRun(ctx, routine, trigger, source, provided, "", missedTicks, claimedTick) } func (s *RoutineService) dispatchRoutineRun( @@ -536,6 +549,7 @@ func (s *RoutineService) dispatchRoutineRun( provided map[string]string, idempotencyKey string, missedTicks int, + claimedTick *time.Time, ) (*RoutineRun, error) { now := time.Now().UTC() defaults := parseDeclaredDefaults(routine.Variables) @@ -573,7 +587,7 @@ func (s *RoutineService) dispatchRoutineRun( return run, nil } - if err := s.materialiseRoutineRun(ctx, routine, run, tmpl, title, description, vars, source, idempotencyKey, missedTicks); err != nil { + if err := s.materialiseRoutineRun(ctx, routine, run, tmpl, title, description, vars, source, idempotencyKey, missedTicks, claimedTick); err != nil { return run, err } @@ -609,11 +623,12 @@ func (s *RoutineService) materialiseRoutineRun( source string, idempotencyKey string, missedTicks int, + claimedTick *time.Time, ) error { if tmpl.Title != "" && s.workflowEnsurer != nil && s.taskCreator != nil { return s.materialiseHeavyRoutineRun(ctx, routine, run, title, description) } - return s.materialiseLightweightRoutineRun(ctx, routine, run, vars, source, idempotencyKey, missedTicks) + return s.materialiseLightweightRoutineRun(ctx, routine, run, vars, source, idempotencyKey, missedTicks, claimedTick) } // materialiseHeavyRoutineRun creates a real task in the routine system @@ -672,11 +687,12 @@ func (s *RoutineService) materialiseLightweightRoutineRun( source string, idempotencyKey string, missedTicks int, + claimedTick *time.Time, ) error { if s.wakeup == nil || routine.AssigneeAgentProfileID == "" { return s.finalizeLightweightRun(ctx, run, models.RoutineRunStatusDone) } - idemKey := buildRoutineIdempotencyKey(routine.ID, run.TriggerID, source, run.ID, idempotencyKey, run.StartedAt) + idemKey := buildRoutineIdempotencyKey(source, routine.ID, run.TriggerID, idempotencyKey, claimedTick, run.ID) payloadStr, _ := marshalRoutinePayload(routine.ID, vars, missedTicks) req := &WakeupRequest{ ID: uuid.New().String(), @@ -729,27 +745,37 @@ func (s *RoutineService) finalizeLightweightRun( } // buildRoutineIdempotencyKey composes the source-level dedup key for a -// routine fire. Cron fires use the trigger and minute bucket. Manual and -// webhook fires use a unique run identity unless the caller supplies an -// explicit request key, so distinct event deliveries never collide. +// routine fire. An explicit request key (a webhook delivery header, say) +// always wins, so distinct event deliveries never collide. Otherwise the +// occurrence identity depends on the source: a cron fire claims a scheduled +// slot off the trigger row (ClaimTrigger's compare-and-swap means exactly +// one RoutineRun exists per slot, so the slot - not the run - is the +// occurrence, and a catch-up collapsing several missed ticks into one fire +// is still one occurrence). A manual or webhook fire claims no slot, so +// RoutineRun.ID is its only durable distinguishing identity, and two such +// fires are two distinct occurrences by design. +// +// A cron source with no claimed tick, or a source this table does not +// recognise, has no occurrence identity to name and goes keyless. func buildRoutineIdempotencyKey( - routineID, triggerID, source, runID, explicitKey string, startedAt *time.Time, + source, routineID, triggerID, explicitKey string, claimedTick *time.Time, routineRunID string, ) string { if explicitKey != "" { return fmt.Sprintf("routine:%s:%s:%s", routineID, source, explicitKey) } - if source != shared.RoutineSourceCron { - return fmt.Sprintf("routine:%s:%s:%s", routineID, source, runID) - } - now := time.Now().UTC() - if startedAt != nil { - now = *startedAt - } - minute := now.Unix() / 60 - if triggerID == "" { - return fmt.Sprintf("routine:%s:%s:%d", routineID, source, minute) + switch source { + case shared.RoutineSourceCron: + if claimedTick == nil { + runsservice.ReportKeylessEnqueue(shared.RoutineDispatchReason(source), runsservice.KeylessCauseUnresolved, "cron_no_claimed_tick") + return "" + } + return fmt.Sprintf("routine:%s:%s:tick:%d", routineID, triggerID, claimedTick.Unix()) + case "manual", "webhook": + return fmt.Sprintf("routine:%s:run:%s", routineID, routineRunID) + default: + runsservice.ReportKeylessEnqueue(shared.RoutineDispatchReason(source), runsservice.KeylessCauseUnresolved, "unrecognised_routine_source") + return "" } - return fmt.Sprintf("routine:%s:%s:%d", routineID, triggerID, minute) } // marshalRoutinePayload renders the wakeup-request payload for a diff --git a/apps/backend/internal/office/runtime/actions.go b/apps/backend/internal/office/runtime/actions.go index bd955da02a0..1a7f6ef6109 100644 --- a/apps/backend/internal/office/runtime/actions.go +++ b/apps/backend/internal/office/runtime/actions.go @@ -10,6 +10,7 @@ import ( "github.com/google/uuid" "github.com/kandev/kandev/internal/office/models" + runsservice "github.com/kandev/kandev/internal/runs/service" ) // CommentWriter is the comment mutation dependency used by runtime actions. @@ -254,7 +255,7 @@ type ApprovalRequester interface { // RunSpawner is the run queue dependency used by runtime actions. type RunSpawner interface { - QueueRun(ctx context.Context, agentInstanceID, reason, payload, idempotencyKey string) error + QueueRun(ctx context.Context, agentInstanceID, reason, payload, idempotencyKey string) (runsservice.QueueOutcome, error) } // AgentModifier is the agent update dependency used by runtime actions. @@ -566,6 +567,15 @@ type SpawnAgentRunInput struct { } // SpawnAgentRun queues a run for an agent in the same workspace. +// +// A non-empty agent-supplied key is prefixed with the calling run's id +// (agent::) so a retry of the same run reuses the run id +// and still dedupes, while a later run gets a different prefix and is not +// suppressed. With no caller run id the request enqueues keyless +// (cause=unresolved) rather than risk colliding across runs. An empty key is +// NOT prefixed: the agent expressed no dedup intent (cause=by_design), and +// prefixing it would collapse every no-dedup-intent call inside one run onto +// a single key, suppressing every call after the first. func (a *Actions) SpawnAgentRun( ctx context.Context, runCtx RunContext, @@ -592,7 +602,17 @@ func (a *Actions) SpawnAgentRun( if err != nil { return err } - return a.deps.Runs.QueueRun(ctx, target.ID, input.Reason, string(payload), input.IdempotencyKey) + key := "" + switch { + case input.IdempotencyKey == "": + runsservice.ReportKeylessEnqueue(input.Reason, runsservice.KeylessCauseByDesign, "") + case runCtx.RunID != "": + key = fmt.Sprintf("agent:%s:%s", runCtx.RunID, input.IdempotencyKey) + default: + runsservice.ReportKeylessEnqueue(input.Reason, runsservice.KeylessCauseUnresolved, "no_caller_run") + } + _, err = a.deps.Runs.QueueRun(ctx, target.ID, input.Reason, string(payload), key) + return err } // ModifyAgentInput contains agent fields an authorized runtime may update. diff --git a/apps/backend/internal/office/runtime/actions_test.go b/apps/backend/internal/office/runtime/actions_test.go index 6115385abc5..9e922ab77da 100644 --- a/apps/backend/internal/office/runtime/actions_test.go +++ b/apps/backend/internal/office/runtime/actions_test.go @@ -10,6 +10,7 @@ import ( "github.com/kandev/kandev/internal/office/models" "github.com/kandev/kandev/internal/office/shared" + runsservice "github.com/kandev/kandev/internal/runs/service" ) func TestCapabilitiesMarshalProjectCapabilityKeys(t *testing.T) { @@ -1221,14 +1222,14 @@ type spawnRunCall struct { func (r *recordingRunSpawner) QueueRun( _ context.Context, agentInstanceID, reason, payload, idempotencyKey string, -) error { +) (runsservice.QueueOutcome, error) { r.calls = append(r.calls, spawnRunCall{ AgentID: agentInstanceID, Reason: reason, Payload: payload, IdempotencyKey: idempotencyKey, }) - return nil + return runsservice.QueueOutcomeQueued, nil } type recordingAgentModifier struct { diff --git a/apps/backend/internal/office/scheduler/approval_adapter.go b/apps/backend/internal/office/scheduler/approval_adapter.go index 3c96ef96aab..f7d414a63cd 100644 --- a/apps/backend/internal/office/scheduler/approval_adapter.go +++ b/apps/backend/internal/office/scheduler/approval_adapter.go @@ -45,7 +45,7 @@ func (a *DashboardApprovalAdapter) QueueApprovalRuns( DecisionComment: w.DecisionComment, IdempotencyKey: w.IdempotencyKey, } - if err := a.scheduler.QueueRunCtx(ctx, w.AgentID, c); err != nil { + if _, err := a.scheduler.QueueRunCtx(ctx, w.AgentID, c); err != nil { a.scheduler.logger.Warn("approval run failed: " + err.Error()) } } diff --git a/apps/backend/internal/office/scheduler/dashboard_adapter.go b/apps/backend/internal/office/scheduler/dashboard_adapter.go index 82187d003a5..e5cdc2e9ab5 100644 --- a/apps/backend/internal/office/scheduler/dashboard_adapter.go +++ b/apps/backend/internal/office/scheduler/dashboard_adapter.go @@ -75,12 +75,13 @@ func (a *DashboardReactivityAdapter) ApplyTaskMutation( // two packages. func convertChangeToMutation(c dashboard.TaskReactivityChange) TaskMutation { out := TaskMutation{ - NewStatus: c.NewStatus, - NewAssigneeID: c.NewAssigneeID, - ReopenIntent: c.ReopenIntent, - ResumeIntent: c.ResumeIntent, - ActorID: c.ActorID, - ActorType: c.ActorType, + NewStatus: c.NewStatus, + NewAssigneeID: c.NewAssigneeID, + AssignmentGeneration: c.AssignmentGeneration, + ReopenIntent: c.ReopenIntent, + ResumeIntent: c.ResumeIntent, + ActorID: c.ActorID, + ActorType: c.ActorType, } if c.Comment != nil { out.Comment = &MutationComment{ diff --git a/apps/backend/internal/office/scheduler/reactivity.go b/apps/backend/internal/office/scheduler/reactivity.go index 0b57aa15897..154c1ea4075 100644 --- a/apps/backend/internal/office/scheduler/reactivity.go +++ b/apps/backend/internal/office/scheduler/reactivity.go @@ -10,6 +10,8 @@ import ( "github.com/kandev/kandev/internal/office/models" "github.com/kandev/kandev/internal/office/repository/sqlite" + "github.com/kandev/kandev/internal/runs/dedupkeys" + runsservice "github.com/kandev/kandev/internal/runs/service" ) // Canonical lowercase status values used inside the pipeline. Backend @@ -46,10 +48,17 @@ type TaskMutation struct { // What's changing. NewStatus *string // nil = unchanged NewAssigneeID *string - NewPriority *string - Comment *MutationComment // user/agent comment if this mutation includes one - ReopenIntent bool // explicit reopen=true (or status moves done|cancelled → todo|in_progress) - ResumeIntent bool // explicit resume=true (validated upstream to require comment) + // AssignmentGeneration is the value the assigning transaction committed, + // carried here rather than re-read (a producer reading the task's + // current generation after the fact can observe a later occurrence's + // value — see docs/specs/office/system-design/run-dedup-generation-01.md + // #carrying-the-generation). Nil means the caller could not supply one; + // reactToAssigneeChange then enqueues keyless rather than guess. + AssignmentGeneration *int64 + NewPriority *string + Comment *MutationComment // user/agent comment if this mutation includes one + ReopenIntent bool // explicit reopen=true (or status moves done|cancelled → todo|in_progress) + ResumeIntent bool // explicit resume=true (validated upstream to require comment) // Who is acting. ActorID string @@ -115,13 +124,17 @@ func (ss *SchedulerService) ApplyTaskMutation( return } seen[key] = struct{}{} - if err := ss.QueueRunCtx(ctx, agentID, c); err != nil { + outcome, err := ss.QueueRunCtx(ctx, agentID, c) + if err != nil { ss.logger.Error("reactivity run failed", zap.String("agent", agentID), zap.String("reason", c.Reason), zap.Error(err)) return } + if outcome != runsservice.QueueOutcomeQueued { + return + } res.Runs = append(res.Runs, QueuedRunSummary{ AgentID: agentID, Reason: c.Reason, TaskID: c.TaskID, }) @@ -133,7 +146,12 @@ func (ss *SchedulerService) ApplyTaskMutation( } // --- Assignee handoff --- - if change.NewAssigneeID != nil && *change.NewAssigneeID != task.AssigneeAgentProfileID { + // Fires on every non-nil NewAssigneeID, including a repeat assignment to + // the agent that already holds the seat: that is a real occurrence (the + // operator is asking for the work again), not a no-op, so this no longer + // gates on the two ids differing. reactToAssigneeChange itself guards the + // session interrupt. + if change.NewAssigneeID != nil { ss.reactToAssigneeChange(task, *change.NewAssigneeID, change, queue, res) } @@ -222,6 +240,13 @@ func (ss *SchedulerService) reactToStatusChange( // // The previous assignee is NOT separately notified — interrupting their // run is the signal that they're no longer in charge. +// +// ApplyTaskMutation now calls this for every non-nil NewAssigneeID, +// including a repeat assignment to the agent that already holds the seat. +// The interrupt must NOT fire for that case — it would hard-cancel the +// agent's own in-flight run — so unlike the removed caller-side equality +// gate, this comparison stays local to the interrupt decision and does not +// also guard the wake. func (ss *SchedulerService) reactToAssigneeChange( task *TaskSnapshot, newAssigneeID string, @@ -229,25 +254,36 @@ func (ss *SchedulerService) reactToAssigneeChange( queue func(string, RunContext), res *ApplyTaskMutationResult, ) { - // Cancel the prior assignee's session if there was one. We re-use - // InterruptSessionID — status→cancelled also sets it; either reason - // for hard-cancelling produces the same downstream call. - if task.AssigneeAgentProfileID != "" && res.InterruptSessionID == "" { + if task.AssigneeAgentProfileID != "" && newAssigneeID != task.AssigneeAgentProfileID { res.InterruptSessionID = task.ID } - // Wake the new assignee. + if newAssigneeID == "" { + // Unassignment: the interrupt above (if any) already fired; queue's + // own empty-agent-id guard would catch this too, but there is no key + // to build or keyless cause to report for a run that will never be + // attempted. + return + } + commentID := "" if change.Comment != nil { commentID = change.Comment.ID } + var key string + if change.AssignmentGeneration != nil { + key = dedupkeys.AssignmentKey(task.ID, newAssigneeID, *change.AssignmentGeneration) + } else { + runsservice.ReportKeylessEnqueue(RunReasonTaskAssigned, runsservice.KeylessCauseUnresolved, "nil_mutation_generation") + } queue(newAssigneeID, RunContext{ - Reason: RunReasonTaskAssigned, - TaskID: task.ID, - WorkspaceID: task.WorkspaceID, - ActorID: change.ActorID, - ActorType: change.ActorType, - CommentID: commentID, + Reason: RunReasonTaskAssigned, + TaskID: task.ID, + WorkspaceID: task.WorkspaceID, + ActorID: change.ActorID, + ActorType: change.ActorType, + CommentID: commentID, + IdempotencyKey: key, }) } @@ -352,19 +388,34 @@ func (ss *SchedulerService) cascadeBlockersResolved( } for _, blockedID := range blockedTaskIDs { // Verify all OTHER blockers are also resolved. - ready, err := ss.allBlockersResolvedExcept(ctx, blockedID, task.ID) - if err != nil || !ready { + ready, blockers, err := ss.allBlockersResolvedExcept(ctx, blockedID, task.ID) + if err != nil { + ss.logger.Error("check blockers resolved failed", + zap.String("task_id", blockedID), zap.Error(err)) + continue + } + if !ready { continue } assignee, err := ss.repo.GetTaskAssignee(ctx, blockedID) if err != nil || assignee == "" { continue } + blockerIDs := make([]string, 0, len(blockers)) + for _, b := range blockers { + blockerIDs = append(blockerIDs, b.BlockerTaskID) + } + if len(blockerIDs) == 0 { + continue + } + key := fmt.Sprintf("%s:%s:%s:%s", + RunReasonTaskBlockersResolved, blockedID, assignee, dedupkeys.BlockerDigest(blockerIDs)) queue(assignee, RunContext{ Reason: RunReasonTaskBlockersResolved, TaskID: blockedID, WorkspaceID: task.WorkspaceID, ResolvedBlockerTaskID: task.ID, + IdempotencyKey: key, }) } } @@ -443,13 +494,17 @@ func childrenCompletedIdempotencyKey(parentID, agentID string, children []sqlite } // allBlockersResolvedExcept returns true if every blocker on `taskID` -// other than `excludeBlockerID` is in a terminal step. +// other than `excludeBlockerID` is in a terminal step, alongside the full +// blocker-task-id set it read to decide — the caller digests that same +// slice for the wake's dedup key rather than re-reading it (AC-001.9 +// applied to a set: a second read could observe a different set than the +// one this readiness decision was actually made against). func (ss *SchedulerService) allBlockersResolvedExcept( ctx context.Context, taskID, excludeBlockerID string, -) (bool, error) { +) (bool, []*models.TaskBlocker, error) { blockers, err := ss.repo.ListTaskBlockers(ctx, taskID) if err != nil { - return false, err + return false, nil, err } for _, b := range blockers { if b.BlockerTaskID == excludeBlockerID { @@ -457,13 +512,13 @@ func (ss *SchedulerService) allBlockersResolvedExcept( } done, err := ss.repo.IsTaskInTerminalStep(ctx, b.BlockerTaskID) if err != nil { - return false, err + return false, nil, err } if !done { - return false, nil + return false, blockers, nil } } - return true, nil + return true, blockers, nil } // normalisedStatus maps both backend uppercase task states (TODO, diff --git a/apps/backend/internal/office/scheduler/reactivity_children_completed_test.go b/apps/backend/internal/office/scheduler/reactivity_children_completed_test.go index 01ae6fbabe7..d80caafc530 100644 --- a/apps/backend/internal/office/scheduler/reactivity_children_completed_test.go +++ b/apps/backend/internal/office/scheduler/reactivity_children_completed_test.go @@ -50,7 +50,7 @@ func createChildrenCompletedAgent(t *testing.T, repo *officesqlite.Repository, i func newChildrenCompletedQueue(t *testing.T, ss *SchedulerService) func(string, RunContext) { t.Helper() return func(agentID string, c RunContext) { - if err := ss.QueueRunCtx(context.Background(), agentID, c); err != nil { + if _, err := ss.QueueRunCtx(context.Background(), agentID, c); err != nil { t.Fatalf("QueueRunCtx: %v", err) } } diff --git a/apps/backend/internal/office/scheduler/run.go b/apps/backend/internal/office/scheduler/run.go index 87d57ab4ec5..47ab506196d 100644 --- a/apps/backend/internal/office/scheduler/run.go +++ b/apps/backend/internal/office/scheduler/run.go @@ -19,6 +19,7 @@ import ( "github.com/kandev/kandev/internal/office/routing" "github.com/kandev/kandev/internal/office/service" "github.com/kandev/kandev/internal/office/shared" + runsservice "github.com/kandev/kandev/internal/runs/service" ) // ErrRoutingNotSupported is returned by TaskStarter.StartTaskWithRoute @@ -107,13 +108,13 @@ type RunContext struct { // task_changes_requested has the context inline. DecisionComment string `json:"decision_comment,omitempty"` - // IdempotencyKey, when non-empty, overrides the default - // "{reason}:{taskID}:{agentID}" key QueueRunCtx mints. The default - // key is permanently unique per (reason, task, agent) — fine for - // reasons that fire at most once per task, wrong for a reason that - // can legitimately recur (e.g. task_children_completed across - // repeated delegation waves). Callers that recur build their own - // key that changes with the thing that makes each occurrence + // IdempotencyKey is the dedup identity QueueRunCtx passes through + // verbatim. Empty means no dedup — the run enqueues keyless — not + // "derive one for me": QueueRunCtx no longer synthesises a + // "{reason}:{taskID}:{agentID}" default, which was permanently unique + // per (reason, task, agent) and silently swallowed every later + // legitimate occurrence for the same triple. Callers that want dedup + // build a key that changes with the thing that makes each occurrence // distinct. Excluded from the JSON payload: it must never change // encodeRunContext's output shape, which CoalesceRun compares for // equality and taskIDFromPayload parses. @@ -248,32 +249,30 @@ func (ss *SchedulerService) SetProjectSkillDirResolver(fn func(agentTypeID strin func (ss *SchedulerService) QueueRun( ctx context.Context, agentInstanceID, reason, payload, idempotencyKey string, -) error { +) (runsservice.QueueOutcome, error) { if err := ss.guardAgentStatus(ctx, agentInstanceID); err != nil { - return err + return runsservice.QueueOutcomeNone, err } if idempotencyKey != "" { dup, err := ss.repo.CheckIdempotencyKey(ctx, idempotencyKey, IdempotencyWindowHours) if err != nil { - return fmt.Errorf("idempotency check: %w", err) + return runsservice.QueueOutcomeNone, fmt.Errorf("idempotency check: %w", err) } if dup { - ss.logger.Debug("run skipped (idempotent)", - zap.String("key", idempotencyKey)) - return nil + return runsservice.ReportWindowedDedup(runsservice.QueueSourceRuns, reason, idempotencyKey), nil } } coalesced, err := ss.repo.CoalesceRun(ctx, agentInstanceID, reason, CoalesceWindowSeconds, payload) if err != nil { - return fmt.Errorf("coalesce check: %w", err) + return runsservice.QueueOutcomeNone, fmt.Errorf("coalesce check: %w", err) } if coalesced { ss.logger.Debug("run coalesced", zap.String("agent", agentInstanceID), zap.String("reason", reason)) - return nil + return runsservice.QueueOutcomeCoalesced, nil } var idemKeyPtr *string @@ -290,36 +289,37 @@ func (ss *SchedulerService) QueueRun( IdempotencyKey: idemKeyPtr, RequestedAt: time.Now().UTC(), } - if err := ss.repo.CreateRun(ctx, req); err != nil { - return fmt.Errorf("enqueue run: %w", err) + insertErr := ss.repo.CreateRun(ctx, req) + outcome, err := runsservice.ReportInsertResult(runsservice.QueueSourceRuns, reason, idempotencyKey, agentInstanceID, insertErr) + if err != nil { + return runsservice.QueueOutcomeNone, fmt.Errorf("enqueue run: %w", err) + } + if outcome == runsservice.QueueOutcomeDeduped { + return outcome, nil } ss.logger.Info("run queued", zap.String("id", req.ID), zap.String("agent", agentInstanceID), zap.String("reason", reason)) - return nil + return runsservice.QueueOutcomeQueued, nil } -// QueueRunCtx is the typed variant of QueueRun that takes a -// structured RunContext. The context is JSON-encoded into the -// payload column so the agent runtime can deserialise it. The -// idempotency key is c.IdempotencyKey when the caller set one; -// otherwise it defaults to "{reason}:{taskID}:{agentID}" so the same -// agent never gets two runs for the same task+reason within the -// idempotency window. +// QueueRunCtx is the typed variant of QueueRun that takes a structured +// RunContext. The context is JSON-encoded into the payload column so the +// agent runtime can deserialise it. The idempotency key is c.IdempotencyKey +// verbatim — an empty key enqueues with no dedup identity rather than +// falling back to a "{reason}:{taskID}:{agentID}" default that would be +// permanently unique per (reason, task, agent) and silently swallow every +// later legitimate occurrence for the same triple. func (ss *SchedulerService) QueueRunCtx( ctx context.Context, agentInstanceID string, c RunContext, -) error { +) (runsservice.QueueOutcome, error) { payload, err := encodeRunContext(c) if err != nil { - return fmt.Errorf("encode run context: %w", err) - } - idempotencyKey := c.IdempotencyKey - if idempotencyKey == "" { - idempotencyKey = fmt.Sprintf("%s:%s:%s", c.Reason, c.TaskID, agentInstanceID) + return runsservice.QueueOutcomeNone, fmt.Errorf("encode run context: %w", err) } - return ss.QueueRun(ctx, agentInstanceID, c.Reason, payload, idempotencyKey) + return ss.QueueRun(ctx, agentInstanceID, c.Reason, payload, c.IdempotencyKey) } func encodeRunContext(c RunContext) (string, error) { diff --git a/apps/backend/internal/office/service/agent_working_status_test.go b/apps/backend/internal/office/service/agent_working_status_test.go index 45922722d61..de77b6e1257 100644 --- a/apps/backend/internal/office/service/agent_working_status_test.go +++ b/apps/backend/internal/office/service/agent_working_status_test.go @@ -40,7 +40,7 @@ func launchedWorkingAgent( svc.ExecSQL(t, `INSERT INTO tasks (id, workspace_id, project_id, title, created_at, updated_at) VALUES (?, 'ws-1', ?, 'Working status task', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)`, taskID, project.ID) - if err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, + if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, `{"task_id":"`+taskID+`"}`, ""); err != nil { t.Fatalf("queue run: %v", err) } diff --git a/apps/backend/internal/office/service/base_test.go b/apps/backend/internal/office/service/base_test.go index cd745005803..9f10932b6fb 100644 --- a/apps/backend/internal/office/service/base_test.go +++ b/apps/backend/internal/office/service/base_test.go @@ -54,6 +54,7 @@ func newTestService(t *testing.T, overrides ...service.ServiceOptions) *service. state TEXT NOT NULL DEFAULT 'TODO', title TEXT DEFAULT '', assignee_user_id TEXT NOT NULL DEFAULT '', + assignment_generation INTEGER NOT NULL DEFAULT 0, description TEXT DEFAULT '', identifier TEXT DEFAULT '', workflow_id TEXT DEFAULT '', diff --git a/apps/backend/internal/office/service/channels.go b/apps/backend/internal/office/service/channels.go index c33297abcd5..7f300d876c6 100644 --- a/apps/backend/internal/office/service/channels.go +++ b/apps/backend/internal/office/service/channels.go @@ -54,7 +54,7 @@ func (s *Service) SetupChannel(ctx context.Context, channel *models.Channel) err // the runner row is keyed against an empty step_id; the office // repo's runner projection still resolves it because it joins // step_id = task.workflow_step_id (also empty). - if err := s.repo.UpdateTaskAssignee(ctx, taskID, channel.AgentProfileID); err != nil { + if _, err := s.repo.UpdateTaskAssignee(ctx, taskID, channel.AgentProfileID); err != nil { _ = s.repo.DeleteChannel(ctx, channel.ID) return fmt.Errorf("set channel task assignee: %w", err) } diff --git a/apps/backend/internal/office/service/continuation_summary_reader_test.go b/apps/backend/internal/office/service/continuation_summary_reader_test.go index cd7329237c8..8d28e07dae1 100644 --- a/apps/backend/internal/office/service/continuation_summary_reader_test.go +++ b/apps/backend/internal/office/service/continuation_summary_reader_test.go @@ -26,7 +26,7 @@ func TestLoadContinuationSummary_AgentScope_RoundTripsThroughRealWriterAndReader createTestAgent(t, svc, "ws-1", "agent-scope-a") - if err := svc.QueueRun( + if _, err := svc.QueueRun( ctx, "agent-scope-a", service.RunReasonTaskAssigned, "{}", "continuation-agent-scope", ); err != nil { t.Fatalf("queue run: %v", err) @@ -147,7 +147,7 @@ func TestLoadContinuationSummary_ScopeSurvivesCoalesceAfterClaim(t *testing.T) { createTestAgent(t, svc, "ws-1", "agent-scope-coalesce") - if err := svc.QueueRun( + if _, err := svc.QueueRun( ctx, "agent-scope-coalesce", service.RunReasonTaskAssigned, "{}", "continuation-scope-coalesce", ); err != nil { t.Fatalf("queue run: %v", err) diff --git a/apps/backend/internal/office/service/event_subscribers.go b/apps/backend/internal/office/service/event_subscribers.go index 8bbe92a8c5e..2c0adb629ce 100644 --- a/apps/backend/internal/office/service/event_subscribers.go +++ b/apps/backend/internal/office/service/event_subscribers.go @@ -21,6 +21,8 @@ import ( "github.com/kandev/kandev/internal/office/repository/sqlite" "github.com/kandev/kandev/internal/office/shared" "github.com/kandev/kandev/internal/runs/commentkeys" + "github.com/kandev/kandev/internal/runs/dedupkeys" + runsservice "github.com/kandev/kandev/internal/runs/service" "github.com/kandev/kandev/internal/workflow/engine" ) @@ -107,12 +109,16 @@ type TaskMovedData struct { SessionID string `json:"session_id"` } -// TaskUpdatedData represents the payload of a task.updated event. +// TaskUpdatedData represents the payload of a task.created / task.updated +// event. AssignmentGeneration is carried on the extra map by +// publishTaskEventWithExtra's caller at task-creation time (never re-read); +// nil means the publishing event predates this field or is not a creation. type TaskUpdatedData struct { TaskID string `json:"task_id"` WorkspaceID string `json:"workspace_id"` AssigneeAgentProfileID string `json:"assignee_agent_profile_id"` Title string `json:"title"` + AssignmentGeneration *int64 `json:"assignment_generation"` } // CommentPostedData represents a comment event payload. @@ -953,22 +959,39 @@ func (s *Service) handleTaskCreated(ctx context.Context, event *bus.Event) error if err != nil { return nil } - return s.queueTaskAssignedRun(ctx, data.TaskID, data.AssigneeAgentProfileID, true) + return s.queueTaskAssignedRun(ctx, data.TaskID, data.AssigneeAgentProfileID, data.AssignmentGeneration, true) } -// handleTaskUpdated fires a task_assigned run when an agent is assigned. +// handleTaskUpdated fires a task_assigned run when an agent is assigned. No +// production update path can put a new agent into the runner seat (see +// office/repository/sqlite/tasks.go's UpdateTaskAssignee vs the four inert +// syncRunnerInTx writers), so this stays subscribed as a redelivery and +// defensive path rather than a live assignment occurrence. func (s *Service) handleTaskUpdated(ctx context.Context, event *bus.Event) error { data, err := decodeEventData[TaskUpdatedData](event) if err != nil { return nil } - return s.queueTaskAssignedRun(ctx, data.TaskID, data.AssigneeAgentProfileID, false) + return s.queueTaskAssignedRun(ctx, data.TaskID, data.AssigneeAgentProfileID, data.AssignmentGeneration, false) } +// queueTaskAssignedRun fires a task_assigned run for the given occurrence. +// +// When fallbackToStoredRunner is set and the event carried no assignee, the +// agent is recovered through a fresh post-commit GetTaskExecutionFields read +// (task.created's payload can omit it) while the generation, if any, still +// describes the payload's own occurrence — not necessarily this freshly-read +// agent's. Task event publication is asynchronous (a per-task FIFO drainer), +// so a reassignment can commit between the event being built and this +// handler running, and the two halves would then name different occurrences. +// Forcing keyless whenever the fallback actually recovers an agent this way +// avoids mis-keying at the cost of a possible duplicate wake, which +// AC-003.2 already accepts. func (s *Service) queueTaskAssignedRun( ctx context.Context, taskID string, agentProfileID string, + assignmentGeneration *int64, fallbackToStoredRunner bool, ) error { if taskID == "" { @@ -984,15 +1007,23 @@ func (s *Service) queueTaskAssignedRun( if fields == nil || !fields.IsFromOffice { return nil } + fellBackToStoredRunner := false if agentProfileID == "" && fallbackToStoredRunner { agentProfileID = fields.AssigneeAgentProfileID + fellBackToStoredRunner = agentProfileID != "" } if agentProfileID == "" { return nil } payload := mustJSON(map[string]string{"task_id": taskID}) - key := fmt.Sprintf("task_assigned:%s:%s", taskID, agentProfileID) - return s.QueueRun(ctx, agentProfileID, RunReasonTaskAssigned, payload, key) + var key string + if fellBackToStoredRunner || assignmentGeneration == nil { + runsservice.ReportKeylessEnqueue(RunReasonTaskAssigned, runsservice.KeylessCauseUnresolved, "event_missing_generation") + } else { + key = dedupkeys.AssignmentKey(taskID, agentProfileID, *assignmentGeneration) + } + _, err = s.QueueRun(ctx, agentProfileID, RunReasonTaskAssigned, payload, key) + return err } // handleTaskMoved keeps the legacy named-step activity fallback and queues @@ -1079,7 +1110,9 @@ func (s *Service) resolveAndWakeIfUnblocked(ctx context.Context, blockedTaskID, if err != nil { return err } + blockerIDs := make([]string, 0, len(blockers)) for _, b := range blockers { + blockerIDs = append(blockerIDs, b.BlockerTaskID) if b.BlockerTaskID == resolvedBlockerID { continue } @@ -1088,7 +1121,10 @@ func (s *Service) resolveAndWakeIfUnblocked(ctx context.Context, blockedTaskID, return err // still blocked } } - key := fmt.Sprintf("blockers_resolved:%s", blockedTaskID) + if len(blockerIDs) == 0 { + return nil + } + key := fmt.Sprintf("blockers_resolved:%s:%s", blockedTaskID, dedupkeys.BlockerDigest(blockerIDs)) return s.dispatchEngineTrigger(ctx, blockedTaskID, engine.TriggerOnBlockerResolved, engine.OnBlockerResolvedPayload{ ResolvedBlockerIDs: []string{resolvedBlockerID}, diff --git a/apps/backend/internal/office/service/event_subscribers_decision_test.go b/apps/backend/internal/office/service/event_subscribers_decision_test.go index d21c25d7000..ecc4148237a 100644 --- a/apps/backend/internal/office/service/event_subscribers_decision_test.go +++ b/apps/backend/internal/office/service/event_subscribers_decision_test.go @@ -39,7 +39,7 @@ func TestHandleAgentCompleted_WarnsWhenReviewDecisionMissing(t *testing.T) { taskID := createOfficeTask(t, svc, "ws-1", "reviewer-1") payload := `{"task_id":"` + taskID + `","stage_type":"review","workflow_step_id":"step-1"}` - if err := svc.QueueRun(ctx, "reviewer-1", service.RunReasonTaskAssigned, payload, "review-no-decision"); err != nil { + if _, err := svc.QueueRun(ctx, "reviewer-1", service.RunReasonTaskAssigned, payload, "review-no-decision"); err != nil { t.Fatalf("queue run: %v", err) } run, err := svc.ClaimNextRun(ctx) @@ -91,7 +91,7 @@ func TestHandleAgentCompleted_WarnsOnLegacyReviewStartedReason(t *testing.T) { taskID := createOfficeTask(t, svc, "ws-1", "reviewer-1") payload := `{"task_id":"` + taskID + `","workflow_step_id":"step-1","agent_profile_id":"reviewer-1"}` - if err := svc.QueueRun(ctx, "reviewer-1", "review_started", payload, "review-started-no-decision"); err != nil { + if _, err := svc.QueueRun(ctx, "reviewer-1", "review_started", payload, "review-started-no-decision"); err != nil { t.Fatalf("queue run: %v", err) } run, err := svc.ClaimNextRun(ctx) @@ -139,7 +139,7 @@ func TestHandleAgentCompleted_WarnsOnStageIDKeyedPayload(t *testing.T) { taskID := createOfficeTask(t, svc, "ws-1", "reviewer-1") payload := `{"task_id":"` + taskID + `","stage_type":"approval","stage_id":"step-1"}` - if err := svc.QueueRun(ctx, "reviewer-1", service.RunReasonTaskAssigned, payload, "approval-stage-id-no-decision"); err != nil { + if _, err := svc.QueueRun(ctx, "reviewer-1", service.RunReasonTaskAssigned, payload, "approval-stage-id-no-decision"); err != nil { t.Fatalf("queue run: %v", err) } run, err := svc.ClaimNextRun(ctx) @@ -179,7 +179,7 @@ func TestHandleAgentCompleted_WarnsOnLegacyApprovalStartedReason(t *testing.T) { createTestAgent(t, svc, "ws-1", "approver-1") taskID := createOfficeTask(t, svc, "ws-1", "approver-1") payload := `{"task_id":"` + taskID + `","workflow_step_id":"step-approval","agent_profile_id":"approver-1"}` - if err := svc.QueueRun(ctx, "approver-1", "approval_started", payload, "approval-started-no-decision"); err != nil { + if _, err := svc.QueueRun(ctx, "approver-1", "approval_started", payload, "approval-started-no-decision"); err != nil { t.Fatalf("queue run: %v", err) } run, err := svc.ClaimNextRun(ctx) @@ -219,7 +219,7 @@ func TestHandleAgentCompleted_WarnsOnTaskReviewRequested(t *testing.T) { svc.ExecSQL(t, `INSERT INTO workflow_steps (id, stage_type) VALUES (?, ?)`, "step-review-requested", "approval") payload := `{"task_id":"` + taskID + `","role":"approver"}` - if err := svc.QueueRun(ctx, "approver-1", service.RunReasonTaskReviewRequested, payload, "task-review-requested-no-decision"); err != nil { + if _, err := svc.QueueRun(ctx, "approver-1", service.RunReasonTaskReviewRequested, payload, "task-review-requested-no-decision"); err != nil { t.Fatalf("queue run: %v", err) } run, err := svc.ClaimNextRun(ctx) @@ -259,7 +259,7 @@ func TestHandleAgentCompleted_UsesAuthoritativeStageTypeWhenPayloadOmitsIt(t *te svc.ExecSQL(t, `INSERT INTO workflow_steps (id, stage_type) VALUES (?, ?)`, "step-authoritative-decision", "approval") payload := `{"task_id":"` + taskID + `","workflow_step_id":"step-authoritative-decision"}` - if err := svc.QueueRun(ctx, "approver-1", service.RunReasonTaskAssigned, payload, "authoritative-decision-no-decision"); err != nil { + if _, err := svc.QueueRun(ctx, "approver-1", service.RunReasonTaskAssigned, payload, "authoritative-decision-no-decision"); err != nil { t.Fatalf("queue run: %v", err) } run, err := svc.ClaimNextRun(ctx) @@ -301,7 +301,7 @@ func TestHandleAgentCompleted_NoWarnWhenReviewDecisionRecorded(t *testing.T) { taskID := createOfficeTask(t, svc, "ws-1", "reviewer-1") payload := `{"task_id":"` + taskID + `","stage_type":"review","workflow_step_id":"step-1"}` - if err := svc.QueueRun(ctx, "reviewer-1", service.RunReasonTaskAssigned, payload, "review-with-decision"); err != nil { + if _, err := svc.QueueRun(ctx, "reviewer-1", service.RunReasonTaskAssigned, payload, "review-with-decision"); err != nil { t.Fatalf("queue run: %v", err) } run, err := svc.ClaimNextRun(ctx) @@ -347,7 +347,7 @@ func TestHandleAgentCompleted_NoWarnForWorkStage(t *testing.T) { taskID := createOfficeTask(t, svc, "ws-1", "builder-1") payload := `{"task_id":"` + taskID + `","stage_type":"work","workflow_step_id":"step-1"}` - if err := svc.QueueRun(ctx, "builder-1", service.RunReasonTaskAssigned, payload, "work-stage"); err != nil { + if _, err := svc.QueueRun(ctx, "builder-1", service.RunReasonTaskAssigned, payload, "work-stage"); err != nil { t.Fatalf("queue run: %v", err) } run, err := svc.ClaimNextRun(ctx) diff --git a/apps/backend/internal/office/service/event_subscribers_engine_test.go b/apps/backend/internal/office/service/event_subscribers_engine_test.go index a63ce9d1cd5..dae39465785 100644 --- a/apps/backend/internal/office/service/event_subscribers_engine_test.go +++ b/apps/backend/internal/office/service/event_subscribers_engine_test.go @@ -322,7 +322,7 @@ func queueTaskAssignedRunForAgentFailedTests( ) *models.Run { t.Helper() ctx := context.Background() - if err := svc.QueueRun( + if _, err := svc.QueueRun( ctx, agentID, service.RunReasonTaskAssigned, `{"task_id":"`+taskID+`"}`, "", ); err != nil { t.Fatalf("queue run: %v", err) @@ -565,7 +565,7 @@ func TestEngineDispatcher_PathBEscalation_DoesNotFireAgentErrorTrigger(t *testin } createTestAgent(t, svc, "ws-1", "worker-pathb") - if err := svc.QueueRun( + if _, err := svc.QueueRun( ctx, "worker-pathb", service.RunReasonTaskAssigned, `{"task_id":"t1"}`, "", ); err != nil { t.Fatalf("queue: %v", err) diff --git a/apps/backend/internal/office/service/event_subscribers_run_output_test.go b/apps/backend/internal/office/service/event_subscribers_run_output_test.go index 48b2a23180b..de5bd17fb43 100644 --- a/apps/backend/internal/office/service/event_subscribers_run_output_test.go +++ b/apps/backend/internal/office/service/event_subscribers_run_output_test.go @@ -70,7 +70,7 @@ func TestHandleAgentCompleted_RecordsFinalAgentMessageAsOutputSummary(t *testing createTestAgent(t, svc, "ws-1", "worker-1") taskID := createOfficeTask(t, svc, "ws-1", "worker-1") - if err := svc.QueueRun( + if _, err := svc.QueueRun( ctx, "worker-1", service.RunReasonTaskAssigned, `{"task_id":"`+taskID+`"}`, "run-output-init", ); err != nil { @@ -127,7 +127,7 @@ func TestHandleAgentCompleted_TruncatesOutputSummaryAt500Chars(t *testing.T) { createTestAgent(t, svc, "ws-1", "worker-1") taskID := createOfficeTask(t, svc, "ws-1", "worker-1") - if err := svc.QueueRun( + if _, err := svc.QueueRun( ctx, "worker-1", service.RunReasonTaskAssigned, `{"task_id":"`+taskID+`"}`, "run-output-truncate", ); err != nil { @@ -175,7 +175,7 @@ func TestHandleAgentCompleted_NoAgentMessageStaysBestEffort(t *testing.T) { createTestAgent(t, svc, "ws-1", "worker-1") taskID := createOfficeTask(t, svc, "ws-1", "worker-1") - if err := svc.QueueRun( + if _, err := svc.QueueRun( ctx, "worker-1", service.RunReasonTaskAssigned, `{"task_id":"`+taskID+`"}`, "run-output-none", ); err != nil { @@ -216,7 +216,7 @@ func TestHandleTasklessAgentCompleted_RecordsOutputSummaryAndKeepsContinuationSu createTestAgent(t, svc, "ws-1", "worker-taskless") - if err := svc.QueueRun( + if _, err := svc.QueueRun( ctx, "worker-taskless", service.RunReasonTaskAssigned, "{}", "run-output-taskless", ); err != nil { t.Fatalf("queue run: %v", err) @@ -268,7 +268,7 @@ func TestHandleAgentCompleted_LastAgentMessageWinsOverLaterUserMessage(t *testin createTestAgent(t, svc, "ws-1", "worker-1") taskID := createOfficeTask(t, svc, "ws-1", "worker-1") - if err := svc.QueueRun( + if _, err := svc.QueueRun( ctx, "worker-1", service.RunReasonTaskAssigned, `{"task_id":"`+taskID+`"}`, "run-output-ordering", ); err != nil { @@ -318,7 +318,7 @@ func TestHandleAgentCompleted_SecondRunOnReusedSessionStaysEmpty(t *testing.T) { // Run 1: queue, claim, produce an agent message, complete. The // session (sess-1) is reused for run 2 below, mirroring how office // task-bound sessions persist across turns. - if err := svc.QueueRun( + if _, err := svc.QueueRun( ctx, "worker-1", service.RunReasonTaskAssigned, `{"task_id":"`+taskID+`"}`, "run-output-reuse-1", ); err != nil { @@ -359,7 +359,7 @@ func TestHandleAgentCompleted_SecondRunOnReusedSessionStaysEmpty(t *testing.T) { // Run 2: reuses sess-1 but produces no new agent message before // completing (e.g. a turn that only made tool calls, or was cut // short). No new task_session_messages row is inserted here. - if err := svc.QueueRun( + if _, err := svc.QueueRun( ctx, "worker-1", service.RunReasonTaskComment, `{"task_id":"`+taskID+`"}`, "run-output-reuse-2", ); err != nil { @@ -390,7 +390,7 @@ func TestHandleAgentCompleted_DoesNotClearExistingSummaryWithoutMessage(t *testi ctx := context.Background() createTestAgent(t, svc, "ws-1", "worker-1") taskID := createOfficeTask(t, svc, "ws-1", "worker-1") - if err := svc.QueueRun(ctx, "worker-1", service.RunReasonTaskAssigned, + if _, err := svc.QueueRun(ctx, "worker-1", service.RunReasonTaskAssigned, `{"task_id":"`+taskID+`"}`, "run-output-preserve"); err != nil { t.Fatalf("queue run: %v", err) } @@ -423,7 +423,7 @@ func TestHandleAgentCompleted_UsesTurnScopedMessageForSharedSession(t *testing.T ctx := context.Background() createTestAgent(t, svc, "ws-1", "worker-1") taskID := createOfficeTask(t, svc, "ws-1", "worker-1") - if err := svc.QueueRun(ctx, "worker-1", service.RunReasonTaskAssigned, + if _, err := svc.QueueRun(ctx, "worker-1", service.RunReasonTaskAssigned, `{"task_id":"`+taskID+`"}`, "run-output-turn"); err != nil { t.Fatalf("queue run: %v", err) } diff --git a/apps/backend/internal/office/service/event_subscribers_test.go b/apps/backend/internal/office/service/event_subscribers_test.go index 43e1b3da5f6..028ca09913a 100644 --- a/apps/backend/internal/office/service/event_subscribers_test.go +++ b/apps/backend/internal/office/service/event_subscribers_test.go @@ -42,7 +42,8 @@ func (d *queueRunDispatcher) HandleTrigger( case engine.TriggerOnComment: p, _ := payload.(engine.OnCommentPayload) body, _ := json.Marshal(map[string]string{"task_id": taskID, "comment_id": p.CommentID}) - return d.svc.QueueRun(ctx, assignee, service.RunReasonTaskComment, string(body), opID) + _, err := d.svc.QueueRun(ctx, assignee, service.RunReasonTaskComment, string(body), opID) + return err case engine.TriggerOnBlockerResolved: p, _ := payload.(engine.OnBlockerResolvedPayload) var resolved string @@ -50,10 +51,12 @@ func (d *queueRunDispatcher) HandleTrigger( resolved = p.ResolvedBlockerIDs[0] } body, _ := json.Marshal(map[string]string{"task_id": taskID, "resolved_blocker_id": resolved}) - return d.svc.QueueRun(ctx, assignee, service.RunReasonTaskBlockersResolved, string(body), opID) + _, err := d.svc.QueueRun(ctx, assignee, service.RunReasonTaskBlockersResolved, string(body), opID) + return err case engine.TriggerOnChildrenCompleted: body, _ := json.Marshal(map[string]string{"task_id": taskID}) - return d.svc.QueueRun(ctx, assignee, service.RunReasonTaskChildrenCompleted, string(body), opID) + _, err := d.svc.QueueRun(ctx, assignee, service.RunReasonTaskChildrenCompleted, string(body), opID) + return err case engine.TriggerOnApprovalResolved: p, _ := payload.(engine.OnApprovalResolvedPayload) body, _ := json.Marshal(map[string]string{ @@ -61,7 +64,8 @@ func (d *queueRunDispatcher) HandleTrigger( "status": p.Status, "decision_note": p.Note, }) - return d.svc.QueueRun(ctx, assignee, service.RunReasonApprovalResolved, string(body), opID) + _, err := d.svc.QueueRun(ctx, assignee, service.RunReasonApprovalResolved, string(body), opID) + return err } return nil } diff --git a/apps/backend/internal/office/service/failure.go b/apps/backend/internal/office/service/failure.go index 45a2c897e35..d213322dc70 100644 --- a/apps/backend/internal/office/service/failure.go +++ b/apps/backend/internal/office/service/failure.go @@ -466,6 +466,9 @@ func (s *Service) autoPauseAgent( return nil } +// requeueRunForTask is a manual resume: the failed run's own id is the +// occurrence identity (falling back to the task id if it is somehow empty), +// so a duplicate "Mark fixed" click dedupes instead of double-queuing. func (s *Service) requeueRunForTask( ctx context.Context, agentID, taskID, failedRunID string, ) error { @@ -475,7 +478,8 @@ func (s *Service) requeueRunForTask( identity = taskID } key := fmt.Sprintf("%s:%s:%s", RunReasonManualResumeAfterFailure, agentID, identity) - return s.QueueRun(ctx, agentID, RunReasonManualResumeAfterFailure, payload, key) + _, err := s.QueueRun(ctx, agentID, RunReasonManualResumeAfterFailure, payload, key) + return err } func (s *Service) publishRunFailed( diff --git a/apps/backend/internal/office/service/failure_test.go b/apps/backend/internal/office/service/failure_test.go index b4f52e90346..6488450dc1d 100644 --- a/apps/backend/internal/office/service/failure_test.go +++ b/apps/backend/internal/office/service/failure_test.go @@ -21,7 +21,7 @@ func queueAndReadRun( ctx := context.Background() payload := mustMarshalJSON(map[string]string{"task_id": taskID}) idem := agentID + ":" + taskID - if err := svc.QueueRun(ctx, agentID, service.RunReasonTaskAssigned, payload, idem); err != nil { + if _, err := svc.QueueRun(ctx, agentID, service.RunReasonTaskAssigned, payload, idem); err != nil { t.Fatalf("queue run: %v", err) } rows, err := svc.ListRuns(ctx, "ws-1") diff --git a/apps/backend/internal/office/service/retry.go b/apps/backend/internal/office/service/retry.go index 01662981cf4..4dc26684ba7 100644 --- a/apps/backend/internal/office/service/retry.go +++ b/apps/backend/internal/office/service/retry.go @@ -205,7 +205,12 @@ func (s *Service) queueCEOAgentError( "run_id": run.ID, "error": errMsg, }) - _ = s.QueueRun(ctx, ceos[0].ID, RunReasonAgentError, payload, "") + // The failed run's own id makes this occurrence identity: one escalation + // per failed run per CEO, but a later run by the same agent that also + // fails escalates again instead of being silently swallowed by a + // permanently-unique-per-pair key. + key := fmt.Sprintf("agent_error:%s:%s", run.ID, ceos[0].ID) + _, _ = s.QueueRun(ctx, ceos[0].ID, RunReasonAgentError, payload, key) } // retryDelayWithJitter returns the base delay for a given retry index diff --git a/apps/backend/internal/office/service/retry_ceo_self_escalation_test.go b/apps/backend/internal/office/service/retry_ceo_self_escalation_test.go index ccbed8bfd49..a030d0f4e81 100644 --- a/apps/backend/internal/office/service/retry_ceo_self_escalation_test.go +++ b/apps/backend/internal/office/service/retry_ceo_self_escalation_test.go @@ -36,7 +36,7 @@ func TestHandleRunFailure_CEOOwnFailure_DoesNotSelfEscalate(t *testing.T) { } for i := 0; i < 3; i++ { - if err := svc.QueueRun( + if _, err := svc.QueueRun( ctx, ceo.ID, service.RunReasonAgentError, `{}`, "", ); err != nil { t.Fatalf("queue ceo run %d: %v", i, err) diff --git a/apps/backend/internal/office/service/retry_ratelimit_test.go b/apps/backend/internal/office/service/retry_ratelimit_test.go index 43b0d7c6d7d..eba2e93583e 100644 --- a/apps/backend/internal/office/service/retry_ratelimit_test.go +++ b/apps/backend/internal/office/service/retry_ratelimit_test.go @@ -226,7 +226,7 @@ func TestHandleRunFailure_RateLimitParsed_UsesResetTime(t *testing.T) { if err := svc.CreateAgentInstance(ctx, agent); err != nil { t.Fatalf("create agent: %v", err) } - if err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, "{}", "k1"); err != nil { + if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, "{}", "k1"); err != nil { t.Fatalf("queue: %v", err) } run, err := svc.ClaimNextRun(ctx) @@ -275,7 +275,7 @@ func TestHandleRunFailure_RateLimitNoParseable_UsesBackoff(t *testing.T) { if err := svc.CreateAgentInstance(ctx, agent); err != nil { t.Fatalf("create agent: %v", err) } - if err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, "{}", "k2"); err != nil { + if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, "{}", "k2"); err != nil { t.Fatalf("queue: %v", err) } run, err := svc.ClaimNextRun(ctx) @@ -325,7 +325,7 @@ func TestHandleRunFailure_NonRateLimit_UsesBackoff(t *testing.T) { if err := svc.CreateAgentInstance(ctx, agent); err != nil { t.Fatalf("create agent: %v", err) } - if err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, "{}", "k3"); err != nil { + if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, "{}", "k3"); err != nil { t.Fatalf("queue: %v", err) } run, err := svc.ClaimNextRun(ctx) diff --git a/apps/backend/internal/office/service/run.go b/apps/backend/internal/office/service/run.go index 92808c1797b..df76afdd23f 100644 --- a/apps/backend/internal/office/service/run.go +++ b/apps/backend/internal/office/service/run.go @@ -85,18 +85,17 @@ const IdempotencyWindowHours = 24 func (s *Service) QueueRun( ctx context.Context, agentInstanceID, reason, payload, idempotencyKey string, -) error { +) (runsservice.QueueOutcome, error) { if err := s.guardAgentStatus(ctx, agentInstanceID); err != nil { - return err + return runsservice.QueueOutcomeNone, err } if s.runsService != nil { - _, err := s.runsService.QueueRun(ctx, runsservice.QueueRunRequest{ + return s.runsService.QueueRun(ctx, runsservice.QueueRunRequest{ Reason: reason, IdempotencyKey: idempotencyKey, Payload: payloadWithAgent(payload, agentInstanceID), }) - return err } return s.queueRunInline(ctx, agentInstanceID, reason, payload, idempotencyKey) } @@ -107,28 +106,26 @@ func (s *Service) QueueRun( func (s *Service) queueRunInline( ctx context.Context, agentInstanceID, reason, payload, idempotencyKey string, -) error { +) (runsservice.QueueOutcome, error) { if idempotencyKey != "" { dup, err := s.repo.CheckIdempotencyKey(ctx, idempotencyKey, IdempotencyWindowHours) if err != nil { - return fmt.Errorf("idempotency check: %w", err) + return runsservice.QueueOutcomeNone, fmt.Errorf("idempotency check: %w", err) } if dup { - s.logger.Debug("run skipped (idempotent)", - zap.String("key", idempotencyKey)) - return nil + return runsservice.ReportWindowedDedup(runsservice.QueueSourceRuns, reason, idempotencyKey), nil } } coalesced, err := s.repo.CoalesceRun(ctx, agentInstanceID, reason, CoalesceWindowSeconds, payload) if err != nil { - return fmt.Errorf("coalesce check: %w", err) + return runsservice.QueueOutcomeNone, fmt.Errorf("coalesce check: %w", err) } if coalesced { s.logger.Debug("run coalesced", zap.String("agent", agentInstanceID), zap.String("reason", reason)) - return nil + return runsservice.QueueOutcomeCoalesced, nil } var idemKeyPtr *string @@ -145,8 +142,13 @@ func (s *Service) queueRunInline( IdempotencyKey: idemKeyPtr, RequestedAt: time.Now().UTC(), } - if err := s.repo.CreateRun(ctx, req); err != nil { - return fmt.Errorf("enqueue run: %w", err) + insertErr := s.repo.CreateRun(ctx, req) + outcome, err := runsservice.ReportInsertResult(runsservice.QueueSourceRuns, reason, idempotencyKey, agentInstanceID, insertErr) + if err != nil { + return runsservice.QueueOutcomeNone, fmt.Errorf("enqueue run: %w", err) + } + if outcome == runsservice.QueueOutcomeDeduped { + return outcome, nil } s.logger.Info("run queued", @@ -155,7 +157,7 @@ func (s *Service) queueRunInline( zap.String("reason", reason)) s.publishRunQueued(ctx, req, idempotencyKey) - return nil + return runsservice.QueueOutcomeQueued, nil } // payloadWithAgent decodes the JSON payload string and adds the diff --git a/apps/backend/internal/office/service/run_lifecycle_events_test.go b/apps/backend/internal/office/service/run_lifecycle_events_test.go index dd818d50d95..36e8d738438 100644 --- a/apps/backend/internal/office/service/run_lifecycle_events_test.go +++ b/apps/backend/internal/office/service/run_lifecycle_events_test.go @@ -30,7 +30,7 @@ func TestRunLifecycle_StepCompleteEventsEmitted(t *testing.T) { // Queue + claim a run for the task so handleAgentTurnMessageSaved // resolves a runID via GetClaimedRunByTaskID. - if err := svc.QueueRun( + if _, err := svc.QueueRun( ctx, "worker-1", service.RunReasonTaskAssigned, `{"task_id":"`+taskID+`"}`, "lifecycle-init", ); err != nil { @@ -102,7 +102,7 @@ func TestRunLifecycle_StepCompleteEventsEmitted(t *testing.T) { // no longer find a claimed run. To exercise the persistence path // directly, we re-queue + claim once more and seed an activity // row tagged with the run id. - if err := svc.QueueRun( + if _, err := svc.QueueRun( ctx, "worker-1", service.RunReasonTaskComment, `{"task_id":"`+taskID+`"}`, "lifecycle-second", ); err != nil { @@ -131,7 +131,7 @@ func TestRunLifecycle_ErrorEventEmitted(t *testing.T) { createTestAgent(t, svc, "ws-1", "worker-1") taskID := createOfficeTask(t, svc, "ws-1", "worker-1") - if err := svc.QueueRun( + if _, err := svc.QueueRun( ctx, "worker-1", service.RunReasonTaskAssigned, `{"task_id":"`+taskID+`"}`, "lifecycle-error", ); err != nil { @@ -249,7 +249,7 @@ func TestQueueRun_PublishesOfficeRunQueued(t *testing.T) { const idemKey = "task_comment:cm-123" payload := `{"task_id":"` + taskID + `","comment_id":"cm-123"}` - if err := svc.QueueRun(ctx, "worker-1", service.RunReasonTaskComment, payload, idemKey); err != nil { + if _, err := svc.QueueRun(ctx, "worker-1", service.RunReasonTaskComment, payload, idemKey); err != nil { t.Fatalf("queue run: %v", err) } @@ -293,7 +293,7 @@ func TestFinishRun_PublishesOfficeRunProcessed(t *testing.T) { createTestAgent(t, svc, "ws-1", "worker-1") taskID := createOfficeTask(t, svc, "ws-1", "worker-1") - if err := svc.QueueRun( + if _, err := svc.QueueRun( ctx, "worker-1", service.RunReasonTaskComment, `{"task_id":"`+taskID+`","comment_id":"cm-1"}`, "task_comment:cm-1", ); err != nil { @@ -353,7 +353,7 @@ func TestFinishRun_PublishesOfficeRunProcessedForSourceCommentTask(t *testing.T) sourceTaskID := "source-task" insertTestTask(t, svc, sourceTaskID, "ws-1") - if err := svc.QueueRun( + if _, err := svc.QueueRun( ctx, "worker-1", service.RunReasonTaskComment, `{"task_id":"`+targetTaskID+`","source_task_id":"`+sourceTaskID+`","comment_id":"cm-source"}`, "task_comment:cm-source:target", @@ -406,7 +406,7 @@ func TestFailRun_PublishesOfficeRunProcessedFailed(t *testing.T) { createTestAgent(t, svc, "ws-1", "worker-1") taskID := createOfficeTask(t, svc, "ws-1", "worker-1") - if err := svc.QueueRun( + if _, err := svc.QueueRun( ctx, "worker-1", service.RunReasonTaskComment, `{"task_id":"`+taskID+`","comment_id":"cm-2"}`, "task_comment:cm-2", ); err != nil { diff --git a/apps/backend/internal/office/service/run_test.go b/apps/backend/internal/office/service/run_test.go index 16da1241bb1..0e87836f683 100644 --- a/apps/backend/internal/office/service/run_test.go +++ b/apps/backend/internal/office/service/run_test.go @@ -17,7 +17,7 @@ func TestQueueRun_Basic(t *testing.T) { t.Fatalf("create agent: %v", err) } - err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, `{"task_id":"t1"}`, "key-1") + _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, `{"task_id":"t1"}`, "key-1") if err != nil { t.Fatalf("queue run: %v", err) } @@ -47,11 +47,11 @@ func TestQueueRun_Idempotency(t *testing.T) { } key := "idem-key-1" - if err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, "{}", key); err != nil { + if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, "{}", key); err != nil { t.Fatalf("first enqueue: %v", err) } // Second enqueue with same key should be silently dropped. - if err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, "{}", key); err != nil { + if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, "{}", key); err != nil { t.Fatalf("second enqueue: %v", err) } @@ -74,7 +74,7 @@ func TestQueueRun_SkipsPausedAgent(t *testing.T) { t.Fatalf("pause agent: %v", err) } - err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, "{}", "") + _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, "{}", "") if err == nil { t.Fatal("expected error for paused agent") } @@ -97,7 +97,7 @@ func TestQueueRun_SkipsStoppedAgent(t *testing.T) { t.Fatalf("stop agent: %v", err) } - err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, "{}", "") + _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, "{}", "") if err == nil { t.Fatal("expected error for stopped agent") } @@ -113,10 +113,10 @@ func TestQueueRun_Coalesce(t *testing.T) { } // Two runs with the same agent + reason within coalesce window should merge. - if err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskComment, `{"task_id":"t1"}`, ""); err != nil { + if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskComment, `{"task_id":"t1"}`, ""); err != nil { t.Fatalf("first: %v", err) } - if err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskComment, `{"task_id":"t1"}`, ""); err != nil { + if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskComment, `{"task_id":"t1"}`, ""); err != nil { t.Fatalf("second: %v", err) } diff --git a/apps/backend/internal/office/service/scheduler_checkout_contention_test.go b/apps/backend/internal/office/service/scheduler_checkout_contention_test.go index 488120052ab..58a4b19761a 100644 --- a/apps/backend/internal/office/service/scheduler_checkout_contention_test.go +++ b/apps/backend/internal/office/service/scheduler_checkout_contention_test.go @@ -42,7 +42,7 @@ func TestSchedulerTick_ContendedCheckoutRequeuesRun(t *testing.T) { t.Fatalf("seed checkout: ok=%v err=%v", ok, err) } - if err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, + if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, `{"task_id":"task-contended-1"}`, ""); err != nil { t.Fatalf("queue: %v", err) } diff --git a/apps/backend/internal/office/service/scheduler_checkout_error_test.go b/apps/backend/internal/office/service/scheduler_checkout_error_test.go index 8fd3cc590a9..1cd259ec9ee 100644 --- a/apps/backend/internal/office/service/scheduler_checkout_error_test.go +++ b/apps/backend/internal/office/service/scheduler_checkout_error_test.go @@ -35,7 +35,7 @@ func TestSchedulerTick_CheckoutErrorRetriesInsteadOfFalseFinish(t *testing.T) { t.Fatalf("create agent: %v", err) } - if err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, + if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, `{"task_id":"task-checkout-db-error-1"}`, ""); err != nil { t.Fatalf("queue: %v", err) } diff --git a/apps/backend/internal/office/service/scheduler_checkout_inactive_agent_test.go b/apps/backend/internal/office/service/scheduler_checkout_inactive_agent_test.go index 65259bc9273..973a5b10849 100644 --- a/apps/backend/internal/office/service/scheduler_checkout_inactive_agent_test.go +++ b/apps/backend/internal/office/service/scheduler_checkout_inactive_agent_test.go @@ -55,7 +55,7 @@ func TestSchedulerTick_InactiveAgentRunDoesNotStealCheckout(t *testing.T) { // Queue while idle (QueueRun itself refuses to queue for a paused // agent), then go paused before the tick runs - reproducing the race // where an agent is deactivated between queueing and processing. - if err := svc.QueueRun(ctx, inactive.ID, service.RunReasonTaskAssigned, + if _, err := svc.QueueRun(ctx, inactive.ID, service.RunReasonTaskAssigned, `{"task_id":"task-inactive-1"}`, ""); err != nil { t.Fatalf("queue: %v", err) } @@ -125,7 +125,7 @@ func TestSchedulerTick_SameAgentPreCheckoutRunDoesNotStealOwnLiveCheckout(t *tes // queued while idle, then the agent goes paused before the tick picks // it up - reproducing the race where the holder is deactivated after // a second run was already queued behind its own live run. - if err := svc.QueueRun(ctx, holder.ID, service.RunReasonTaskAssigned, + if _, err := svc.QueueRun(ctx, holder.ID, service.RunReasonTaskAssigned, `{"task_id":"task-same-1"}`, ""); err != nil { t.Fatalf("queue: %v", err) } diff --git a/apps/backend/internal/office/service/scheduler_checkout_release_test.go b/apps/backend/internal/office/service/scheduler_checkout_release_test.go index c5ed401244b..7dd1607308f 100644 --- a/apps/backend/internal/office/service/scheduler_checkout_release_test.go +++ b/apps/backend/internal/office/service/scheduler_checkout_release_test.go @@ -64,7 +64,7 @@ func TestSchedulerTick_AgentCompletedReleasesTaskCheckout(t *testing.T) { svc.ExecSQL(t, `INSERT INTO tasks (id, workspace_id, title, description, created_at, updated_at) VALUES ('task-checkout-release-1', 'ws-1', 'Build API', 'Implement endpoint', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)`) - if err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, + if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, `{"task_id":"task-checkout-release-1"}`, ""); err != nil { t.Fatalf("queue: %v", err) } @@ -148,7 +148,7 @@ func TestSchedulerTick_AgentFailedReleasesTaskCheckout(t *testing.T) { svc.ExecSQL(t, `INSERT INTO tasks (id, workspace_id, title, description, created_at, updated_at) VALUES ('task-checkout-fail-1', 'ws-1', 'Build API', 'Implement endpoint', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)`) - if err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, + if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, `{"task_id":"task-checkout-fail-1"}`, ""); err != nil { t.Fatalf("queue: %v", err) } @@ -213,7 +213,7 @@ func TestSchedulerTick_TasklessAgentCompletedStampsRuntime(t *testing.T) { t.Fatalf("create agent: %v", err) } - if err := svc.QueueRun(ctx, agent.ID, service.RunReasonHeartbeat, `{}`, ""); err != nil { + if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonHeartbeat, `{}`, ""); err != nil { t.Fatalf("queue: %v", err) } claimed, err := svc.ClaimNextRun(ctx) diff --git a/apps/backend/internal/office/service/scheduler_features_test.go b/apps/backend/internal/office/service/scheduler_features_test.go index c996f229053..e7fa5fbbfe9 100644 --- a/apps/backend/internal/office/service/scheduler_features_test.go +++ b/apps/backend/internal/office/service/scheduler_features_test.go @@ -29,7 +29,7 @@ func TestCooldown_RecentFinish_GuardAllows(t *testing.T) { } // Queue a run. - if err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, `{"task_id":"t1"}`, ""); err != nil { + if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, `{"task_id":"t1"}`, ""); err != nil { t.Fatalf("queue: %v", err) } @@ -69,7 +69,7 @@ func TestCooldown_PastCooldown_ClaimedNormally(t *testing.T) { } // Queue a run. - if err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, `{"task_id":"t1"}`, ""); err != nil { + if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, `{"task_id":"t1"}`, ""); err != nil { t.Fatalf("queue: %v", err) } @@ -97,7 +97,7 @@ func TestRetry_FailedRun_RetriedWithBackoff(t *testing.T) { t.Fatalf("create agent: %v", err) } - if err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, `{"task_id":"t1"}`, ""); err != nil { + if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, `{"task_id":"t1"}`, ""); err != nil { t.Fatalf("queue: %v", err) } @@ -156,7 +156,7 @@ func TestRetry_FifthFailure_MarkedFailed(t *testing.T) { t.Fatalf("create agent: %v", err) } - if err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, `{"task_id":"t1"}`, ""); err != nil { + if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, `{"task_id":"t1"}`, ""); err != nil { t.Fatalf("queue: %v", err) } @@ -402,7 +402,7 @@ func TestIdleSkip_HeartbeatNoTasks_Skipped(t *testing.T) { } // Worker defaults to skip_idle_runs=true, no tasks assigned. - if err := svc.QueueRun(ctx, agent.ID, service.RunReasonHeartbeat, `{}`, ""); err != nil { + if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonHeartbeat, `{}`, ""); err != nil { t.Fatalf("queue: %v", err) } @@ -457,7 +457,7 @@ func TestIdleSkip_HeartbeatWithActionableTasks_Proceeds(t *testing.T) { // Assign an IN_PROGRESS task to this agent. insertActionableTask(t, svc, "task-inprog-1", agent.ID, "IN_PROGRESS") - if err := svc.QueueRun(ctx, agent.ID, service.RunReasonHeartbeat, `{}`, ""); err != nil { + if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonHeartbeat, `{}`, ""); err != nil { t.Fatalf("queue: %v", err) } @@ -499,7 +499,7 @@ func TestIdleSkip_HeartbeatSkipDisabled_Proceeds(t *testing.T) { `UPDATE agent_profiles SET skip_idle_runs = 0 WHERE id = ?`, agent.ID) // No tasks assigned. - if err := svc.QueueRun(ctx, agent.ID, service.RunReasonHeartbeat, `{}`, ""); err != nil { + if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonHeartbeat, `{}`, ""); err != nil { t.Fatalf("queue: %v", err) } @@ -535,7 +535,7 @@ func TestIdleSkip_NonHeartbeatRun_NotSkipped(t *testing.T) { svc.ExecSQL(t, `INSERT INTO tasks (id, workspace_id, title, created_at, updated_at) VALUES ('task-event-1', 'ws-1', 'Event Task', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)`) - if err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, `{"task_id":"task-event-1"}`, ""); err != nil { + if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, `{"task_id":"task-event-1"}`, ""); err != nil { t.Fatalf("queue: %v", err) } @@ -569,7 +569,7 @@ func TestIdleSkip_CEODefaultFalse_NotSkipped(t *testing.T) { } // CEO has no tasks, but skip_idle_runs defaults to false. - if err := svc.QueueRun(ctx, ceo.ID, service.RunReasonHeartbeat, `{}`, ""); err != nil { + if _, err := svc.QueueRun(ctx, ceo.ID, service.RunReasonHeartbeat, `{}`, ""); err != nil { t.Fatalf("queue: %v", err) } diff --git a/apps/backend/internal/office/service/scheduler_integration_routing_test.go b/apps/backend/internal/office/service/scheduler_integration_routing_test.go index cbdd70ad6e5..bc857f11f47 100644 --- a/apps/backend/internal/office/service/scheduler_integration_routing_test.go +++ b/apps/backend/internal/office/service/scheduler_integration_routing_test.go @@ -109,7 +109,7 @@ func TestSchedulerIntegration_RoutingReceivesBuiltPromptAndEnv(t *testing.T) { svc.ExecSQL(t, `INSERT INTO tasks (id, workspace_id, title, description, priority, created_at, updated_at) VALUES ('task-routing-1', 'ws-1', 'ROUTING_PROMPT_SENTINEL_TITLE', 'Implement endpoint', 'medium', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)`) - if err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, + if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, `{"task_id":"task-routing-1"}`, ""); err != nil { t.Fatalf("queue: %v", err) } @@ -317,7 +317,7 @@ func TestSchedulerIntegration_RoutingFallThrough_FallsBackToLegacy(t *testing.T) } svc.ExecSQL(t, `INSERT INTO tasks (id, workspace_id, title, description, created_at, updated_at) VALUES ('task-ft-1', 'ws-1', 'Fall-through Task', 'desc', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)`) - if err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, + if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, `{"task_id":"task-ft-1"}`, ""); err != nil { t.Fatalf("queue: %v", err) } @@ -361,7 +361,7 @@ func TestSchedulerIntegration_RoutingParked_LeavesAgentIdle(t *testing.T) { } svc.ExecSQL(t, `INSERT INTO tasks (id, workspace_id, title, description, created_at, updated_at) VALUES ('task-parked-1', 'ws-1', 'Parked Task', 'desc', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)`) - if err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, + if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, `{"task_id":"task-parked-1"}`, ""); err != nil { t.Fatalf("queue: %v", err) } @@ -402,7 +402,7 @@ func TestSchedulerIntegration_RoutingDispatchError_LeavesAgentIdle(t *testing.T) } svc.ExecSQL(t, `INSERT INTO tasks (id, workspace_id, title, description, created_at, updated_at) VALUES ('task-routing-err-1', 'ws-1', 'Routing Error Task', 'desc', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)`) - if err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, + if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, `{"task_id":"task-routing-err-1"}`, ""); err != nil { t.Fatalf("queue: %v", err) } diff --git a/apps/backend/internal/office/service/scheduler_integration_test.go b/apps/backend/internal/office/service/scheduler_integration_test.go index 5fa4309b1b4..d79d98fe46a 100644 --- a/apps/backend/internal/office/service/scheduler_integration_test.go +++ b/apps/backend/internal/office/service/scheduler_integration_test.go @@ -20,7 +20,7 @@ func TestSchedulerIntegration_TickProcessesRun(t *testing.T) { t.Fatalf("create agent: %v", err) } - if err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, `{"task_id":"t1"}`, ""); err != nil { + if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, `{"task_id":"t1"}`, ""); err != nil { t.Fatalf("queue run: %v", err) } @@ -72,7 +72,7 @@ func TestSchedulerIntegration_CancelsRunForMovedWorkflowStep(t *testing.T) { // The run was queued while the task was on step-old. A later workflow // move must prevent the scheduler from launching that stale step. - if err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, + if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, `{"task_id":"task-moved-step","workflow_step_id":"step-old"}`, ""); err != nil { t.Fatalf("queue run: %v", err) } @@ -116,7 +116,7 @@ func TestSchedulerIntegration_ResolvesExecutorFromTaskProject(t *testing.T) { VALUES ('task-project-exec', 'ws-1', ?, 'Project executor task', 'desc', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)`, project.ID) - if err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, + if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, `{"task_id":"task-project-exec"}`, ""); err != nil { t.Fatalf("queue run: %v", err) } @@ -171,7 +171,7 @@ func TestSchedulerIntegration_PausedAgentSkipped(t *testing.T) { } // Queue while agent is active. - if err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, `{"task_id":"t1"}`, ""); err != nil { + if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, `{"task_id":"t1"}`, ""); err != nil { t.Fatalf("queue: %v", err) } @@ -210,10 +210,10 @@ func TestSchedulerIntegration_AtCapacityStaysQueued(t *testing.T) { } // Queue two runs. - if err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, `{"task_id":"t1"}`, "k1"); err != nil { + if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, `{"task_id":"t1"}`, "k1"); err != nil { t.Fatalf("queue first: %v", err) } - if err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskComment, `{"task_id":"t2"}`, "k2"); err != nil { + if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskComment, `{"task_id":"t2"}`, "k2"); err != nil { t.Fatalf("queue second: %v", err) } @@ -296,7 +296,7 @@ func TestSchedulerIntegration_PromptBuiltCorrectly(t *testing.T) { t.Fatalf("create agent: %v", err) } - if err := svc.QueueRun(ctx, agent.ID, tt.reason, tt.payload, ""); err != nil { + if _, err := svc.QueueRun(ctx, agent.ID, tt.reason, tt.payload, ""); err != nil { t.Fatalf("queue: %v", err) } diff --git a/apps/backend/internal/office/service/scheduler_recovery.go b/apps/backend/internal/office/service/scheduler_recovery.go index 3f8a73b24c1..e464f567dec 100644 --- a/apps/backend/internal/office/service/scheduler_recovery.go +++ b/apps/backend/internal/office/service/scheduler_recovery.go @@ -7,6 +7,7 @@ import ( "go.uber.org/zap" "github.com/kandev/kandev/internal/common/logger" + runsservice "github.com/kandev/kandev/internal/runs/service" ) // maxRecoveryPerTick caps the number of unstarted tasks recovered in one tick. @@ -69,7 +70,8 @@ func (si *SchedulerIntegration) recoverUnstartedTasks(ctx context.Context, log * zap.String("agent_profile_id", t.AssigneeAgentProfileID)) payload := mustJSON(map[string]string{"task_id": t.ID}) - if err := si.svc.QueueRun(ctx, t.AssigneeAgentProfileID, + runsservice.ReportKeylessEnqueue(RunReasonTaskAssigned, runsservice.KeylessCauseByDesign, "") + if _, err := si.svc.QueueRun(ctx, t.AssigneeAgentProfileID, RunReasonTaskAssigned, payload, ""); err != nil { if ctx.Err() != nil { return diff --git a/apps/backend/internal/office/service/scheduler_run_outcome_test.go b/apps/backend/internal/office/service/scheduler_run_outcome_test.go index 21d4a430599..944407f8075 100644 --- a/apps/backend/internal/office/service/scheduler_run_outcome_test.go +++ b/apps/backend/internal/office/service/scheduler_run_outcome_test.go @@ -64,7 +64,7 @@ func TestSchedulerOutcome_AgentInactive_WritesOutcome(t *testing.T) { if err := svc.CreateAgentInstance(ctx, agent); err != nil { t.Fatalf("create agent: %v", err) } - if err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, `{"task_id":"t1"}`, ""); err != nil { + if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, `{"task_id":"t1"}`, ""); err != nil { t.Fatalf("queue: %v", err) } @@ -106,7 +106,7 @@ func TestSchedulerOutcome_IdleSkipped_WritesOutcome(t *testing.T) { } // Worker defaults to skip_idle_runs=true, no tasks assigned. - if err := svc.QueueRun(ctx, agent.ID, service.RunReasonHeartbeat, `{}`, ""); err != nil { + if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonHeartbeat, `{}`, ""); err != nil { t.Fatalf("queue: %v", err) } @@ -134,7 +134,7 @@ func TestSchedulerOutcome_TaskTreeHeld_WritesOutcome(t *testing.T) { if err := svc.CreateAgentInstance(ctx, agent); err != nil { t.Fatalf("create agent: %v", err) } - if err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, + if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, `{"task_id":"task-tree-outcome"}`, ""); err != nil { t.Fatalf("queue: %v", err) } @@ -184,7 +184,7 @@ func TestSchedulerOutcome_CheckoutError_RetriesInsteadOfFinishing(t *testing.T) if err := svc.CreateAgentInstance(ctx, agent); err != nil { t.Fatalf("create agent: %v", err) } - if err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, + if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, `{"task_id":"task-checkout-error"}`, ""); err != nil { t.Fatalf("queue: %v", err) } @@ -250,7 +250,7 @@ func TestSchedulerOutcome_CheckoutUnavailable_RetriesInsteadOfFinishing(t *testi if err := svc.CreateAgentInstance(ctx, agent); err != nil { t.Fatalf("create agent: %v", err) } - if err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, + if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, `{"task_id":"task-checkout-unavail"}`, ""); err != nil { t.Fatalf("queue: %v", err) } @@ -286,7 +286,7 @@ func TestSchedulerOutcome_BudgetBlocked_WritesOutcome(t *testing.T) { insertTestTask(t, svc, "task-budget-outcome", "ws-1") insertTestCostEvent(t, svc, agent.ID, "task-budget-outcome", int64(600)) - if err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, + if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, `{"task_id":"task-budget-outcome"}`, ""); err != nil { t.Fatalf("queue: %v", err) } @@ -312,7 +312,7 @@ func TestSchedulerOutcome_NoTaskStarter_FailsRun(t *testing.T) { if err := svc.CreateAgentInstance(ctx, agent); err != nil { t.Fatalf("create agent: %v", err) } - if err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, + if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, `{"task_id":"task-no-launch"}`, ""); err != nil { t.Fatalf("queue: %v", err) } @@ -353,7 +353,7 @@ func TestSchedulerOutcome_Processed_WritesOutcome(t *testing.T) { createTestAgent(t, svc, "ws-1", "worker-processed") taskID := createOfficeTask(t, svc, "ws-1", "worker-processed") - if err := svc.QueueRun( + if _, err := svc.QueueRun( ctx, "worker-processed", service.RunReasonTaskAssigned, `{"task_id":"`+taskID+`"}`, "processed-outcome", ); err != nil { diff --git a/apps/backend/internal/office/service/scheduler_runs_test.go b/apps/backend/internal/office/service/scheduler_runs_test.go index 56218742e28..01a8df7163c 100644 --- a/apps/backend/internal/office/service/scheduler_runs_test.go +++ b/apps/backend/internal/office/service/scheduler_runs_test.go @@ -30,7 +30,7 @@ func TestClaimNextRun_ClaimsQueued(t *testing.T) { t.Fatalf("create agent: %v", err) } - if err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, "{}", ""); err != nil { + if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, "{}", ""); err != nil { t.Fatalf("queue: %v", err) } @@ -96,7 +96,7 @@ func TestFinishAndFailRun(t *testing.T) { t.Fatalf("create agent: %v", err) } - if err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, "{}", "k1"); err != nil { + if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, "{}", "k1"); err != nil { t.Fatalf("queue: %v", err) } @@ -132,7 +132,7 @@ func TestFailRun_WritesFailedStatusAndNullOutcome(t *testing.T) { t.Fatalf("create agent: %v", err) } - if err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, "{}", "k1"); err != nil { + if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, "{}", "k1"); err != nil { t.Fatalf("queue: %v", err) } @@ -173,7 +173,7 @@ func TestTransitionRunTerminal_LastWriterWinsOnStatusAndOutcome(t *testing.T) { if err := svc.CreateAgentInstance(ctx, agent); err != nil { t.Fatalf("create agent: %v", err) } - if err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, "{}", "k1"); err != nil { + if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, "{}", "k1"); err != nil { t.Fatalf("queue: %v", err) } req, _ := svc.ClaimNextRun(ctx) diff --git a/apps/backend/internal/office/service/scheduler_taskless_launch_test.go b/apps/backend/internal/office/service/scheduler_taskless_launch_test.go index 7fba442bf39..e3025dd7204 100644 --- a/apps/backend/internal/office/service/scheduler_taskless_launch_test.go +++ b/apps/backend/internal/office/service/scheduler_taskless_launch_test.go @@ -45,7 +45,7 @@ func TestSchedulerTick_TasklessRunFailsInsteadOfFinishing(t *testing.T) { // Mirrors wakeup/dispatcher.go's createFreshRun: reason from the // routine trigger, payload literally "{}" (no task_id). - if err := svc.QueueRun(ctx, agent.ID, service.RunReasonRoutineTrigger, `{}`, ""); err != nil { + if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonRoutineTrigger, `{}`, ""); err != nil { t.Fatalf("queue: %v", err) } @@ -99,7 +99,7 @@ func TestSchedulerTick_TaskBoundRunStillLaunches(t *testing.T) { svc.ExecSQL(t, `INSERT INTO tasks (id, workspace_id, title, description, created_at, updated_at) VALUES ('task-wo35-1', 'ws-1', 'Build API', 'Implement endpoint', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)`) - if err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, `{"task_id":"task-wo35-1"}`, ""); err != nil { + if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, `{"task_id":"task-wo35-1"}`, ""); err != nil { t.Fatalf("queue: %v", err) } @@ -159,7 +159,7 @@ func TestSchedulerTick_TasklessRunsDoNotAutoPauseAgent(t *testing.T) { // mirrors 3 ticks of the pre-installed coordinator heartbeat routine. const firesAtThreshold = 3 for i := 0; i < firesAtThreshold; i++ { - if err := svc.QueueRun(ctx, agent.ID, service.RunReasonRoutineTrigger, `{}`, ""); err != nil { + if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonRoutineTrigger, `{}`, ""); err != nil { t.Fatalf("queue taskless run %d: %v", i, err) } service.RunSchedulerTick(svc, ctx) @@ -184,7 +184,7 @@ func TestSchedulerTick_TasklessRunsDoNotAutoPauseAgent(t *testing.T) { // failures: a task-bound run queued afterwards must still launch. svc.ExecSQL(t, `INSERT INTO tasks (id, workspace_id, title, description, created_at, updated_at) VALUES ('task-wo35-pause-1', 'ws-1', 'Build API', 'Implement endpoint', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)`) - if err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, `{"task_id":"task-wo35-pause-1"}`, ""); err != nil { + if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, `{"task_id":"task-wo35-pause-1"}`, ""); err != nil { t.Fatalf("queue task-bound run: %v", err) } service.RunSchedulerTick(svc, ctx) @@ -233,7 +233,7 @@ func TestSchedulerTick_TasklessRunFailure_PublishesResolvableWorkspaceEvent(t *t } t.Cleanup(func() { _ = sub.Unsubscribe() }) - if err := svc.QueueRun(ctx, agent.ID, service.RunReasonRoutineTrigger, `{}`, ""); err != nil { + if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonRoutineTrigger, `{}`, ""); err != nil { t.Fatalf("queue: %v", err) } service.RunSchedulerTick(svc, ctx) @@ -294,7 +294,7 @@ func TestSchedulerTick_UnlaunchableRun_PublishesResolvableWorkspaceEvent(t *test } t.Cleanup(func() { _ = sub.Unsubscribe() }) - if err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, + if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, `{"task_id":"task-wo35-unlaunchable"}`, ""); err != nil { t.Fatalf("queue: %v", err) } @@ -349,7 +349,7 @@ func TestSchedulerTick_RepeatTasklessFailures_OnlyFirstStaysInInbox(t *testing.T const fires = 3 var firstRunID string for i := 0; i < fires; i++ { - if err := svc.QueueRun(ctx, agent.ID, service.RunReasonRoutineTrigger, `{}`, ""); err != nil { + if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonRoutineTrigger, `{}`, ""); err != nil { t.Fatalf("queue taskless run %d: %v", i, err) } service.RunSchedulerTick(svc, ctx) @@ -415,7 +415,7 @@ func TestSchedulerTick_RepeatTasklessFailures_StayVisiblePerRoutineScope(t *test queueRoutineFailure := func(scope, idempotencyKey string) string { t.Helper() - if err := svc.QueueRun(ctx, agent.ID, service.RunReasonRoutineTrigger, `{}`, idempotencyKey); err != nil { + if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonRoutineTrigger, `{}`, idempotencyKey); err != nil { t.Fatalf("queue routine %s: %v", scope, err) } runs, err := svc.ListRuns(ctx, "ws-1") diff --git a/apps/backend/internal/office/service/task_assignee.go b/apps/backend/internal/office/service/task_assignee.go index 9c5e7a196fb..0f2e5362560 100644 --- a/apps/backend/internal/office/service/task_assignee.go +++ b/apps/backend/internal/office/service/task_assignee.go @@ -38,7 +38,8 @@ const participantTypeAgent = "agent" // by tests and a couple of internal callers; the dashboard's // permissioned variant is SetTaskAssigneeAsAgent. func (s *Service) SetTaskAssignee(ctx context.Context, taskID, assigneeID string) error { - return s.repo.UpdateTaskAssignee(ctx, taskID, assigneeID) + _, err := s.repo.UpdateTaskAssignee(ctx, taskID, assigneeID) + return err } // SetTaskAssigneeAsAgent checks can_assign_tasks for the given caller @@ -55,5 +56,6 @@ func (s *Service) SetTaskAssigneeAsAgent(ctx context.Context, callerAgentID, tas return shared.ErrForbidden } } - return s.repo.UpdateTaskAssignee(ctx, taskID, assigneeID) + _, err := s.repo.UpdateTaskAssignee(ctx, taskID, assigneeID) + return err } diff --git a/apps/backend/internal/office/service/task_starter_test.go b/apps/backend/internal/office/service/task_starter_test.go index 7b11a589b9b..37cd086d79e 100644 --- a/apps/backend/internal/office/service/task_starter_test.go +++ b/apps/backend/internal/office/service/task_starter_test.go @@ -99,7 +99,7 @@ func TestSchedulerTick_LaunchesAgent(t *testing.T) { svc.ExecSQL(t, `INSERT INTO tasks (id, workspace_id, title, description, priority, created_at, updated_at) VALUES ('task-launch-1', 'ws-1', 'Build API', 'Implement endpoint', 'medium', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)`) - if err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, `{"task_id":"task-launch-1"}`, ""); err != nil { + if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, `{"task_id":"task-launch-1"}`, ""); err != nil { t.Fatalf("queue: %v", err) } @@ -145,7 +145,7 @@ func TestSchedulerTick_LaunchIncludesRuntimeTokenEnv(t *testing.T) { } svc.ExecSQL(t, `INSERT INTO tasks (id, workspace_id, title, description, created_at, updated_at) VALUES ('task-token-1', 'ws-1', 'Build API', 'Implement endpoint', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)`) - if err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, `{"task_id":"task-token-1","session_id":"sess-token"}`, ""); err != nil { + if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, `{"task_id":"task-token-1","session_id":"sess-token"}`, ""); err != nil { t.Fatalf("queue: %v", err) } @@ -192,7 +192,7 @@ func TestSchedulerTick_SnapshotsRunSkills(t *testing.T) { } svc.ExecSQL(t, `INSERT INTO tasks (id, workspace_id, title, description, created_at, updated_at) VALUES ('task-skill-1', 'ws-1', 'Review API', 'Review endpoint', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)`) - if err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, `{"task_id":"task-skill-1"}`, ""); err != nil { + if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, `{"task_id":"task-skill-1"}`, ""); err != nil { t.Fatalf("queue: %v", err) } @@ -239,7 +239,7 @@ func TestSchedulerTick_KeepsRunClaimedUntilAgentCompletes(t *testing.T) { } svc.ExecSQL(t, `INSERT INTO tasks (id, workspace_id, title, description, created_at, updated_at) VALUES ('task-life-1', 'ws-1', 'Build API', 'Implement endpoint', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)`) - if err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, `{"task_id":"task-life-1"}`, ""); err != nil { + if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, `{"task_id":"task-life-1"}`, ""); err != nil { t.Fatalf("queue: %v", err) } @@ -301,7 +301,7 @@ func TestSchedulerTick_AgentStoppedFinishesRun(t *testing.T) { } svc.ExecSQL(t, `INSERT INTO tasks (id, workspace_id, title, description, created_at, updated_at) VALUES ('task-stop-1', 'ws-1', 'Stop handler test', 'desc', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)`) - if err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, `{"task_id":"task-stop-1"}`, ""); err != nil { + if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, `{"task_id":"task-stop-1"}`, ""); err != nil { t.Fatalf("queue: %v", err) } @@ -386,7 +386,7 @@ func TestSchedulerTick_StartTaskError_TriggersRetry(t *testing.T) { svc.ExecSQL(t, `INSERT INTO tasks (id, workspace_id, title, created_at, updated_at) VALUES ('task-fail-1', 'ws-1', 'Failing Task', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)`) - if err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, `{"task_id":"task-fail-1"}`, ""); err != nil { + if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, `{"task_id":"task-fail-1"}`, ""); err != nil { t.Fatalf("queue: %v", err) } @@ -439,7 +439,7 @@ func TestSchedulerTick_NoTaskStarter_FailsRunLoudly(t *testing.T) { svc.ExecSQL(t, `INSERT INTO tasks (id, workspace_id, title, created_at, updated_at) VALUES ('task-noop-1', 'ws-1', 'NoOp Task', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)`) - if err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, `{"task_id":"task-noop-1"}`, ""); err != nil { + if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, `{"task_id":"task-noop-1"}`, ""); err != nil { t.Fatalf("queue: %v", err) } @@ -494,7 +494,7 @@ func TestSchedulerTick_NoTaskStarter_UpdatesFailureAccounting(t *testing.T) { for i := 0; i < threshold; i++ { idempotencyKey := fmt.Sprintf("task-noop-accounting-%d", i) - if err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, + if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, `{"task_id":"task-noop-accounting"}`, idempotencyKey); err != nil { t.Fatalf("queue failure %d: %v", i, err) } diff --git a/apps/backend/internal/office/service/wo46_idle_skip_routine_dispatch_test.go b/apps/backend/internal/office/service/wo46_idle_skip_routine_dispatch_test.go index 2268840c0eb..1fdf6b20500 100644 --- a/apps/backend/internal/office/service/wo46_idle_skip_routine_dispatch_test.go +++ b/apps/backend/internal/office/service/wo46_idle_skip_routine_dispatch_test.go @@ -32,7 +32,7 @@ func TestIdleSkip_RoutineDispatchNoTasks_Skipped(t *testing.T) { } // Worker defaults to skip_idle_runs=true, no tasks assigned. - if err := svc.QueueRun(ctx, agent.ID, shared.RunReasonRoutineDispatchCron, `{}`, ""); err != nil { + if _, err := svc.QueueRun(ctx, agent.ID, shared.RunReasonRoutineDispatchCron, `{}`, ""); err != nil { t.Fatalf("queue: %v", err) } @@ -98,7 +98,7 @@ func TestIdleSkip_RoutineDispatch_CoordinatorNotSkipped(t *testing.T) { t.Fatalf("CEO role should default to SkipIdleRuns=false") } - if err := svc.QueueRun(ctx, agent.ID, shared.RunReasonRoutineDispatchCron, `{}`, ""); err != nil { + if _, err := svc.QueueRun(ctx, agent.ID, shared.RunReasonRoutineDispatchCron, `{}`, ""); err != nil { t.Fatalf("queue: %v", err) } diff --git a/apps/backend/internal/office/shared/interfaces.go b/apps/backend/internal/office/shared/interfaces.go index f99aeb6cf15..40f2938fb18 100644 --- a/apps/backend/internal/office/shared/interfaces.go +++ b/apps/backend/internal/office/shared/interfaces.go @@ -6,6 +6,7 @@ import ( "time" "github.com/kandev/kandev/internal/office/models" + runsservice "github.com/kandev/kandev/internal/runs/service" "github.com/kandev/kandev/internal/workflow/engine" ) @@ -39,8 +40,11 @@ type AgentWriter interface { // Implemented by the run feature (and transitionally by office/service.Service). type RunQueuer interface { // QueueRun enqueues a run for agentInstanceID with the given reason, payload, - // and optional idempotency key (empty string disables deduplication). - QueueRun(ctx context.Context, agentInstanceID, reason, payload, idempotencyKey string) error + // and optional idempotency key (empty string disables deduplication). The + // returned QueueOutcome reports what actually happened (queued / deduped / + // coalesced / none-on-error) so callers that need to distinguish a fresh + // insert from a no-op don't have to infer it from side effects. + QueueRun(ctx context.Context, agentInstanceID, reason, payload, idempotencyKey string) (runsservice.QueueOutcome, error) } // WorkflowEngineDispatcher routes typed office task events through the diff --git a/apps/backend/internal/orchestrator/event_handlers_workflow.go b/apps/backend/internal/orchestrator/event_handlers_workflow.go index ed2b9cb233a..477eae680c4 100644 --- a/apps/backend/internal/orchestrator/event_handlers_workflow.go +++ b/apps/backend/internal/orchestrator/event_handlers_workflow.go @@ -22,6 +22,7 @@ import ( "github.com/kandev/kandev/internal/orchestrator/executor" "github.com/kandev/kandev/internal/orchestrator/messagequeue" "github.com/kandev/kandev/internal/orchestrator/watcher" + runsservice "github.com/kandev/kandev/internal/runs/service" "github.com/kandev/kandev/internal/steptelemetry" "github.com/kandev/kandev/internal/sysprompt" "github.com/kandev/kandev/internal/task/models" @@ -1711,13 +1712,16 @@ func (s *Service) queueOfficeAutoStartRun(ctx context.Context, task *models.Task } // officeAutoStartIdempotencyKey uses the immutable workflow-step transition -// row as the per-entry component. A legacy event without that field uses the -// task timestamp as a compatibility fallback until the event is republished. +// row as the per-entry component. A zero stepTransitionID means the +// per-occurrence identity is unavailable, so the enqueue goes keyless rather +// than falling back to a time-derived key that would never suppress a +// redelivery. func officeAutoStartIdempotencyKey(task *models.Task, agentProfileID, stepID string, stepTransitionID int64) string { - entryID := strconv.FormatInt(stepTransitionID, 10) if stepTransitionID == 0 { - entryID = "legacy:" + task.UpdatedAt.UTC().Format(time.RFC3339Nano) + runsservice.ReportKeylessEnqueue(officeAutoStartRunReason, runsservice.KeylessCauseUnresolved, "zero_step_transition") + return "" } + entryID := strconv.FormatInt(stepTransitionID, 10) return fmt.Sprintf("%s:%s:%s:%s:%s", officeAutoStartRunReason, task.ID, agentProfileID, stepID, entryID) } diff --git a/apps/backend/internal/orchestrator/event_handlers_workflow_office_autostart_test.go b/apps/backend/internal/orchestrator/event_handlers_workflow_office_autostart_test.go index 5b75c2e4667..5827a696374 100644 --- a/apps/backend/internal/orchestrator/event_handlers_workflow_office_autostart_test.go +++ b/apps/backend/internal/orchestrator/event_handlers_workflow_office_autostart_test.go @@ -36,6 +36,20 @@ func TestOfficeAutoStartIdempotencyKey(t *testing.T) { } } +// TestOfficeAutoStartIdempotencyKey_ZeroStepTransitionGoesKeyless pins that a +// zero StepTransitionID (no per-occurrence identity available) enqueues with +// no key at all, rather than falling back to a time-derived key that would +// never suppress a genuine redelivery. +func TestOfficeAutoStartIdempotencyKey_ZeroStepTransitionGoesKeyless(t *testing.T) { + task := &models.Task{ID: "t1", UpdatedAt: time.Now().UTC(), WorkflowStepTransitionID: 0} + + key := officeAutoStartIdempotencyKey(task, "agent-1", "step2", task.WorkflowStepTransitionID) + + if key != "" { + t.Errorf("key = %q, want empty (keyless) for zero StepTransitionID", key) + } +} + // TestAutoStartOfficeTaskLogsQueuedOutcomeAtInfo pins that a real insert // (QueueOutcomeQueued) is logged as an info-level "queued" message, and the // log is only emitted after the queue attempt resolves — not asserted diff --git a/apps/backend/internal/runs/dedupkeys/dedupkeys.go b/apps/backend/internal/runs/dedupkeys/dedupkeys.go new file mode 100644 index 00000000000..95da27cf1d0 --- /dev/null +++ b/apps/backend/internal/runs/dedupkeys/dedupkeys.go @@ -0,0 +1,38 @@ +// Package dedupkeys centralizes the shared dedup-key builders that more than +// one office producer must derive identically for the same occurrence +// (docs/specs/office/system-design/run-dedup-generation-01.md#convergent-producers, +// #the-shared-key-builders). Placed beside internal/runs/commentkeys as a +// sibling package so the office service, the office scheduler and the +// orchestrator can all reach it without an import cycle. +package dedupkeys + +import ( + "crypto/sha256" + "fmt" + "sort" + "strings" +) + +// AssignmentKey builds the canonical task_assigned dedup key. Both +// task_assigned producers (office/service's queueTaskAssignedRun and +// office/scheduler's reactToAssigneeChange) call this rather than +// formatting the string themselves, which is what makes their convergence +// structural instead of coincidental (AC-OFFICE-RUN-DEDUP-002.1). +func AssignmentKey(taskID, agentProfileID string, generation int64) string { + return fmt.Sprintf("task_assigned:%s:%s:%d", taskID, agentProfileID, generation) +} + +// BlockerDigest computes the shared digest over a blocker-task-id set. Both +// blocker producers (office/service's resolveAndWakeIfUnblocked and +// office/scheduler's cascadeBlockersResolved) call this so a set observed in +// different read order still converges on the same digest. The encoding is +// binding: sort ascending by byte value (NOT ListTaskBlockers's created_at, +// which is not unique), join with a single comma, then SHA-256 the UTF-8 +// bytes and render the full digest as lowercase hex. +func BlockerDigest(blockerTaskIDs []string) string { + sorted := make([]string, len(blockerTaskIDs)) + copy(sorted, blockerTaskIDs) + sort.Strings(sorted) + sum := sha256.Sum256([]byte(strings.Join(sorted, ","))) + return fmt.Sprintf("%x", sum) +} diff --git a/apps/backend/internal/runs/dedupkeys/dedupkeys_test.go b/apps/backend/internal/runs/dedupkeys/dedupkeys_test.go new file mode 100644 index 00000000000..7e6867670cf --- /dev/null +++ b/apps/backend/internal/runs/dedupkeys/dedupkeys_test.go @@ -0,0 +1,43 @@ +package dedupkeys + +import "testing" + +func TestAssignmentKey(t *testing.T) { + got := AssignmentKey("task-1", "agent-1", 3) + want := "task_assigned:task-1:agent-1:3" + if got != want { + t.Fatalf("AssignmentKey() = %q, want %q", got, want) + } +} + +func TestAssignmentKey_VariesByGeneration(t *testing.T) { + a := AssignmentKey("task-1", "agent-1", 1) + b := AssignmentKey("task-1", "agent-1", 2) + if a == b { + t.Fatalf("expected distinct keys for distinct generations, got %q for both", a) + } +} + +func TestBlockerDigest_OrderIndependent(t *testing.T) { + a := BlockerDigest([]string{"b1", "b2", "b3"}) + b := BlockerDigest([]string{"b3", "b1", "b2"}) + if a != b { + t.Fatalf("BlockerDigest should be order-independent: %q != %q", a, b) + } +} + +func TestBlockerDigest_VariesBySet(t *testing.T) { + a := BlockerDigest([]string{"b1", "b2"}) + b := BlockerDigest([]string{"b1", "b2", "b3"}) + if a == b { + t.Fatalf("expected distinct digests for distinct sets, got %q for both", a) + } +} + +func TestBlockerDigest_DoesNotMutateInput(t *testing.T) { + input := []string{"b3", "b1", "b2"} + _ = BlockerDigest(input) + if input[0] != "b3" || input[1] != "b1" || input[2] != "b2" { + t.Fatalf("BlockerDigest mutated its input slice: %v", input) + } +} diff --git a/apps/backend/internal/runs/service/dedup.go b/apps/backend/internal/runs/service/dedup.go new file mode 100644 index 00000000000..ce9d9f0f8ee --- /dev/null +++ b/apps/backend/internal/runs/service/dedup.go @@ -0,0 +1,116 @@ +package service + +import ( + "go.uber.org/zap" + + "github.com/kandev/kandev/internal/common/logger" + runssqlite "github.com/kandev/kandev/internal/runs/repository/sqlite" +) + +// QueueOutcomeNone means no enqueue was attempted, or the attempt returned +// an error. It is the zero value, so it is what a widened signature yields +// on any path that returns before deciding an outcome. +// +// It is declared identically in internal/workflow/engine/adapters.go — both +// declarations MUST match, the same invariant QueueOutcome itself already +// carries. +const QueueOutcomeNone QueueOutcome = "" + +// QueueSource distinguishes which queue implementation made a dedup +// decision, for the "queue" label on office_run_dedup_total. +type QueueSource string + +const ( + // QueueSourceRuns identifies a suppression on the runs table (the + // idx_run_idempotency windowed lookup or durable unique index). + QueueSourceRuns QueueSource = "runs" + // QueueSourceWakeup identifies a suppression on the wakeup-request + // table. That table has no windowed lookup, so its kind is always + // "durable". + QueueSourceWakeup QueueSource = "wakeup" +) + +// KeylessCause discriminates why a producer enqueued a wake with no dedup +// key: a generation that should have been available and was not +// (KeylessCauseUnresolved), or a producer that never had an occurrence to +// name (KeylessCauseByDesign). See +// docs/specs/office/system-design/run-dedup-generation-01.md#unresolvable-generation. +type KeylessCause string + +const ( + // KeylessCauseUnresolved means the occurrence has a generation that this + // producer simply was not handed or could not resolve. + KeylessCauseUnresolved KeylessCause = "unresolved" + // KeylessCauseByDesign means the producer never had an occurrence + // identity to name — a status transition with no redelivery path, an + // agent that expressed no dedup intent, and similar. + KeylessCauseByDesign KeylessCause = "by_design" +) + +func dedupLogger() *logger.Logger { + return logger.Default().WithFields(zap.String("component", "runs-dedup")) +} + +// ReportWindowedDedup records a suppression caught by the recent-duplicate +// lookup (a windowed dedup hit): counts kind="windowed", logs at Info, and +// returns QueueOutcomeDeduped. +func ReportWindowedDedup(q QueueSource, reason, key string) QueueOutcome { + incRunDedup(q, reason, "windowed") + dedupLogger().Info("run deduplicated (windowed)", + zap.String("queue", string(q)), + zap.String("reason", reason), + zap.String("key", key)) + return QueueOutcomeDeduped +} + +// ReportInsertResult classifies the error from an insert attempt. A unique +// violation on idx_run_idempotency counts kind="durable", logs at Warn with +// key, reason and agent, and returns (QueueOutcomeDeduped, nil). Any other +// non-nil error is returned UNCHANGED with QueueOutcomeNone and moves no +// counter — a disk error is not a dedup decision. A nil error returns +// (QueueOutcomeQueued, nil). +func ReportInsertResult( + q QueueSource, reason, key, agentProfileID string, err error, +) (QueueOutcome, error) { + if err == nil { + return QueueOutcomeQueued, nil + } + if runssqlite.IsIdempotencyKeyUniqueViolation(err) { + return ReportDurableDedup(q, reason, key, agentProfileID), nil + } + return QueueOutcomeNone, err +} + +// ReportDurableDedup records a durable conflict the caller has already +// classified (the wakeup path's ErrWakeupIdempotencyConflict, or +// ReportInsertResult's own classification): counts kind="durable", logs at +// Warn, and returns QueueOutcomeDeduped. +func ReportDurableDedup(q QueueSource, reason, key, agentProfileID string) QueueOutcome { + incRunDedup(q, reason, "durable") + dedupLogger().Warn("run deduplicated (durable index)", + zap.String("queue", string(q)), + zap.String("reason", reason), + zap.String("key", key), + zap.String("agent_profile_id", agentProfileID)) + return QueueOutcomeDeduped +} + +// ReportKeylessEnqueue records a producer's decision to enqueue a wake with +// no dedup key, attributed to reason and discriminated by cause so a +// resolution failure (unresolved) is countable separately from expected, +// by-design keyless traffic. Called by the producer at its own decision +// site, immediately before enqueuing — the queue itself never infers a +// cause from an empty key. detail names what failed to resolve (a fixed +// lowercase snake_case constant, never a formatted message); it is a log +// field only, never a counter label, and is empty for cause=by_design, +// which is counted but never logged. +func ReportKeylessEnqueue(reason string, cause KeylessCause, detail string) { + incRunDedupKeyless(reason, cause) + if cause != KeylessCauseUnresolved { + return + } + dedupLogger().Info("run enqueued with no dedup key", + zap.String("reason", reason), + zap.String("cause", string(cause)), + zap.String("detail", detail)) +} diff --git a/apps/backend/internal/runs/service/dedup_test.go b/apps/backend/internal/runs/service/dedup_test.go new file mode 100644 index 00000000000..4fdc8fbabb0 --- /dev/null +++ b/apps/backend/internal/runs/service/dedup_test.go @@ -0,0 +1,127 @@ +package service_test + +import ( + "errors" + "expvar" + "strings" + "testing" + + runsservice "github.com/kandev/kandev/internal/runs/service" +) + +// counterHasLabel reports whether the named expvar.Map has an entry whose +// key contains every given substring. Counters are process-global expvar +// state (no reset hook), so tests assert presence/growth rather than exact +// values that could collide with other tests in the same binary run. +func counterHasLabel(t *testing.T, mapName string, substrs ...string) bool { + t.Helper() + v := expvar.Get(mapName) + if v == nil { + t.Fatalf("expvar map %q not registered", mapName) + } + m, ok := v.(*expvar.Map) + if !ok { + t.Fatalf("expvar %q is not a *expvar.Map", mapName) + } + found := false + m.Do(func(kv expvar.KeyValue) { + matches := true + for _, s := range substrs { + if !strings.Contains(kv.Key, s) { + matches = false + break + } + } + if matches { + found = true + } + }) + return found +} + +func TestQueueOutcomeNone_IsZeroValue(t *testing.T) { + if runsservice.QueueOutcomeNone != "" { + t.Fatalf("QueueOutcomeNone = %q, want empty string", runsservice.QueueOutcomeNone) + } + var zero runsservice.QueueOutcome + if zero != runsservice.QueueOutcomeNone { + t.Fatalf("zero QueueOutcome %q != QueueOutcomeNone", zero) + } +} + +func TestReportWindowedDedup(t *testing.T) { + reason := "test_windowed_" + t.Name() + outcome := runsservice.ReportWindowedDedup(runsservice.QueueSourceRuns, reason, "some-key") + if outcome != runsservice.QueueOutcomeDeduped { + t.Fatalf("outcome = %q, want deduped", outcome) + } + if !counterHasLabel(t, "office_run_dedup_total", "reason="+reason, "kind=windowed", "queue=runs") { + t.Fatal("expected office_run_dedup_total to carry a windowed/runs entry for this reason") + } +} + +func TestReportInsertResult_NilError(t *testing.T) { + outcome, err := runsservice.ReportInsertResult(runsservice.QueueSourceRuns, "r", "k", "agent", nil) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if outcome != runsservice.QueueOutcomeQueued { + t.Fatalf("outcome = %q, want queued", outcome) + } +} + +func TestReportInsertResult_UniqueViolation(t *testing.T) { + reason := "test_durable_" + t.Name() + violation := errors.New("UNIQUE constraint failed: runs.idempotency_key") + outcome, err := runsservice.ReportInsertResult(runsservice.QueueSourceRuns, reason, "k", "agent-1", violation) + if err != nil { + t.Fatalf("expected nil error on classified conflict, got %v", err) + } + if outcome != runsservice.QueueOutcomeDeduped { + t.Fatalf("outcome = %q, want deduped", outcome) + } + if !counterHasLabel(t, "office_run_dedup_total", "reason="+reason, "kind=durable", "queue=runs") { + t.Fatal("expected office_run_dedup_total to carry a durable/runs entry for this reason") + } +} + +func TestReportInsertResult_OtherError_PassesThrough(t *testing.T) { + reason := "test_other_" + t.Name() + diskErr := errors.New("disk I/O error") + outcome, err := runsservice.ReportInsertResult(runsservice.QueueSourceRuns, reason, "k", "agent-1", diskErr) + if !errors.Is(err, diskErr) { + t.Fatalf("expected the original error to pass through unchanged, got %v", err) + } + if outcome != runsservice.QueueOutcomeNone { + t.Fatalf("outcome = %q, want none", outcome) + } + if counterHasLabel(t, "office_run_dedup_total", "reason="+reason) { + t.Fatal("a non-conflict error must not move the dedup counter") + } +} + +func TestReportDurableDedup_Wakeup(t *testing.T) { + reason := "test_wakeup_" + t.Name() + outcome := runsservice.ReportDurableDedup(runsservice.QueueSourceWakeup, reason, "k", "agent-1") + if outcome != runsservice.QueueOutcomeDeduped { + t.Fatalf("outcome = %q, want deduped", outcome) + } + if !counterHasLabel(t, "office_run_dedup_total", "reason="+reason, "kind=durable", "queue=wakeup") { + t.Fatal("expected office_run_dedup_total to carry a durable/wakeup entry for this reason") + } +} + +func TestReportKeylessEnqueue_CountsBothCauses(t *testing.T) { + reasonUnresolved := "test_keyless_unresolved_" + t.Name() + reasonByDesign := "test_keyless_by_design_" + t.Name() + + runsservice.ReportKeylessEnqueue(reasonUnresolved, runsservice.KeylessCauseUnresolved, "some_detail") + runsservice.ReportKeylessEnqueue(reasonByDesign, runsservice.KeylessCauseByDesign, "") + + if !counterHasLabel(t, "office_run_dedup_keyless_total", "reason="+reasonUnresolved, "cause=unresolved") { + t.Fatal("expected office_run_dedup_keyless_total to carry the unresolved entry") + } + if !counterHasLabel(t, "office_run_dedup_keyless_total", "reason="+reasonByDesign, "cause=by_design") { + t.Fatal("expected office_run_dedup_keyless_total to carry the by_design entry") + } +} diff --git a/apps/backend/internal/runs/service/metrics_vars.go b/apps/backend/internal/runs/service/metrics_vars.go new file mode 100644 index 00000000000..4a74c439bd4 --- /dev/null +++ b/apps/backend/internal/runs/service/metrics_vars.go @@ -0,0 +1,34 @@ +package service + +import ( + "expvar" + "strings" +) + +// expvar maps published at package init, exposed via stdlib's /debug/vars +// handler. Mirrors internal/office/scheduler/metrics_vars.go's "k=v;k=v" +// label model and counters-only rule. +var ( + runDedupTotal = expvar.NewMap("office_run_dedup_total") + runDedupKeylessTotal = expvar.NewMap("office_run_dedup_keyless_total") +) + +// metricLabel builds a "k1=v1;k2=v2;..." label string for an expvar map key. +func metricLabel(pairs ...string) string { + if len(pairs)%2 != 0 { + return "" + } + parts := make([]string, 0, len(pairs)/2) + for i := 0; i < len(pairs); i += 2 { + parts = append(parts, pairs[i]+"="+pairs[i+1]) + } + return strings.Join(parts, ";") +} + +func incRunDedup(q QueueSource, reason, kind string) { + runDedupTotal.Add(metricLabel("reason", reason, "kind", kind, "queue", string(q)), 1) +} + +func incRunDedupKeyless(reason string, cause KeylessCause) { + runDedupKeylessTotal.Add(metricLabel("reason", reason, "cause", string(cause)), 1) +} diff --git a/apps/backend/internal/runs/service/queue_outcome_none_test.go b/apps/backend/internal/runs/service/queue_outcome_none_test.go new file mode 100644 index 00000000000..d1edb0eb409 --- /dev/null +++ b/apps/backend/internal/runs/service/queue_outcome_none_test.go @@ -0,0 +1,22 @@ +package service_test + +import ( + "testing" + + "github.com/kandev/kandev/internal/workflow/engine" + + runsservice "github.com/kandev/kandev/internal/runs/service" +) + +// TestQueueOutcomeNone_MatchesEngineDeclaration pins the "both MUST match" +// invariant both QueueOutcomeNone doc comments carry: runs/service and +// workflow/engine declare the same string type and zero-value constant +// independently, and a change to one without the other would silently break +// callers comparing an outcome received from one package against a constant +// imported from the other. +func TestQueueOutcomeNone_MatchesEngineDeclaration(t *testing.T) { + if string(runsservice.QueueOutcomeNone) != string(engine.QueueOutcomeNone) { + t.Fatalf("runsservice.QueueOutcomeNone = %q, engine.QueueOutcomeNone = %q; both MUST match", + runsservice.QueueOutcomeNone, engine.QueueOutcomeNone) + } +} diff --git a/apps/backend/internal/runs/service/service.go b/apps/backend/internal/runs/service/service.go index ea0c42e449f..43d9349c530 100644 --- a/apps/backend/internal/runs/service/service.go +++ b/apps/backend/internal/runs/service/service.go @@ -12,7 +12,6 @@ package service import ( "context" "encoding/json" - "errors" "fmt" "time" @@ -27,18 +26,6 @@ import ( runssqlite "github.com/kandev/kandev/internal/runs/repository/sqlite" ) -// errIdempotencyKeyConflict signals that insertRun's CreateRun failed -// because idx_run_idempotency rejected a duplicate idempotency_key — -// distinct from the earlier CheckIdempotencyKey miss, which only looks -// within IdempotencyWindowHours. Two independent producers deriving the -// same operation id for the same event can both pass that fast-path check -// before either commits (or the colliding row can simply be older than the -// window); the unique index is what actually stops the second insert. -// QueueRun treats this as a durable dedupe hit: QueueOutcomeDeduped, not an -// error, so the losing producer's caller does not abort or log a spurious -// failure for what is really a no-op. -var errIdempotencyKeyConflict = errors.New("idempotency key conflict") - // RunQueueAdapter is the interface the workflow engine uses to enqueue // runs from queue_run actions. Phase 2 final's parallel agent // declares the same shape inside internal/workflow/engine; both @@ -180,34 +167,32 @@ func (s *Service) SubscribeSignal() <-chan struct{} { return s.signalCh } func (s *Service) QueueRun(ctx context.Context, req QueueRunRequest) (QueueOutcome, error) { agentInstanceID, err := s.resolveAgentInstance(ctx, req) if err != nil { - return "", err + return QueueOutcomeNone, err } if agentInstanceID == "" { - return "", fmt.Errorf("queue run: agent_profile_id is required") + return QueueOutcomeNone, fmt.Errorf("queue run: agent_profile_id is required") } if req.IdempotencyKey != "" { dup, err := s.repo.CheckIdempotencyKey(ctx, req.IdempotencyKey, IdempotencyWindowHours) if err != nil { - return "", fmt.Errorf("idempotency check: %w", err) + return QueueOutcomeNone, fmt.Errorf("idempotency check: %w", err) } if dup { - s.log.Debug("run skipped (idempotent)", - zap.String("key", req.IdempotencyKey)) - return QueueOutcomeDeduped, nil + return ReportWindowedDedup(QueueSourceRuns, req.Reason, req.IdempotencyKey), nil } } payloadMap := runPayload(req, agentInstanceID) payload, err := encodePayload(payloadMap) if err != nil { - return "", fmt.Errorf("encode payload: %w", err) + return QueueOutcomeNone, fmt.Errorf("encode payload: %w", err) } if shouldCoalesceRun(req) { coalesced, err := s.repo.CoalesceRun(ctx, agentInstanceID, req.Reason, CoalesceWindowSeconds, payload) if err != nil { - return "", fmt.Errorf("coalesce check: %w", err) + return QueueOutcomeNone, fmt.Errorf("coalesce check: %w", err) } if coalesced { s.log.Debug("run coalesced", @@ -220,20 +205,17 @@ func (s *Service) QueueRun(ctx context.Context, req QueueRunRequest) (QueueOutco } } - row, err := s.insertRun(ctx, agentInstanceID, req, payload) + row, insertErr := s.insertRun(ctx, agentInstanceID, req, payload) + // idx_run_idempotency has no time bound, so a conflict here can come + // from a row older than IdempotencyWindowHours, not just the windowed + // race CheckIdempotencyKey guards against above. ReportInsertResult + // classifies that as a no-op dedupe rather than a hard error. + outcome, err := ReportInsertResult(QueueSourceRuns, req.Reason, req.IdempotencyKey, agentInstanceID, insertErr) if err != nil { - // idx_run_idempotency has no time bound, so a conflict here can - // come from a row older than IdempotencyWindowHours, not just the - // windowed race CheckIdempotencyKey guards against above. Either - // way the existing row is definitionally the same operation this - // key identifies, so treat it as a no-op dedupe rather than a hard - // error (see errIdempotencyKeyConflict's doc comment). - if errors.Is(err, errIdempotencyKeyConflict) { - s.log.Debug("run skipped (idempotency index race)", - zap.String("key", req.IdempotencyKey)) - return QueueOutcomeDeduped, nil - } - return "", err + return QueueOutcomeNone, err + } + if outcome == QueueOutcomeDeduped { + return outcome, nil } s.log.Info("run queued", @@ -247,7 +229,10 @@ func (s *Service) QueueRun(ctx context.Context, req QueueRunRequest) (QueueOutco } // insertRun creates the runs row and returns it. Pulled out of -// QueueRun to keep the latter under the funlen budget. +// QueueRun to keep the latter under the funlen budget. The returned error is +// the raw CreateRun error (wrapped for context, never reclassified) — the +// caller runs it through ReportInsertResult, which does its own unique-index +// classification against the original error chain. func (s *Service) insertRun( ctx context.Context, agentInstanceID string, req QueueRunRequest, payload string, ) (*models.Run, error) { @@ -267,9 +252,6 @@ func (s *Service) insertRun( RequestedAt: time.Now().UTC(), } if err := s.repo.CreateRun(ctx, row); err != nil { - if runssqlite.IsIdempotencyKeyUniqueViolation(err) { - return nil, errIdempotencyKeyConflict - } return nil, fmt.Errorf("enqueue run: %w", err) } return row, nil diff --git a/apps/backend/internal/task/repository/sqlite/base_migrations.go b/apps/backend/internal/task/repository/sqlite/base_migrations.go index 7c79c494dcf..36f93dd2543 100644 --- a/apps/backend/internal/task/repository/sqlite/base_migrations.go +++ b/apps/backend/internal/task/repository/sqlite/base_migrations.go @@ -114,6 +114,12 @@ func (r *Repository) runMigrations() error { if err := r.migrateTasksRemoveWorkflowFK(); err != nil { return err } + // Must run AFTER migrateTasksRemoveWorkflowFK: that migration recreates + // tasks from an explicit column list. Adding this column beforehand would + // have it silently dropped by the recreate on any database still carrying + // the legacy FK, leaving it absent for the remainder of that boot (the + // same hazard class as the task_sessions.name comment above). + r.migrate.Apply("tasks.assignment_generation", `ALTER TABLE tasks ADD COLUMN assignment_generation INTEGER NOT NULL DEFAULT 0`) if err := r.dropRetiredSlackIntegration(); err != nil { return err } diff --git a/apps/backend/internal/task/repository/sqlite/task.go b/apps/backend/internal/task/repository/sqlite/task.go index 91d07c8d6c4..40ea124b152 100644 --- a/apps/backend/internal/task/repository/sqlite/task.go +++ b/apps/backend/internal/task/repository/sqlite/task.go @@ -388,6 +388,14 @@ func (r *Repository) insertTaskTx(ctx context.Context, tx *sql.Tx, task *models. if err := upsertRunnerInTx(ctx, tx, r.db.Rebind, task.WorkflowStepID, task.ID, task.AssigneeAgentProfileID); err != nil { return "", err } + // A task created already assigned starts at generation 1, the same + // value UpdateTaskAssignee would commit for a first assignment - a + // creation and a following first assignment must not both mint + // generation 1. + if _, err := tx.ExecContext(ctx, r.db.Rebind( + `UPDATE tasks SET assignment_generation = 1 WHERE id = ?`), task.ID); err != nil { + return "", err + } } return entryID, nil } diff --git a/apps/backend/internal/task/service/service_tasks.go b/apps/backend/internal/task/service/service_tasks.go index b89a9d56cfa..2b228dbf926 100644 --- a/apps/backend/internal/task/service/service_tasks.go +++ b/apps/backend/internal/task/service/service_tasks.go @@ -440,7 +440,8 @@ func (s *Service) finalizeCreatedTask(ctx context.Context, prepared *preparedTas task.Repositories = repos } - s.publishTaskEvent(ctx, events.TaskCreated, task, nil) + s.publishTaskEventWithExtra(ctx, events.TaskCreated, task, nil, + map[string]interface{}{"assignment_generation": assignmentGenerationForCreate(task)}) s.pullTasksFromNewFeederWork(ctx, task.WorkflowID, task.WorkflowStepID) if refreshed, err := s.tasks.GetTask(ctx, task.ID); err != nil { s.logger.Warn("failed to refresh task after feeder pull", zap.String("task_id", task.ID), zap.Error(err)) @@ -453,6 +454,17 @@ func (s *Service) finalizeCreatedTask(ctx context.Context, prepared *preparedTas return CreateTaskResult{Task: task, Outcome: CreateTaskOutcomeCreated}, nil } +// assignmentGenerationForCreate mirrors insertTaskTx's runner-row guard in +// memory rather than re-reading the row it just wrote: a task created +// already assigned starts at generation 1 (matching the value insertTaskTx +// committed), everything else starts at 0 (never assigned). +func assignmentGenerationForCreate(task *models.Task) int64 { + if task.AssigneeAgentProfileID != "" && task.WorkflowStepID != "" { + return 1 + } + return 0 +} + func (s *Service) prepareWorkspacePolicyForCreation(ctx context.Context, req *CreateTaskRequest) error { policy := WorkspacePolicy{} if req.WorkspacePolicy != nil { diff --git a/apps/backend/internal/task/service/service_tasks_assignment_generation_test.go b/apps/backend/internal/task/service/service_tasks_assignment_generation_test.go new file mode 100644 index 00000000000..5ed8c72fa8d --- /dev/null +++ b/apps/backend/internal/task/service/service_tasks_assignment_generation_test.go @@ -0,0 +1,119 @@ +package service + +import ( + "context" + "testing" + + "github.com/kandev/kandev/internal/task/models" + sqliterepo "github.com/kandev/kandev/internal/task/repository/sqlite" +) + +// setupOfficeTestWithBus mirrors setupOfficeTest but keeps the MockEventBus +// so a test can inspect the payload of a published task.created event. +func setupOfficeTestWithBus(t *testing.T) (*Service, *MockEventBus, *sqliterepo.Repository) { + t.Helper() + svc, bus, repo := createTestService(t) + ctx := context.Background() + + _, err := repo.DB().Exec(` + CREATE TABLE IF NOT EXISTS workflow_steps ( + id TEXT PRIMARY KEY, + workflow_id TEXT NOT NULL, + name TEXT NOT NULL, + position INTEGER NOT NULL, + color TEXT DEFAULT '', + prompt TEXT DEFAULT '', + events TEXT DEFAULT '{}', + allow_manual_move INTEGER DEFAULT 1, + is_start_step INTEGER DEFAULT 0, + show_in_command_panel INTEGER DEFAULT 1, + auto_archive_after_hours INTEGER DEFAULT 0, + agent_profile_id TEXT DEFAULT '', + created_at TIMESTAMP NOT NULL, + updated_at TIMESTAMP NOT NULL, + FOREIGN KEY (workflow_id) REFERENCES workflows(id) ON DELETE CASCADE + )`) + if err != nil { + t.Fatalf("create workflow_steps: %v", err) + } + + if err := repo.CreateWorkspace(ctx, &models.Workspace{ID: "ws-1", Name: "Workspace"}); err != nil { + t.Fatalf("create workspace: %v", err) + } + if _, err := repo.EnsureOfficeWorkflow(ctx, "ws-1"); err != nil { + t.Fatalf("EnsureOfficeWorkflow: %v", err) + } + svc.SetStartStepResolver(&dbStepResolver{repo: repo}) + svc.SetWorkspacePolicyAttacher(noOpWorkspacePolicyAttacher{}) + return svc, bus, repo +} + +// AC-OFFICE-RUN-DEDUP-001.9: a task created already assigned publishes +// assignment_generation 1 on its task.created event, matching the value +// insertTaskTx committed - a creation and the first following assignment +// must not both mint generation 1. +func TestCreateTask_Assigned_PublishesAssignmentGenerationOne(t *testing.T) { + svc, bus, repo := setupOfficeTestWithBus(t) + ctx := context.Background() + + result, err := svc.CreateTask(ctx, &CreateTaskRequest{ + WorkspaceID: "ws-1", + Title: "Agent Task", + Origin: models.TaskOriginAgentCreated, + AssigneeAgentProfileID: "agent-1", + }) + if err != nil { + t.Fatalf("CreateTask: %v", err) + } + + data := singlePublishedEventData(t, bus) + gen, ok := data["assignment_generation"].(int64) + if !ok || gen != 1 { + t.Fatalf("assignment_generation = %#v, want int64(1)", data["assignment_generation"]) + } + + stored, err := repo.GetTask(ctx, result.Task.ID) + if err != nil { + t.Fatalf("GetTask: %v", err) + } + var persistedGen int64 + if err := repo.DB().QueryRowContext(ctx, + `SELECT assignment_generation FROM tasks WHERE id = ?`, stored.ID).Scan(&persistedGen); err != nil { + t.Fatalf("query assignment_generation: %v", err) + } + if persistedGen != 1 { + t.Fatalf("persisted assignment_generation = %d, want 1", persistedGen) + } +} + +// AC-OFFICE-RUN-DEDUP-001.9 boundary: a task created UNASSIGNED publishes +// assignment_generation 0, never 1 - the boundary that keeps a creation and a +// first assignment from both minting generation 1. +func TestCreateTask_Unassigned_PublishesAssignmentGenerationZero(t *testing.T) { + svc, bus, repo := setupOfficeTestWithBus(t) + ctx := context.Background() + + result, err := svc.CreateTask(ctx, &CreateTaskRequest{ + WorkspaceID: "ws-1", + Title: "Unassigned Task", + ProjectID: "proj-1", + }) + if err != nil { + t.Fatalf("CreateTask: %v", err) + } + + data := singlePublishedEventData(t, bus) + gen, ok := data["assignment_generation"].(int64) + if !ok || gen != 0 { + t.Fatalf("assignment_generation = %#v, want int64(0)", data["assignment_generation"]) + } + + var persistedGen int64 + if err := repo.DB().QueryRowContext(ctx, + `SELECT assignment_generation FROM tasks WHERE id = ?`, result.Task.ID).Scan(&persistedGen); err != nil { + t.Fatalf("query assignment_generation: %v", err) + } + if persistedGen != 0 { + t.Fatalf("persisted assignment_generation = %d, want 0", persistedGen) + } +} diff --git a/apps/backend/internal/workflow/engine/adapters.go b/apps/backend/internal/workflow/engine/adapters.go index c0334b8db27..98ae260eff4 100644 --- a/apps/backend/internal/workflow/engine/adapters.go +++ b/apps/backend/internal/workflow/engine/adapters.go @@ -34,6 +34,12 @@ const ( // queued row for the same agent + reason within the coalescing // window, so nothing new was inserted. QueueOutcomeCoalesced QueueOutcome = "coalesced" + // QueueOutcomeNone means no enqueue was attempted, or the attempt + // returned an error. It is the zero value, so it is what a widened + // signature yields on any path that returns before deciding an + // outcome. Declared identically in internal/runs/service — both + // declarations MUST match. + QueueOutcomeNone QueueOutcome = "" ) // QueueRunRequest is the typed payload the engine hands to RunQueueAdapter. diff --git a/docs/specs/office/requirements/run-dedup-generation.md b/docs/specs/office/requirements/run-dedup-generation.md new file mode 100644 index 00000000000..460c680a8bb --- /dev/null +++ b/docs/specs/office/requirements/run-dedup-generation.md @@ -0,0 +1,337 @@ +--- +status: draft +system: office +created: 2026-09-07 +owners: + - kandev +--- + +# Office Run Deduplication — Generation Identity + +## Overview + +Every autonomous wake in Office is enqueued through the shared runs queue with an +`idempotency_key`. The key exists so that a *redelivery* of one occurrence (a +replayed event, a retried handler, two producers reacting to the same fact) does +not launch the same agent twice. + +Several producers instead mint a key that is **permanently unique per (reason, +task, agent)**. Such a key cannot distinguish a redelivery from a genuine repeat +of the same kind of work, so the second legitimate occurrence is suppressed. The +suppression is total and leaves no trace: no run row, no inbox item, no wire +event, and a `Debug`-level log line. + +The canonical case is assignment. `task_assigned::` is stable for +the life of that pair, so both of these are permanent, invisible no-ops: + +- Reassigning a task back to an agent that previously held it (A -> B -> A), + suppressed by the 24-hour lookup. +- Reassigning the same task to the same agent more than 24 hours later, + suppressed by the unbounded `UNIQUE(idempotency_key)` index after the lookup + has already passed. + +The second case is the more damaging of the two: the fast lookup reports no +duplicate, the producer proceeds, and the durable index rejects the insert. The +queue reports that rejection as a successful no-op. + +Two producers already avoid the trap by varying their key with something that +changes per occurrence: the workflow auto-start key carries the step-transition +row id, and the step-entry key carries the entry id and position. Re-running an +epic by moving it through a step therefore works, while re-running it by +re-assignment does not. This capability generalises what those two do into a +contract that every producer must satisfy, and makes the queue's dedup decisions +observable. + +This capability owns the *contract on the key*, and the *observability of the +outcome*. It does not change the 24-hour lookup, the durable unique index, the +coalescing window, or the run lifecycle. + +## Terminology + +- **Occurrence:** one real-world fact that justifies waking an agent — this + assignment, this comment, this decision, this cron fire. Two occurrences of + the same kind on the same (task, agent) are distinct work. Two occurrences are + **distinguishable** when some durable row the producer can read differs between + them; where nothing does, they collapse to one, and every case that collapses is + named under `## Out of scope`. +- **Redelivery:** a second attempt to enqueue a wake for an occurrence that has + already been enqueued — an event replayed, a handler retried, or a second + producer reacting to the same fact. +- **Dedup key:** the value persisted in `runs.idempotency_key` or + `agent_wakeup_requests.idempotency_key`. +- **Generation component:** the part of a dedup key that changes between two + occurrences and stays equal across redeliveries of one occurrence. +- **Windowed dedup hit:** the queue's recent-duplicate lookup matched an + existing row inside its lookback window. +- **Durable dedup hit:** the recent-duplicate lookup found nothing, and the + unbounded unique index then rejected the insert. After this capability ships + this means one of three things: a genuine race between two producers; a producer + that minted a colliding key; or a **legitimate late redelivery** — one occurrence + re-enqueued more than the lookback window after the original, which a + level-triggered reconciler that re-derives the same key on every tick produces + as normal operation rather than as a fault. Only the first two are anomalies. + +## Prior art + +### Our own recorded reasoning (wiki) + +**Searched:** resolved `OBSIDIAN_VAULT_PATH=/Users/henry/Documents/henry/wiki` +and `QMD_WIKI_COLLECTION=wiki` from `~/.obsidian-wiki/config` (symlink to +`config.henry`). The leg then **did not run**: neither `obsidian-wiki` nor `qmd` +is on this executor's PATH, no qmd MCP server is exposed to this session, and +the vault directory itself is unreadable here (`ls` returns `Operation not +permitted` for `~/Documents` both inside and outside the sandbox, which is a +macOS privacy restriction on this process, not a permission-gate denial). This +is a skipped step, not an empty result: the vault is configured and may well +hold relevant prior positions that this specification therefore did not consult. + +### What other products shipped (saas-kb) + +**Searched:** nothing. The `saas-kb` MCP server and its `search_fsm_docs` tool +are not exposed to this session; the only MCP servers available are `codex` and +`kandev`. Skipped step, not an empty result. + +### Prior art inside this repository + +This leg did run, and it is the one that shaped the design. + +- **Two producers already solve this problem**, which is why the defect is + visible as an asymmetry rather than a total outage. + `officeAutoStartIdempotencyKey` carries the step-transition row id, and the + step-entry key carries the entry id and position. Re-running an epic by + moving it through a step works; re-running it by re-assignment does not. This + capability generalises their shape rather than inventing one. +- **`childrenCompletedIdempotencyKey`** already argues, in a written comment, + the exact principle this specification makes general: a key that is + "permanently unique per (parent, agent)" would wake a parent only for its + first delegation wave, so the key digests the child set instead. It also + already made the judgment that the digest covers *which* children exist + rather than each child's state, to avoid spurious wakes. That reasoning is + adopted here, not re-derived. +- **`task_sessions.route_generation`** is a monotonic counter already in the + tree, bumped on a routing decision and compared for staleness. The assignment + generation this capability needs is the same pattern, so it follows that + precedent rather than introducing a new one. + +**What we are doing differently:** the existing solutions are per-producer, each +one discovered after its own bug report. This capability makes the generation +component a property of the key contract itself (AC-OFFICE-RUN-DEDUP-001.1), +enforced by a test over every producer, so the next producer added cannot +reintroduce a permanent key silently. It also inverts the failure direction: +where `officeAutoStartIdempotencyKey` falls back to a time-derived legacy key +when its generation is missing, this contract falls back to *no key at all*, +because a duplicate wake is recoverable and a suppressed one is not. + +## Requirements + +### REQ-OFFICE-RUN-DEDUP-001: Generation identity in every persisted dedup key + +**Intent:** A dedup key must suppress a redelivery and must not suppress a +repeat. A key built only from durable identifiers that never change (reason, +task, agent) cannot do both, and today it silently chooses the wrong one. + +**User story:** As an Office operator, I want re-assigning a task to an agent +that already held it to actually wake that agent, so that a re-run is not a +silent no-op. + +#### Acceptance criteria + +- **AC-OFFICE-RUN-DEDUP-001.1:** When a producer enqueues a wake with a + non-empty dedup key, that key shall contain a generation component that is + equal for two enqueue attempts describing the same occurrence, and different + for two enqueue attempts describing occurrences that differ in some durable row + the producer can read. Occurrences that no durable row distinguishes collapse + to one key; each such case shall be named under `## Out of scope`. +- **AC-OFFICE-RUN-DEDUP-001.2:** When a task is assigned to an agent, then to a + different agent, then back to the first agent, the system shall wake the first + agent for the third assignment — by inserting a run row, or, when that + assignment lands within the coalescing window of a still-queued run for the + same agent and task, by merging into that row. It shall not report a + deduplicated outcome, which is the only outcome that produces no run at all. +- **AC-OFFICE-RUN-DEDUP-001.3:** When a task is assigned to the same agent a + second time and the earlier assignment's run is older than the queue's + recent-duplicate lookback window, the system shall insert a run row. That + earlier run is necessarily outside the far shorter coalescing window, so the + outcome here is queued and never coalesced. +- **AC-OFFICE-RUN-DEDUP-001.4:** When the same assignment occurrence is + delivered to a producer more than once and the first delivery persisted a run + row bearing its key, the system shall queue exactly one run for it. A delivery + that coalesced persisted no such row, so a redelivery after the coalescing + window queues a second run; see `## Out of scope`. +- **AC-OFFICE-RUN-DEDUP-001.5:** When a wake is triggered by an occurrence that a + durable row identifies, its generation component shall be derived from that row + — its primary key, or a persisted column whose value is fixed for that + occurrence — and shall not be read from the clock at the moment the wake is + produced. A persisted scheduled time is a stored column, not a clock read, and + satisfies this criterion; the processing time at which a producer happens to run + does not. +- **AC-OFFICE-RUN-DEDUP-001.6:** When a wake is triggered by a clock tick that no + durable row identifies, its generation component can be derived from the tick's + scheduled time. A tick whose due time is persisted and claimed on a durable row + before dispatch is identified by that row, and is governed by + AC-OFFICE-RUN-DEDUP-001.5 instead of by this criterion. +- **AC-OFFICE-RUN-DEDUP-001.9:** When a durable row identifies an occurrence but + the producer would have to re-read that row after the occurrence has advanced, + the identifying value shall be captured at the point the occurrence is committed + or claimed and carried to the producer. A producer shall not recover a + generation component by re-reading a row that a later occurrence has since + moved. +- **AC-OFFICE-RUN-DEDUP-001.7:** When a dedup key is supplied by an agent + through a runtime action, the system shall scope that key to the supplying + run so a value the agent reuses across two of its own runs does not suppress + the second. +- **AC-OFFICE-RUN-DEDUP-001.8:** When a run row persisted before this capability + shipped carries a key in the previous format, the system shall leave that row + unchanged and shall not treat it as a duplicate of a key in the new format. + +### REQ-OFFICE-RUN-DEDUP-002: One occurrence yields one key across producers + +**Intent:** Several occurrences are observed by more than one producer. Assignment +is observed by both the task-event subscriber and the reactivity pipeline, and +those two producers converge on one key today. That convergence is what makes a +reassignment wake an agent once rather than twice. Adding a generation component +independently in each producer would break it. + +#### Acceptance criteria + +- **AC-OFFICE-RUN-DEDUP-002.1:** When two producers that each derive a dedup key + enqueue a wake for the same occurrence with the same reason, task, and agent, + they shall derive an identical key. A producer enqueueing that occurrence with + no key is governed by REQ-OFFICE-RUN-DEDUP-003 and does not violate this + criterion. +- **AC-OFFICE-RUN-DEDUP-002.2:** When an assignment is observed by both the + task-event subscriber and the reactivity pipeline, the system shall queue + exactly one run. +- **AC-OFFICE-RUN-DEDUP-002.3:** When two producers attempt to enqueue the same + key concurrently, each shall either persist a run or observe a suppression + outcome — deduplicated, or coalesced when a coalescible run was queued for that + agent, reason and task at its coalesce check — without returning an error to + its caller. At most one may persist, and where such a run was queued for both, + neither does. No outcome adds a second row bearing the key; none is a failure. +- **AC-OFFICE-RUN-DEDUP-002.4:** When two producers attempt to enqueue the same + key concurrently, at most one row shall ever bear that key — never two — and + neither producer shall abort. Whether it is one row or none depends on whether + a coalescible run was queued at each producer's coalesce check, not on which + insert committed first; nothing may branch on which outcome a producer saw. + +### REQ-OFFICE-RUN-DEDUP-003: Never suppress silently when the generation is unknown + +**Intent:** A generation component can be unresolvable — a legacy event without +the field, an occurrence row not yet committed, a read that failed. Falling back +to a permanently-unique key in that case reintroduces exactly the defect this +capability removes, and does so in the paths that are hardest to observe. A +duplicate wake costs one redundant agent turn; a suppressed wake costs the work. + +#### Acceptance criteria + +- **AC-OFFICE-RUN-DEDUP-003.1:** When a producer cannot resolve a generation + component for an occurrence, it shall not enqueue the wake under a key whose + only components are the reason, task, and agent. +- **AC-OFFICE-RUN-DEDUP-003.2:** When a producer cannot resolve a generation + component, it shall enqueue the wake with no dedup key, accepting a possible + duplicate run rather than a possible suppression. +- **AC-OFFICE-RUN-DEDUP-003.3:** When a producer enqueues a wake with no dedup + key, the system shall record that event in telemetry, attributed to the run + reason and discriminated by whether the key was omitted because a generation + was unresolvable or because that reason is keyless by design, so that a + resolution failure is countable on its own. +- **AC-OFFICE-RUN-DEDUP-003.4:** When a wake is enqueued with an empty dedup + key, the system shall not apply idempotency suppression to it. + +### REQ-OFFICE-RUN-DEDUP-004: Deduplication is observable + +**Intent:** A dedup is a decision not to do requested work. Today it is reported +at `Debug`, is not counted, and is reported to the office producer path as an +undifferentiated success. An operator asking "why did nothing happen when I +reassigned this" has nothing to read. + +**User story:** As an Office operator, I want to see that a wake was suppressed +and why, so that a no-op is diagnosable without attaching a debugger. + +#### Acceptance criteria + +- **AC-OFFICE-RUN-DEDUP-004.1:** When the queue suppresses a wake, it shall + increment a counter exposed through the process metrics endpoint, labelled by + run reason and by whether the suppression was a windowed or a durable dedup + hit. +- **AC-OFFICE-RUN-DEDUP-004.2:** When the queue suppresses a wake through the + durable unique index after the recent-duplicate lookup reported no duplicate, + it shall emit a structured log record at a level an operator sees by default, + carrying the dedup key, the run reason, and the resolved agent. +- **AC-OFFICE-RUN-DEDUP-004.3:** When the queue suppresses a wake through the + recent-duplicate lookup, it shall emit a structured log record carrying the + dedup key and the run reason. +- **AC-OFFICE-RUN-DEDUP-004.4:** When a caller enqueues a wake through the + **run queue**, the run queue shall report which of queued, deduplicated, or + coalesced occurred, and shall report it through every run-queue enqueue + interface Office producers use. This criterion does not reach the + wakeup-request table, which has neither a windowed lookup nor coalescing and so + has no third outcome to report; AC-OFFICE-RUN-DEDUP-004.6 covers it in full. +- **AC-OFFICE-RUN-DEDUP-004.5:** When the **run queue** reports a deduplicated or + coalesced outcome, it shall not return an error, and the caller shall not treat + the outcome as a failure. The wakeup-request table is exempt from the first + clause only: it reports its one suppression as a sentinel error, and a caller + receiving that sentinel shall still not treat it as a failure. +- **AC-OFFICE-RUN-DEDUP-004.6:** When a wakeup request is rejected by the wakeup + table's unique index, the system shall increment the same counter family with + the wakeup queue identified as the source. + +## Out of scope + +- **Rewriting persisted keys on existing rows.** A run row is a historical + record of a wake that happened. Backfilling it to the new key format would + restate history and could itself collide with a key a live producer is about + to mint. AC-OFFICE-RUN-DEDUP-001.8 makes the old rows inert instead. +- **Changing the 24-hour recent-duplicate lookback window.** The window is a + performance affordance on the fast path; the unbounded unique index is what + actually enforces identity. Neither is the defect, and both are unchanged. +- **Changing the 5-second coalescing window or which reasons coalesce.** + Coalescing merges wakes that are already agreed to be redundant within seconds; + it is a separate mechanism from identity-based dedup and is out of scope here. + It is deliberately *not* a suppression: a coalesced wake merges into a run row + that is still queued for that agent and task, carrying the newer payload, so the + agent is woken. Only a deduplicated outcome produces no run at all. That is why + AC-OFFICE-RUN-DEDUP-001.2 accepts a coalesced outcome and forbids only a + deduplicated one, and why the coalescing predicate is left exactly as it stands. + Two distinct assignment generations landing inside the same five seconds + therefore yield one wake, not two: the operator asked twice in five seconds and + gets one launch carrying the second request's payload. A wake that coalesces + also leaves no row bearing its own key, so a redelivery of that occurrence + after the window queues a second run. AC-OFFICE-RUN-DEDUP-003.2 elects a + duplicate over a suppression, so that is the accepted direction; closing it + would mean keying the merged row, which changes the coalescing mechanism. + +- **Distinguishing a blocker set that resolves, un-resolves and re-resolves + identically.** `task_blockers_resolved` identifies its occurrence by digesting + the blocker id set, and blocker edges survive resolution — the readiness check + reads each blocker task's state, not the edge's existence. So if every blocker + resolves, one is reopened, and it then completes again with no blocker added or + removed, the second wave digests to the same key as the first and is suppressed + once past the lookback window. This is the one case + AC-OFFICE-RUN-DEDUP-001.1 requires be named here: no durable row distinguishes + the two waves. Both candidates were rejected — the resolving task's id is + identical in both waves, because the readiness check fires only on the last + blocker to complete; and a blocker task's `updated_at` is exactly the + time-derived shape AC-OFFICE-RUN-DEDUP-001.5 disallows, besides re-introducing + the spurious wakes `childrenCompletedIdempotencyKey` digests the child *set* to + avoid. A durable per-wave row would close this; creating one is separate work. +- **A user-facing surface for a suppressed wake.** Deduplication is a + high-frequency, expected outcome on the engine redelivery paths; an inbox + item or a task-timeline entry per occurrence would be noise that trains + operators to ignore the surface. REQ-OFFICE-RUN-DEDUP-004 places the + observability floor at operator telemetry and logs. If a user-visible surface + is wanted later, the counter added here is the evidence needed to size it. +- **Retiring the duplicate `RunReason*` constant blocks** in + `internal/office/scheduler` and `internal/office/service`. The audit crosses + them, but consolidating them is an unrelated refactor. +- **The order in which several wakes from one mutation are enqueued.** A single + task mutation can fan out to an assignee and several reviewers. This + capability changes each wake's key, not the sequence the reactivity pipeline + emits them in, and defines no ordering or tiebreak over that fan-out. +- **Changing the `heartbeat` wake source's key.** `fireHeartbeat` still mints a + live engine operation id (`heartbeat:::`) that the + engine expands into a persisted key, so heartbeat *is* a producer and is + covered by the audit. It is already generational on its tick time, which + AC-OFFICE-RUN-DEDUP-001.6 permits, so no change is required to it. Retiring + the source itself, and the duplicate `heartbeat` constants left behind, are + separate work. diff --git a/docs/specs/office/system-design/run-dedup-generation-01.md b/docs/specs/office/system-design/run-dedup-generation-01.md new file mode 100644 index 00000000000..dde2341fd33 --- /dev/null +++ b/docs/specs/office/system-design/run-dedup-generation-01.md @@ -0,0 +1,568 @@ +--- +status: draft +system: office +requirements: + - REQ-OFFICE-RUN-DEDUP-001 + - REQ-OFFICE-RUN-DEDUP-002 + - REQ-OFFICE-RUN-DEDUP-003 +--- + +# Office Run Deduplication — Generation Identity System Design Part 1: the key contract + +Part 1 covers what goes *into* a dedup key: the producer audit, the assignment +generation and how it is carried, the per-producer generation sources, producer +convergence, and the keyless path. Observability, failure and recovery, and the +test plan are in +[Part 2](run-dedup-generation-02.md). The full producer inventory and the exact Go +signatures this capability adds or widens are in +[Part 3](run-dedup-generation-03.md). + +## Purpose and boundaries + +This design changes what goes *into* a dedup key and what the queue *reports* +when it acts on one. It changes none of the queue's suppression mechanisms: +`CheckIdempotencyKey` (the 24-hour recent-duplicate lookup), `idx_run_idempotency` +and `idx_wakeup_idempotency` (the unbounded partial unique indexes), and +`CoalesceRun` / `shouldCoalesceRun` (the 5-second merge) are all unchanged. The +last is deliberate; see [Concurrency](#concurrency). + +Adjacent contracts read and constrained but not owned: + +- `internal/runs/service` — `QueueRun`, `QueueOutcome`, + `errIdempotencyKeyConflict`. +- `internal/workflow/engine` — `idempotencyKey` and its `EntryID` / + `OperationID` precedence; already generational, unchanged. +- `internal/orchestrator/event_handlers_workflow.go` — + `officeAutoStartIdempotencyKey`; generational, unchanged apart from its legacy + fallback. +- `internal/task/repository/sqlite` — the `tasks` table and the + `workflow_step_participants` runner seat. + +## Requirement mapping + +| Requirement | Design section | +| --- | --- | +| `REQ-OFFICE-RUN-DEDUP-001` | [Generation sources per producer](#generation-sources-per-producer), [Assignment generation](#assignment-generation) | +| `REQ-OFFICE-RUN-DEDUP-002` | [Convergent producers](#convergent-producers) | +| `REQ-OFFICE-RUN-DEDUP-003` | [Unresolvable generation](#unresolvable-generation), [Keyless causes per producer](run-dedup-generation-03.md#keyless-causes-per-producer) (Part 3) | +| `REQ-OFFICE-RUN-DEDUP-004` | [Observability](run-dedup-generation-02.md#observability) (Part 2) | + +## Producer audit + +The full inventory — every producer that reaches a run queue, including the four +that pass no key at all — moved to +[Part 3](run-dedup-generation-03.md#producer-audit) when it grew a fourth table. +Two entries are load-bearing here and are repeated rather than chased: +`queueTaskAssignedRun` and `QueueRunCtx`'s default are the non-generational +producers this design replaces, and `QueueRunCtx`'s default reaches every +reactivity wake except children-completed. + +## Assignment generation + +Assignment is the only occurrence in the audit with no durable identity to +borrow. `UpdateTaskAssignee` updates the runner seat in place, so +`workflow_step_participants.id` does not change across a reassignment, and a +plain reassignment writes no `task_step_transitions` row. + +Add a monotonic counter to the task: + +```text +tasks.assignment_generation INTEGER NOT NULL DEFAULT 0 +``` + +Applied through the existing additive migration mechanism +(`internal/task/repository/sqlite/base_migrations.go`), mirroring +`task_sessions.route_generation`, the same pattern already in the tree. + +### The two bump sites + +The runner seat has **two** writers, not one, and both bump: + +| Writer | Reached from | Bump | +| --- | --- | --- | +| `office/repository/sqlite/tasks.go` `UpdateTaskAssignee` | every reassignment and unassignment | increment, inside its existing transaction | +| `task/repository/sqlite/task.go` `insertTaskTx` -> `upsertRunnerInTx` | `CreateTask` | insert `assignment_generation = 1` **under exactly the guard that writes the runner row** (`AssigneeAgentProfileID != "" && WorkflowStepID != ""`); any other create inserts `0` | + +Both already run in a transaction. Each bumps on **every committed assignment +write**, including one setting the agent that already held the seat. That +unconditional rule is load-bearing: a guard of "only when the resolved runner +changes" would leave a repeat assignment minting an identical key, the permanent +no-op AC-001.3 forbids. Re-assigning a task to the agent that already holds it is +a real occurrence — the operator is asking for the work again. + +An unassignment (`assigneeID == ""`, deleting the runner row) is an assignment +write and bumps too, so A -> unassigned -> A yields three distinct generations +and the final assignment is not suppressed by the first. + +The occurrence's generation is stamped into: + +```text +task_assigned::: +``` + +A different shape from the pre-existing `task_assigned::` rows, so +AC-001.8 holds without a backfill: an old row can never equal a new key. The +counter is task-level, not per-agent — A -> B -> A produces generations 1, 2, 3, +so the third assignment's key differs from the first's. + +### Four writers of the same row that must NOT bump + +`task/repository/sqlite/task.go` also calls `syncRunnerInTx` from `updateTaskTx`, +`UpdateTaskIfWorkflowStepHasCapacity`, +`PromoteQueuedTaskIfWorkflowStepHasCapacity` and +`RestoreTaskMessageRollbackIfSessionState`, each passing +`task.AssigneeAgentProfileID`. They write the same `role='runner'` row and are +provably inert as assignments: the field is **not a stored column** — `tasks` has +had none since ADR 0005 Wave F, every read derives it through `runnerProjection` +from the seat — so each of these paths writes back the value it just read. + +No production caller can put a *new* agent into the seat through these four. +`task/service.UpdateTaskRequest` and `task/dto.UpdateTaskRequest` have no +`AssigneeAgentProfileID`, only `AssigneeUserID`, which its own doc comment +records as independent of the agent assignee. **Package-qualify that claim +wherever it is cited:** `office/dashboard.UpdateTaskRequest` is a different type +that *does* carry the field, but it routes to +`DashboardService.SetTaskAssigneeAsAgent` -> `UpdateTaskAssignee` — bump site one +— and never reaches `syncRunnerInTx`. Every production site supplying a new value +reaches either that path or `CreateTaskRequest` -> `insertTaskTx`, the second +bump site (`task/service/service_tasks.go`, `service_child_task.go` inheriting +the parent's, `backendapp/adapters_office.go`, `mcp/handlers/handlers.go`). Every +other non-test mention of the field is a read, a comparison, or a DTO projection. + +Two of the four are step moves, which re-key the seat to a new `step_id`. That is +a seat migration, not an assignment; see +[What the counter does not cover](#what-the-counter-does-not-cover). + +### Carrying the generation + +**It is captured inside the assigning transaction and carried to the producers. +A producer never re-reads it.** The `UpdateTaskAssignee` bump site reads the new +value back inside its existing transaction, before `Commit`, and returns it; the +caller passes it to each producer explicitly. The create-path bump site returns +nothing and changes no signature: its generation is a constant the runner guard +already fixes (Part 3). On the dashboard path that caller is +`DashboardService.SetTaskAssigneeAsAgent` (`office/dashboard/service_tasks.go`). +Note that `office/service` declares a same-named `SetTaskAssigneeAsAgent` and a +`SetTaskAssignee` which also reach `UpdateTaskAssignee` but publish nothing and +run no reactivity; they are not producers, may discard the value, and neither +signature changes. Two channel-setup sites reach it as well and are likewise not +producers. + +`UpdateTaskAssignee`'s widened signature, all five of its non-test call sites and +what each does with the value, and the behaviour when the read-back itself fails, +are specified in +[Part 3](run-dedup-generation-03.md#updatetaskassignee-and-the-bump-sites). The +wire encoding of the event field is specified in +[Part 3](run-dedup-generation-03.md#the-event-payload-field); `0` is a legal +generation and is never a sentinel for absent. + +Two carriers, both **new fields this design adds** (neither exists today): + +- `TaskReactivityChange.AssignmentGeneration` (`office/dashboard/service.go`), + copied by `convertChangeToMutation` (`office/scheduler/dashboard_adapter.go`) + onto a matching `TaskMutation.AssignmentGeneration` + (`office/scheduler/reactivity.go`). Both `*int64`; nil means "not supplied". No + struct is needed — the mutation already carries the assignee as the bare + `NewAssigneeID *string` and the generation follows that shape. +- `assignment_generation` on the `task.created` / `task.updated` payload, decoded + by `TaskUpdatedData` (`office/service/event_subscribers.go`) and published by + `task/service/service_events.go`. + +The field exists because the read is unsound: a producer reading +`assignment_generation` after the commit reads *the task's current* generation, +not *its occurrence's*. Given A -> B (1), B -> A (2), A -> B (3), a producer +still on occurrence 1 reads 3 and mints occurrence 3's key byte for byte; one is +then suppressed by the unique index, the direction REQ-OFFICE-RUN-DEDUP-003 +forbids. A producer handed no generation takes the keyless path of +[Unresolvable generation](#unresolvable-generation) and does **not** read the +task row to recover one. + +**The one producer that already re-reads stays keyless.** `handleTaskCreated` +calls `queueTaskAssignedRun` with `fallbackToStoredRunner=true`, so when the +`task.created` payload omits the assignee that producer recovers it through +`GetTaskExecutionFields` — a fresh read after the commit. Do **not** extend +`TaskExecutionFields` with the generation; that would make the unsound read above +a supported path. When the event carries no `assignment_generation` the producer +goes keyless with `cause=unresolved` even though it did recover an agent id: it +has an agent to wake and no occurrence identity to name. + +### The reactivity gate + +A new key alone is not sufficient. `ApplyTaskMutation` +(`office/scheduler/reactivity.go`) reaches `reactToAssigneeChange` only when the +agent differs — `change.NewAssigneeID != nil && *change.NewAssigneeID != +task.AssigneeAgentProfileID` — and `DashboardReactivityAdapter` substitutes the +*pre-update* assignee into the snapshot. On a repeat assignment to the agent +already holding the seat both sides are equal, so **no `task_assigned` wake is +produced at all** and AC-001.3 fails before any key is built. + +The gate drops the equality test and fires whenever `change.NewAssigneeID` is +non-nil. That is safe by the same call-site argument the bump sites use: of the +three `TaskReactivityChange` constructors in `office/dashboard/service_tasks.go`, +only `runReactivityForAssigneeChange` sets `NewAssigneeID`; the status and +comment constructors leave it nil, so a non-nil value already means "this +mutation is an assignment". An unassignment (non-nil but `""`) needs no new +guard: the pipeline's `queue` closure already returns early on an empty agent id. + +**The interrupt needs a guard of its own, and does not have one today.** +`reactToAssigneeChange` also sets `res.InterruptSessionID`, hard-cancelling the +seat's previous session — and it does so on `task.AssigneeAgentProfileID != ""` +alone, never comparing `newAssigneeID`. That was safe only because the caller's +equality gate never delivered it a same-agent repeat. Removing that gate exposes +it, and because the adapter substitutes the pre-update assignee, on an A -> A +repeat the previous assignee is non-empty and the interrupt would cancel the +agent's own in-flight run. Move the comparison inside: set `InterruptSessionID` +only when `task.AssigneeAgentProfileID != ""` **and** `newAssigneeID != +task.AssigneeAgentProfileID`. Only the wake becomes unconditional — this +capability changes dedup identity, not session lifecycle. + +### What the counter does not cover + +Three cases a builder would otherwise guess at. Ordering note: +`assignment_generation` is compared only for equality, never `>`/`<`. Nothing +reads it as a sequence, so no ordering or tiebreak over generations is defined. + +**Two concurrent reassignments of one task.** Each bump site increments inside +its own transaction and writes are serialised, so the commits take distinct +generations N and N+1 — two keys, two runs, one per assignment. No extra locking. + +**A producer handed no generation.** Because the value is carried, not read, the +only "stale value" case left is *absent* value — an event minted before this +shipped, or a nil `AssignmentGeneration`. It goes keyless. + +**A step move.** It re-keys the seat row to the new `step_id`, and because +`runnerProjection` falls back to `workflow_steps.agent_profile_id` it can both +change the resolved runner with no participant write and materialise a per-task +row where none existed. Neither is an assignment: the occurrence is already +covered by `officeAutoStartIdempotencyKey`'s `stepTransitionID`. Do not bump to +compensate — that would re-wake an agent auto-start has already woken. + +## Generation sources per producer + +The replacement for each non-generational producer. + +| Reason | New key | Generation source | +| --- | --- | --- | +| `task_assigned` | `task_assigned:::` | `tasks.assignment_generation` | +| `task_comment` (reactivity, assignee) | `task_comment::` | comment row id, plus the recipient | +| `task_mentioned` | `task_mentioned::` | comment row id, plus the recipient | +| `task_blockers_resolved` | `task_blockers_resolved:::` | blocker-set digest | +| `blockers_resolved` operation id | `blockers_resolved::` | same digest, same sort rule | +| `task_unblocked` | none (keyless) | no durable occurrence row | +| `task_reopened` | none (keyless) | no durable occurrence row | +| `task_reopened_via_comment` | `task_reopened_via_comment::` | comment row id | +| `task_review_requested` | none (keyless) | no durable occurrence row | + +`task_mentioned` must carry the recipient: the mention wake and the assignee +comment wake describe the same comment but address different agents, and +including the agent keeps the fan-out independent when a comment mentions several +agents and one is later re-resolved. `task_comment` carries it for the same +reason — see [Convergent producers](#convergent-producers). + +**AC-001.8 for the reshaped comment keys.** Only `task_assigned` gains a segment. +`task_comment`, `task_mentioned` and `task_reopened_via_comment` keep three +segments and merely swap the task id for a comment id, so the "different shape" +argument above does not carry over; all three hold instead because comment ids +and task ids come from disjoint uuid-keyed tables, so +`::` can never equal `::` +for a real pair. `task_comment`'s superseded form is the +`{reason}:{taskID}:{agentID}` default this design removes from `QueueRunCtx`. + +`task_blockers_resolved` uses a set digest rather than the resolving blocker's +id, matching `childrenCompletedIdempotencyKey`'s already-argued shape: a wave is +identified by which blockers were in play, so a second wave that adds blockers +gets a distinct key. A wave re-resolving the *same* set digests identically and +is suppressed — the one collapse the requirements name under `## Out of scope`. +No producer may improvise around it. + +Both blocker producers must sort the ids **lexicographically by +`blocker_task_id`** before digesting, and must not inherit `ListTaskBlockers`'s +ordering. That query is `ORDER BY created_at`, which is not unique: two blockers +added in one timestamp tick have undefined relative order, so two reads of one +unchanged set can digest differently and mint two keys for one occurrence. +`childrenCompletedIdempotencyKey` is safe only because `ListChildStates` orders +by `id`; the blocker path must impose its own. + +Sorting is necessary but not sufficient: the two blocker producers live in +different packages and AC-002.1 requires them to derive an identical key, so they +call **one shared builder** rather than each implementing the same convention. +The delimiter, the digest encoding and the empty-set rule are binding and are +specified in +[Part 3](run-dedup-generation-03.md#the-shared-key-builders). + +Three status-driven reasons are keyless. `ApplyTaskMutation` is called +synchronously after the mutation commits, never from the event bus, so a status +transition has no redelivery path and the key guarded nothing. Two guards +suffice: the pipeline's per-mutation `seen` map (`{agentID}:{taskID}:{reason}`) +stops a double-queue inside one mutation, and the 5-second coalescing window +absorbs a rapid repeat across mutations. A status-transition ledger is not +justified by the defect, and AC-003.2 elects a duplicate over a suppression. + +`QueueRunCtx`'s `idempotencyKey == ""` default is **removed**, not replaced. Its +existence is what silently gave every future caller a permanent key. After this +change a `RunContext` with no `IdempotencyKey` is enqueued keyless, the safe +direction. Producers that want dedup set the field explicitly, and the +`RunContext.IdempotencyKey` doc comment records that empty now means "no dedup" +rather than "derive one for me". + +### Agent-supplied keys + +`SpawnAgentRun` prefixes a **non-empty** agent value with the calling run's id: + +```text +agent:: +``` + +A retry of the same run reuses the run id and still dedupes, which is the case +the agent is protecting against; a later run gets a different prefix and is not +suppressed. With no caller run id the request is enqueued keyless. + +An **empty** `input.IdempotencyKey` is not prefixed — it stays empty and goes +keyless. Prefixing it would collapse every no-dedup-intent `SpawnAgentRun` inside +one run onto the single key `agent::`, suppressing the second such +call: the run-scoped form of the exact defect this capability removes. + +### Routine keys + +Every routine fire commits a `RoutineRun` row (`dispatchRoutineRun` -> +`CreateRoutineRun`) before `materialiseLightweightRoutineRun` builds the key, so a +durable row is always available. But **that row identifies the dispatch attempt, +not always the occurrence**, and which is which depends on the source. The +discriminator is `RoutineRun.Source` (`shared.RoutineSourceCron` = `"cron"`, +`"manual"`, `"webhook"`), which `buildRoutineIdempotencyKey`'s current +`(routineID, triggerID string, startedAt *time.Time)` signature does not receive +and must gain. + +| `run.Source` | New key | Generation source | +| --- | --- | --- | +| `cron` | `routine:::tick:` | the scheduled tick claimed off `office_routine_triggers.next_run_at` | +| `manual`, `webhook` | `routine::run:` | `RoutineRun.ID` | + +**Why cron does not use `RoutineRun.ID`.** `processCronTrigger` opens with +`ClaimTrigger`, a compare-and-swap (`SET next_run_at = NULL WHERE id = ? AND +next_run_at = ?`). Exactly one caller wins a given scheduled tick and the loser +returns before any row is written, so there is one `RoutineRun` per *slot*: the +row is the attempt, the slot is the occurrence. Keying on `run.ID` would give a +redelivery of one slot a fresh key, the direction AC-001.4 forbids. Catch-up does +not change this - `computeRoutineMissed` collapses N missed ticks into **one** +fire carrying `missedTicks`, so a catch-up is still one occurrence. + +**Why manual and webhook do use `RoutineRun.ID`.** Neither claims a slot. Two +manual fires are two distinct intents, an operator asking twice, and +`RoutineRun.ID` is their only durable distinguishing identity. This is also the +collision the audit table names: both carry `triggerID == ""`, so today they +collapse onto one `routine::` key. + +The consequence, stated rather than left to be inferred: a **retried** manual or +webhook request commits a second `RoutineRun` and therefore wakes twice, because +nothing upstream of `CreateRoutineRun` deduplicates the request itself. That is +the direction AC-003.2 elects, a duplicate wake over a suppressed one, and +narrowing it would need a request-level identity this capability does not +introduce. Two concurrent manual fires behave the same way: two rows, two keys, +two wakes, no locking. Cron is the only source where the CAS makes redelivery +converge. + +**The claimed tick must be CARRIED, not re-read (AC-001.9).** `processCronTrigger` +calls `UpdateTriggerNextRun(advanceTo)` *before* dispatching, so by the time the +key is built the trigger row's `next_run_at` already names the **next** slot; +only the in-memory `*trigger.NextRunAt` still holds the claimed value. Thread it +from `processCronTrigger` through `DispatchRoutineRunWithMissed` -> +`dispatchRoutineRun` -> `materialiseRoutineRun` -> +`materialiseLightweightRoutineRun`. A producer that re-reads the trigger row +mints the next occurrence's key, the same unsound-read failure +[Carrying the generation](#carrying-the-generation) rules out for assignment. + +`trigger` reaches `dispatchRoutineRun` and stops there, so the claimed tick +travels the last two hops as its own `*time.Time` parameter, exactly as +`missedTicks` already does. `materialiseRoutineRun` branches on `tmpl.Title`; +`materialiseHeavyRoutineRun` ignores the parameter, the treatment `missedTicks` +already receives on that branch, because the heavy path creates a real task and +builds no wakeup key. + +**Nil and boundary.** `processCronTrigger` returns early when +`trigger.NextRunAt == nil`, so a live cron dispatch always has a claimed tick. The +exported `DispatchRoutineRunWithMissed` can still be called with +`source == "cron"` and a nil trigger or nil claimed tick; that enqueues +**keyless** with `cause=unresolved` (see +[Unresolvable generation](#unresolvable-generation)) rather than falling back to a +minute. Unix seconds suffice because cron granularity is one minute at finest, so +two distinct slots of one trigger can never share a value. + +A `RoutineRun.Source` that is neither `cron` nor `manual`/`webhook` matches no +branch of the table above. It enqueues **keyless** with `cause=unresolved`, the +same direction as a cron dispatch with no claimed tick: an unrecognised source is +a source whose occurrence this design cannot name, and inventing a key for it +would be guessing at an identity rather than reading one. + +`unix_minute` is removed from both branches; no fire keeps a processing-clock +key. + +## Convergent producers + +Two producers observe an assignment on the paths this section governs: +`queueTaskAssignedRun` +(`office/service/event_subscribers.go`, from `task.created` / `task.updated`) and +`reactToAssigneeChange` (`office/scheduler/reactivity.go`, through +`ApplyTaskMutation`). They converge on one key today by accident of format — +`fmt.Sprintf("task_assigned:%s:%s", …)` and `QueueRunCtx`'s +`fmt.Sprintf("%s:%s:%s", reason, taskID, agentID)` produce the same string — and +that convergence must survive (AC-002.2). + +**They are driven by different entry paths, and no decision here may assume one +call sequence reaches both.** Verified: + +| Entry path | Publishes / calls | Producer that fires | +| --- | --- | --- | +| Office dashboard `SetTaskAssigneeAsAgent` | `publishTaskUpdated` -> `events.OfficeTaskUpdated`, then `runReactivityForAssigneeChange` | reactivity **only** | +| Task creation with an assignee | `events.TaskCreated` | `queueTaskAssignedRun` **only** | +| Task service update | `events.TaskUpdated` | `queueTaskAssignedRun`, but see below | + +`handleTaskUpdated` subscribes to `events.TaskUpdated`; `publishTaskUpdated` +emits `events.OfficeTaskUpdated`. Different subjects, so the dashboard assignment +path does **not** reach `queueTaskAssignedRun`. + +**The `task.updated` row cannot carry an assignment *change*.** As +[Four writers](#four-writers-of-the-same-row-that-must-not-bump) establishes, no +production update path reaching `syncRunnerInTx` can put a new agent into the +seat. So `queueTaskAssignedRun`'s live assignment occurrence is task +**creation**; `handleTaskUpdated` stays subscribed as a redelivery and defensive +path, and if it fires without a carried generation it goes keyless with +`cause=unresolved`. Nothing here may assume a task-service update produces an +assignment occurrence. + +**A third producer exists on the onboarding path and does not converge.** +`office/onboarding` creates a task with an assignee and enqueues its own +`task_assigned` run, while the same creation's `task.created` event drives +`queueTaskAssignedRun`. The onboarding producer is never handed the generation, +so it goes keyless rather than minting a divergent key. AC-002.1 binds only +producers that each derive a key, so a keyless third producer owes no +convergence; see +[Part 3](run-dedup-generation-03.md#onboarding-is-a-third-task_assigned-producer). + +AC-002.2 therefore requires something narrower than "both always fire": *when one +assignment occurrence is observed by both producers — a redelivery, or a future +path that drives both — they must derive the same key.* Carrying the generation +delivers that: both stamp the value the assigning transaction committed rather +than each re-deriving one. Part 2's test case drives both producers directly +rather than through a dashboard call, because on that path only one fires. + +To make the convergence structural rather than coincidental, both producers call +one exported builder instead of formatting the string themselves. Place it beside +`commentkeys` as a sibling package so the office service, the office scheduler +and the orchestrator all reach it without an import cycle; +`officeAutoStartIdempotencyKey` keeps its own shape, a step-driven auto-start +being a different occurrence from a reassignment. + +**The comment wakes do not converge, and do not need to.** `queueCommentRun` +passes `commentkeys.TaskComment(commentID)` to `dispatchEngineTrigger` as an +*operation id*, which the engine expands to +`::::` before persisting, so the shapes can +never be equal. They are already mutually exclusive by construction: +`runReactivityForComment` (`office/dashboard/service_tasks.go` — **not** +`handleCommentCreated`, which never touches the field) sets +`SkipAssigneeCommentWake` from `dispatchCommentEngineTrigger`'s return, and the +adapter carries it across as `SkipAssigneeWake`, so when the engine handled the +comment the reactivity assignee wake is skipped. One comment, one assignee wake, +enforced by a branch rather than a key collision. The engine is unchanged. + +Consequently `task_comment` **does** carry the recipient. The occurrence is +"(this comment, this recipient)", not "this comment": one comment fans out to an +assignee and every mentioned agent, and a bare `task_comment:` would +collide two recipients' wakes and suppress one. The `task_comment:` prefix is +preserved, so `shouldCoalesceRun`'s prefix test still excludes these from +coalescing, and `commentkeys` already parses salted keys (`CommentIDFromKey` cuts +at the first colon), so the extra segment needs no parser change. + +### Concurrency + +Two producers racing on one key resolve without locking, and this design adds +none. Both call `CheckIdempotencyKey` and may see no duplicate; both proceed; +`idx_run_idempotency` admits one; the loser's `CreateRun` returns a unique +violation, mapped to `errIdempotencyKeyConflict` and reported as +`QueueOutcomeDeduped` with a nil error. + +That last step is today true **only** in `runs/service`. `queueRunInline` and +`SchedulerService.QueueRun` return the raw violation as an error, so a race on +the reactivity path surfaces as a failure rather than a dedupe. Both gain the +mapping via the shared helper (see +[Counters](run-dedup-generation-02.md#counters)); without it AC-002.3 does not +hold on the path this capability fixes. + +**The loser can observe either suppression outcome, and AC-002.3 accepts both.** +The queue checks the key, *then* coalesces, *then* inserts. A loser whose lookup +ran after the winner's insert sees `Deduped`; one whose lookup ran before that +insert and whose coalesce check ran after it sees `Coalesced`, because the +winner's row is queued for the same agent, reason and task inside the 5-second +window. Both leave exactly one run row and neither returns an error. A test for +AC-002.3 must accept either and must not pin `Deduped`. Note also that +`SchedulerService.QueueRun` coalesces unconditionally where `runs/service` first +consults `shouldCoalesceRun`; neither is changed, and this outcome contract is +written to hold for both. + +Neither caller aborts, and the observable result is the same whichever insert +committed first (AC-002.4): at most one row ever bears that key, and none does +when a coalescible run was queued for that agent, reason and task at each +producer's coalesce check, because a coalesced enqueue never reaches +`insertRun`. See +[Key durability](run-dedup-generation-02.md#key-durability), which owns that +boundary. No ordering or tiebreak is defined: the losing path has no output to +order. The run `id` is a fresh UUID per attempt, so nothing may depend on which +uuid won. + +## Unresolvable generation + +A producer that cannot resolve its generation component enqueues with an empty +key. It does **not** fall back to `::` (AC-003.1), because +that is the defect. + +Concretely: an event carrying no `assignment_generation`, or a nil +`AssignmentGeneration` on a mutation; `officeAutoStartIdempotencyKey`'s zero +`stepTransitionID`; `SpawnAgentRun` with no caller run id; a reactivity comment +wake whose `MutationComment` is nil; and a `source == "cron"` routine dispatch +reaching [Routine keys](#routine-keys) with no claimed tick carried. + +`SpawnAgentRun` with an **empty agent-supplied key** is deliberately absent from +that list: nothing failed to resolve, the agent expressed no dedup intent, so it +is `by_design`. Part 3's +[keyless causes](run-dedup-generation-03.md#keyless-causes-per-producer) assigns +every producer its side and agrees. + +An empty key already means "no dedup" throughout the queue: `QueueRun` skips +`CheckIdempotencyKey`, `insertRun` writes a NULL `idempotency_key`, and both +unique indexes are partial and ignore NULL (AC-003.4). Coalescing still applies — +`shouldCoalesceRun` only excludes the `task_comment:` prefix. + +**Where the keyless counter is incremented, and how `cause` reaches it.** The +cause is a property of the producer's *decision*, not of the request: by the time +an enqueue reaches a queue, "I never needed a key" and "I tried to build one and +failed" are both an empty string, and `RunContext` carries nothing separating +them. So the queue does **not** report keyless enqueues and nothing infers a cause +from an empty key. Instead `internal/runs/service` exports a reporter beside the +suppression helper of [Counters](run-dedup-generation-02.md#counters): + +```text +runsservice.ReportKeylessEnqueue(reason string, cause KeylessCause, detail string) +``` + +with `KeylessCauseUnresolved` and `KeylessCauseByDesign`. Each producer calls it +where it decides to go keyless, naming its own cause, immediately before +enqueuing. `detail` names **what failed to resolve** — the short constant +[Part 3](run-dedup-generation-03.md#keyless-causes-per-producer) assigns each +producer — so the `Info` record [Part 2](run-dedup-generation-02.md#logs) +requires for `cause=unresolved` separates two producers that go keyless under one +reason. It is `""` for `cause=by_design`, which is not logged. Counters stay declared once in `internal/runs/service` — no parallel +map — and the import direction already holds: `runs/service` imports +`office/models` only, so `office/service`, `office/scheduler` and +`office/routines` can all call it without a cycle. + +`cause=unresolved` is every site enumerated in *this* section. `cause=by_design` +is the three status reasons and any other producer that never had an occurrence to +name. **The earlier rule — that any `RunContext` setting no key is `by_design` — +is withdrawn**: with `QueueRunCtx`'s default removed a producer that failed to +resolve also arrives with no key, so that rule counted precisely the failures +AC-003.3 exists to isolate as normal traffic. A producer reaching a queue keyless +without having called the reporter is a bug; the key-format table test in +[Part 2](run-dedup-generation-02.md#testing) is where it is caught. + +The counters themselves are defined in +[Part 2](run-dedup-generation-02.md#counters). diff --git a/docs/specs/office/system-design/run-dedup-generation-02.md b/docs/specs/office/system-design/run-dedup-generation-02.md new file mode 100644 index 00000000000..8888343aff1 --- /dev/null +++ b/docs/specs/office/system-design/run-dedup-generation-02.md @@ -0,0 +1,393 @@ +--- +status: draft +system: office +requirements: + - REQ-OFFICE-RUN-DEDUP-001 + - REQ-OFFICE-RUN-DEDUP-002 + - REQ-OFFICE-RUN-DEDUP-003 + - REQ-OFFICE-RUN-DEDUP-004 +--- + +# Office Run Deduplication — Generation Identity System Design Part 2: observability and verification + +Part 2 covers what the queue *reports* when it acts on a dedup key +(`REQ-OFFICE-RUN-DEDUP-004`), the upgrade and failure behaviour, and the test +plan for all four requirements. The key contract itself — the producer audit, +the assignment generation, the per-producer sources, convergence and the keyless +path — is in [Part 1](run-dedup-generation-01.md). The full producer inventory +and the exact Go signatures named below are in +[Part 3](run-dedup-generation-03.md). + +## Observability + +### Counters + +New expvar maps, following `internal/office/scheduler/metrics_vars.go`'s +`k=v;k=v` label model and the counters-only rule stated there. + +**Three queue implementations make a dedup decision, not one, and all three are +in scope:** + +| Queue | Reached from | Today | +| --- | --- | --- | +| `runs/service.Service.QueueRun` | `office/service.Service.QueueRun` when a runs service is wired | checks the key; maps the unique violation to `Deduped`; logs `Debug` | +| `office/service.Service.queueRunInline` | same, when no runs service is wired (older tests, transitional deployments) | checks the key; returns the raw `CreateRun` error; logs `Debug` | +| `office/scheduler.SchedulerService.QueueRun` | `QueueRunCtx`, and so every reactivity wake | checks the key; returns the raw `CreateRun` error; logs `Debug` | + +The third most needs instrumenting: `ApplyTaskMutation` -> `QueueRunCtx` -> +`SchedulerService.QueueRun` is the path a reassignment travels and is the *least* +observable today. Instrumenting only `internal/runs/service` would leave +AC-004.1 / .2 / .3 unmet on the flow the card reports. + +To keep one behaviour rather than three, the *reporting* is one exported helper +in `internal/runs/service`: it classifies a suppression as `windowed` or +`durable`, increments the counter, emits the log, and returns the `QueueOutcome`. +All three queues call it. Counters are declared once there; nothing declares a +parallel map. Its name, its two entry points, and what it does with an error that +is *not* a unique violation are specified in +[Part 3](run-dedup-generation-03.md#the-suppression-reporter-and-the-no-op-outcome). + +The keyless counter is **not** driven from the queues. It has its own exported +reporter in the same package, `ReportKeylessEnqueue(reason, cause, detail)`, called by the +producer at its own decision site, because only the producer knows whether the key +was never needed or could not be resolved — see +[Unresolvable generation](run-dedup-generation-01.md#unresolvable-generation). + +| Name | Labels | Incremented when | +| --- | --- | --- | +| `office_run_dedup_total` | `reason`, `kind` (`windowed` or `durable`), `queue` (`runs` or `wakeup`) | the queue suppresses a wake | +| `office_run_dedup_keyless_total` | `reason`, `cause` (`unresolved` or `by_design`) | a producer calls `ReportKeylessEnqueue` before enqueuing with no dedup key | + +`kind` earns the metric: `windowed` is the expected, high-frequency outcome on +the engine redelivery paths, while `durable` means the 24-hour lookup found +nothing and the unbounded index still rejected the insert. After this ships that +is one of three things — a genuine race, a producer minting a colliding key, or a +**legitimate late redelivery** of one occurrence past the lookback window. The +third is not a fault and is not rare: `ParentWakeReconciler` is a level-triggered +backstop that re-derives the identical `wakeOperationID` from current task state +on every tick, with no age bound, so a parent stuck for more than a day produces a +durable hit as normal operation. `Warn` is still the right level — a durable hit +always means the fast path missed something an operator may want to see — but +neither the log text nor any alert built on this counter may call a durable hit an +anomaly on its own. One undifferentiated counter would bury the first two. +`cause` earns its place the same way: `by_design` is high-frequency and expected, +`unresolved` is the generation that should have been carried and was not +(AC-003.3), and by `reason` alone the two are indistinguishable. Cardinality is +the run-reason enum times a small constant, the same order as `routing_*`. + +`office_run_dedup_total{queue="wakeup"}` increments where `CreateWakeupRequest` +returns `ErrWakeupIdempotencyConflict` (AC-004.6); that path has no windowed +lookup, so its `kind` is always `durable`. Its `reason` label is +`WakeupRequest.Reason`, the field already on the row being inserted, not the +coarser `Source`, which would collapse every wakeup onto one series. + +### Logs + +- Windowed hit: `Info`, carrying key and reason. Currently `Debug`. +- Durable hit: `Warn`, carrying key, reason, and resolved agent. Currently + `Debug`, on both the `runs` and wakeup paths. +- Keyless enqueue with `cause=unresolved`: `Info`, carrying `reason` and the + `detail` constant naming what failed to resolve — `ReportKeylessEnqueue`'s + third argument, whose per-producer values + [Part 3](run-dedup-generation-03.md#keyless-causes-per-producer) fixes. Without + it the four producers that go keyless under `reason=task_assigned` are + indistinguishable in the log, which is what AC-003.3 exists to prevent. + `detail` is a log field only, never a counter label. A `cause=by_design` + keyless enqueue is counted but **not** logged — it is the normal path for + several reasons and would be pure noise. + +### Outcome propagation + +`runs/service.QueueRun` already returns `QueueOutcome`. Office's own enqueue +interfaces discard it: `office/shared.RunQueuer.QueueRun`, +`office/service.Service.QueueRun`, `office/scheduler.SchedulerService.QueueRun` +and `QueueRunCtx`, plus the duplicate declarations in `office/runtime/actions.go` +and `office/approvals/service.go`, all return only `error`. Widen them to +`(runsservice.QueueOutcome, error)` (AC-004.4). + +The three queues under [Counters](#counters) produce that outcome, and +`office/service.Service.QueueRun` forwards whichever of its two it used. +`queueRunInline` and `SchedulerService.QueueRun` return no `QueueOutcome` today +and map no unique violation; both gain the mapping from the shared helper, so a +durable conflict on the reactivity path becomes `Deduped` with a nil error +instead of today's `"reactivity run failed"` `Error` log (AC-002.3). + +**The wakeup enqueue is outside this widening**, and AC-004.4 now says so in its +own text rather than leaving it to be inferred. `CreateWakeupRequest` +(`office/repository/sqlite/wakeup_requests.go`, interface in +`office/routines/service.go`) keeps its `error` signature: no coalescing and no +windowed lookup, so there is no third outcome to report, and its one suppression +is the sentinel `ErrWakeupIdempotencyConflict`, which AC-004.5 exempts by name. +The exemption covers the signature only — a caller receiving that sentinel must +still not treat it as a failure, which the one live caller already does by +logging and continuing. + +**Sizing.** This widening is the highest-diff item here — `.QueueRun(` appears +~112 times across 33 files in `internal/office`, ~9 of them production call +sites. Size it as its own task rather than folding it into "observability". + +**A path that enqueues nothing reports `QueueOutcomeNone`.** `QueueOutcome`'s +zero value is the empty string, which is none of `queued`, `deduped` or +`coalesced`. Part 3 gives it a name and a meaning +([the no-op outcome](run-dedup-generation-03.md#the-suppression-reporter-and-the-no-op-outcome)). +This is additive and changes no acceptance criterion: AC-004.4 obliges the queue +to report one of three outcomes *when a caller enqueues a wake*, and a widened +signature returning a non-nil error never meets that condition. The reactivity +pipeline's empty-agent-id guard is **not** an instance of it: that guard lives in +the `queue` closure and returns *before* calling `QueueRunCtx`, so it yields no +outcome at all rather than `QueueOutcomeNone`. `QueueRunCtx` itself has no such +early return. + +Callers that have nothing to decide assign the outcome to `_`. No caller may +treat `Deduped` or `Coalesced` as an error (AC-004.5); the reactivity pipeline's +`queue` closure in particular must keep appending to `res.Runs` only for +`QueueOutcomeQueued`, so `ApplyTaskMutationResult.Runs` stops reporting +suppressed wakes as queued ones. + +## Key durability + +A dedup key is durable only for an enqueue that reached `insertRun`. All three +queues run `CheckIdempotencyKey` -> `CoalesceRun` -> `insertRun`, and +`CoalesceRun` (`internal/runs/repository/sqlite/runs.go`) selects its neighbour +on `agent_profile_id`, `reason`, `status='queued'`, `requested_at` inside the +window, — for `task_assigned` only — the payload's `task_id`, **and +`(idempotency_key IS NULL OR idempotency_key NOT LIKE 'task_comment:%')`**. + +That last predicate is the one to be exact about, because two different keys are +in play and only one of them is read: + +- **The incoming enqueue's key is neither read nor written here.** The `UPDATE` + sets `coalesced_count` and `payload` only, so the merged row keeps its own key + and the incoming key is persisted nowhere. That is what makes a coalesced + enqueue **persist no row bearing its own key**; neither unique index ever sees + it. Whether the incoming enqueue may coalesce at all is decided earlier and + elsewhere, by `shouldCoalesceRun` (`internal/runs/service`), which excludes + only a `task_comment:`-prefixed key. +- **The predicate reads the candidate *neighbour's* key**, keeping a + `task_comment:`-keyed row from absorbing another reason's wake. A neighbour is + therefore matchable when its key is NULL **or** any non-`task_comment:` key, + including a `task_assigned:::` row. A test that needs + a matchable neighbour must not construct a `task_comment:`-keyed one — it would + never match, and the test would pass while asserting nothing. + +Two consequences the contract states rather than leaving to be discovered: + +- Two producers racing on one key may **both** coalesce into a queued neighbour, + leaving **zero** rows bearing that key. If the neighbour is claimed between + their two coalesce checks, the later one inserts and exactly one row bears it. + AC-002.4 is therefore written as *at most one, never two* rather than *exactly + one*: that is the only invariant true in every interleaving, and a test must + not pin the count to one. +- An occurrence that coalesced and is then redelivered *after* the coalescing + window has no durable key to match, and queues a second run (AC-001.4). That + duplicate is accepted, not closed: AC-003.2 elects a duplicate over a + suppression, and keying the merged row would change `CoalesceRun`, which + [Part 1](run-dedup-generation-01.md#purpose-and-boundaries) holds unchanged. + +This is a property of the pre-existing coalescing mechanism, not of the +generation component — but the generation component makes it *reachable* for +`task_assigned`, which is why it is stated here. Before this capability two +assignments inside one window shared a permanent key, so the second was stopped +by `CheckIdempotencyKey` and never reached `CoalesceRun`. With distinct +generations the windowed lookup misses and the coalesce arm is live. + +## Failure and recovery + +- A producer handed no `assignment_generation` goes keyless, never permanent: + the wake happens, dedup for that one enqueue does not. The only degraded case. +- A counter increment is best-effort and never fails an enqueue. +- The migration is additive with a `0` default and runs at startup, so both bump + sites always have the column. Every task starts at `0` and the first assignment + after upgrade commits `1`, so no task's first new-format key is `...:0`. +- **A second migration rebuilds `tasks` and must carry the column through it.** + `taskPriorityMigrationStatements` + (`office/repository/sqlite/base_migrations.go`) recreates `tasks` from an + **explicit column list** to change `priority` from INTEGER to TEXT. That list + would silently drop `assignment_generation`, so the column joins it the same + way eight other columns already do in that file — **its established idiom for + this exact hazard, followed rather than replaced**: + 1. a defensive `ALTER TABLE tasks ADD COLUMN assignment_generation INTEGER NOT + NULL DEFAULT 0` before the recreate, with the error swallowed like its + neighbours (`archived_by_cascade_id`, `wip_admitted`, `external_id` and the + rest), so the `SELECT` can reference the column on a legacy fixture that + predates it; + 2. the column declared in the `tasks_priority_new` definition; + 3. `COALESCE(assignment_generation,0)` in the recreate `SELECT`, so values are + **preserved**, not reset. + Copying rather than defaulting is what makes this order-independent: it holds + whether or not the task repository's own migration ran first, so nothing rests + on an argument about which repository initialises when. + Left undone the defect is not a hard failure, which is why it is easy to miss. + The task repository initialises first (`backendapp/storage.go`), so the column + is added, then dropped by the rebuild, absent for the remainder of that boot, + then re-added at `0` on the next one — `db.MigrateLogger.Apply` + (`internal/db/migratelog.go`) is not ledger-backed, so the ALTER re-runs. A + task previously at generation 3 re-mints `...:1` and collides with its own + historical key on the unbounded index: this capability's own defect, + resurrected. +- Events in flight across the upgrade carry no `assignment_generation`. Their + producer goes keyless with `cause=unresolved` rather than minting a + legacy-shaped key: one event's lifetime, in the direction AC-003.2 elects. +- No runtime feature flag. A key-format change plus telemetry has no + partially-migrated state to guard — a mixed fleet writing old and new formats + does not collide, the property AC-001.8 relies on. + +## Testing + +Backend `*_test.go` beside each source. Behaviours with no equivalent test today: + +- A -> B -> A reassignment wakes the first agent again on the third assignment + (AC-001.2): three distinct keys, and the third enqueue reports queued or + coalesced, never deduped. Assert the **outcome**, not a row count — a test + driving all three inside the 5-second window legitimately gets two rows. +- A repeat assignment whose prior run's `requested_at` predates the window queues + a run (AC-001.3), driven by writing an aged row. That row is also outside the + coalescing window, so this case asserts `Queued` exactly. +- Both producers, driven directly with the same task, agent and carried + generation, derive one key and queue one run (AC-002.1 / .2). At the producers, + not through a dashboard call — only one fires there. +- A repeat assignment to the **same** agent passes the relaxed reactivity gate, + bumps, and wakes the agent (AC-001.3), including when the prior run is aged + past the window; and it does **not** set `InterruptSessionID`. A reassignment + to a *different* agent still does — both branches of the guard added in + [The reactivity gate](run-dedup-generation-01.md#the-reactivity-gate). +- A non-assignment task update (title, priority) reaches `syncRunnerInTx` but + does not bump and queues no `task_assigned` run — the inert-writer claim in + [Four writers](run-dedup-generation-01.md#four-writers-of-the-same-row-that-must-not-bump). +- A task created with an assignee whose `task.created` event carries + `assignment_generation` commits generation `1` and queues one run; the same + creation with the field absent goes keyless with `cause=unresolved` and still + queues, exercising `fallbackToStoredRunner`. +- **A task created UNASSIGNED publishes `assignment_generation` `0`**, carried on + `publishTaskEventWithExtra`'s `extra` map, and queues no `task_assigned` run. + Assigning it afterwards commits `1` and queues one. This pins the boundary that + keeps a creation and a first assignment from both minting `...:1`, and it is + the only assertion that catches a create path that hardcodes `1`. +- A producer handed a generation does not read the task row for one: drive + occurrence 1's producer after occurrence 3 has committed and assert it still + mints occurrence 1's key. +- A durable conflict on the **reactivity** path (`SchedulerService.QueueRun`, and + the same for `queueRunInline`) returns `Deduped` with a nil error and does not + log `"reactivity run failed"` (AC-002.3). +- A key-format table test over **every row of the + [producer audit](run-dedup-generation-03.md#producer-audit)**, with the + assertion chosen by what the row produces. **Two assertions, not one:** + - A row that mints a key asserts its output has a segment that varies across + two constructed occurrences (AC-001.1), so a new producer reintroducing a + permanent key fails a test rather than shipping. + - A row that is **keyless by contract** has no key and therefore no segment, + so the varying-segment assertion is unsatisfiable and is not applied to it. + It asserts instead that the enqueue carries an **empty** key and that the + producer reported the `cause` and `detail` + [Part 3](run-dedup-generation-03.md#keyless-causes-per-producer) assigns it. + These are the three permanently-keyless rows of + [Part 3](run-dedup-generation-03.md#keyless-today--must-report-a-cause) — + the fourth, `agent_error`, becomes generational here and takes the first + assertion — plus the three status-driven reactivity reasons. + The same-set blocker re-resolution named in the requirements' `## Out of scope` + is the one row that mints a key and is asserted *stable* rather than varying, + pinning the collapse rather than letting it drift. +- Durable-index conflict increments `office_run_dedup_total` with `kind=durable` + and returns `Deduped` with a nil error. +- An unresolvable generation enqueues keyless with + `office_run_dedup_keyless_total{cause=unresolved}`; a status-driven reason + enqueues keyless with `cause=by_design` and emits no log record. +- `SpawnAgentRun` with an empty agent key and a live caller run id enqueues + keyless, and two such calls in one run are **not deduplicated**. Assert the + outcome, not a row count: an empty key passes `shouldCoalesceRun`, and the + first call's own row — which persisted a NULL key — satisfies the + `idempotency_key IS NULL` arm of the neighbour predicate in + [Key durability](#key-durability), whose task-scoping clause applies only to + `task_assigned`, so two calls to one agent with one reason inside the 5-second + window legitimately coalesce. Drive them with distinct reasons, or + outside the window, to assert `Queued` twice. +- Two recipients' wakes for one comment (assignee plus a mentioned agent) mint + distinct keys and both queue. +- **AC-001.7's run-id prefix, on the non-empty agent key** — the case the + empty-key bullet above does not reach. Two `SpawnAgentRun` calls passing the + *same* literal key inside one caller run dedupe; the same literal key from a + *second* caller run queues, because the prefix differs. A test that drives only + the empty-key path leaves the prefix logic unpinned. +- **The wakeup queue increments its own counter (AC-004.6).** A + `CreateWakeupRequest` rejected with `ErrWakeupIdempotencyConflict` moves + `office_run_dedup_total{queue="wakeup",kind="durable"}` **through + `ReportDurableDedup`, not `ReportInsertResult`** — the caller does its own + `errors.Is` on the sentinel, so assert the counter moves without + `runs/service` ever being handed that error. Labelled by + `WakeupRequest.Reason` and not by `Source`, and the caller does not treat the + sentinel as a failure. Nothing else in this plan exercises that file, so + without this the label can be unwired or wired to the wrong `queue` value with + the suite green. +- **The four direct keyless producers each report their assigned cause** — the + onboarding wake and `handleTaskCreated`'s fallback as `unresolved`, the + recovery sweep and `requeueRunForTask` as `by_design` — per + [Part 3](run-dedup-generation-03.md#keyless-causes-per-producer). The recovery + sweep additionally asserts it does **not** mint the assignment key: drive a + sweep over a task whose assignment run already persisted + `task_assigned:::` and assert the sweep still queues, + rather than being suppressed by the durable index. +- **`agent_error` is generational on the failed run id.** Two distinct run + failures for one agent mint different keys and both queue; a redelivery of one + failure escalation is suppressed. There is no two-CEO case to assert: a + workspace admits at most one CEO (`ErrAgentCEOAlreadyExists`) and + `queueCEOAgentError` escalates to `ceos[0]` alone. +- **Both blocker producers derive a byte-identical digest** for one blocker set, + driven through the shared builder from both packages, including a set whose ids + were returned in a different order. An empty blocker set enqueues nothing and + moves no keyless counter. +- **A path that enqueues nothing returns `QueueOutcomeNone`**, not + `QueueOutcomeQueued`: a widened signature returning an error. Assert + `ApplyTaskMutationResult.Runs` gains no entry. The `queue` closure's + empty-agent-id guard is a **separate** case with a different assertion — it + reaches no queue at all, so assert `res.Runs` gains no entry and that + `QueueRunCtx` was never called, not that it returned an outcome. +- **`QueueOutcomeNone` is declared identically in both packages.** A compile-time + or table assertion that `runsservice.QueueOutcomeNone` and + `engine.QueueOutcomeNone` hold the same value, pinning the "both MUST match" + invariant those two declarations already carry. +- **`ReportInsertResult` passes a non-conflict error through unchanged**, with + `QueueOutcomeNone` and no counter movement — a disk error is not a dedup + decision and must not be counted or swallowed as one. +- **The `tasks` rebuild preserves the column and its values.** Drive + `taskPriorityMigrationStatements` over a legacy fixture whose `tasks.priority` + is still INTEGER, seeded with one row at `assignment_generation` 3 and one at + `0`, and assert both survive the recreate unchanged. Also drive it over a + fixture predating the column entirely and assert the defensive ALTER makes the + `SELECT` succeed at `0`. Without this the loss is invisible: the column is + re-added on the next boot, so nothing fails and only the resurrected key + collision would ever show it. +- Routine keys, per source: two cron fires of one trigger on **different claimed + ticks** mint different keys; one slot re-dispatched with the same claimed tick + mints the identical key; two **manual** fires of one routine inside one minute + mint different keys (today they collide); and a `source == "cron"` dispatch with + no claimed tick carried enqueues keyless with `cause=unresolved`. The cron case + is driven by supplying the claimed tick directly, not by advancing a clock. +- A cron producer does not recover the tick by re-reading the trigger row + (AC-001.9): drive a dispatch after `UpdateTriggerNextRun` has advanced + `next_run_at` and assert the key still names the claimed slot, not the next one. +- `ReportKeylessEnqueue` is what moves `office_run_dedup_keyless_total`: a + producer that goes keyless increments its own `cause` and logs its own + `detail` constant — assert two producers going keyless under + `reason=task_assigned` are distinguishable in the log record, which is the + whole reason the argument exists; and an enqueue with an + empty key that did **not** call the reporter moves no counter — the assertion + that keeps the withdrawn "empty key means `by_design`" inference from returning. +- An occurrence that coalesced into a queued neighbour and is then redelivered + after the coalescing window queues a **second** run (AC-001.4, see + [Key durability](#key-durability)): enqueue generation N, enqueue generation + N+1 inside the window and assert `Coalesced`, age the merged row past the + window, then re-enqueue generation N+1's key and assert `Queued`. This pins the + accepted duplicate — a test asserting one run here is asserting the wrong + contract. +- Two producers enqueuing one key concurrently **with** a coalescible run already + queued for that agent, reason and task: both observe a suppression, neither + errors, and no row bears the new key (AC-002.3, AC-002.4). The no-neighbour + arrangement is the separate case the same ACs bound, and needs its own test. +- A durable-index conflict produced by re-deriving one occurrence's key past the + lookback window (the `ParentWakeReconciler` shape) increments + `office_run_dedup_total{kind=durable}`, returns `Deduped` with a nil error, and + is not reported as a failure anywhere. + +No Playwright coverage: the observable surfaces are `/debug/vars` and backend +logs, and the one user-visible consequence (a run row appearing for a repeat +assignment) is asserted at the queue in Go. diff --git a/docs/specs/office/system-design/run-dedup-generation-03.md b/docs/specs/office/system-design/run-dedup-generation-03.md new file mode 100644 index 00000000000..7054d409cf9 --- /dev/null +++ b/docs/specs/office/system-design/run-dedup-generation-03.md @@ -0,0 +1,505 @@ +--- +status: draft +system: office +requirements: + - REQ-OFFICE-RUN-DEDUP-001 + - REQ-OFFICE-RUN-DEDUP-002 + - REQ-OFFICE-RUN-DEDUP-003 + - REQ-OFFICE-RUN-DEDUP-004 +created: 2026-09-07 +owners: + - kandev +--- + +# Office Run Deduplication — Generation Identity System Design Part 3: producer inventory and API contracts + +Part 3 owns two things Parts 1 and 2 refer to but do not spell out: the complete +inventory of every producer that reaches a run queue, and the exact Go contracts +this capability adds or widens. The conceptual key contract is in +[Part 1](run-dedup-generation-01.md); observability, key durability and the test +plan are in [Part 2](run-dedup-generation-02.md). + +Everything here is a contract a builder would otherwise have to invent. Where a +name or a byte encoding is given, it is binding — two producers that must agree +(AC-OFFICE-RUN-DEDUP-002.1) cannot agree on a convention nobody wrote down. + +## Producer audit + +Every site that reaches a run queue. "Generational" means the key already varies +with the occurrence. The fourth table is the one Parts 1 and 2 previously +omitted: producers that pass no key at all, which +AC-OFFICE-RUN-DEDUP-003.3 nonetheless obliges to report a cause. + +### Already generational — no change + +| Producer | Key shape | Generation component | +| --- | --- | --- | +| `office/service` `queueCommentRun` | op id `task_comment:`, engine-expanded on persist | comment row id | +| `office/service` `handleApprovalResolved` | `approval_resolved:` | approval row id | +| `office/approvals/service.go` `queueApprovalRun` | `approval:` | approval row id | +| `office/dashboard/decisions.go` `decisionRunIdempotencyKey` | `decision:` | decision row id | +| `office/scheduler` `childrenCompletedIdempotencyKey` | `task_children_completed:::` | child-set digest | +| `workflow/engine/phase2_callbacks.go` `idempotencyKey` | `::::` | step-entry / operation id | +| `orchestrator` `officeAutoStartIdempotencyKey` | `task_assigned::::` | step-transition row id | +| `orchestrator` step-entry operation id | `step_entry::` | entry id and position | +| `scheduler/cron/heartbeat.go` operation id | `heartbeat:::` | tick time (AC-001.6) | +| `office/service/scheduler_wake_reconciler.go` `wakeOperationID` | `task_children_completed::` op id, shared with `queueChildrenCompletedRun` | child-set digest | + +`officeAutoStartIdempotencyKey`'s `legacy:` branch fires when +`stepTransitionID` is zero. It is time-derived, which AC-001.5 disallows for an +occurrence that has a durable row. Replace it with the keyless path of +[Unresolvable generation](run-dedup-generation-01.md#unresolvable-generation); do +not otherwise touch this producer. + +### Not generational — must change + +| Producer | Key today | Failure | +| --- | --- | --- | +| `office/service` `queueTaskAssignedRun` | `task_assigned::` | permanent per pair | +| `office/scheduler/run.go` `QueueRunCtx` default | `::` | permanent per (reason, task, agent) | +| `office/service` blocker-resolved dispatch | op id `blockers_resolved:` | permanent per blocked task | + +`QueueRunCtx`'s default is the larger of the two: every `RunContext` that does +not set `IdempotencyKey` inherits it — every reactivity wake except +children-completed. + +### Unconstrained — must be scoped + +| Producer | Key today | Failure | +| --- | --- | --- | +| `office/runtime/actions.go` `SpawnAgentRun` | verbatim `input.IdempotencyKey` | an agent reusing a literal permanently suppresses its own later wakes | +| `office/routines/service.go` `buildRoutineIdempotencyKey` | `routine:[:]:` | the minute comes from the **processing** clock (`run.StartedAt`, set to `time.Now()` inside `dispatchRoutineRun`), not from the occurrence. A cron fire's occurrence is its claimed scheduled tick; a manual or webhook fire's is `RoutineRun.ID`. Both are durable and neither is what the key uses. The reachable collision today is two manual fires of one routine inside one minute — they share `triggerID == ""` and the same minute | + +### Keyless today — must report a cause + +These four call a run queue **directly** with an empty key. None of them passes +through `QueueRunCtx`, so the blanket entry in the second table does not reach +them, and none was previously enumerated anywhere in this design. Each is +covered by [Keyless causes per producer](#keyless-causes-per-producer) below, and +each is a row the key-format table test of +[Part 2](run-dedup-generation-02.md#testing) must iterate — under that test's +keyless assertion (empty key, plus the reported `cause` and `detail`), never its +varying-segment one, which a producer with no key cannot satisfy. + +| Producer | Reason | Key today | Disposition | +| --- | --- | --- | --- | +| `office/onboarding/service.go` `maybeCreateOnboardingTask` | `task_assigned` | `""` | stays keyless, `cause=unresolved` | +| `office/service/scheduler_recovery.go` recovery sweep | `task_assigned` | `""` | stays keyless, `cause=by_design` | +| `office/service/retry.go` CEO error escalation | `agent_error` | `""` | **becomes generational** on the failed run id | +| `office/service/failure.go` `requeueRunForTask` | `manual_resume_after_failure` | `""` | stays keyless, `cause=by_design` | + +`office/onboarding` also declares its own `runReasonTaskAssigned` constant, a +third copy of the same string. Consolidating it is out of scope for the same +reason the requirements already give for the `office/scheduler` and +`office/service` blocks. + +### Writers that reach no queue at all + +`UpdateTaskAssignee` has five non-test call sites and only one of them is a +producer. The other four build no key and wake nobody, so they never appear in +the tables above; they are enumerated in +[UpdateTaskAssignee and the bump sites](#updatetaskassignee-and-the-bump-sites) +because they still take the widened signature. + +## Keyless causes per producer + +AC-OFFICE-RUN-DEDUP-003.3 splits keyless enqueues into `unresolved` — a +generation that should have been available and was not — and `by_design` — a +producer that never had an occurrence to name. The split only pays for itself if +each producer is assigned a side deliberately, so each is assigned one here. + +| Producer | Reason | `cause` | Why | +| --- | --- | --- | --- | +| reactivity status wakes | `task_unblocked`, `task_reopened`, `task_review_requested` | `by_design` | a status transition has no redelivery path; see Part 1 | +| `office/service/scheduler_recovery.go` | `task_assigned` | `by_design` | see [The recovery sweep](#the-recovery-sweep-must-not-borrow-the-assignment-generation) | +| `office/service/failure.go` | `manual_resume_after_failure` | `by_design` | an operator resume; the call site holds only a task id, and no row records the resume | +| `office/onboarding/service.go` | `task_assigned` | `unresolved` | the occurrence **does** have a generation; this producer simply is not handed it | +| `handleTaskCreated` with `fallbackToStoredRunner` | `task_assigned` | `unresolved` | Part 1 | +| `officeAutoStartIdempotencyKey`, zero `stepTransitionID` | `task_assigned` | `unresolved` | Part 1 | +| `SpawnAgentRun`, no caller run id | agent-supplied | `unresolved` | Part 1 | +| `SpawnAgentRun`, empty agent key | agent-supplied | `by_design` | the agent expressed no dedup intent; nothing failed to resolve | +| routine dispatch, `cron` with no claimed tick | routine reasons | `unresolved` | Part 1 | +| routine dispatch, unrecognised `Source` | routine reasons | `unresolved` | Part 1 | +| reactivity comment wake, nil `MutationComment` | `task_comment` | `unresolved` | Part 1 | + +**The `detail` argument, per producer.** Part 1's +[`ReportKeylessEnqueue`](run-dedup-generation-01.md#unresolvable-generation) +takes `detail` so the `cause=unresolved` log record names *what* failed to +resolve, which `reason` alone cannot: four distinct producers above go keyless +under `reason=task_assigned`. Each `unresolved` row reports a fixed lowercase +snake_case constant naming the producer's own failure, not a formatted message: + +| Producer | `detail` | +| --- | --- | +| `office/onboarding/service.go` | `onboarding_no_generation` | +| `handleTaskCreated` with `fallbackToStoredRunner` | `event_missing_generation` | +| `officeAutoStartIdempotencyKey`, zero `stepTransitionID` | `zero_step_transition` | +| `SpawnAgentRun`, no caller run id | `no_caller_run` | +| routine dispatch, `cron` with no claimed tick | `cron_no_claimed_tick` | +| routine dispatch, unrecognised `Source` | `unrecognised_routine_source` | +| reactivity comment wake, nil `MutationComment` | `nil_mutation_comment` | + +Every `by_design` row passes `""`: that cause is counted and never logged, so it +has nothing to distinguish. `detail` is a log field only and **not** a counter +label — these seven values would multiply the keyless counter's cardinality for +no operational question that `cause` and `reason` do not already answer. + +### The recovery sweep must not borrow the assignment generation + +`office/service/scheduler_recovery.go` re-queues tasks it finds unstarted. It is +level-triggered from current task state, the same shape as `ParentWakeReconciler`, +and by the time it runs, `tasks.assignment_generation` for that task is readable. +Reading it is nonetheless **forbidden**, and not only because AC-001.9 rules out +recovering a generation by re-reading a row. + +The decisive argument is that it would defeat the sweep. The sweep exists to +recover a task whose assignment wake was lost or whose run never started. If it +minted `task_assigned:::` from the current row, that is +byte-for-byte the key the original assignment already persisted on the run that +failed. `idx_run_idempotency` is unbounded, so the recovery enqueue would be +rejected as a durable dedup hit and the sweep would recover nothing — it would be +a permanent no-op with a counter, which is the defect this whole capability +exists to remove, reintroduced in the one path meant to be the backstop. + +So the sweep enqueues keyless with `cause=by_design`. Its occurrence is "this +task was still unstarted at this sweep", which no durable row records. The cost +is a possible duplicate wake if the sweep runs twice before a run is claimed; +AC-003.2 elects exactly that trade. The 5-second coalescing window absorbs the +common case. + +### Onboarding is a third `task_assigned` producer + +`maybeCreateOnboardingTask` calls `CreateOfficeTask` -> `taskservice.CreateTask` +with `AssigneeAgentProfileID` set, then enqueues a `task_assigned` run itself. +Because that creation publishes `events.TaskCreated`, `queueTaskAssignedRun` also +observes the same occurrence and — after this capability — mints the keyed +`task_assigned:::1`. + +Part 1's [Convergent producers](run-dedup-generation-01.md#convergent-producers) +says two producers observe an assignment. For a creation occurrence reached +through onboarding there are **three**, and the third cannot converge with the +other two: `CreateOfficeTask` returns only the task id, so onboarding never +receives the generation the creating transaction committed. Widening that +adapter is not undertaken here. + +Onboarding therefore enqueues keyless with `cause=unresolved` — `unresolved` +rather than `by_design` precisely because the occurrence *does* have a +generation and this producer merely is not handed it, which is the distinction +AC-003.3 exists to make countable. This does not violate AC-002.1: that criterion +binds two producers that **each derive a key**, and a producer enqueueing with no +key is governed by REQ-003 instead. The scoping is in the criterion's own text, +so the onboarding pair needs no exemption of its own. The consequence is stated rather than left to +be discovered: onboarding's wake and `queueTaskAssignedRun`'s wake are two +enqueues for one occurrence, and only one carries a key. Both fire on one call +path within milliseconds, for one agent, reason and task, so `CoalesceRun` merges +them into a single queued run in the ordinary case. If they are ever separated by +more than the coalescing window, the operator gets a duplicate onboarding wake. +That is AC-003.2's elected direction, and the counter makes it visible. + +### `agent_error` becomes generational + +`office/service/retry.go` escalates a run failure to the workspace CEO. Its +payload already carries `run_id`, so the occurrence has a durable identity and +the audit's own rule applies: a producer with an occurrence to name must name it. + +```text +agent_error:: +``` + +The CEO agent id is included to keep the key's shape uniform with the other +recipient-addressed keys (`task_comment`, `task_mentioned`), which is the whole +of its justification. **It is not doing collision-avoidance work**, and a builder +must not go looking for the fan-out that would make it necessary: a workspace has +at most one CEO — `office/agents/service.go` and `office/service/agents.go` both +reject a second with `ErrAgentCEOAlreadyExists` — and `queueCEOAgentError` +(`office/service/retry.go`) escalates to `ceos[0]` alone, with no fan-out. +`agent_error:` would be equally correct today; the salt is retained +because it costs nothing and survives a CEO seat being replaced. + +A redelivery of one failure escalation is suppressed; a second, distinct failure +of the same agent produces a different `run_id` and wakes again. + +This producer moves out of the keyless path entirely and takes no `cause`. AC-001.8 +needs no argument here: every `agent_error` row persisted before this capability +carries a NULL `idempotency_key`, and both unique indexes are partial and ignore +NULL, so an old row can never collide with the new key. + +## Go API contracts + +### `UpdateTaskAssignee` and the bump sites + +`office/repository/sqlite/tasks.go` and the interface declaration in +`office/dashboard/service.go` both widen to return the generation the +transaction committed: + +```go +UpdateTaskAssignee(ctx context.Context, taskID, assigneeID string) (int64, error) +``` + +The returned value is read back **inside the existing transaction, after the +UPDATE and before `Commit`**. It is the generation of *this* assignment +occurrence, which is the only value a producer may key on. + +All five non-test call sites, and what each does with the value: + +| Call site | Role | Disposition | +| --- | --- | --- | +| `office/dashboard/service_tasks.go` (reached from `DashboardService.SetTaskAssigneeAsAgent`) | **producer** | carries the value into `TaskReactivityChange.AssignmentGeneration` | +| `office/service/task_assignee.go` `SetTaskAssignee` | non-producer | discards it (`_`); its own signature is unchanged | +| `office/service/task_assignee.go` `SetTaskAssigneeAsAgent` | non-producer | discards it (`_`); its own signature is unchanged | +| `office/service/channels.go` channel setup | non-producer | discards it (`_`) | +| `office/channels/service.go` channel setup | non-producer | discards it (`_`) | + +**The channel-task flow is explicitly out of scope as a producer, and in scope as +a bump.** Both channel sites assign a long-lived channel task's runner seat +through `UpdateTaskAssignee`, so both bump the counter — that is correct, a +channel task's runner genuinely is being assigned. Neither publishes an +assignment event nor runs the reactivity pipeline, so neither builds a key and +neither wakes anybody. They take the widened signature and discard the value. No +`ReportKeylessEnqueue` call is made, because no enqueue happens. + +**A read-back failure fails the assignment.** The read is a `SELECT` of one +column on the row just written, in the same transaction, so it can only fail if +that transaction is already unusable. When it does fail, the transaction rolls +back and `UpdateTaskAssignee` returns the error with a zero generation: the +assignment does **not** commit. Committing an assignment whose generation could +not be read would produce an occurrence that no producer can ever name, which is +strictly worse than a failed call the caller can retry. + +The create-path bump site, `task/repository/sqlite/task.go` `insertTaskTx` -> +`upsertRunnerInTx`, inserts the literal `1` under the guard that writes the +runner row — `task.AssigneeAgentProfileID != "" && task.WorkflowStepID != ""` — +so it needs no read-back and neither signature changes. + +**What carries the generation onto `task.created`.** Not a post-commit re-read; +AC-001.9 forbids it and it is not needed, because the value is a constant the +guard above already determines: + +- the guard fired -> a runner row committed at generation `1`; +- the guard did not fire -> no runner row, and the task keeps the column default, + generation `0`. + +`CreateTask` (`task/service/service_tasks.go`) holds the same two fields the +guard reads, on the very `*models.Task` it handed the transaction, so it +re-evaluates that condition without a query and publishes the result through +`publishTaskEventWithExtra` +(`task/service/service_events.go`), whose `extra map[string]interface{}` is the +carrier. The create site calls `publishTaskEvent` today and moves to the +`WithExtra` form; `publishTaskEvent` already delegates to it, so no new plumbing +is introduced. + +**A task created unassigned publishes generation `0`, not `1`.** `0` means "never +assigned" and can never name an assignment occurrence, so no `task_assigned` +producer keys on it — a creation that assigned nobody produces no assignment +occurrence to wake for, and the task's first real assignment goes through +`UpdateTaskAssignee` and commits `1`. Publishing `1` for an unassigned creation +would let one task mint `task_assigned:::1` twice — once at a +creation that assigned nobody, once at its genuine first assignment — recreating +the durable collision this capability exists to remove. + +This satisfies AC-001.9 rather than bending it. The prohibition is on recovering +a generation by re-reading a row a later occurrence has since moved; nothing is +re-read here. The value is derived from in-memory fields of the struct the +creating transaction was given, so it cannot observe a later occurrence, and for +a creation it is a constant either way. + +### The event payload field + +`assignment_generation` on the `task.created` / `task.updated` payload is a +**nullable** JSON number, decoded into the `*int64` that Part 1 specifies: + +- absent, or JSON `null` -> `nil` -> the producer goes keyless with + `cause=unresolved`; +- a JSON number -> that generation, used verbatim. + +`0` on the wire is a literal generation zero and **is not** a sentinel for +"absent". A task at generation `0` has never been assigned, so it can never be +the subject of a `task_assigned` occurrence; but a decoder that mapped `0` to +"absent" would silently convert a malformed payload into a keyless enqueue, and +one that mapped "absent" to `0` would mint `...:0` keys that collide across every +unassigned task. Neither is permitted: the field is nullable on the wire and +optional in the decoder, and the two states stay distinct. + +### The shared key builders + +Two exported builders live beside `internal/runs/commentkeys`, reachable from +`office/service`, `office/scheduler` and `orchestrator` without an import cycle. + +**The assignment key.** Both `task_assigned` producers call it rather than +formatting the string themselves, which is what makes their convergence +structural instead of coincidental (AC-002.1). + +**The blocker-set digest.** Both blocker producers — `task_blockers_resolved` in +`office/scheduler` and the `blockers_resolved` operation id in `office/service` — +call **one shared builder**. They live in different packages, and +AC-002.1 requires them to derive an identical key for one occurrence, so a +convention each implements separately is not sufficient: two packages choosing +`,` and `:` as a delimiter produce different digests for the same blocker set and +fail invisibly, presenting as a durable dedup miss rather than as an error. + +The encoding is binding, and is the encoding +`childrenCompletedIdempotencyKey` already uses, so the two agree by construction: + +1. take the `blocker_task_id` of each blocker in the wave; +2. sort ascending by byte value (**not** by `ListTaskBlockers`'s `created_at`, + which is not unique); +3. join with a single `,` (U+002C); +4. SHA-256 over the UTF-8 bytes of that string; +5. render the full 32-byte digest as lowercase hex (Go `%x`), never truncated. + +No de-duplication step is needed before sorting: `task_blockers` is +`PRIMARY KEY (task_id, blocker_task_id)`, so one task cannot list the same +blocker twice and two reads of an unchanged set always digest the same input. + +**An empty blocker set is not an occurrence and is not enqueued.** A wave with no +blockers has nothing to have resolved, and the alternative is worse than a +missing wake: the digest of the empty string is a constant, so an empty-set key +would be `task_blockers_resolved:::` — permanently +unique per (task, agent), which is precisely the defect this capability removes. +A producer that finds an empty set returns without enqueuing and without calling +`ReportKeylessEnqueue`, because no enqueue was attempted. + +**The set is digested at the readiness read, and is never re-read to build the +key.** Each blocker producer already reads the blocker rows to decide whether the +task is ready; that read is the snapshot the digest uses. This is the +capture-and-carry AC-001.9 imposes on the assignment generation and the cron +claimed tick, applied to a set rather than a scalar, and it is what stops a +producer digesting a set that a later wave has already moved. + +- `office/service` `resolveAndWakeIfUnblocked` (`event_subscribers.go`) already + holds the slice it read from `ListTaskBlockers`, and digests that slice. +- `office/scheduler` `allBlockersResolvedExcept` (`reactivity.go`) reads the same + rows and currently **discards** them, returning only a boolean. It returns the + slice alongside that boolean, and `cascadeBlockersResolved` passes it to the + shared builder rather than issuing a second read. + +Two producers observing one occurrence therefore digest what each saw at its own +readiness decision, and the consequence is stated rather than left to be +inferred. A blocker edge **added** between the two reads makes the later producer +find the task not ready at all, so it enqueues nothing. An edge **removed** +between them makes it digest a smaller set and mint a second key, which is a +duplicate wake — the direction AC-003.2 elects. A set that differs between the +two reads is a different wave, so AC-002.1's "same occurrence" premise does not +hold across it and no convergence is owed. Nothing here may lock the blocker +table or re-read to reconcile the two. + +**A failed blocker read is not an empty set, and the two must not collapse.** An +empty set returns without enqueuing because there was no occurrence; a failed +read means the producer does not know whether there was one. On a read error the +producer propagates the error and enqueues nothing — it does **not** fall through +to the empty-set arm, does not enqueue keyless, and does not call +`ReportKeylessEnqueue`, because no enqueue was attempted in either case. + +It must nonetheless be visible, and on one of the two paths it is not today. +`resolveAndWakeIfUnblocked` returns the error and `queueBlockersResolvedRuns` +logs it at `Error` with the blocked task id. `cascadeBlockersResolved` instead +drops it with a bare `continue` on `err != nil || !ready`, recording nothing, so +a wake lost to a read failure is indistinguishable from a task that was +legitimately still blocked. That arm gains an `Error` log naming the blocked task +and the error. This is the only behaviour change this section makes to the +readiness check itself; the check's own logic is unchanged. + +### The suppression reporter and the no-op outcome + +Part 2 requires one shared helper so three queue implementations report one +behaviour. It is exported from `internal/runs/service`, beside +`ReportKeylessEnqueue`, as three functions rather than one — the suppressions +are reached at different points and carry different evidence, and one of them is +classified by its caller rather than here: + +```go +type QueueSource string + +const ( + QueueSourceRuns QueueSource = "runs" + QueueSourceWakeup QueueSource = "wakeup" +) + +// The recent-duplicate lookup matched. Counts kind="windowed", logs at Info, +// and returns QueueOutcomeDeduped. +func ReportWindowedDedup(q QueueSource, reason, key string) QueueOutcome + +// Classifies the error from an insert. A unique violation on idempotency_key +// counts kind="durable", logs at Warn with key, reason and agent, and returns +// (QueueOutcomeDeduped, nil). Any other non-nil error is returned UNCHANGED +// with QueueOutcomeNone and moves no counter. A nil error returns +// (QueueOutcomeQueued, nil). +func ReportInsertResult( + q QueueSource, reason, key, agentProfileID string, err error, +) (QueueOutcome, error) + +// A durable conflict the CALLER has already classified. Counts kind="durable", +// logs at Warn, and returns QueueOutcomeDeduped. ReportInsertResult delegates +// here once its own classification succeeds. +func ReportDurableDedup(q QueueSource, reason, key, agentProfileID string) QueueOutcome +``` + +`ReportInsertResult` is the single place the *runs* unique violation is +recognised and mapped, which is what makes AC-002.3 hold on `queueRunInline` and +`SchedulerService.QueueRun` as well as in `runs/service`. Neither function ever +returns `QueueOutcomeCoalesced`: all three queues order +`CheckIdempotencyKey` -> `CoalesceRun` -> `insertRun`, so an enqueue that +coalesced returns from the coalesce arm and never reaches an insert to classify. A non-conflict error is +deliberately passed through untouched: a disk error is not a dedup decision and +must not be counted as one, nor swallowed into a successful-looking outcome. + +**The wakeup path classifies its own conflict and calls `ReportDurableDedup`**, +not `ReportInsertResult` (AC-004.6). `ReportInsertResult` recognises a conflict +with `runssqlite.IsIdempotencyKeyUniqueViolation` +(`internal/runs/repository/sqlite`), which matches `idx_run_idempotency` on the +`runs` table and is already the classifier `runs/service` uses. It does **not** +recognise `ErrWakeupIdempotencyConflict` +(`office/repository/sqlite/wakeup_requests.go`) — a wrapped sentinel raised by a +different table's index, in the office tree — and it must not be taught to: +importing that sentinel would reverse the import direction this design pins +(`runs/service` imports only `office/models` from the office tree). + +So the classification stays where the sentinel already is. The wakeup caller +lives in the office tree, already receives the error, does its own +`errors.Is(err, ErrWakeupIdempotencyConflict)`, and calls `ReportDurableDedup` +with `QueueSourceWakeup`. Its `kind` is always `durable`: that table has no +windowed lookup. This is reporting only; per AC-004.4 and AC-004.5 the wakeup +signature itself is not widened and keeps returning that sentinel, and the caller +keeps treating it as a suppression rather than a failure. + +**The no-op outcome.** `QueueOutcome` is a string type whose zero value is `""`, +which is not one of `queued`, `deduped` or `coalesced`. That zero value is given +a name and a meaning rather than being left for a builder to rediscover: + +```go +// QueueOutcomeNone means no enqueue was attempted, or the attempt returned an +// error. It is the zero value, so it is what a widened signature yields on any +// path that returns before deciding an outcome. +const QueueOutcomeNone QueueOutcome = "" +``` + +**It is declared in both copies of the type.** `QueueOutcome` exists twice — +`internal/runs/service/service.go` and `internal/workflow/engine/adapters.go` — +and each declaration carries the doc invariant "both MUST match". +`QueueOutcomeNone` is therefore added to **both**, with the same value and the +same doc comment. Part 1 lists `internal/workflow/engine` as already generational +and unchanged; that is a statement about its *keys*, and adding this constant is +the only edit this capability makes to that package. Adding it to one side alone +would break an invariant the code asserts in its own comments. + +This is additive and changes no acceptance criterion. AC-004.4's obligation to +report queued, deduplicated or coalesced is conditioned on a caller enqueuing a +wake; a path that enqueues nothing never meets that condition. The paths that +return `QueueOutcomeNone` are the widened signatures returning a non-nil error, +including `ReportInsertResult`'s non-conflict arm. + +**`QueueRunCtx` is not one of them.** It has no early return on an empty resolved +agent id. That guard sits one level up, in the reactivity pipeline's `queue` +closure (`office/scheduler/reactivity.go`), which returns *before* calling +`QueueRunCtx` at all — so an empty agent id yields no outcome to report rather +than `QueueOutcomeNone`, and `ApplyTaskMutationResult.Runs` gains nothing because +no enqueue was attempted. Part 1's +[reactivity gate](run-dedup-generation-01.md#the-reactivity-gate) states this +correctly and is the reference. + +No caller may treat `QueueOutcomeNone` as success. In particular the reactivity +pipeline's `queue` closure appends to `ApplyTaskMutationResult.Runs` only for +`QueueOutcomeQueued`, so neither a suppression nor a no-op is reported as a +queued run. Returning `QueueOutcomeQueued` from a path that inserted nothing +would put a phantom run in that slice, which is the specific mis-report Part 2 +added the `Queued`-only rule to prevent. From 5adf12f9b26fa52e768242a92968614b29292b81 Mon Sep 17 00:00:00 2001 From: nova28 <17953305+nova28@users.noreply.github.com> Date: Wed, 9 Sep 2026 02:57:50 +0800 Subject: [PATCH 02/15] test(office): close coverage gaps in run-dedup-generation key producers Adds regression tests found missing during the testing pass: a replay test for the legacy FK-removal migration sequencing hazard (TRAP 23, previously zero coverage), a test proving task reassignment across assignment_generation no longer collides with the durable idempotency index (the reported defect), and coverage for the office/scheduler blocker-resolution producer's shared digest key (previously untested). No production code changes; all three gaps were missing tests for already-correct behavior. --- .../reactivity_blockers_resolved_test.go | 117 ++++++++++++ .../task_assigned_generation_key_test.go | 180 ++++++++++++++++++ ...w_fk_removal_assignment_generation_test.go | 117 ++++++++++++ 3 files changed, 414 insertions(+) create mode 100644 apps/backend/internal/office/scheduler/reactivity_blockers_resolved_test.go create mode 100644 apps/backend/internal/office/service/task_assigned_generation_key_test.go create mode 100644 apps/backend/internal/task/repository/sqlite/tasks_workflow_fk_removal_assignment_generation_test.go diff --git a/apps/backend/internal/office/scheduler/reactivity_blockers_resolved_test.go b/apps/backend/internal/office/scheduler/reactivity_blockers_resolved_test.go new file mode 100644 index 00000000000..380bc5310e4 --- /dev/null +++ b/apps/backend/internal/office/scheduler/reactivity_blockers_resolved_test.go @@ -0,0 +1,117 @@ +package scheduler + +import ( + "context" + "fmt" + "testing" + + officesqlite "github.com/kandev/kandev/internal/office/repository/sqlite" + "github.com/kandev/kandev/internal/runs/dedupkeys" +) + +// insertBlockerRelationship records that taskID is blocked by +// blockerTaskID, and gives blockerTaskID the given (already-terminal or +// not) state. +func insertBlockerRelationship( + t *testing.T, repo *officesqlite.Repository, taskID, blockerTaskID, blockerState string, +) { + t.Helper() + ctx := context.Background() + if _, err := repo.ExecRaw(ctx, + `INSERT INTO tasks (id, workspace_id, state) VALUES (?, 'ws-1', ?)`, + blockerTaskID, blockerState, + ); err != nil { + t.Fatalf("insert blocker task %s: %v", blockerTaskID, err) + } + if _, err := repo.ExecRaw(ctx, + `INSERT INTO task_blockers (task_id, blocker_task_id, created_at) VALUES (?, ?, CURRENT_TIMESTAMP)`, + taskID, blockerTaskID, + ); err != nil { + t.Fatalf("insert task_blockers row (%s blocked by %s): %v", taskID, blockerTaskID, err) + } +} + +// TestCascadeBlockersResolved_QueuesRunWithSharedDigestKey is the +// regression test for both blocker-resolution producers converging on the +// same key shape (AC-OFFICE-RUN-DEDUP-002.1, SR-59's resolution: only the +// digest need be byte-identical, not the whole key/opID — see +// docs/specs/office/system-design/run-dedup-generation-03.md#the-shared-key-builders). +// Before this test, office/scheduler.cascadeBlockersResolved and +// allBlockersResolvedExcept had zero coverage: nothing exercised the +// blocked task's wake at all, let alone the specific +// dedupkeys.BlockerDigest call. +func TestCascadeBlockersResolved_QueuesRunWithSharedDigestKey(t *testing.T) { + repo := newReactivityTestRepo(t) + ss := newChildrenCompletedTestScheduler(t, repo) + createChildrenCompletedAgent(t, repo, "agent-1") + queue := newChildrenCompletedQueue(t, ss) + ctx := context.Background() + + // blocked-1 is blocked by blocker-a (already done) and blocker-b (the + // one resolving now). setupChildrenCompletedParent gives blocked-1 a + // runner participant row so GetTaskAssignee resolves it to agent-1 — + // reused here even though it's named for the children-completed tests, + // because it does exactly what a blocked task's assignee wiring needs. + setupChildrenCompletedParent(t, ss, "blocked-1", "agent-1") + insertBlockerRelationship(t, repo, "blocked-1", "blocker-a", "COMPLETED") + insertBlockerRelationship(t, repo, "blocked-1", "blocker-b", "IN_PROGRESS") + + // blocker-b finishes: it's the second and last outstanding blocker. + if _, err := repo.ExecRaw(ctx, `UPDATE tasks SET state = 'COMPLETED' WHERE id = 'blocker-b'`); err != nil { + t.Fatalf("complete blocker-b: %v", err) + } + ss.cascadeBlockersResolved(ctx, &TaskSnapshot{ID: "blocker-b", WorkspaceID: "ws-1"}, queue) + + runs, err := repo.ListRuns(ctx, "ws-1") + if err != nil { + t.Fatalf("list runs: %v", err) + } + var got *string + for _, run := range runs { + if run.Reason == RunReasonTaskBlockersResolved && run.AgentProfileID == "agent-1" { + got = run.IdempotencyKey + break + } + } + if got == nil { + t.Fatalf("no %s run persisted for agent-1: runs=%#v", RunReasonTaskBlockersResolved, runs) + } + + // The key's digest segment must be exactly what the OTHER blocker + // producer (office/service.resolveAndWakeIfUnblocked) would derive + // for the identical blocker set, in either read order — that's the + // structural convergence AC-002.1 requires. + digestAscending := dedupkeys.BlockerDigest([]string{"blocker-a", "blocker-b"}) + digestDescending := dedupkeys.BlockerDigest([]string{"blocker-b", "blocker-a"}) + if digestAscending != digestDescending { + t.Fatalf("BlockerDigest is not order-independent: %q vs %q", digestAscending, digestDescending) + } + want := fmt.Sprintf("%s:blocked-1:agent-1:%s", RunReasonTaskBlockersResolved, digestAscending) + if *got != want { + t.Fatalf("idempotency key = %q, want %q", *got, want) + } +} + +// TestCascadeBlockersResolved_StillBlocked_DoesNotQueue proves the +// negative: cascadeBlockersResolved must not wake the blocked task while +// another blocker is still outstanding. +func TestCascadeBlockersResolved_StillBlocked_DoesNotQueue(t *testing.T) { + repo := newReactivityTestRepo(t) + ss := newChildrenCompletedTestScheduler(t, repo) + createChildrenCompletedAgent(t, repo, "agent-1") + queue := newChildrenCompletedQueue(t, ss) + ctx := context.Background() + + setupChildrenCompletedParent(t, ss, "blocked-2", "agent-1") + insertBlockerRelationship(t, repo, "blocked-2", "blocker-c", "IN_PROGRESS") // never finishes + insertBlockerRelationship(t, repo, "blocked-2", "blocker-d", "IN_PROGRESS") + + if _, err := repo.ExecRaw(ctx, `UPDATE tasks SET state = 'COMPLETED' WHERE id = 'blocker-d'`); err != nil { + t.Fatalf("complete blocker-d: %v", err) + } + ss.cascadeBlockersResolved(ctx, &TaskSnapshot{ID: "blocker-d", WorkspaceID: "ws-1"}, queue) + + if got := runsCountForReason(t, ss, RunReasonTaskBlockersResolved); got != 0 { + t.Fatalf("persisted runs = %d, want 0 (blocker-c is still unresolved)", got) + } +} diff --git a/apps/backend/internal/office/service/task_assigned_generation_key_test.go b/apps/backend/internal/office/service/task_assigned_generation_key_test.go new file mode 100644 index 00000000000..13e0ac6c430 --- /dev/null +++ b/apps/backend/internal/office/service/task_assigned_generation_key_test.go @@ -0,0 +1,180 @@ +package service_test + +// This file proves the specific defect run-dedup-generation fixes: a +// task_assigned dedup key that used to be stable for the life of a +// (task, agent) pair (docs/specs/office/requirements/run-dedup-generation.md, +// AC-OFFICE-SCHEDULER-001.7). idx_run_idempotency has no time bound, so a +// stable key collided with the UNIQUE index forever, making a re-assignment +// of the same task to the same agent a silent, permanent no-op. These tests +// exercise the real bus -> handleTaskUpdated -> queueTaskAssignedRun path +// (not just the dedupkeys.AssignmentKey unit) and assert on the persisted +// idempotency_key, which none of the existing task_assigned tests do. + +import ( + "context" + "encoding/json" + "testing" + + "github.com/kandev/kandev/internal/events" + "github.com/kandev/kandev/internal/events/bus" + "github.com/kandev/kandev/internal/office/service" + "github.com/kandev/kandev/internal/runs/dedupkeys" +) + +func publishTaskAssigned( + t *testing.T, ctx context.Context, eb bus.EventBus, taskID, agentID string, generation *int64, +) { + t.Helper() + data := map[string]any{ + "task_id": taskID, + "assignee_agent_profile_id": agentID, + } + if generation != nil { + data["assignment_generation"] = *generation + } + event := bus.NewEvent(events.TaskUpdated, "test", data) + if err := eb.Publish(ctx, events.TaskUpdated, event); err != nil { + t.Fatalf("publish task updated event: %v", err) + } +} + +func gen(n int64) *int64 { return &n } + +// taskAssignedRunsFor returns every task_assigned run queued for agentID on +// the given task, in insertion order (ListRuns has no ordering guarantee +// this test relies on beyond "all rows present"). +func taskAssignedRunsFor(t *testing.T, svc *service.Service, wsID, taskID, agentID string) []string { + t.Helper() + runs, err := svc.ListRuns(context.Background(), wsID) + if err != nil { + t.Fatalf("list runs: %v", err) + } + var keys []string + for _, run := range runs { + if run.AgentProfileID != agentID || run.Reason != service.RunReasonTaskAssigned { + continue + } + var payload struct { + TaskID string `json:"task_id"` + } + if err := json.Unmarshal([]byte(run.Payload), &payload); err != nil { + t.Fatalf("decode run payload: %v", err) + } + if payload.TaskID != taskID { + continue + } + if run.IdempotencyKey == nil { + keys = append(keys, "") + continue + } + keys = append(keys, *run.IdempotencyKey) + } + return keys +} + +func TestTaskAssigned_SameGenerationRedelivery_DedupesForever(t *testing.T) { + svc, eb := newTestServiceWithBus(t) + ctx := context.Background() + + createTestAgent(t, svc, "ws-1", "worker-gen") + insertTestTask(t, svc, "task-gen-redeliver", "ws-1") + svc.ExecSQL(t, `UPDATE tasks SET project_id = 'office-project' WHERE id = ?`, "task-gen-redeliver") + + publishTaskAssigned(t, ctx, eb, "task-gen-redeliver", "worker-gen", gen(0)) + publishTaskAssigned(t, ctx, eb, "task-gen-redeliver", "worker-gen", gen(0)) + + keys := taskAssignedRunsFor(t, svc, "ws-1", "task-gen-redeliver", "worker-gen") + if len(keys) != 1 { + t.Fatalf("redelivering the same generation queued %d runs, want exactly 1 (deduped): keys=%#v", len(keys), keys) + } + want := dedupkeys.AssignmentKey("task-gen-redeliver", "worker-gen", 0) + if keys[0] != want { + t.Fatalf("idempotency key = %q, want %q", keys[0], want) + } +} + +// TestTaskAssigned_GenerationBump_AvoidsPermanentCollision is the direct +// regression test for the reported defect: reassigning the same task to the +// same agent must NOT be a permanent no-op once the durable unique index +// already holds a row for an earlier generation's key. Unlike +// TestTaskAssigned_ReassignmentUsesAgentScopedIdempotency (which reassigns to +// a *different* agent and never sets assignment_generation, so it never +// exercised the keyed path at all), this drives the same agent through two +// generations to prove the key itself changed. +func TestTaskAssigned_GenerationBump_AvoidsPermanentCollision(t *testing.T) { + svc, eb := newTestServiceWithBus(t) + ctx := context.Background() + + createTestAgent(t, svc, "ws-1", "worker-regen") + insertTestTask(t, svc, "task-regen", "ws-1") + svc.ExecSQL(t, `UPDATE tasks SET project_id = 'office-project' WHERE id = ?`, "task-regen") + + // Occurrence 1: original assignment at generation 0. In production this + // row's idempotency_key (task_assigned:task-regen:worker-regen:0) is + // exactly the durable UNIQUE row that used to collide forever under the + // pre-fix key format (task_assigned::, no generation). + publishTaskAssigned(t, ctx, eb, "task-regen", "worker-regen", gen(0)) + + // The first run is claimed and finishes (agent did the work) long before + // the reassignment — CoalesceRun only merges into a still-'queued' row, + // so this ensures the second occurrence exercises the durable + // idx_run_idempotency INSERT path, not the 5-second coalesce window. + // This is exactly the "more than 24 hours later" gap in the bug report. + svc.ExecSQL(t, `UPDATE runs SET status = 'completed' WHERE status = 'queued'`) + + // Simulate a real reassignment: UpdateTaskAssignee bumps the stored + // generation before the task.updated event carrying it is published. + svc.ExecSQL(t, `UPDATE tasks SET assignment_generation = 1 WHERE id = ?`, "task-regen") + publishTaskAssigned(t, ctx, eb, "task-regen", "worker-regen", gen(1)) + + keys := taskAssignedRunsFor(t, svc, "ws-1", "task-regen", "worker-regen") + if len(keys) != 2 { + t.Fatalf("reassigning the same agent at a new generation queued %d runs, want 2 (no false collision): keys=%#v", len(keys), keys) + } + wantGen0 := dedupkeys.AssignmentKey("task-regen", "worker-regen", 0) + wantGen1 := dedupkeys.AssignmentKey("task-regen", "worker-regen", 1) + matchesInOrder := keys[0] == wantGen0 && keys[1] == wantGen1 + matchesReversed := keys[0] == wantGen1 && keys[1] == wantGen0 + if !matchesInOrder && !matchesReversed { + t.Fatalf("keys = %#v, want %q and %q (one run per generation)", keys, wantGen0, wantGen1) + } + if keys[0] == keys[1] { + t.Fatalf("both runs share idempotency key %q — this is the reported defect: a reassignment collided with the durable unique index", keys[0]) + } + + // Redelivering generation 1 again — past the coalesce window and after + // the generation-1 run has itself been claimed — must still dedupe via + // the durable unique index: the fix must not turn every event into a + // fresh key, only a genuine generation change. + svc.ExecSQL(t, `UPDATE runs SET status = 'completed' WHERE status = 'queued'`) + publishTaskAssigned(t, ctx, eb, "task-regen", "worker-regen", gen(1)) + keysAfterRedeliver := taskAssignedRunsFor(t, svc, "ws-1", "task-regen", "worker-regen") + if len(keysAfterRedeliver) != 2 { + t.Fatalf("redelivering generation 1 queued a 3rd run: keys=%#v", keysAfterRedeliver) + } +} + +// TestTaskAssigned_MissingGeneration_EnqueuesKeyless proves SR-55's +// resolution: an event carrying no assignment_generation must never mint a +// key with a guessed or zero generation — it goes keyless (empty +// idempotency_key) so a possible duplicate wake is accepted (AC-003.2) +// instead of a mis-keyed row that could collide with an unrelated +// occurrence. +func TestTaskAssigned_MissingGeneration_EnqueuesKeyless(t *testing.T) { + svc, eb := newTestServiceWithBus(t) + ctx := context.Background() + + createTestAgent(t, svc, "ws-1", "worker-nogen") + insertTestTask(t, svc, "task-nogen", "ws-1") + svc.ExecSQL(t, `UPDATE tasks SET project_id = 'office-project' WHERE id = ?`, "task-nogen") + + publishTaskAssigned(t, ctx, eb, "task-nogen", "worker-nogen", nil) + + keys := taskAssignedRunsFor(t, svc, "ws-1", "task-nogen", "worker-nogen") + if len(keys) != 1 { + t.Fatalf("got %d runs, want 1: keys=%#v", len(keys), keys) + } + if keys[0] != "" { + t.Fatalf("idempotency key = %q, want empty (keyless enqueue)", keys[0]) + } +} diff --git a/apps/backend/internal/task/repository/sqlite/tasks_workflow_fk_removal_assignment_generation_test.go b/apps/backend/internal/task/repository/sqlite/tasks_workflow_fk_removal_assignment_generation_test.go new file mode 100644 index 00000000000..8df37ce5fa6 --- /dev/null +++ b/apps/backend/internal/task/repository/sqlite/tasks_workflow_fk_removal_assignment_generation_test.go @@ -0,0 +1,117 @@ +package sqlite + +import ( + "strings" + "testing" + + "github.com/jmoiron/sqlx" + _ "github.com/mattn/go-sqlite3" +) + +// TestMigrateTasksRemoveWorkflowFK_AssignmentGenerationSurvivesRebuild is the +// replay regression test flagged (but not written) in the run-dedup-generation +// build: migrateTasksRemoveWorkflowFK (base_migrations.go) recreates `tasks` +// from an explicit 19-column list that predates assignment_generation. The +// tasks.assignment_generation ADD COLUMN is sequenced strictly AFTER that +// rebuild in runMigrations() precisely so a database still carrying the +// legacy `FOREIGN KEY (workflow_id)` DDL does not have the new column +// silently dropped for the remainder of that boot (the same hazard class as +// the task_sessions.name comment in the same file). This had zero coverage +// before this test: nothing exercised migrateTasksRemoveWorkflowFK's rebuild +// path at all. +func TestMigrateTasksRemoveWorkflowFK_AssignmentGenerationSurvivesRebuild(t *testing.T) { + db, err := sqlx.Open("sqlite3", ":memory:") + if err != nil { + t.Fatalf("open sqlite: %v", err) + } + t.Cleanup(func() { _ = db.Close() }) + + // Seed a pre-migration `tasks` table: the legacy FK clause that triggers + // the rebuild, the 19 columns migrateTasksRemoveWorkflowFK's SELECT list + // expects, and no assignment_generation column at all — matching a real + // database that predates this feature. + if _, err := db.Exec(` + CREATE TABLE tasks ( + id TEXT PRIMARY KEY, + workspace_id TEXT NOT NULL DEFAULT '', + workflow_id TEXT NOT NULL DEFAULT '', + workflow_step_id TEXT NOT NULL DEFAULT '', + title TEXT NOT NULL, + description TEXT DEFAULT '', + state TEXT DEFAULT 'TODO', + priority INTEGER DEFAULT 0, + position INTEGER DEFAULT 0, + wip_admitted INTEGER NOT NULL DEFAULT 1, + queued_for_step_id TEXT NOT NULL DEFAULT '', + queued_at TIMESTAMP, + metadata TEXT DEFAULT '{}', + is_ephemeral INTEGER NOT NULL DEFAULT 0, + parent_id TEXT DEFAULT '', + autopilot_enabled INTEGER NOT NULL DEFAULT 0, + archived_at TIMESTAMP, + created_at TIMESTAMP NOT NULL, + updated_at TIMESTAMP NOT NULL, + FOREIGN KEY (workflow_id) REFERENCES workflows(id) + ); + INSERT INTO tasks (id, workspace_id, title, created_at, updated_at) + VALUES ('task-legacy-fk', 'ws-legacy', 'Legacy FK task', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP); + `); err != nil { + t.Fatalf("seed legacy schema: %v", err) + } + + if _, err := NewWithDB(db, db, nil); err != nil { + t.Fatalf("init task repo (run migrations): %v", err) + } + + // The rebuild must have fired: the legacy FK clause is gone. + var tableSQL string + if err := db.QueryRow( + `SELECT sql FROM sqlite_master WHERE type='table' AND name='tasks'`, + ).Scan(&tableSQL); err != nil { + t.Fatalf("read tasks schema: %v", err) + } + if strings.Contains(tableSQL, "FOREIGN KEY (workflow_id)") { + t.Fatalf("tasks table still carries the legacy workflow_id FK after migration: %s", tableSQL) + } + + // assignment_generation must be present post-rebuild, not silently + // dropped by the rebuild's pre-feature explicit column list. + rows, err := db.Queryx(`PRAGMA table_info(tasks)`) + if err != nil { + t.Fatalf("pragma table_info: %v", err) + } + found := false + for rows.Next() { + var cid int + var name, ctype string + var notnull, pk int + var dflt *string + if err := rows.Scan(&cid, &name, &ctype, ¬null, &dflt, &pk); err != nil { + _ = rows.Close() + t.Fatalf("scan pragma: %v", err) + } + if name == "assignment_generation" { + found = true + } + } + _ = rows.Close() + if !found { + t.Fatalf("assignment_generation column missing after migrating a legacy FK-bearing tasks table") + } + + // The pre-existing row must survive the rebuild, with the new column + // defaulted to 0. + var gotTitle string + var gotGeneration int64 + if err := db.QueryRow( + `SELECT title, assignment_generation FROM tasks WHERE id = ?`, "task-legacy-fk", + ).Scan(&gotTitle, &gotGeneration); err != nil { + t.Fatalf("query migrated row: %v", err) + } + if gotTitle != "Legacy FK task" { + t.Fatalf("title = %q, want %q (row lost during rebuild)", gotTitle, "Legacy FK task") + } + if gotGeneration != 0 { + t.Fatalf("assignment_generation = %d, want 0 for a pre-existing row", gotGeneration) + } +} From 08b7a060b9ac7d662f2a86149bf9f6f8e6a1bf2b Mon Sep 17 00:00:00 2001 From: nova28 <17953305+nova28@users.noreply.github.com> Date: Wed, 9 Sep 2026 04:27:00 +0800 Subject: [PATCH 03/15] fix(office): add missing dedup keys and keyless telemetry to reactivity producers Review round 1 found that removing QueueRunCtx's dangerous default key fallback exposed 6 of reactivity.go's 8 run reasons (task_comment, task_mentioned, task_reopened_via_comment, task_unblocked, task_reopened, task_review_requested) as never having had their own dedup key or keyless-telemetry logic, since they were silently relying on that removed fallback the whole time. Adds the spec's exact key shape for the three comment-carrying reasons and ReportKeylessEnqueue(..., KeylessCauseByDesign) for the three status-only reasons, plus the test coverage flagged alongside it: the producer-audit key-format table test, ApplyTaskMutation coverage for the same-agent-repeat-wake gate and relocated interrupt guard, the office-side priority-rebuild migration's assignment_generation COALESCE copy expression, and SpawnAgentRun's caller-run-id key prefix. --- ...ons_priority_assignment_generation_test.go | 79 ++++++ .../runtime/spawn_agent_run_key_test.go | 180 ++++++++++++ .../internal/office/scheduler/reactivity.go | 57 ++-- .../reactivity_apply_mutation_test.go | 138 +++++++++ .../reactivity_producer_key_audit_test.go | 261 ++++++++++++++++++ 5 files changed, 695 insertions(+), 20 deletions(-) create mode 100644 apps/backend/internal/office/repository/sqlite/migrations_priority_assignment_generation_test.go create mode 100644 apps/backend/internal/office/runtime/spawn_agent_run_key_test.go create mode 100644 apps/backend/internal/office/scheduler/reactivity_apply_mutation_test.go create mode 100644 apps/backend/internal/office/scheduler/reactivity_producer_key_audit_test.go diff --git a/apps/backend/internal/office/repository/sqlite/migrations_priority_assignment_generation_test.go b/apps/backend/internal/office/repository/sqlite/migrations_priority_assignment_generation_test.go new file mode 100644 index 00000000000..f1cc53e87e8 --- /dev/null +++ b/apps/backend/internal/office/repository/sqlite/migrations_priority_assignment_generation_test.go @@ -0,0 +1,79 @@ +package sqlite_test + +import ( + "path/filepath" + "testing" + + "github.com/jmoiron/sqlx" + _ "github.com/mattn/go-sqlite3" + + "github.com/kandev/kandev/internal/office/repository/sqlite" +) + +// TestMigrate_PriorityRebuildPreservesAssignmentGeneration is the regression +// test for the same rebuild hazard TestMigrate_PriorityRebuildPreservesAssignee +// covers one column over: taskPriorityMigrationStatements' INSERT ... SELECT +// carries tasks.assignment_generation through +// COALESCE(assignment_generation,0) (docs/specs/office/system-design/ +// run-dedup-generation-02.md#failure-and-recovery). That expression had zero +// coverage before this test — the sibling assignee test seeds a table where +// runTaskPriorityRecreate's own defensive ALTER TABLE ADD COLUMN supplies the +// column fresh (every row defaults to 0), which never exercises copying a +// genuinely non-zero value through the recreate. Here the legacy table +// already carries the column with a non-zero value, as an install upgrading +// mid-feature (assignment_generation shipped, priority rebuild still pending) +// would. +func TestMigrate_PriorityRebuildPreservesAssignmentGeneration(t *testing.T) { + dbPath := filepath.Join(t.TempDir(), "test.db") + "?_journal_mode=WAL" + db, err := sqlx.Open("sqlite3", dbPath) + if err != nil { + t.Fatalf("open sqlite: %v", err) + } + t.Cleanup(func() { _ = db.Close() }) + + if _, err := db.Exec(` + CREATE TABLE tasks ( + id TEXT PRIMARY KEY, + workspace_id TEXT NOT NULL DEFAULT '', + workflow_id TEXT NOT NULL DEFAULT '', + workflow_step_id TEXT NOT NULL DEFAULT '', + title TEXT NOT NULL, + description TEXT DEFAULT '', + state TEXT DEFAULT 'TODO', + priority INTEGER DEFAULT 0, + position INTEGER DEFAULT 0, + metadata TEXT DEFAULT '{}', + is_ephemeral INTEGER NOT NULL DEFAULT 0, + parent_id TEXT DEFAULT '', + archived_at TIMESTAMP, + created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + origin TEXT DEFAULT 'manual', + project_id TEXT DEFAULT '', + labels TEXT DEFAULT '[]', + identifier TEXT, + assignee_user_id TEXT NOT NULL DEFAULT '', + assignment_generation INTEGER NOT NULL DEFAULT 0 + ); + `); err != nil { + t.Fatalf("seed legacy schema: %v", err) + } + if _, err := db.Exec(` + INSERT INTO tasks (id, workspace_id, title, assignment_generation) + VALUES ('task-1', 'ws-1', 'reassigned task', 3) + `); err != nil { + t.Fatalf("seed reassigned task: %v", err) + } + + if _, err := sqlite.NewWithDB(db, db, nil); err != nil { + t.Fatalf("init office repo (run migrations): %v", err) + } + + var generation int64 + if err := db.Get(&generation, `SELECT assignment_generation FROM tasks WHERE id = 'task-1'`); err != nil { + t.Fatalf("tasks.assignment_generation missing after priority rebuild: %v", err) + } + if generation != 3 { + t.Fatalf("assignment_generation lost by priority rebuild: got %d, want 3", generation) + } +} diff --git a/apps/backend/internal/office/runtime/spawn_agent_run_key_test.go b/apps/backend/internal/office/runtime/spawn_agent_run_key_test.go new file mode 100644 index 00000000000..6f1dd76f035 --- /dev/null +++ b/apps/backend/internal/office/runtime/spawn_agent_run_key_test.go @@ -0,0 +1,180 @@ +package runtime + +import ( + "context" + "expvar" + "strings" + "testing" + + "github.com/kandev/kandev/internal/office/models" +) + +// spawnAgentRunKeylessCounterHasLabel reports whether the process-global +// office_run_dedup_keyless_total expvar map carries an entry for the given +// reason and cause. Mirrors internal/runs/service's own counterHasLabel test +// helper — that one lives in a different package (service_test) and cannot +// be imported here. +func spawnAgentRunKeylessCounterHasLabel(t *testing.T, reason, cause string) bool { + t.Helper() + v := expvar.Get("office_run_dedup_keyless_total") + if v == nil { + t.Fatalf("expvar map office_run_dedup_keyless_total not registered") + } + m, ok := v.(*expvar.Map) + if !ok { + t.Fatalf("office_run_dedup_keyless_total is not a *expvar.Map") + } + found := false + m.Do(func(kv expvar.KeyValue) { + if strings.Contains(kv.Key, "reason="+reason) && strings.Contains(kv.Key, "cause="+cause) { + found = true + } + }) + return found +} + +// TestActionsSpawnAgentRun_PrefixesKeyWithCallerRunID is the AC-001.7 +// regression: a non-empty agent-supplied key is prefixed with the calling +// run's id (agent::) before reaching QueueRun, so a retry +// of the same run reuses the run id and still dedupes, while a later run +// (different RunID) produces a different key and is not suppressed. Before +// this test, the only coverage touching SpawnAgentRun's key logic was the +// unrelated cross-workspace denial test — the prefix itself was unpinned. +func TestActionsSpawnAgentRun_PrefixesKeyWithCallerRunID(t *testing.T) { + agents := &recordingAgentModifier{ + agents: map[string]*models.AgentInstance{ + "agent-2": {ID: "agent-2", WorkspaceID: "ws-1"}, + }, + } + runs := &recordingRunSpawner{} + actions := NewActions(ActionDependencies{Runs: runs, AgentModifier: agents}) + runCtx := RunContext{ + WorkspaceID: "ws-1", + RunID: "run-caller-1", + Capabilities: Capabilities{ + CanSpawnAgentRun: true, + }, + } + + if err := actions.SpawnAgentRun(context.Background(), runCtx, SpawnAgentRunInput{ + AgentID: "agent-2", + Reason: "custom_reason", + IdempotencyKey: "custom-key", + }); err != nil { + t.Fatalf("SpawnAgentRun: %v", err) + } + + if len(runs.calls) != 1 { + t.Fatalf("run spawner calls = %d, want 1", len(runs.calls)) + } + want := "agent:run-caller-1:custom-key" + if got := runs.calls[0].IdempotencyKey; got != want { + t.Fatalf("idempotency key = %q, want %q", got, want) + } +} + +// TestActionsSpawnAgentRun_DifferentCallerRun_ProducesDifferentKey proves +// the prefix's other half: the SAME agent-supplied key from a DIFFERENT +// calling run does not collide, because the caller run id varies the key. +func TestActionsSpawnAgentRun_DifferentCallerRun_ProducesDifferentKey(t *testing.T) { + agents := &recordingAgentModifier{ + agents: map[string]*models.AgentInstance{ + "agent-2": {ID: "agent-2", WorkspaceID: "ws-1"}, + }, + } + runs := &recordingRunSpawner{} + actions := NewActions(ActionDependencies{Runs: runs, AgentModifier: agents}) + caps := Capabilities{CanSpawnAgentRun: true} + + for _, callerRunID := range []string{"run-a", "run-b"} { + runCtx := RunContext{WorkspaceID: "ws-1", RunID: callerRunID, Capabilities: caps} + if err := actions.SpawnAgentRun(context.Background(), runCtx, SpawnAgentRunInput{ + AgentID: "agent-2", + Reason: "custom_reason", + IdempotencyKey: "same-key", + }); err != nil { + t.Fatalf("SpawnAgentRun (caller %s): %v", callerRunID, err) + } + } + + if len(runs.calls) != 2 { + t.Fatalf("run spawner calls = %d, want 2", len(runs.calls)) + } + if runs.calls[0].IdempotencyKey == runs.calls[1].IdempotencyKey { + t.Fatalf("two different caller runs produced the same key %q", runs.calls[0].IdempotencyKey) + } +} + +// TestActionsSpawnAgentRun_EmptyKey_NotPrefixed_ByDesign proves the third +// branch: the agent expressed no dedup intent (empty IdempotencyKey), so the +// call enqueues keyless without a prefix — prefixing an empty key would +// collapse every no-dedup-intent call inside one run onto a single key, +// suppressing every call after the first. +func TestActionsSpawnAgentRun_EmptyKey_NotPrefixed_ByDesign(t *testing.T) { + agents := &recordingAgentModifier{ + agents: map[string]*models.AgentInstance{ + "agent-2": {ID: "agent-2", WorkspaceID: "ws-1"}, + }, + } + runs := &recordingRunSpawner{} + actions := NewActions(ActionDependencies{Runs: runs, AgentModifier: agents}) + runCtx := RunContext{ + WorkspaceID: "ws-1", + RunID: "run-caller-2", + Capabilities: Capabilities{CanSpawnAgentRun: true}, + } + + if err := actions.SpawnAgentRun(context.Background(), runCtx, SpawnAgentRunInput{ + AgentID: "agent-2", + Reason: "test_spawn_empty_key", + }); err != nil { + t.Fatalf("SpawnAgentRun: %v", err) + } + + if len(runs.calls) != 1 { + t.Fatalf("run spawner calls = %d, want 1", len(runs.calls)) + } + if got := runs.calls[0].IdempotencyKey; got != "" { + t.Fatalf("idempotency key = %q, want empty (keyless by design)", got) + } + if !spawnAgentRunKeylessCounterHasLabel(t, "test_spawn_empty_key", "by_design") { + t.Fatal("expected office_run_dedup_keyless_total to carry a by_design entry for this reason") + } +} + +// TestActionsSpawnAgentRun_NoCallerRunID_EnqueuesKeylessUnresolved proves +// the fourth branch: a non-empty agent-supplied key with no caller run id to +// prefix it enqueues keyless (cause=unresolved) rather than risk colliding +// across unrelated runs. +func TestActionsSpawnAgentRun_NoCallerRunID_EnqueuesKeylessUnresolved(t *testing.T) { + agents := &recordingAgentModifier{ + agents: map[string]*models.AgentInstance{ + "agent-2": {ID: "agent-2", WorkspaceID: "ws-1"}, + }, + } + runs := &recordingRunSpawner{} + actions := NewActions(ActionDependencies{Runs: runs, AgentModifier: agents}) + runCtx := RunContext{ + WorkspaceID: "ws-1", + RunID: "", + Capabilities: Capabilities{CanSpawnAgentRun: true}, + } + + if err := actions.SpawnAgentRun(context.Background(), runCtx, SpawnAgentRunInput{ + AgentID: "agent-2", + Reason: "test_spawn_no_caller_run", + IdempotencyKey: "custom-key", + }); err != nil { + t.Fatalf("SpawnAgentRun: %v", err) + } + + if len(runs.calls) != 1 { + t.Fatalf("run spawner calls = %d, want 1", len(runs.calls)) + } + if got := runs.calls[0].IdempotencyKey; got != "" { + t.Fatalf("idempotency key = %q, want empty (keyless, no caller run to prefix with)", got) + } + if !spawnAgentRunKeylessCounterHasLabel(t, "test_spawn_no_caller_run", "unresolved") { + t.Fatal("expected office_run_dedup_keyless_total to carry an unresolved entry") + } +} diff --git a/apps/backend/internal/office/scheduler/reactivity.go b/apps/backend/internal/office/scheduler/reactivity.go index 154c1ea4075..1d1ce274b43 100644 --- a/apps/backend/internal/office/scheduler/reactivity.go +++ b/apps/backend/internal/office/scheduler/reactivity.go @@ -10,6 +10,7 @@ import ( "github.com/kandev/kandev/internal/office/models" "github.com/kandev/kandev/internal/office/repository/sqlite" + "github.com/kandev/kandev/internal/runs/commentkeys" "github.com/kandev/kandev/internal/runs/dedupkeys" runsservice "github.com/kandev/kandev/internal/runs/service" ) @@ -198,7 +199,8 @@ func (ss *SchedulerService) reactToStatusChange( ss.cascadeReviewRequested(ctx, task, change, queue) case prev == statusBlocked && next != statusBlocked: - // Unblocked — wake assignee with task_unblocked. + // Unblocked — no durable occurrence row to key on: keyless by design. + runsservice.ReportKeylessEnqueue(RunReasonTaskUnblocked, runsservice.KeylessCauseByDesign, "") queue(task.AssigneeAgentProfileID, RunContext{ Reason: RunReasonTaskUnblocked, TaskID: task.ID, @@ -208,7 +210,7 @@ func (ss *SchedulerService) reactToStatusChange( }) case (prev == statusDone || prev == statusCancelled) && (next == statusTodo || next == statusInProgress): - // Reopen — different reason if a comment was attached. + // Reopen — different reason (and dedup identity) if a comment was attached. reason := RunReasonTaskReopened commentID := "" if change.Comment != nil { @@ -219,13 +221,21 @@ func (ss *SchedulerService) reactToStatusChange( // Explicit resume always uses the comment-flavoured reason. reason = RunReasonTaskReopenedComment } + var key string + if reason == RunReasonTaskReopenedComment { + key = fmt.Sprintf("%s:%s:%s", RunReasonTaskReopenedComment, commentID, task.AssigneeAgentProfileID) + } else { + // Silent reopen (no comment) — no durable occurrence row: keyless by design. + runsservice.ReportKeylessEnqueue(RunReasonTaskReopened, runsservice.KeylessCauseByDesign, "") + } queue(task.AssigneeAgentProfileID, RunContext{ - Reason: reason, - TaskID: task.ID, - WorkspaceID: task.WorkspaceID, - ActorID: change.ActorID, - ActorType: change.ActorType, - CommentID: commentID, + Reason: reason, + TaskID: task.ID, + WorkspaceID: task.WorkspaceID, + ActorID: change.ActorID, + ActorType: change.ActorType, + CommentID: commentID, + IdempotencyKey: key, }) } } @@ -298,13 +308,15 @@ func (ss *SchedulerService) reactToComment( // Assignee wake — skip if self-comment or task is closed. if !comment.SkipAssigneeWake && !selfComment && !closed { + key := commentkeys.TaskComment(comment.ID) + ":" + task.AssigneeAgentProfileID queue(task.AssigneeAgentProfileID, RunContext{ - Reason: RunReasonTaskComment, - TaskID: task.ID, - WorkspaceID: task.WorkspaceID, - ActorID: comment.AuthorID, - ActorType: comment.AuthorType, - CommentID: comment.ID, + Reason: RunReasonTaskComment, + TaskID: task.ID, + WorkspaceID: task.WorkspaceID, + ActorID: comment.AuthorID, + ActorType: comment.AuthorType, + CommentID: comment.ID, + IdempotencyKey: key, }) } @@ -322,13 +334,15 @@ func (ss *SchedulerService) reactToComment( if agentID == comment.AuthorID { continue } + key := fmt.Sprintf("%s:%s:%s", RunReasonTaskMentioned, comment.ID, agentID) queue(agentID, RunContext{ - Reason: RunReasonTaskMentioned, - TaskID: task.ID, - WorkspaceID: task.WorkspaceID, - ActorID: comment.AuthorID, - ActorType: comment.AuthorType, - CommentID: comment.ID, + Reason: RunReasonTaskMentioned, + TaskID: task.ID, + WorkspaceID: task.WorkspaceID, + ActorID: comment.AuthorID, + ActorType: comment.AuthorType, + CommentID: comment.ID, + IdempotencyKey: key, }) } } @@ -363,6 +377,9 @@ func (ss *SchedulerService) cascadeReviewRequested( if p.Role != models.ParticipantRoleReviewer && p.Role != models.ParticipantRoleApprover { continue } + // No durable occurrence row to key on: keyless by design, reported + // per recipient so the fan-out's full volume is countable. + runsservice.ReportKeylessEnqueue(RunReasonTaskReviewRequested, runsservice.KeylessCauseByDesign, "") queue(p.AgentProfileID, RunContext{ Reason: RunReasonTaskReviewRequested, TaskID: task.ID, diff --git a/apps/backend/internal/office/scheduler/reactivity_apply_mutation_test.go b/apps/backend/internal/office/scheduler/reactivity_apply_mutation_test.go new file mode 100644 index 00000000000..1c82bce74be --- /dev/null +++ b/apps/backend/internal/office/scheduler/reactivity_apply_mutation_test.go @@ -0,0 +1,138 @@ +package scheduler + +import ( + "context" + "testing" +) + +// TestApplyTaskMutation_SameAgentRepeatAssignment_QueuesWithoutInterrupt +// pins the reactivity gate relaxation this feature made: ApplyTaskMutation +// now calls reactToAssigneeChange for EVERY non-nil NewAssigneeID, including +// a repeat assignment to the agent that already holds the seat, because that +// is a real occurrence (the operator asking for the work again) rather than +// a no-op. Before this feature the caller-side equality gate suppressed it +// entirely. The relocated interrupt-comparison guard must still hold: a +// same-agent repeat must NOT hard-cancel the agent's own in-flight session. +// Before this test, ApplyTaskMutation/reactToStatusChange/ +// reactToAssigneeChange were at 0.0% coverage. +func TestApplyTaskMutation_SameAgentRepeatAssignment_QueuesWithoutInterrupt(t *testing.T) { + repo := newReactivityTestRepo(t) + ss := newChildrenCompletedTestScheduler(t, repo) + createChildrenCompletedAgent(t, repo, "agent-repeat") + + task := &TaskSnapshot{ + ID: "task-repeat-assign", + WorkspaceID: "ws-1", + AssigneeAgentProfileID: "agent-repeat", // already holds the seat + } + newAssignee := "agent-repeat" + gen := int64(1) + change := TaskMutation{ + NewAssigneeID: &newAssignee, + AssignmentGeneration: &gen, + ActorID: "user-1", + ActorType: "user", + } + + res, err := ss.ApplyTaskMutation(context.Background(), task, change) + if err != nil { + t.Fatalf("ApplyTaskMutation: %v", err) + } + + if res.InterruptSessionID != "" { + t.Fatalf("same-agent repeat assignment set InterruptSessionID = %q, want empty (must not cancel its own run)", res.InterruptSessionID) + } + found := false + for _, r := range res.Runs { + if r.AgentID == "agent-repeat" && r.Reason == RunReasonTaskAssigned { + found = true + } + } + if !found { + t.Fatalf("expected a task_assigned run queued for agent-repeat, got %+v", res.Runs) + } +} + +// TestApplyTaskMutation_DifferentAgentReassignment_InterruptsPreviousAssignee +// is the positive counterpart: reassigning to a DIFFERENT agent must still +// hard-cancel the previous assignee's session — the interrupt guard's +// comparison (moved inside reactToAssigneeChange by this feature) must not +// have accidentally suppressed the case it was already handling. +func TestApplyTaskMutation_DifferentAgentReassignment_InterruptsPreviousAssignee(t *testing.T) { + repo := newReactivityTestRepo(t) + ss := newChildrenCompletedTestScheduler(t, repo) + createChildrenCompletedAgent(t, repo, "agent-old") + createChildrenCompletedAgent(t, repo, "agent-new") + + task := &TaskSnapshot{ + ID: "task-reassign", + WorkspaceID: "ws-1", + AssigneeAgentProfileID: "agent-old", + } + newAssignee := "agent-new" + gen := int64(1) + change := TaskMutation{ + NewAssigneeID: &newAssignee, + AssignmentGeneration: &gen, + ActorID: "user-1", + ActorType: "user", + } + + res, err := ss.ApplyTaskMutation(context.Background(), task, change) + if err != nil { + t.Fatalf("ApplyTaskMutation: %v", err) + } + + if res.InterruptSessionID != task.ID { + t.Fatalf("InterruptSessionID = %q, want %q (previous assignee's session must be cancelled)", res.InterruptSessionID, task.ID) + } + found := false + for _, r := range res.Runs { + if r.AgentID == "agent-new" && r.Reason == RunReasonTaskAssigned { + found = true + } + } + if !found { + t.Fatalf("expected a task_assigned run queued for agent-new, got %+v", res.Runs) + } +} + +// TestApplyTaskMutation_StatusChangeAndCommentDispatch drives +// ApplyTaskMutation's NewStatus and Comment branches together (an unblock +// plus an attached comment), covering the dispatcher itself rather than only +// the reactToStatusChange/reactToComment helpers it delegates to. +func TestApplyTaskMutation_StatusChangeAndCommentDispatch(t *testing.T) { + repo := newReactivityTestRepo(t) + ss := newChildrenCompletedTestScheduler(t, repo) + createChildrenCompletedAgent(t, repo, "agent-dispatch") + + task := &TaskSnapshot{ + ID: "task-dispatch", + WorkspaceID: "ws-1", + State: "BLOCKED", + AssigneeAgentProfileID: "agent-dispatch", + } + newStatus := "todo" + change := TaskMutation{ + NewStatus: &newStatus, + Comment: &MutationComment{ID: "comment-dispatch", AuthorType: "user", AuthorID: "user-1"}, + ActorID: "user-1", + ActorType: "user", + } + + res, err := ss.ApplyTaskMutation(context.Background(), task, change) + if err != nil { + t.Fatalf("ApplyTaskMutation: %v", err) + } + + reasons := map[string]bool{} + for _, r := range res.Runs { + reasons[r.Reason] = true + } + if !reasons[RunReasonTaskUnblocked] { + t.Fatalf("expected %s among dispatched runs, got %+v", RunReasonTaskUnblocked, res.Runs) + } + if !reasons[RunReasonTaskComment] { + t.Fatalf("expected %s among dispatched runs, got %+v", RunReasonTaskComment, res.Runs) + } +} diff --git a/apps/backend/internal/office/scheduler/reactivity_producer_key_audit_test.go b/apps/backend/internal/office/scheduler/reactivity_producer_key_audit_test.go new file mode 100644 index 00000000000..5435ca78c69 --- /dev/null +++ b/apps/backend/internal/office/scheduler/reactivity_producer_key_audit_test.go @@ -0,0 +1,261 @@ +package scheduler + +import ( + "context" + "expvar" + "strings" + "testing" + + officesqlite "github.com/kandev/kandev/internal/office/repository/sqlite" + "github.com/kandev/kandev/internal/runs/commentkeys" + "github.com/kandev/kandev/internal/runs/dedupkeys" +) + +// newAuditScheduler wires a real repo + real *service.Service (reused from +// reactivity_children_completed_test.go's newChildrenCompletedTestScheduler) +// so every producer under audit — including the two (task_blockers_resolved, +// task_review_requested) that read participants/blockers off the repo, and +// task_mentioned, which resolves mentions via ss.svc.ListAgentInstances — +// can run unmodified. +func newAuditScheduler(t *testing.T) (*SchedulerService, *officesqlite.Repository) { + t.Helper() + repo := newReactivityTestRepo(t) + return newChildrenCompletedTestScheduler(t, repo), repo +} + +// requireOneCallKey finds the single recorded queue call for (reason, agentID) +// and returns its IdempotencyKey, failing the test if no such call exists. +func requireOneCallKey(t *testing.T, calls []recordedQueueCall, reason, agentID string) string { + t.Helper() + for _, c := range calls { + if c.agentID == agentID && c.ctx.Reason == reason { + return c.ctx.IdempotencyKey + } + } + t.Fatalf("no %s call recorded for agent %q: calls=%#v", reason, agentID, calls) + return "" +} + +// requireOneCallKeyless is requireOneCallKey's keyless counterpart: it +// asserts the matching call carries NO idempotency key, rather than +// asserting a value. +func requireOneCallKeyless(t *testing.T, calls []recordedQueueCall, reason, agentID string) { + t.Helper() + for _, c := range calls { + if c.agentID == agentID && c.ctx.Reason == reason { + if c.ctx.IdempotencyKey != "" { + t.Fatalf("%s call for agent %q carries key %q, want keyless", reason, agentID, c.ctx.IdempotencyKey) + } + return + } + } + t.Fatalf("no %s call recorded for agent %q: calls=%#v", reason, agentID, calls) +} + +// auditKeylessCounterHasLabel reports whether the process-global +// office_run_dedup_keyless_total expvar map (internal/runs/service) carries +// an entry for the given reason and cause. Duplicated locally rather than +// imported: the counter map is unexported and the sibling helper in +// internal/runs/service/dedup_test.go lives in a different package. +func auditKeylessCounterHasLabel(t *testing.T, reason, cause string) bool { + t.Helper() + v := expvar.Get("office_run_dedup_keyless_total") + if v == nil { + t.Fatalf("expvar map office_run_dedup_keyless_total not registered") + } + m, ok := v.(*expvar.Map) + if !ok { + t.Fatalf("office_run_dedup_keyless_total is not a *expvar.Map") + } + found := false + m.Do(func(kv expvar.KeyValue) { + if strings.Contains(kv.Key, "reason="+reason) && strings.Contains(kv.Key, "cause="+cause) { + found = true + } + }) + return found +} + +// TestReactivityProducerKeyAudit is the Part 2-required "producer-audit +// key-format table test" (docs/specs/office/system-design/ +// run-dedup-generation-02.md#test-strategy): a table over every reactivity +// producer named in the "Generation sources per producer" table +// (…-01.md#generation-sources-per-producer), asserting each one derives the +// spec's exact key shape, or reports the spec's exact keyless cause when the +// reason has no durable occurrence to key on. Review round 1 found 6 of this +// file's 8 run reasons silently doing neither — this is the design's own +// named enforcement mechanism whose absence is why that gap shipped +// undetected, and what stops the next reason added to reactivity.go from +// reintroducing it silently. +func TestReactivityProducerKeyAudit(t *testing.T) { + t.Run("task_assigned", func(t *testing.T) { + ss, _ := newAuditScheduler(t) + var calls []recordedQueueCall + queue := func(agentID string, c RunContext) { + calls = append(calls, recordedQueueCall{agentID: agentID, ctx: c}) + } + task := &TaskSnapshot{ID: "task-audit-assigned", WorkspaceID: "ws-1"} + gen := int64(3) + res := &ApplyTaskMutationResult{} + ss.reactToAssigneeChange(task, "agent-assigned", TaskMutation{ + AssignmentGeneration: &gen, ActorID: "user-1", ActorType: "user", + }, queue, res) + + key := requireOneCallKey(t, calls, RunReasonTaskAssigned, "agent-assigned") + if want := dedupkeys.AssignmentKey(task.ID, "agent-assigned", gen); key != want { + t.Fatalf("task_assigned key = %q, want %q", key, want) + } + }) + + t.Run("task_blockers_resolved", func(t *testing.T) { + ss, repo := newAuditScheduler(t) + setupChildrenCompletedParent(t, ss, "blocked-audit", "agent-blocked") + insertBlockerRelationship(t, repo, "blocked-audit", "blocker-audit-a", "COMPLETED") + insertBlockerRelationship(t, repo, "blocked-audit", "blocker-audit-b", "COMPLETED") + + var calls []recordedQueueCall + queue := func(agentID string, c RunContext) { + calls = append(calls, recordedQueueCall{agentID: agentID, ctx: c}) + } + ss.cascadeBlockersResolved(context.Background(), + &TaskSnapshot{ID: "blocker-audit-b", WorkspaceID: "ws-1"}, queue) + + key := requireOneCallKey(t, calls, RunReasonTaskBlockersResolved, "agent-blocked") + digest := dedupkeys.BlockerDigest([]string{"blocker-audit-a", "blocker-audit-b"}) + want := RunReasonTaskBlockersResolved + ":blocked-audit:agent-blocked:" + digest + if key != want { + t.Fatalf("task_blockers_resolved key = %q, want %q", key, want) + } + }) + + t.Run("task_comment", func(t *testing.T) { + ss, _ := newAuditScheduler(t) + task := &TaskSnapshot{ID: "task-audit-comment", WorkspaceID: "ws-1", AssigneeAgentProfileID: "agent-assignee"} + comment := &MutationComment{ID: "comment-audit-1", Body: "status update", AuthorType: "user", AuthorID: "user-1"} + var calls []recordedQueueCall + queue := func(agentID string, c RunContext) { + calls = append(calls, recordedQueueCall{agentID: agentID, ctx: c}) + } + ss.reactToComment(context.Background(), task, comment, queue) + + key := requireOneCallKey(t, calls, RunReasonTaskComment, "agent-assignee") + want := commentkeys.TaskComment("comment-audit-1") + ":agent-assignee" + if key != want { + t.Fatalf("task_comment key = %q, want %q", key, want) + } + }) + + t.Run("task_mentioned", func(t *testing.T) { + ss, repo := newAuditScheduler(t) + createChildrenCompletedAgent(t, repo, "agent-mentioned") + task := &TaskSnapshot{ID: "task-audit-mentioned", WorkspaceID: "ws-1"} + comment := &MutationComment{ + ID: "comment-audit-2", AuthorType: "user", AuthorID: "user-1", + // The mention token regex allows internal spaces (for multi-word + // agent names), so it greedily consumes everything after the "@" + // up to end of string or a disallowed character. Keeping the + // mention last with nothing trailing bounds the capture to + // exactly the agent name. + Body: "cc @agent-mentioned", + // Isolate the mention path from the assignee-wake path (task has + // no assignee here, but SkipAssigneeWake documents the intent). + SkipAssigneeWake: true, + } + var calls []recordedQueueCall + queue := func(agentID string, c RunContext) { + calls = append(calls, recordedQueueCall{agentID: agentID, ctx: c}) + } + ss.reactToComment(context.Background(), task, comment, queue) + + key := requireOneCallKey(t, calls, RunReasonTaskMentioned, "agent-mentioned") + want := RunReasonTaskMentioned + ":comment-audit-2:agent-mentioned" + if key != want { + t.Fatalf("task_mentioned key = %q, want %q", key, want) + } + }) + + t.Run("task_reopened_via_comment", func(t *testing.T) { + ss, _ := newAuditScheduler(t) + task := &TaskSnapshot{ID: "task-audit-reopen-comment", WorkspaceID: "ws-1", State: "DONE", AssigneeAgentProfileID: "agent-reopen"} + res := &ApplyTaskMutationResult{} + var calls []recordedQueueCall + queue := func(agentID string, c RunContext) { + calls = append(calls, recordedQueueCall{agentID: agentID, ctx: c}) + } + change := TaskMutation{ + Comment: &MutationComment{ID: "comment-audit-3", AuthorType: "user", AuthorID: "user-1"}, + ActorID: "user-1", + ActorType: "user", + } + ss.reactToStatusChange(context.Background(), task, "todo", change, queue, res) + + key := requireOneCallKey(t, calls, RunReasonTaskReopenedComment, "agent-reopen") + want := RunReasonTaskReopenedComment + ":comment-audit-3:agent-reopen" + if key != want { + t.Fatalf("task_reopened_via_comment key = %q, want %q", key, want) + } + }) + + t.Run("task_unblocked", func(t *testing.T) { + ss, _ := newAuditScheduler(t) + task := &TaskSnapshot{ID: "task-audit-unblocked", WorkspaceID: "ws-1", State: "BLOCKED", AssigneeAgentProfileID: "agent-unblocked"} + res := &ApplyTaskMutationResult{} + var calls []recordedQueueCall + queue := func(agentID string, c RunContext) { + calls = append(calls, recordedQueueCall{agentID: agentID, ctx: c}) + } + change := TaskMutation{ActorID: "user-1", ActorType: "user"} + ss.reactToStatusChange(context.Background(), task, "todo", change, queue, res) + + requireOneCallKeyless(t, calls, RunReasonTaskUnblocked, "agent-unblocked") + if !auditKeylessCounterHasLabel(t, RunReasonTaskUnblocked, "by_design") { + t.Fatal("expected office_run_dedup_keyless_total to carry task_unblocked/by_design") + } + }) + + t.Run("task_reopened", func(t *testing.T) { + ss, _ := newAuditScheduler(t) + task := &TaskSnapshot{ID: "task-audit-reopen-silent", WorkspaceID: "ws-1", State: "CANCELLED", AssigneeAgentProfileID: "agent-silent-reopen"} + res := &ApplyTaskMutationResult{} + var calls []recordedQueueCall + queue := func(agentID string, c RunContext) { + calls = append(calls, recordedQueueCall{agentID: agentID, ctx: c}) + } + change := TaskMutation{ActorID: "user-1", ActorType: "user"} + ss.reactToStatusChange(context.Background(), task, "in_progress", change, queue, res) + + requireOneCallKeyless(t, calls, RunReasonTaskReopened, "agent-silent-reopen") + if !auditKeylessCounterHasLabel(t, RunReasonTaskReopened, "by_design") { + t.Fatal("expected office_run_dedup_keyless_total to carry task_reopened/by_design") + } + }) + + t.Run("task_review_requested", func(t *testing.T) { + ss, repo := newAuditScheduler(t) + ctx := context.Background() + if _, err := repo.ExecRaw(ctx, ` + INSERT INTO tasks (id, workspace_id, workflow_step_id) VALUES ('task-audit-review', 'ws-1', 'step-audit') + `); err != nil { + t.Fatalf("insert task: %v", err) + } + if _, err := repo.ExecRaw(ctx, ` + INSERT INTO workflow_step_participants + (id, step_id, task_id, role, agent_profile_id, decision_required, position) + VALUES ('p-audit-reviewer', 'step-audit', 'task-audit-review', 'reviewer', 'agent-reviewer-audit', 1, 0) + `); err != nil { + t.Fatalf("insert participant: %v", err) + } + + task := &TaskSnapshot{ID: "task-audit-review", WorkspaceID: "ws-1"} + var calls []recordedQueueCall + queue := func(agentID string, c RunContext) { + calls = append(calls, recordedQueueCall{agentID: agentID, ctx: c}) + } + ss.cascadeReviewRequested(ctx, task, TaskMutation{ActorID: "user-1", ActorType: "user"}, queue) + + requireOneCallKeyless(t, calls, RunReasonTaskReviewRequested, "agent-reviewer-audit") + if !auditKeylessCounterHasLabel(t, RunReasonTaskReviewRequested, "by_design") { + t.Fatal("expected office_run_dedup_keyless_total to carry task_review_requested/by_design") + } + }) +} From 9180ba3cc1262b1b20a38d5cd262536f34fcf7d6 Mon Sep 17 00:00:00 2001 From: nova28 <17953305+nova28@users.noreply.github.com> Date: Wed, 9 Sep 2026 06:16:32 +0800 Subject: [PATCH 04/15] fix(office): stop counting keyless enqueues that never happen reactToStatusChange reported task_unblocked and silent task_reopened as keyless-by-design before checking whether the task had an assignee. The queue closure silently drops an empty agent id, so an unassigned task still incremented office_run_dedup_keyless_total for an enqueue attempt that never occurred. --- .../internal/office/scheduler/reactivity.go | 13 ++- .../reactivity_producer_key_audit_test.go | 97 +++++++++++++++---- 2 files changed, 87 insertions(+), 23 deletions(-) diff --git a/apps/backend/internal/office/scheduler/reactivity.go b/apps/backend/internal/office/scheduler/reactivity.go index 1d1ce274b43..9920b2240a8 100644 --- a/apps/backend/internal/office/scheduler/reactivity.go +++ b/apps/backend/internal/office/scheduler/reactivity.go @@ -200,7 +200,12 @@ func (ss *SchedulerService) reactToStatusChange( case prev == statusBlocked && next != statusBlocked: // Unblocked — no durable occurrence row to key on: keyless by design. - runsservice.ReportKeylessEnqueue(RunReasonTaskUnblocked, runsservice.KeylessCauseByDesign, "") + // Reported only when there's an assignee to enqueue for — queue() + // silently drops an empty agent id, so reporting unconditionally + // would count an enqueue that never happened. + if task.AssigneeAgentProfileID != "" { + runsservice.ReportKeylessEnqueue(RunReasonTaskUnblocked, runsservice.KeylessCauseByDesign, "") + } queue(task.AssigneeAgentProfileID, RunContext{ Reason: RunReasonTaskUnblocked, TaskID: task.ID, @@ -224,8 +229,10 @@ func (ss *SchedulerService) reactToStatusChange( var key string if reason == RunReasonTaskReopenedComment { key = fmt.Sprintf("%s:%s:%s", RunReasonTaskReopenedComment, commentID, task.AssigneeAgentProfileID) - } else { - // Silent reopen (no comment) — no durable occurrence row: keyless by design. + } else if task.AssigneeAgentProfileID != "" { + // Silent reopen (no comment) — no durable occurrence row: keyless + // by design. Reported only when there's an assignee to enqueue + // for, matching the task_unblocked case above. runsservice.ReportKeylessEnqueue(RunReasonTaskReopened, runsservice.KeylessCauseByDesign, "") } queue(task.AssigneeAgentProfileID, RunContext{ diff --git a/apps/backend/internal/office/scheduler/reactivity_producer_key_audit_test.go b/apps/backend/internal/office/scheduler/reactivity_producer_key_audit_test.go index 5435ca78c69..69e8361b816 100644 --- a/apps/backend/internal/office/scheduler/reactivity_producer_key_audit_test.go +++ b/apps/backend/internal/office/scheduler/reactivity_producer_key_audit_test.go @@ -3,7 +3,6 @@ package scheduler import ( "context" "expvar" - "strings" "testing" officesqlite "github.com/kandev/kandev/internal/office/repository/sqlite" @@ -52,12 +51,16 @@ func requireOneCallKeyless(t *testing.T, calls []recordedQueueCall, reason, agen t.Fatalf("no %s call recorded for agent %q: calls=%#v", reason, agentID, calls) } -// auditKeylessCounterHasLabel reports whether the process-global -// office_run_dedup_keyless_total expvar map (internal/runs/service) carries -// an entry for the given reason and cause. Duplicated locally rather than -// imported: the counter map is unexported and the sibling helper in -// internal/runs/service/dedup_test.go lives in a different package. -func auditKeylessCounterHasLabel(t *testing.T, reason, cause string) bool { +// auditKeylessCounterValue returns the current value of the process-global +// office_run_dedup_keyless_total expvar map (internal/runs/service) for the +// exact "reason=;cause=" label, or 0 if that label has never +// been incremented. Duplicated locally rather than imported: the counter map +// is unexported and the sibling helper in internal/runs/service/dedup_test.go +// lives in a different package. Callers snapshot before and after a producer +// call and assert the delta, rather than merely that the label exists — a +// mere-existence check would pass even if this specific call site failed to +// report, as long as some other test already set that label. +func auditKeylessCounterValue(t *testing.T, reason, cause string) int64 { t.Helper() v := expvar.Get("office_run_dedup_keyless_total") if v == nil { @@ -67,13 +70,15 @@ func auditKeylessCounterHasLabel(t *testing.T, reason, cause string) bool { if !ok { t.Fatalf("office_run_dedup_keyless_total is not a *expvar.Map") } - found := false - m.Do(func(kv expvar.KeyValue) { - if strings.Contains(kv.Key, "reason="+reason) && strings.Contains(kv.Key, "cause="+cause) { - found = true - } - }) - return found + iv := m.Get("reason=" + reason + ";cause=" + cause) + if iv == nil { + return 0 + } + i, ok := iv.(*expvar.Int) + if !ok { + t.Fatalf("counter value for reason=%s;cause=%s is not *expvar.Int", reason, cause) + } + return i.Value() } // TestReactivityProducerKeyAudit is the Part 2-required "producer-audit @@ -204,12 +209,39 @@ func TestReactivityProducerKeyAudit(t *testing.T) { queue := func(agentID string, c RunContext) { calls = append(calls, recordedQueueCall{agentID: agentID, ctx: c}) } + before := auditKeylessCounterValue(t, RunReasonTaskUnblocked, "by_design") change := TaskMutation{ActorID: "user-1", ActorType: "user"} ss.reactToStatusChange(context.Background(), task, "todo", change, queue, res) requireOneCallKeyless(t, calls, RunReasonTaskUnblocked, "agent-unblocked") - if !auditKeylessCounterHasLabel(t, RunReasonTaskUnblocked, "by_design") { - t.Fatal("expected office_run_dedup_keyless_total to carry task_unblocked/by_design") + if got, want := auditKeylessCounterValue(t, RunReasonTaskUnblocked, "by_design"), before+1; got != want { + t.Fatalf("office_run_dedup_keyless_total{task_unblocked,by_design} = %d, want %d", got, want) + } + }) + + t.Run("task_unblocked_no_assignee_does_not_report", func(t *testing.T) { + // Regression test for Review round 2 Finding 3: an unassigned task + // must not increment the keyless counter for an enqueue that never + // happens. Goes through ApplyTaskMutation (not reactToStatusChange + // directly) so the real queue closure's empty-agent-id guard + // (reactivity.go's `if agentID == "" { return }`) is exercised — + // that guard is exactly what silently swallowed the enqueue while + // FINDING 3's ReportKeylessEnqueue call still fired unconditionally. + ss, _ := newAuditScheduler(t) + task := &TaskSnapshot{ID: "task-audit-unblocked-noassignee", WorkspaceID: "ws-1", State: "BLOCKED"} + before := auditKeylessCounterValue(t, RunReasonTaskUnblocked, "by_design") + newStatus := "todo" + change := TaskMutation{NewStatus: &newStatus, ActorID: "user-1", ActorType: "user"} + res, err := ss.ApplyTaskMutation(context.Background(), task, change) + if err != nil { + t.Fatalf("ApplyTaskMutation: %v", err) + } + + if len(res.Runs) != 0 { + t.Fatalf("expected no queued runs for an unassigned task_unblocked, got %#v", res.Runs) + } + if got := auditKeylessCounterValue(t, RunReasonTaskUnblocked, "by_design"); got != before { + t.Fatalf("office_run_dedup_keyless_total{task_unblocked,by_design} = %d, want unchanged %d (no enqueue attempted)", got, before) } }) @@ -221,12 +253,36 @@ func TestReactivityProducerKeyAudit(t *testing.T) { queue := func(agentID string, c RunContext) { calls = append(calls, recordedQueueCall{agentID: agentID, ctx: c}) } + before := auditKeylessCounterValue(t, RunReasonTaskReopened, "by_design") change := TaskMutation{ActorID: "user-1", ActorType: "user"} ss.reactToStatusChange(context.Background(), task, "in_progress", change, queue, res) requireOneCallKeyless(t, calls, RunReasonTaskReopened, "agent-silent-reopen") - if !auditKeylessCounterHasLabel(t, RunReasonTaskReopened, "by_design") { - t.Fatal("expected office_run_dedup_keyless_total to carry task_reopened/by_design") + if got, want := auditKeylessCounterValue(t, RunReasonTaskReopened, "by_design"), before+1; got != want { + t.Fatalf("office_run_dedup_keyless_total{task_reopened,by_design} = %d, want %d", got, want) + } + }) + + t.Run("task_reopened_no_assignee_does_not_report", func(t *testing.T) { + // Regression test for Review round 2 Finding 3, silent-reopen branch. + // Goes through ApplyTaskMutation for the same reason as the + // task_unblocked case above: the empty-agent-id guard lives in the + // real queue closure, not in reactToStatusChange itself. + ss, _ := newAuditScheduler(t) + task := &TaskSnapshot{ID: "task-audit-reopen-silent-noassignee", WorkspaceID: "ws-1", State: "CANCELLED"} + before := auditKeylessCounterValue(t, RunReasonTaskReopened, "by_design") + newStatus := "in_progress" + change := TaskMutation{NewStatus: &newStatus, ActorID: "user-1", ActorType: "user"} + res, err := ss.ApplyTaskMutation(context.Background(), task, change) + if err != nil { + t.Fatalf("ApplyTaskMutation: %v", err) + } + + if len(res.Runs) != 0 { + t.Fatalf("expected no queued runs for an unassigned silent reopen, got %#v", res.Runs) + } + if got := auditKeylessCounterValue(t, RunReasonTaskReopened, "by_design"); got != before { + t.Fatalf("office_run_dedup_keyless_total{task_reopened,by_design} = %d, want unchanged %d (no enqueue attempted)", got, before) } }) @@ -251,11 +307,12 @@ func TestReactivityProducerKeyAudit(t *testing.T) { queue := func(agentID string, c RunContext) { calls = append(calls, recordedQueueCall{agentID: agentID, ctx: c}) } + before := auditKeylessCounterValue(t, RunReasonTaskReviewRequested, "by_design") ss.cascadeReviewRequested(ctx, task, TaskMutation{ActorID: "user-1", ActorType: "user"}, queue) requireOneCallKeyless(t, calls, RunReasonTaskReviewRequested, "agent-reviewer-audit") - if !auditKeylessCounterHasLabel(t, RunReasonTaskReviewRequested, "by_design") { - t.Fatal("expected office_run_dedup_keyless_total to carry task_review_requested/by_design") + if got, want := auditKeylessCounterValue(t, RunReasonTaskReviewRequested, "by_design"), before+1; got != want { + t.Fatalf("office_run_dedup_keyless_total{task_review_requested,by_design} = %d, want %d", got, want) } }) } From 9aaa48e5f27735113c99275d9666b977d59b2963 Mon Sep 17 00:00:00 2001 From: nova28 <17953305+nova28@users.noreply.github.com> Date: Wed, 9 Sep 2026 06:25:10 +0800 Subject: [PATCH 05/15] test(office): prove blocker-resolved digest convergence across producers AC-002.1 required office/scheduler.cascadeBlockersResolved and office/service.resolveAndWakeIfUnblocked to derive the same dedup key for an equivalent blocker set, but no test drove the second producer directly. Add coverage exercising resolveAndWakeIfUnblocked end to end and asserting its persisted operation id matches the digest the other producer would derive for the same occurrence. --- .../blockers_resolved_key_convergence_test.go | 192 ++++++++++++++++++ 1 file changed, 192 insertions(+) create mode 100644 apps/backend/internal/office/service/blockers_resolved_key_convergence_test.go diff --git a/apps/backend/internal/office/service/blockers_resolved_key_convergence_test.go b/apps/backend/internal/office/service/blockers_resolved_key_convergence_test.go new file mode 100644 index 00000000000..487a2281fea --- /dev/null +++ b/apps/backend/internal/office/service/blockers_resolved_key_convergence_test.go @@ -0,0 +1,192 @@ +package service + +import ( + "context" + "sync" + "testing" + + "github.com/jmoiron/sqlx" + _ "github.com/mattn/go-sqlite3" + + settingsstore "github.com/kandev/kandev/internal/agent/settings/store" + "github.com/kandev/kandev/internal/common/logger" + "github.com/kandev/kandev/internal/office/models" + "github.com/kandev/kandev/internal/office/repository/sqlite" + "github.com/kandev/kandev/internal/runs/dedupkeys" + "github.com/kandev/kandev/internal/workflow/engine" +) + +// newBlockersConvergenceTestRepo builds a real office repository (real +// migrations, so task_blockers is the production schema) plus the minimal +// hand-rolled tasks table resolveAndWakeIfUnblocked reads state from — +// tasks is owned by the task package's schema, which office's migrations +// only reference by foreign key. +func newBlockersConvergenceTestRepo(t *testing.T) *sqlite.Repository { + t.Helper() + db, err := sqlx.Open("sqlite3", ":memory:") + if err != nil { + t.Fatalf("open sqlite: %v", err) + } + db.SetMaxOpenConns(1) + t.Cleanup(func() { _ = db.Close() }) + if _, _, err := settingsstore.Provide(db, db, nil); err != nil { + t.Fatalf("settings store: %v", err) + } + repo, err := sqlite.NewWithDB(db, db, nil) + if err != nil { + t.Fatalf("new repo: %v", err) + } + ctx := context.Background() + if _, err := repo.ExecRaw(ctx, ` + CREATE TABLE IF NOT EXISTS tasks ( + id TEXT PRIMARY KEY, + workspace_id TEXT DEFAULT '', + state TEXT DEFAULT '' + ) + `); err != nil { + t.Fatalf("create tasks table: %v", err) + } + return repo +} + +// convergenceDispatcher records every HandleTrigger call so the test can +// assert the exact operation id resolveAndWakeIfUnblocked derived. +type convergenceDispatcher struct { + mu sync.Mutex + calls []convergenceCall +} + +type convergenceCall struct { + taskID string + trigger engine.Trigger + payload any + opID string +} + +func (d *convergenceDispatcher) HandleTrigger( + _ context.Context, taskID string, trigger engine.Trigger, payload any, opID string, +) error { + d.mu.Lock() + defer d.mu.Unlock() + d.calls = append(d.calls, convergenceCall{taskID, trigger, payload, opID}) + return nil +} + +func newConvergenceTestService(t *testing.T, repo *sqlite.Repository, dispatcher *convergenceDispatcher) *Service { + t.Helper() + log, err := logger.NewLogger(logger.LoggingConfig{Level: "error", Format: "console"}) + if err != nil { + t.Fatalf("logger: %v", err) + } + s := &Service{repo: repo, logger: log} + s.SetWorkflowEngineDispatcher(dispatcher) + return s +} + +// TestResolveAndWakeIfUnblocked_DerivesSameDigestAsCascadeBlockersResolved +// is the AC-OFFICE-RUN-DEDUP-002.1 convergence test for the second blocker +// producer. office/scheduler.cascadeBlockersResolved (the other half of the +// same AC) already has a regression test +// (reactivity_blockers_resolved_test.go), but that test never exercises +// this package's resolveAndWakeIfUnblocked — it only proves +// dedupkeys.BlockerDigest is order-independent in isolation. Review round 2 +// found the AC's own explicit test requirement ("both producers derive a +// byte-identical digest, driven through the shared builder from both +// packages") unproven by any executing test. This test drives +// resolveAndWakeIfUnblocked directly and asserts its persisted operation id +// carries the exact digest office/scheduler's producer would derive for the +// identical blocker set. +func TestResolveAndWakeIfUnblocked_DerivesSameDigestAsCascadeBlockersResolved(t *testing.T) { + repo := newBlockersConvergenceTestRepo(t) + ctx := context.Background() + + for _, row := range []struct{ id, state string }{ + {"blocked-1", "IN_PROGRESS"}, + {"blocker-conv-a", "COMPLETED"}, + {"blocker-conv-b", "COMPLETED"}, + } { + if _, err := repo.ExecRaw(ctx, + `INSERT INTO tasks (id, workspace_id, state) VALUES (?, 'ws-1', ?)`, + row.id, row.state, + ); err != nil { + t.Fatalf("insert task %s: %v", row.id, err) + } + } + for _, blockerID := range []string{"blocker-conv-a", "blocker-conv-b"} { + if err := repo.CreateTaskBlocker(ctx, &models.TaskBlocker{ + TaskID: "blocked-1", BlockerTaskID: blockerID, + }); err != nil { + t.Fatalf("create blocker relationship for %s: %v", blockerID, err) + } + } + + dispatcher := &convergenceDispatcher{} + s := newConvergenceTestService(t, repo, dispatcher) + + if err := s.resolveAndWakeIfUnblocked(ctx, "blocked-1", "blocker-conv-b"); err != nil { + t.Fatalf("resolveAndWakeIfUnblocked: %v", err) + } + + dispatcher.mu.Lock() + calls := append([]convergenceCall{}, dispatcher.calls...) + dispatcher.mu.Unlock() + if len(calls) != 1 { + t.Fatalf("HandleTrigger calls = %d, want 1: %#v", len(calls), calls) + } + call := calls[0] + if call.taskID != "blocked-1" { + t.Fatalf("taskID = %q, want blocked-1", call.taskID) + } + if call.trigger != engine.TriggerOnBlockerResolved { + t.Fatalf("trigger = %q, want %q", call.trigger, engine.TriggerOnBlockerResolved) + } + + // The digest office/scheduler.cascadeBlockersResolved would derive for + // the identical blocker set — proving the two independently-implemented + // producers converge on the same occurrence identity, per AC-002.1. + digest := dedupkeys.BlockerDigest([]string{"blocker-conv-a", "blocker-conv-b"}) + want := "blockers_resolved:blocked-1:" + digest + if call.opID != want { + t.Fatalf("operation id = %q, want %q", call.opID, want) + } +} + +// TestResolveAndWakeIfUnblocked_StillBlocked_DoesNotDispatch proves the +// negative: a task with another outstanding blocker must not wake. +func TestResolveAndWakeIfUnblocked_StillBlocked_DoesNotDispatch(t *testing.T) { + repo := newBlockersConvergenceTestRepo(t) + ctx := context.Background() + + for _, row := range []struct{ id, state string }{ + {"blocked-2", "IN_PROGRESS"}, + {"blocker-conv-c", "IN_PROGRESS"}, // never finishes + {"blocker-conv-d", "COMPLETED"}, + } { + if _, err := repo.ExecRaw(ctx, + `INSERT INTO tasks (id, workspace_id, state) VALUES (?, 'ws-1', ?)`, + row.id, row.state, + ); err != nil { + t.Fatalf("insert task %s: %v", row.id, err) + } + } + for _, blockerID := range []string{"blocker-conv-c", "blocker-conv-d"} { + if err := repo.CreateTaskBlocker(ctx, &models.TaskBlocker{ + TaskID: "blocked-2", BlockerTaskID: blockerID, + }); err != nil { + t.Fatalf("create blocker relationship for %s: %v", blockerID, err) + } + } + + dispatcher := &convergenceDispatcher{} + s := newConvergenceTestService(t, repo, dispatcher) + + if err := s.resolveAndWakeIfUnblocked(ctx, "blocked-2", "blocker-conv-d"); err != nil { + t.Fatalf("resolveAndWakeIfUnblocked: %v", err) + } + + dispatcher.mu.Lock() + defer dispatcher.mu.Unlock() + if len(dispatcher.calls) != 0 { + t.Fatalf("expected no HandleTrigger calls while blocker-conv-c is still outstanding, got %#v", dispatcher.calls) + } +} From 4b6d4924e7f3e82ceb34f11d01508993037945e6 Mon Sep 17 00:00:00 2001 From: nova28 <17953305+nova28@users.noreply.github.com> Date: Wed, 9 Sep 2026 06:27:28 +0800 Subject: [PATCH 06/15] test(office): cover requeueRunForTask's keyless telemetry report Manual resume (Mark fixed / unpause) has no prior occurrence row to key on and reports itself keyless by design, but no test exercised that call site. Drive it through the public MarkAgentRunFailedFixed entry point and assert the counter's exact delta. --- .../service/manual_resume_keyless_test.go | 83 +++++++++++++++++++ 1 file changed, 83 insertions(+) create mode 100644 apps/backend/internal/office/service/manual_resume_keyless_test.go diff --git a/apps/backend/internal/office/service/manual_resume_keyless_test.go b/apps/backend/internal/office/service/manual_resume_keyless_test.go new file mode 100644 index 00000000000..e73294cf149 --- /dev/null +++ b/apps/backend/internal/office/service/manual_resume_keyless_test.go @@ -0,0 +1,83 @@ +package service_test + +import ( + "context" + "expvar" + "testing" + + "github.com/kandev/kandev/internal/office/service" + runsservice "github.com/kandev/kandev/internal/runs/service" +) + +// keylessCounterValue reads the current value of one +// office_run_dedup_keyless_total{reason,cause} label, or 0 if the label +// has never been reported. Callers snapshot before/after and assert the +// exact delta, so a call site that stops reporting (or a passing test +// that never actually reaches the report) can't hide behind a value some +// other test already set. +func keylessCounterValue(t *testing.T, reason string, cause runsservice.KeylessCause) int64 { + t.Helper() + v := expvar.Get("office_run_dedup_keyless_total") + if v == nil { + t.Fatalf("expvar map office_run_dedup_keyless_total not registered") + } + m, ok := v.(*expvar.Map) + if !ok { + t.Fatalf("office_run_dedup_keyless_total is not a *expvar.Map") + } + iv := m.Get("reason=" + reason + ";cause=" + string(cause)) + if iv == nil { + return 0 + } + i, ok := iv.(*expvar.Int) + if !ok { + t.Fatalf("counter value for reason=%s;cause=%s is not *expvar.Int", reason, cause) + } + return i.Value() +} + +// TestMarkAgentRunFailedFixed_ReportsKeylessRequeue pins +// requeueRunForTask's ReportKeylessEnqueue call (failure.go) — a manual +// resume has no prior occurrence row to key on, so it is keyless by +// design like the reactivity producers, but before this test that call +// site had zero coverage anywhere in the suite. +func TestMarkAgentRunFailedFixed_ReportsKeylessRequeue(t *testing.T) { + svc, _ := newTestServiceWithBus(t) + ctx := context.Background() + + createTestAgent(t, svc, "ws-1", "agent-manual-resume") + taskID := "task-manual-resume-1" + insertSyntheticTask(t, svc, taskID, "ws-1", "agent-manual-resume") + w := queueAndReadRun(t, svc, "agent-manual-resume", taskID) + if err := svc.HandleAgentFailure(ctx, w, "boom"); err != nil { + t.Fatalf("handle failure: %v", err) + } + + before := keylessCounterValue(t, service.RunReasonManualResumeAfterFailure, runsservice.KeylessCauseByDesign) + + if err := svc.MarkAgentRunFailedFixed(ctx, "user-1", w.ID); err != nil { + t.Fatalf("mark fixed: %v", err) + } + + after := keylessCounterValue(t, service.RunReasonManualResumeAfterFailure, runsservice.KeylessCauseByDesign) + if after != before+1 { + t.Fatalf("office_run_dedup_keyless_total{%s,%s} = %d, want %d (one manual requeue)", + service.RunReasonManualResumeAfterFailure, runsservice.KeylessCauseByDesign, after, before+1) + } + + rows, err := svc.ListRuns(ctx, "ws-1") + if err != nil { + t.Fatalf("list runs: %v", err) + } + found := false + for _, r := range rows { + if r.AgentProfileID == "agent-manual-resume" && r.Reason == service.RunReasonManualResumeAfterFailure && + taskIDFromPayload(t, r.Payload) == taskID { + found = true + } + } + if !found { + t.Fatalf("expected a %s run queued for agent-manual-resume/%s, got %+v", + service.RunReasonManualResumeAfterFailure, taskID, rows) + } +} From 18e0145dc13723b54603a0463586c51e74bdf23d Mon Sep 17 00:00:00 2001 From: nova28 <17953305+nova28@users.noreply.github.com> Date: Wed, 9 Sep 2026 07:36:36 +0800 Subject: [PATCH 07/15] fix(office): report review-request keyless telemetry once per recipient cascadeReviewRequested reported ReportKeylessEnqueue once per workflow_step_participants row, but the queue closure dedupes enqueue attempts by agent, so an agent seated as both reviewer and approver was counted twice for one actual enqueue attempt. --- .../internal/office/scheduler/reactivity.go | 10 +++- .../reactivity_producer_key_audit_test.go | 54 +++++++++++++++++++ 2 files changed, 62 insertions(+), 2 deletions(-) diff --git a/apps/backend/internal/office/scheduler/reactivity.go b/apps/backend/internal/office/scheduler/reactivity.go index 9920b2240a8..fc4d10f8fff 100644 --- a/apps/backend/internal/office/scheduler/reactivity.go +++ b/apps/backend/internal/office/scheduler/reactivity.go @@ -377,6 +377,7 @@ func (ss *SchedulerService) cascadeReviewRequested( zap.String("task_id", task.ID), zap.Error(err)) return } + reported := map[string]struct{}{} for _, p := range parts { if p.AgentProfileID == "" { continue @@ -385,8 +386,13 @@ func (ss *SchedulerService) cascadeReviewRequested( continue } // No durable occurrence row to key on: keyless by design, reported - // per recipient so the fan-out's full volume is countable. - runsservice.ReportKeylessEnqueue(RunReasonTaskReviewRequested, runsservice.KeylessCauseByDesign, "") + // once per distinct agent rather than per role — an agent seated as + // both reviewer and approver is one recipient, and queue()'s own + // seen-map dedup only attempts one enqueue for it. + if _, dup := reported[p.AgentProfileID]; !dup { + reported[p.AgentProfileID] = struct{}{} + runsservice.ReportKeylessEnqueue(RunReasonTaskReviewRequested, runsservice.KeylessCauseByDesign, "") + } queue(p.AgentProfileID, RunContext{ Reason: RunReasonTaskReviewRequested, TaskID: task.ID, diff --git a/apps/backend/internal/office/scheduler/reactivity_producer_key_audit_test.go b/apps/backend/internal/office/scheduler/reactivity_producer_key_audit_test.go index 69e8361b816..1dd0e3944be 100644 --- a/apps/backend/internal/office/scheduler/reactivity_producer_key_audit_test.go +++ b/apps/backend/internal/office/scheduler/reactivity_producer_key_audit_test.go @@ -315,4 +315,58 @@ func TestReactivityProducerKeyAudit(t *testing.T) { t.Fatalf("office_run_dedup_keyless_total{task_review_requested,by_design} = %d, want %d", got, want) } }) + + t.Run("task_review_requested_dual_role_recipient_counts_once", func(t *testing.T) { + // Regression test for Review round 3 Finding 4: an agent seated as + // both reviewer and approver on the same task is one recipient, not + // two. Driven through the real ApplyTaskMutation (not a bare + // closure) so the real queue closure's seen-map dedup — which + // silently absorbs the second queue() call for the same agent — + // is what proves the counter must match the number of enqueue + // attempts actually made, not the number of participant rows read. + ss, repo := newAuditScheduler(t) + ctx := context.Background() + createChildrenCompletedAgent(t, repo, "agent-dual-role") + if _, err := repo.ExecRaw(ctx, ` + INSERT INTO tasks (id, workspace_id, workflow_step_id) VALUES ('task-audit-review-dual', 'ws-1', 'step-audit') + `); err != nil { + t.Fatalf("insert task: %v", err) + } + if _, err := repo.ExecRaw(ctx, ` + INSERT INTO workflow_step_participants + (id, step_id, task_id, role, agent_profile_id, decision_required, position) + VALUES ('p-audit-dual-reviewer', 'step-audit', 'task-audit-review-dual', 'reviewer', 'agent-dual-role', 1, 0) + `); err != nil { + t.Fatalf("insert reviewer participant: %v", err) + } + if _, err := repo.ExecRaw(ctx, ` + INSERT INTO workflow_step_participants + (id, step_id, task_id, role, agent_profile_id, decision_required, position) + VALUES ('p-audit-dual-approver', 'step-audit', 'task-audit-review-dual', 'approver', 'agent-dual-role', 1, 1) + `); err != nil { + t.Fatalf("insert approver participant: %v", err) + } + + task := &TaskSnapshot{ID: "task-audit-review-dual", WorkspaceID: "ws-1", State: statusTodo} + before := auditKeylessCounterValue(t, RunReasonTaskReviewRequested, "by_design") + newStatus := statusInReview + change := TaskMutation{NewStatus: &newStatus, ActorID: "user-1", ActorType: "user"} + res, err := ss.ApplyTaskMutation(ctx, task, change) + if err != nil { + t.Fatalf("ApplyTaskMutation: %v", err) + } + + queued := 0 + for _, r := range res.Runs { + if r.Reason == RunReasonTaskReviewRequested && r.AgentID == "agent-dual-role" { + queued++ + } + } + if queued != 1 { + t.Fatalf("expected exactly 1 queued task_review_requested run for the dual-role agent, got %d (%#v)", queued, res.Runs) + } + if got, want := auditKeylessCounterValue(t, RunReasonTaskReviewRequested, "by_design"), before+1; got != want { + t.Fatalf("office_run_dedup_keyless_total{task_review_requested,by_design} = %d, want %d (one recipient, not one row per role)", got, want) + } + }) } From 85681d821eb60e0ed2c8d14ee8ad348a033e599b Mon Sep 17 00:00:00 2001 From: nova28 <17953305+nova28@users.noreply.github.com> Date: Wed, 9 Sep 2026 07:39:49 +0800 Subject: [PATCH 08/15] fix(office): do not terminate an agent's own session on self-reassignment runReactivityForAssigneeChange flipped the prior assignee's office session row to COMPLETED whenever a previous assignee existed, with no comparison against the new assignee. A same-agent repeat assignment is not a handoff (the reactivity pipeline already declines to interrupt it), so terminating the persisted session row there risked a duplicate session on the next EnsureSessionForAgent call. --- .../office/dashboard/service_tasks.go | 5 +++- .../dashboard/session_termination_test.go | 26 +++++++++++++++++++ 2 files changed, 30 insertions(+), 1 deletion(-) diff --git a/apps/backend/internal/office/dashboard/service_tasks.go b/apps/backend/internal/office/dashboard/service_tasks.go index 775e7fe1ddf..5fd0a58b99c 100644 --- a/apps/backend/internal/office/dashboard/service_tasks.go +++ b/apps/backend/internal/office/dashboard/service_tasks.go @@ -915,7 +915,10 @@ func (s *DashboardService) runReactivityForAssigneeChange( // Flip the prev assignee's office session row to COMPLETED so it leaves // the active sessions list. The reactivity pipeline already hard-cancels // the running execution above; this is the persistent-row counterpart. - if prevAssigneeID != "" && s.sessionTerm != nil { + // A same-agent reassignment is not a handoff — the pipeline above never + // interrupts it — so this must not terminate the agent's own live + // session out from under it. + if prevAssigneeID != "" && prevAssigneeID != newAssigneeID && s.sessionTerm != nil { if err := s.sessionTerm.TerminateOfficeSession(ctx, taskID, prevAssigneeID, sessionTermReasonReassigned); err != nil { s.logger.Warn("terminate prev-assignee office session failed", zap.String("task_id", taskID), diff --git a/apps/backend/internal/office/dashboard/session_termination_test.go b/apps/backend/internal/office/dashboard/session_termination_test.go index df96ce381fa..e89d7c267d6 100644 --- a/apps/backend/internal/office/dashboard/session_termination_test.go +++ b/apps/backend/internal/office/dashboard/session_termination_test.go @@ -106,3 +106,29 @@ func TestSetTaskAssignee_TerminatesPrevSession(t *testing.T) { t.Errorf("term call: got %+v", got) } } + +// TestSetTaskAssignee_SameAgent_DoesNotTerminateSession is the regression +// test for Review round 3 Finding 5: a repeat assignment to the agent that +// already holds the seat must not flip that agent's own live session row to +// COMPLETED. The reactivity pipeline correctly declines to hard-cancel this +// case (AC-OFFICE-RUN-DEDUP-001.3 treats it as a real occurrence, not an +// interrupt), and the persisted-row side effect must agree. +func TestSetTaskAssignee_SameAgent_DoesNotTerminateSession(t *testing.T) { + deps := newTestDeps(t) + rt := &recordingTerminator{} + deps.svc.SetSessionTerminator(rt) + deps.svc.SetReactivityApplier(&recordingReactivity{result: &dashboard.TaskReactivityResult{}}) + + insertTestTask(t, deps.db, "task-same", "ws-r", "Reassign", "todo", 2) + if _, err := deps.repo.UpdateTaskAssignee(context.Background(), "task-same", "agent-x"); err != nil { + t.Fatalf("seed prev assignee: %v", err) + } + + if err := deps.svc.SetTaskAssigneeAsAgent(context.Background(), "", "task-same", "agent-x"); err != nil { + t.Fatalf("set assignee: %v", err) + } + + if len(rt.calls) != 0 { + t.Fatalf("same-agent reassignment must not terminate the agent's own session, got %d calls (%+v)", len(rt.calls), rt.calls) + } +} From 8f5ac3b38f43f87d5ec1eab74b177f190aa982e5 Mon Sep 17 00:00:00 2001 From: nova28 <17953305+nova28@users.noreply.github.com> Date: Wed, 9 Sep 2026 07:44:13 +0800 Subject: [PATCH 09/15] test(office): assert windowed-dedup outcome on both office QueueRun paths Existing coverage only asserted the resulting row count on a redelivered key, so a regression that dropped ReportWindowedDedup from either office-owned QueueRun implementation would have gone undetected. Adds an exact-outcome and exact-counter-delta assertion for both. --- .../scheduler/queue_run_dedup_outcome_test.go | 72 +++++++++++++++++++ .../internal/office/service/run_test.go | 67 +++++++++++++++++ 2 files changed, 139 insertions(+) create mode 100644 apps/backend/internal/office/scheduler/queue_run_dedup_outcome_test.go diff --git a/apps/backend/internal/office/scheduler/queue_run_dedup_outcome_test.go b/apps/backend/internal/office/scheduler/queue_run_dedup_outcome_test.go new file mode 100644 index 00000000000..2524711d79b --- /dev/null +++ b/apps/backend/internal/office/scheduler/queue_run_dedup_outcome_test.go @@ -0,0 +1,72 @@ +package scheduler + +import ( + "context" + "expvar" + "testing" + + runsservice "github.com/kandev/kandev/internal/runs/service" +) + +// windowedDedupCounterValue reads the current value of one +// office_run_dedup_total{reason,kind=windowed,queue=runs} label, or 0 if the +// label has never been reported. Callers snapshot before/after and assert +// the exact delta. +func windowedDedupCounterValue(t *testing.T, reason string) int64 { + t.Helper() + v := expvar.Get("office_run_dedup_total") + if v == nil { + t.Fatalf("expvar map office_run_dedup_total not registered") + } + m, ok := v.(*expvar.Map) + if !ok { + t.Fatalf("office_run_dedup_total is not a *expvar.Map") + } + iv := m.Get("reason=" + reason + ";kind=windowed;queue=runs") + if iv == nil { + return 0 + } + i, ok := iv.(*expvar.Int) + if !ok { + t.Fatalf("counter value for reason=%s;kind=windowed;queue=runs is not *expvar.Int", reason) + } + return i.Value() +} + +// TestSchedulerService_QueueRun_ReportsWindowedDedupOutcome closes a +// phantom-green gap on the office-owned queue implementation the design +// names as least observable (run-dedup-generation-02.md#test-strategy): +// no test asserted SchedulerService.QueueRun actually calls +// ReportWindowedDedup, so deleting that call would leave every existing +// test green while AC-OFFICE-RUN-DEDUP-004.1 silently went unmet on the +// path a reassignment travels. +func TestSchedulerService_QueueRun_ReportsWindowedDedupOutcome(t *testing.T) { + repo := newReactivityTestRepo(t) + ss := newChildrenCompletedTestScheduler(t, repo) + createChildrenCompletedAgent(t, repo, "agent-queue-run-outcome") + ctx := context.Background() + + key := "idem-key-scheduler-outcome" + first, err := ss.QueueRun(ctx, "agent-queue-run-outcome", RunReasonTaskAssigned, "{}", key) + if err != nil { + t.Fatalf("first enqueue: %v", err) + } + if first != runsservice.QueueOutcomeQueued { + t.Fatalf("first outcome = %v, want QueueOutcomeQueued", first) + } + + before := windowedDedupCounterValue(t, RunReasonTaskAssigned) + + second, err := ss.QueueRun(ctx, "agent-queue-run-outcome", RunReasonTaskAssigned, "{}", key) + if err != nil { + t.Fatalf("second enqueue: %v", err) + } + if second != runsservice.QueueOutcomeDeduped { + t.Fatalf("second outcome = %v, want QueueOutcomeDeduped", second) + } + + after := windowedDedupCounterValue(t, RunReasonTaskAssigned) + if after != before+1 { + t.Fatalf("office_run_dedup_total{%s,windowed,runs} = %d, want %d", RunReasonTaskAssigned, after, before+1) + } +} diff --git a/apps/backend/internal/office/service/run_test.go b/apps/backend/internal/office/service/run_test.go index 0e87836f683..e0e2257d934 100644 --- a/apps/backend/internal/office/service/run_test.go +++ b/apps/backend/internal/office/service/run_test.go @@ -2,12 +2,39 @@ package service_test import ( "context" + "expvar" "testing" "github.com/kandev/kandev/internal/office/models" "github.com/kandev/kandev/internal/office/service" + runsservice "github.com/kandev/kandev/internal/runs/service" ) +// windowedDedupCounterValue reads the current value of one +// office_run_dedup_total{reason,kind=windowed,queue=runs} label, or 0 if the +// label has never been reported. Callers snapshot before/after and assert +// the exact delta. +func windowedDedupCounterValue(t *testing.T, reason string) int64 { + t.Helper() + v := expvar.Get("office_run_dedup_total") + if v == nil { + t.Fatalf("expvar map office_run_dedup_total not registered") + } + m, ok := v.(*expvar.Map) + if !ok { + t.Fatalf("office_run_dedup_total is not a *expvar.Map") + } + iv := m.Get("reason=" + reason + ";kind=windowed;queue=runs") + if iv == nil { + return 0 + } + i, ok := iv.(*expvar.Int) + if !ok { + t.Fatalf("counter value for reason=%s;kind=windowed;queue=runs is not *expvar.Int", reason) + } + return i.Value() +} + func TestQueueRun_Basic(t *testing.T) { svc := newTestService(t) ctx := context.Background() @@ -61,6 +88,46 @@ func TestQueueRun_Idempotency(t *testing.T) { } } +// TestQueueRun_Idempotency_ReportsWindowedDedupOutcome closes a phantom-green +// gap: TestQueueRun_Idempotency above only asserted the resulting row count, +// so deleting queueRunInline's ReportWindowedDedup call would leave it green +// while AC-OFFICE-RUN-DEDUP-004.1's observability contract silently went +// unmet on this queue implementation. Asserts the outcome value and the +// office_run_dedup_total counter directly. +func TestQueueRun_Idempotency_ReportsWindowedDedupOutcome(t *testing.T) { + svc := newTestService(t) + ctx := context.Background() + + agent := makeAgent("worker-1", models.AgentRoleWorker) + if err := svc.CreateAgentInstance(ctx, agent); err != nil { + t.Fatalf("create agent: %v", err) + } + + key := "idem-key-outcome" + first, err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, "{}", key) + if err != nil { + t.Fatalf("first enqueue: %v", err) + } + if first != runsservice.QueueOutcomeQueued { + t.Fatalf("first outcome = %v, want QueueOutcomeQueued", first) + } + + before := windowedDedupCounterValue(t, service.RunReasonTaskAssigned) + + second, err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, "{}", key) + if err != nil { + t.Fatalf("second enqueue: %v", err) + } + if second != runsservice.QueueOutcomeDeduped { + t.Fatalf("second outcome = %v, want QueueOutcomeDeduped", second) + } + + after := windowedDedupCounterValue(t, service.RunReasonTaskAssigned) + if after != before+1 { + t.Fatalf("office_run_dedup_total{%s,windowed,runs} = %d, want %d", service.RunReasonTaskAssigned, after, before+1) + } +} + func TestQueueRun_SkipsPausedAgent(t *testing.T) { svc := newTestService(t) ctx := context.Background() From 9c0143dbe9a0d083b958273c1d74c7c2c3245b91 Mon Sep 17 00:00:00 2001 From: nova28 <17953305+nova28@users.noreply.github.com> Date: Wed, 9 Sep 2026 11:21:23 +0800 Subject: [PATCH 10/15] fix(office): reconcile run-dedup generation with rebased main MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Rebasing onto main pulled in an independent fix (#3464) that gave manual resume ("Mark fixed") a real dedup key derived from the failed run's id instead of the keyless-by-design placeholder this branch had chosen for the same call site — keep the real key, since it is strictly better, and drop the now-obsolete test asserting the superseded keyless behavior. Also merges the routine dedup key builder with a webhook explicit-key feature that landed independently: an explicit request key still wins over the claimed-tick cron identity, with a new test pinning that priority. Fixes the remaining QueueRun call sites broken by its now-two-value return signature, and wraps both wakeup-conflict sentinels in routineWakeupAdapter's error so errors.Is still matches the sqlite-layer ErrWakeupIdempotencyConflict alongside the routines-layer ErrWakeupAlreadyRequested. Co-Authored-By: Claude Sonnet 5 --- .../internal/backendapp/adapters_office.go | 6 +- .../run_dedup_generation_keys_test.go | 38 +++++++-- .../internal/office/service/failure_test.go | 2 +- .../service/manual_resume_keyless_test.go | 83 ------------------- .../scheduler_integration_routing_test.go | 2 +- .../task/repository/sqlite/base_migrations.go | 2 +- 6 files changed, 37 insertions(+), 96 deletions(-) delete mode 100644 apps/backend/internal/office/service/manual_resume_keyless_test.go diff --git a/apps/backend/internal/backendapp/adapters_office.go b/apps/backend/internal/backendapp/adapters_office.go index 378f5fd3cb1..2a0c0bdcdb0 100644 --- a/apps/backend/internal/backendapp/adapters_office.go +++ b/apps/backend/internal/backendapp/adapters_office.go @@ -268,7 +268,11 @@ func (a *routineWakeupAdapter) CreateWakeupRequest( if err := a.repo.CreateWakeupRequest(ctx, row); err != nil { if errors.Is(err, officesqlite.ErrWakeupIdempotencyConflict) { runsservice.ReportDurableDedup(runsservice.QueueSourceWakeup, req.Reason, req.IdempotencyKey, req.AgentProfileID) - return officeroutines.ErrWakeupAlreadyRequested + // Wraps both sentinels so a caller can check either: routines + // callers key off ErrWakeupAlreadyRequested to treat this as + // success by another route, while errors.Is against the + // sqlite-layer ErrWakeupIdempotencyConflict still matches. + return fmt.Errorf("%w: %w", officeroutines.ErrWakeupAlreadyRequested, err) } return err } diff --git a/apps/backend/internal/office/routines/run_dedup_generation_keys_test.go b/apps/backend/internal/office/routines/run_dedup_generation_keys_test.go index 567c6fbf9bc..97cae86d23b 100644 --- a/apps/backend/internal/office/routines/run_dedup_generation_keys_test.go +++ b/apps/backend/internal/office/routines/run_dedup_generation_keys_test.go @@ -14,7 +14,7 @@ import ( func TestBuildRoutineIdempotencyKey_Cron_UsesClaimedTick(t *testing.T) { tick := time.Date(2026, 5, 10, 12, 0, 0, 0, time.UTC) - key := buildRoutineIdempotencyKey(shared.RoutineSourceCron, "routine-1", "trigger-1", &tick, "run-ignored") + key := buildRoutineIdempotencyKey(shared.RoutineSourceCron, "routine-1", "trigger-1", "", &tick, "run-ignored") want := fmt.Sprintf("routine:%s:%s:tick:%d", "routine-1", "trigger-1", tick.Unix()) if key != want { @@ -22,7 +22,7 @@ func TestBuildRoutineIdempotencyKey_Cron_UsesClaimedTick(t *testing.T) { } // A second claim of the identical slot (redelivery) mints the same key. - repeat := buildRoutineIdempotencyKey(shared.RoutineSourceCron, "routine-1", "trigger-1", &tick, "run-different") + repeat := buildRoutineIdempotencyKey(shared.RoutineSourceCron, "routine-1", "trigger-1", "", &tick, "run-different") if repeat != key { t.Fatalf("redelivery of the same slot produced %q, want %q (identical to the first)", repeat, key) } @@ -34,8 +34,8 @@ func TestBuildRoutineIdempotencyKey_Cron_DistinctTicksDiffer(t *testing.T) { tickOne := time.Date(2026, 5, 10, 12, 0, 0, 0, time.UTC) tickTwo := tickOne.Add(time.Minute) - keyOne := buildRoutineIdempotencyKey(shared.RoutineSourceCron, "routine-1", "trigger-1", &tickOne, "run-a") - keyTwo := buildRoutineIdempotencyKey(shared.RoutineSourceCron, "routine-1", "trigger-1", &tickTwo, "run-b") + keyOne := buildRoutineIdempotencyKey(shared.RoutineSourceCron, "routine-1", "trigger-1", "", &tickOne, "run-a") + keyTwo := buildRoutineIdempotencyKey(shared.RoutineSourceCron, "routine-1", "trigger-1", "", &tickTwo, "run-b") if keyOne == keyTwo { t.Fatalf("distinct cron slots must mint distinct keys, both got %q", keyOne) @@ -47,7 +47,7 @@ func TestBuildRoutineIdempotencyKey_Cron_DistinctTicksDiffer(t *testing.T) { // live processCronTrigger guard) has no occurrence identity and goes // keyless with cause=unresolved. func TestBuildRoutineIdempotencyKey_Cron_NoClaimedTickGoesKeyless(t *testing.T) { - key := buildRoutineIdempotencyKey(shared.RoutineSourceCron, "routine-1", "trigger-1", nil, "run-1") + key := buildRoutineIdempotencyKey(shared.RoutineSourceCron, "routine-1", "trigger-1", "", nil, "run-1") if key != "" { t.Fatalf("key = %q, want empty (keyless) for a cron fire with no claimed tick", key) } @@ -57,7 +57,7 @@ func TestBuildRoutineIdempotencyKey_Cron_NoClaimedTickGoesKeyless(t *testing.T) // two distinct occurrences by design: each RoutineRun.ID mints its own key. func TestBuildRoutineIdempotencyKey_ManualAndWebhook_UseRoutineRunID(t *testing.T) { for _, source := range []string{"manual", "webhook"} { - key := buildRoutineIdempotencyKey(source, "routine-1", "", nil, "run-1") + key := buildRoutineIdempotencyKey(source, "routine-1", "", "", nil, "run-1") want := "routine:routine-1:run:run-1" if key != want { t.Errorf("source=%q key = %q, want %q", source, key, want) @@ -66,8 +66,8 @@ func TestBuildRoutineIdempotencyKey_ManualAndWebhook_UseRoutineRunID(t *testing. // Two distinct manual fires -> two distinct keys (the collision the // old unix-minute key format could reach). - first := buildRoutineIdempotencyKey("manual", "routine-1", "", nil, "run-a") - second := buildRoutineIdempotencyKey("manual", "routine-1", "", nil, "run-b") + first := buildRoutineIdempotencyKey("manual", "routine-1", "", "", nil, "run-a") + second := buildRoutineIdempotencyKey("manual", "routine-1", "", "", nil, "run-b") if first == second { t.Fatalf("two distinct manual fires must mint distinct keys, both got %q", first) } @@ -77,8 +77,28 @@ func TestBuildRoutineIdempotencyKey_ManualAndWebhook_UseRoutineRunID(t *testing. // and goes keyless with cause=unresolved, the same direction as a cron // fire with no claimed tick. func TestBuildRoutineIdempotencyKey_UnrecognisedSourceGoesKeyless(t *testing.T) { - key := buildRoutineIdempotencyKey("some_future_source", "routine-1", "", nil, "run-1") + key := buildRoutineIdempotencyKey("some_future_source", "routine-1", "", "", nil, "run-1") if key != "" { t.Fatalf("key = %q, want empty (keyless) for an unrecognised source", key) } } + +// An explicit request key (a webhook delivery header, say) always wins, +// even over a claimed cron tick, so a caller-supplied idempotency key never +// collides with the source-derived identity scheme. +func TestBuildRoutineIdempotencyKey_ExplicitKeyTakesPriority(t *testing.T) { + tick := time.Date(2026, 5, 10, 12, 0, 0, 0, time.UTC) + + key := buildRoutineIdempotencyKey(shared.RoutineSourceCron, "routine-1", "trigger-1", "delivery-abc", &tick, "run-1") + want := fmt.Sprintf("routine:%s:%s:%s", "routine-1", shared.RoutineSourceCron, "delivery-abc") + if key != want { + t.Fatalf("key = %q, want %q (explicit key must take priority over claimedTick)", key, want) + } + + // Two webhook deliveries with distinct explicit keys mint distinct keys. + first := buildRoutineIdempotencyKey("webhook", "routine-1", "", "delivery-1", nil, "run-a") + second := buildRoutineIdempotencyKey("webhook", "routine-1", "", "delivery-2", nil, "run-a") + if first == second { + t.Fatalf("two distinct explicit keys must mint distinct keys, both got %q", first) + } +} diff --git a/apps/backend/internal/office/service/failure_test.go b/apps/backend/internal/office/service/failure_test.go index 6488450dc1d..60c05f21a74 100644 --- a/apps/backend/internal/office/service/failure_test.go +++ b/apps/backend/internal/office/service/failure_test.go @@ -247,7 +247,7 @@ func TestMarkAgentPausedFixed_RecoversAgent(t *testing.T) { // Proves the guardAgentStatus rejection is gone: QueueRun must // succeed now that the agent is actually idle. - if err := svc.QueueRun(ctx, "agent-recover", service.RunReasonTaskAssigned, + if _, err := svc.QueueRun(ctx, "agent-recover", service.RunReasonTaskAssigned, mustMarshalJSON(map[string]string{"task_id": "agent-recover-task-a"}), "agent-recover:post-fix"); err != nil { t.Fatalf("expected QueueRun to succeed after mark fixed, got: %v", err) diff --git a/apps/backend/internal/office/service/manual_resume_keyless_test.go b/apps/backend/internal/office/service/manual_resume_keyless_test.go deleted file mode 100644 index e73294cf149..00000000000 --- a/apps/backend/internal/office/service/manual_resume_keyless_test.go +++ /dev/null @@ -1,83 +0,0 @@ -package service_test - -import ( - "context" - "expvar" - "testing" - - "github.com/kandev/kandev/internal/office/service" - runsservice "github.com/kandev/kandev/internal/runs/service" -) - -// keylessCounterValue reads the current value of one -// office_run_dedup_keyless_total{reason,cause} label, or 0 if the label -// has never been reported. Callers snapshot before/after and assert the -// exact delta, so a call site that stops reporting (or a passing test -// that never actually reaches the report) can't hide behind a value some -// other test already set. -func keylessCounterValue(t *testing.T, reason string, cause runsservice.KeylessCause) int64 { - t.Helper() - v := expvar.Get("office_run_dedup_keyless_total") - if v == nil { - t.Fatalf("expvar map office_run_dedup_keyless_total not registered") - } - m, ok := v.(*expvar.Map) - if !ok { - t.Fatalf("office_run_dedup_keyless_total is not a *expvar.Map") - } - iv := m.Get("reason=" + reason + ";cause=" + string(cause)) - if iv == nil { - return 0 - } - i, ok := iv.(*expvar.Int) - if !ok { - t.Fatalf("counter value for reason=%s;cause=%s is not *expvar.Int", reason, cause) - } - return i.Value() -} - -// TestMarkAgentRunFailedFixed_ReportsKeylessRequeue pins -// requeueRunForTask's ReportKeylessEnqueue call (failure.go) — a manual -// resume has no prior occurrence row to key on, so it is keyless by -// design like the reactivity producers, but before this test that call -// site had zero coverage anywhere in the suite. -func TestMarkAgentRunFailedFixed_ReportsKeylessRequeue(t *testing.T) { - svc, _ := newTestServiceWithBus(t) - ctx := context.Background() - - createTestAgent(t, svc, "ws-1", "agent-manual-resume") - taskID := "task-manual-resume-1" - insertSyntheticTask(t, svc, taskID, "ws-1", "agent-manual-resume") - w := queueAndReadRun(t, svc, "agent-manual-resume", taskID) - if err := svc.HandleAgentFailure(ctx, w, "boom"); err != nil { - t.Fatalf("handle failure: %v", err) - } - - before := keylessCounterValue(t, service.RunReasonManualResumeAfterFailure, runsservice.KeylessCauseByDesign) - - if err := svc.MarkAgentRunFailedFixed(ctx, "user-1", w.ID); err != nil { - t.Fatalf("mark fixed: %v", err) - } - - after := keylessCounterValue(t, service.RunReasonManualResumeAfterFailure, runsservice.KeylessCauseByDesign) - if after != before+1 { - t.Fatalf("office_run_dedup_keyless_total{%s,%s} = %d, want %d (one manual requeue)", - service.RunReasonManualResumeAfterFailure, runsservice.KeylessCauseByDesign, after, before+1) - } - - rows, err := svc.ListRuns(ctx, "ws-1") - if err != nil { - t.Fatalf("list runs: %v", err) - } - found := false - for _, r := range rows { - if r.AgentProfileID == "agent-manual-resume" && r.Reason == service.RunReasonManualResumeAfterFailure && - taskIDFromPayload(t, r.Payload) == taskID { - found = true - } - } - if !found { - t.Fatalf("expected a %s run queued for agent-manual-resume/%s, got %+v", - service.RunReasonManualResumeAfterFailure, taskID, rows) - } -} diff --git a/apps/backend/internal/office/service/scheduler_integration_routing_test.go b/apps/backend/internal/office/service/scheduler_integration_routing_test.go index bc857f11f47..a5444ba1be3 100644 --- a/apps/backend/internal/office/service/scheduler_integration_routing_test.go +++ b/apps/backend/internal/office/service/scheduler_integration_routing_test.go @@ -171,7 +171,7 @@ func TestSchedulerIntegration_SeatActionFlowsToPromptAndLaunch(t *testing.T) { svc.ExecSQL(t, `INSERT INTO tasks (id, workspace_id, workflow_step_id, title, description, created_at, updated_at) VALUES (?, ?, ?, ?, ?, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)`, "task-decision", "ws-1", "step-decision", "Decision task", "Review the change") - if err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, + if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, `{"task_id":"task-decision"}`, ""); err != nil { t.Fatalf("queue: %v", err) } diff --git a/apps/backend/internal/task/repository/sqlite/base_migrations.go b/apps/backend/internal/task/repository/sqlite/base_migrations.go index 36f93dd2543..3fb6186e42b 100644 --- a/apps/backend/internal/task/repository/sqlite/base_migrations.go +++ b/apps/backend/internal/task/repository/sqlite/base_migrations.go @@ -119,7 +119,7 @@ func (r *Repository) runMigrations() error { // have it silently dropped by the recreate on any database still carrying // the legacy FK, leaving it absent for the remainder of that boot (the // same hazard class as the task_sessions.name comment above). - r.migrate.Apply("tasks.assignment_generation", `ALTER TABLE tasks ADD COLUMN assignment_generation INTEGER NOT NULL DEFAULT 0`) + _ = r.migrate.Apply("tasks.assignment_generation", `ALTER TABLE tasks ADD COLUMN assignment_generation INTEGER NOT NULL DEFAULT 0`) if err := r.dropRetiredSlackIntegration(); err != nil { return err } From 6929ad112cd2bcb4577f78edc097deed77c5e479 Mon Sep 17 00:00:00 2001 From: nova28 <17953305+nova28@users.noreply.github.com> Date: Wed, 9 Sep 2026 11:39:22 +0800 Subject: [PATCH 11/15] docs(office): drop personal machine details from a spec's prior-art note The "prior art" search log recorded a specific local wiki vault path and config filename from the authoring sandbox. Neither is durable content the spec needs; generalize the note so it keeps the skipped-step reasoning without the personal path. Co-Authored-By: Claude Sonnet 5 --- .../office/requirements/run-dedup-generation.md | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/docs/specs/office/requirements/run-dedup-generation.md b/docs/specs/office/requirements/run-dedup-generation.md index 460c680a8bb..e61c23ea062 100644 --- a/docs/specs/office/requirements/run-dedup-generation.md +++ b/docs/specs/office/requirements/run-dedup-generation.md @@ -75,15 +75,14 @@ coalescing window, or the run lifecycle. ### Our own recorded reasoning (wiki) -**Searched:** resolved `OBSIDIAN_VAULT_PATH=/Users/henry/Documents/henry/wiki` -and `QMD_WIKI_COLLECTION=wiki` from `~/.obsidian-wiki/config` (symlink to -`config.henry`). The leg then **did not run**: neither `obsidian-wiki` nor `qmd` -is on this executor's PATH, no qmd MCP server is exposed to this session, and -the vault directory itself is unreadable here (`ls` returns `Operation not -permitted` for `~/Documents` both inside and outside the sandbox, which is a -macOS privacy restriction on this process, not a permission-gate denial). This -is a skipped step, not an empty result: the vault is configured and may well -hold relevant prior positions that this specification therefore did not consult. +**Searched:** resolved a configured personal wiki vault path and collection +name from the local `obsidian-wiki` config. The leg then **did not run**: +neither `obsidian-wiki` nor `qmd` is on this executor's PATH, no qmd MCP server +is exposed to this session, and the vault directory itself is unreadable here +(a macOS privacy restriction on this process, not a permission-gate denial). +This is a skipped step, not an empty result: the vault is configured and may +well hold relevant prior positions that this specification therefore did not +consult. ### What other products shipped (saas-kb) From 29098b0a6c53030cdc39d44acb6ac347bbaab49a Mon Sep 17 00:00:00 2001 From: nova28 <17953305+nova28@users.noreply.github.com> Date: Fri, 11 Sep 2026 00:29:52 +0800 Subject: [PATCH 12/15] fix(office): bound SpawnAgentRun's agent-supplied metric label Greptile (PR #3533) found that SpawnAgentRunInput.Reason reached office_run_dedup_total / office_run_dedup_keyless_total (expvar.Map, never evicts) verbatim as a label with no length bound, letting an agent with CapabilitySpawnAgentRun grow those process-global maps without limit. Reject an over-length Reason before it reaches either the run spawner or the metric label. Also: apply github-actions' suggested comments documenting the intentional generation-discard in SetTaskAssignee / SetTaskAssigneeAsAgent, and reconcile the design doc's stale manual-resume classification (requeueRunForTask picked up a real generational key from upstream #3464 during this branch's rebase, which Part 2/3 hadn't caught up to). Co-Authored-By: Claude Sonnet 5 --- .../internal/office/runtime/actions.go | 9 +++++ .../backend/internal/office/runtime/errors.go | 6 ++++ .../internal/office/runtime/handler.go | 2 +- .../runtime/spawn_agent_run_key_test.go | 33 +++++++++++++++++++ .../internal/office/service/task_assignee.go | 4 +++ .../system-design/run-dedup-generation-02.md | 12 ++++--- .../system-design/run-dedup-generation-03.md | 3 +- 7 files changed, 61 insertions(+), 8 deletions(-) diff --git a/apps/backend/internal/office/runtime/actions.go b/apps/backend/internal/office/runtime/actions.go index 1a7f6ef6109..b046f3c61bb 100644 --- a/apps/backend/internal/office/runtime/actions.go +++ b/apps/backend/internal/office/runtime/actions.go @@ -566,6 +566,12 @@ type SpawnAgentRunInput struct { IdempotencyKey string `json:"idempotency_key"` } +// maxSpawnAgentRunReasonLength bounds SpawnAgentRunInput.Reason. Reason is +// agent-supplied and reaches office_run_dedup_total / +// office_run_dedup_keyless_total as an expvar.Map label; those maps never +// evict, so an unbounded Reason would let a caller grow them without limit. +const maxSpawnAgentRunReasonLength = 100 + // SpawnAgentRun queues a run for an agent in the same workspace. // // A non-empty agent-supplied key is prefixed with the calling run's id @@ -587,6 +593,9 @@ func (a *Actions) SpawnAgentRun( if a.deps.Runs == nil || a.deps.AgentModifier == nil { return fmt.Errorf("%w: runs", ErrRuntimeDependencyMissing) } + if len(input.Reason) > maxSpawnAgentRunReasonLength { + return ErrReasonTooLong + } target, err := a.deps.AgentModifier.GetAgentInstance(ctx, input.AgentID) if err != nil { return err diff --git a/apps/backend/internal/office/runtime/errors.go b/apps/backend/internal/office/runtime/errors.go index 1e61e69cd71..d08f4c38234 100644 --- a/apps/backend/internal/office/runtime/errors.go +++ b/apps/backend/internal/office/runtime/errors.go @@ -20,4 +20,10 @@ var ( // either) in a workspace that has at least one project to choose from. // Caller-correctable: the caller should retry with an explicit project_id. ErrProjectRequired = fmt.Errorf("project_id is required") + // ErrReasonTooLong is returned when SpawnAgentRunInput.Reason exceeds + // maxSpawnAgentRunReasonLength. Reason is agent-supplied and becomes a + // label on the process-global office_run_dedup_total / + // office_run_dedup_keyless_total expvar maps, which never evict entries; + // an unbounded value lets a caller grow those maps without limit. + ErrReasonTooLong = fmt.Errorf("reason exceeds max length of %d characters", maxSpawnAgentRunReasonLength) ) diff --git a/apps/backend/internal/office/runtime/handler.go b/apps/backend/internal/office/runtime/handler.go index c315fb866b4..2d779c40264 100644 --- a/apps/backend/internal/office/runtime/handler.go +++ b/apps/backend/internal/office/runtime/handler.go @@ -478,7 +478,7 @@ func (h *Handler) respondRuntimeError( targetID string, err error, ) { - if errors.Is(err, errTaskTitleRequired) || errors.Is(err, ErrProjectRequired) { + if errors.Is(err, errTaskTitleRequired) || errors.Is(err, ErrProjectRequired) || errors.Is(err, ErrReasonTooLong) { h.appendDeniedRunEvent(c.Request.Context(), runCtx, action, targetType, targetID, err) c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()}) return diff --git a/apps/backend/internal/office/runtime/spawn_agent_run_key_test.go b/apps/backend/internal/office/runtime/spawn_agent_run_key_test.go index 6f1dd76f035..2fce20e77d5 100644 --- a/apps/backend/internal/office/runtime/spawn_agent_run_key_test.go +++ b/apps/backend/internal/office/runtime/spawn_agent_run_key_test.go @@ -2,6 +2,7 @@ package runtime import ( "context" + "errors" "expvar" "strings" "testing" @@ -178,3 +179,35 @@ func TestActionsSpawnAgentRun_NoCallerRunID_EnqueuesKeylessUnresolved(t *testing t.Fatal("expected office_run_dedup_keyless_total to carry an unresolved entry") } } + +// TestActionsSpawnAgentRun_ReasonTooLong_Rejected proves Reason is bounded +// before it can reach office_run_dedup_total / +// office_run_dedup_keyless_total as an expvar.Map label: those maps never +// evict entries, so an unvalidated agent-supplied Reason would let a caller +// grow them without limit. An over-length Reason is rejected with +// ErrReasonTooLong and never reaches the run spawner. +func TestActionsSpawnAgentRun_ReasonTooLong_Rejected(t *testing.T) { + agents := &recordingAgentModifier{ + agents: map[string]*models.AgentInstance{ + "agent-2": {ID: "agent-2", WorkspaceID: "ws-1"}, + }, + } + runs := &recordingRunSpawner{} + actions := NewActions(ActionDependencies{Runs: runs, AgentModifier: agents}) + runCtx := RunContext{ + WorkspaceID: "ws-1", + RunID: "run-caller-3", + Capabilities: Capabilities{CanSpawnAgentRun: true}, + } + + err := actions.SpawnAgentRun(context.Background(), runCtx, SpawnAgentRunInput{ + AgentID: "agent-2", + Reason: strings.Repeat("a", maxSpawnAgentRunReasonLength+1), + }) + if !errors.Is(err, ErrReasonTooLong) { + t.Fatalf("SpawnAgentRun error = %v, want ErrReasonTooLong", err) + } + if len(runs.calls) != 0 { + t.Fatalf("run spawner calls = %d, want 0 (rejected before enqueue)", len(runs.calls)) + } +} diff --git a/apps/backend/internal/office/service/task_assignee.go b/apps/backend/internal/office/service/task_assignee.go index 0f2e5362560..94ef5d6a0be 100644 --- a/apps/backend/internal/office/service/task_assignee.go +++ b/apps/backend/internal/office/service/task_assignee.go @@ -38,6 +38,8 @@ const participantTypeAgent = "agent" // by tests and a couple of internal callers; the dashboard's // permissioned variant is SetTaskAssigneeAsAgent. func (s *Service) SetTaskAssignee(ctx context.Context, taskID, assigneeID string) error { + // generation discarded — this caller does not carry it to the event bus; + // queueTaskAssignedRun falls through to keyless enqueue (AC-OFFICE-RUN-DEDUP-003). _, err := s.repo.UpdateTaskAssignee(ctx, taskID, assigneeID) return err } @@ -56,6 +58,8 @@ func (s *Service) SetTaskAssigneeAsAgent(ctx context.Context, callerAgentID, tas return shared.ErrForbidden } } + // generation discarded — this caller does not carry it to the event bus; + // queueTaskAssignedRun falls through to keyless enqueue (AC-OFFICE-RUN-DEDUP-003). _, err := s.repo.UpdateTaskAssignee(ctx, taskID, assigneeID) return err } diff --git a/docs/specs/office/system-design/run-dedup-generation-02.md b/docs/specs/office/system-design/run-dedup-generation-02.md index 8888343aff1..1b3b21c3666 100644 --- a/docs/specs/office/system-design/run-dedup-generation-02.md +++ b/docs/specs/office/system-design/run-dedup-generation-02.md @@ -318,17 +318,19 @@ Backend `*_test.go` beside each source. Behaviours with no equivalent test today sentinel as a failure. Nothing else in this plan exercises that file, so without this the label can be unwired or wired to the wrong `queue` value with the suite green. -- **The four direct keyless producers each report their assigned cause** — the +- **The three direct keyless producers each report their assigned cause** — the onboarding wake and `handleTaskCreated`'s fallback as `unresolved`, the - recovery sweep and `requeueRunForTask` as `by_design` — per + recovery sweep as `by_design` — per [Part 3](run-dedup-generation-03.md#keyless-causes-per-producer). The recovery sweep additionally asserts it does **not** mint the assignment key: drive a sweep over a task whose assignment run already persisted `task_assigned:::` and assert the sweep still queues, rather than being suppressed by the durable index. -- **`agent_error` is generational on the failed run id.** Two distinct run - failures for one agent mint different keys and both queue; a redelivery of one - failure escalation is suppressed. There is no two-CEO case to assert: a +- **`agent_error` and `manual_resume_after_failure` are generational** on the + failed run id (`manual_resume_after_failure` falls back to the task id when + the failed run id is empty). Two distinct run failures for one agent mint + different keys and both queue; a redelivery of one failure escalation is + suppressed. There is no two-CEO case to assert for `agent_error`: a workspace admits at most one CEO (`ErrAgentCEOAlreadyExists`) and `queueCEOAgentError` escalates to `ceos[0]` alone. - **Both blocker producers derive a byte-identical digest** for one blocker set, diff --git a/docs/specs/office/system-design/run-dedup-generation-03.md b/docs/specs/office/system-design/run-dedup-generation-03.md index 7054d409cf9..48d792a4700 100644 --- a/docs/specs/office/system-design/run-dedup-generation-03.md +++ b/docs/specs/office/system-design/run-dedup-generation-03.md @@ -86,7 +86,7 @@ varying-segment one, which a producer with no key cannot satisfy. | `office/onboarding/service.go` `maybeCreateOnboardingTask` | `task_assigned` | `""` | stays keyless, `cause=unresolved` | | `office/service/scheduler_recovery.go` recovery sweep | `task_assigned` | `""` | stays keyless, `cause=by_design` | | `office/service/retry.go` CEO error escalation | `agent_error` | `""` | **becomes generational** on the failed run id | -| `office/service/failure.go` `requeueRunForTask` | `manual_resume_after_failure` | `""` | stays keyless, `cause=by_design` | +| `office/service/failure.go` `requeueRunForTask` | `manual_resume_after_failure` | `""` | **becomes generational** on the failed run id (task id if empty) | `office/onboarding` also declares its own `runReasonTaskAssigned` constant, a third copy of the same string. Consolidating it is out of scope for the same @@ -112,7 +112,6 @@ each producer is assigned a side deliberately, so each is assigned one here. | --- | --- | --- | --- | | reactivity status wakes | `task_unblocked`, `task_reopened`, `task_review_requested` | `by_design` | a status transition has no redelivery path; see Part 1 | | `office/service/scheduler_recovery.go` | `task_assigned` | `by_design` | see [The recovery sweep](#the-recovery-sweep-must-not-borrow-the-assignment-generation) | -| `office/service/failure.go` | `manual_resume_after_failure` | `by_design` | an operator resume; the call site holds only a task id, and no row records the resume | | `office/onboarding/service.go` | `task_assigned` | `unresolved` | the occurrence **does** have a generation; this producer simply is not handed it | | `handleTaskCreated` with `fallbackToStoredRunner` | `task_assigned` | `unresolved` | Part 1 | | `officeAutoStartIdempotencyKey`, zero `stepTransitionID` | `task_assigned` | `unresolved` | Part 1 | From 2499733dcbd11c0e0272e97e40d8bec58d61c1ce Mon Sep 17 00:00:00 2001 From: nova28 <17953305+nova28@users.noreply.github.com> Date: Fri, 11 Sep 2026 02:04:54 +0800 Subject: [PATCH 13/15] fix(office): update QueueRun call sites picked up by the rebase onto main Rebasing onto origin/main (57 commits ahead, including #3517's own CoalesceRun task-scoping and #3520's budget-admission test suite) brought in more test call sites still assuming office/service.QueueRun's old single-return-value signature. Update them to the two-value (QueueOutcome, error) signature this branch already widened it to. Also resolved during the rebase (folded into the replayed feature commit, not a separate commit here): tasks.go's UpdateTaskAssignee INSERT picked up upstream's independent created_at column addition to workflow_step_participants alongside this branch's widened int64 return; run_test.go's TestQueueRun_Coalesce now asserts same-task coalescing (task_comment became task-scoped like every other reason under #3517, so the old cross-task assertion no longer holds). Co-Authored-By: Claude Sonnet 5 --- .../office/service/budget_admission_test.go | 46 +++++++++---------- .../office/service/budget_metrics_test.go | 22 ++++----- .../scheduler_integration_routing_test.go | 2 +- 3 files changed, 35 insertions(+), 35 deletions(-) diff --git a/apps/backend/internal/office/service/budget_admission_test.go b/apps/backend/internal/office/service/budget_admission_test.go index a1b7d07f868..67e1d2d855f 100644 --- a/apps/backend/internal/office/service/budget_admission_test.go +++ b/apps/backend/internal/office/service/budget_admission_test.go @@ -101,7 +101,7 @@ func TestAdmitRun_NoEvaluatorWired_UnattendedCancels(t *testing.T) { if err := svc.CreateAgentInstance(ctx, agent); err != nil { t.Fatalf("create agent: %v", err) } - if err := svc.QueueRun(ctx, agent.ID, service.RunReasonRoutineTrigger, `{}`, ""); err != nil { + if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonRoutineTrigger, `{}`, ""); err != nil { t.Fatalf("queue: %v", err) } @@ -128,7 +128,7 @@ func TestAdmitRun_NoEvaluatorWired_AttendedLaunches(t *testing.T) { t.Fatalf("create agent: %v", err) } insertTestTask(t, svc, "task-no-evaluator-attended", "ws-1") - if err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, + if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, `{"task_id":"task-no-evaluator-attended"}`, ""); err != nil { t.Fatalf("queue: %v", err) } @@ -163,7 +163,7 @@ func TestAdmitRun_EvaluatorFault_DefersThenFailsWithoutEscalation(t *testing.T) t.Fatalf("create agent: %v", err) } insertTestTask(t, svc, "task-evaluator-fault", "ws-1") - if err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, + if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, `{"task_id":"task-evaluator-fault"}`, ""); err != nil { t.Fatalf("queue: %v", err) } @@ -222,7 +222,7 @@ func TestAdmitRun_UnevaluatedPolicy_NamesPolicyInDeferral(t *testing.T) { t.Fatalf("create agent: %v", err) } insertTestTask(t, svc, "task-unevaluated-policy", "ws-1") - if err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, + if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, `{"task_id":"task-unevaluated-policy"}`, ""); err != nil { t.Fatalf("queue: %v", err) } @@ -272,7 +272,7 @@ func TestFinishPolicyBlock_ClearsStaleAgentWorkingStatus(t *testing.T) { if err := svc.CreateAgentInstance(ctx, agent); err != nil { t.Fatalf("create agent: %v", err) } - if err := svc.QueueRun(ctx, agent.ID, service.RunReasonRoutineTrigger, `{}`, ""); err != nil { + if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonRoutineTrigger, `{}`, ""); err != nil { t.Fatalf("queue: %v", err) } run, err := svc.ClaimNextRun(ctx) @@ -301,7 +301,7 @@ func TestCancelBudgetRun_ClearsStaleAgentWorkingStatus(t *testing.T) { if err := svc.CreateAgentInstance(ctx, agent); err != nil { t.Fatalf("create agent: %v", err) } - if err := svc.QueueRun(ctx, agent.ID, service.RunReasonRoutineTrigger, `{}`, ""); err != nil { + if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonRoutineTrigger, `{}`, ""); err != nil { t.Fatalf("queue: %v", err) } run, err := svc.ClaimNextRun(ctx) @@ -338,7 +338,7 @@ func TestFailRunNoEscalation_ClearsStaleAgentWorkingStatus(t *testing.T) { if err := svc.CreateAgentInstance(ctx, agent); err != nil { t.Fatalf("create agent: %v", err) } - if err := svc.QueueRun(ctx, agent.ID, service.RunReasonRoutineTrigger, `{}`, ""); err != nil { + if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonRoutineTrigger, `{}`, ""); err != nil { t.Fatalf("queue: %v", err) } run, err := svc.ClaimNextRun(ctx) @@ -381,7 +381,7 @@ func TestProcessRun_MissingAgentReleasesPriorCheckout(t *testing.T) { INSERT INTO tasks (id, workspace_id, title, created_at, updated_at) VALUES ('task-routed-missing', 'ws-1', 'Routed task', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP) `) - if err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, + if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, `{"task_id":"task-routed-missing"}`, ""); err != nil { t.Fatalf("queue: %v", err) } @@ -438,7 +438,7 @@ func TestProcessRun_WorkspaceLookupErrorReleasesPriorCheckout(t *testing.T) { INSERT INTO tasks (id, workspace_id, title, created_at, updated_at) VALUES ('task-routed-lookup-error', 'ws-1', 'Routed task', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP) `) - if err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, + if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, `{"task_id":"task-routed-lookup-error"}`, ""); err != nil { t.Fatalf("queue: %v", err) } @@ -489,7 +489,7 @@ func TestAdmitRun_RepeatedDeferral_AttemptIncrementsPerActivity(t *testing.T) { t.Fatalf("create agent: %v", err) } insertTestTask(t, svc, "task-attempt-increments", "ws-1") - if err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, + if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, `{"task_id":"task-attempt-increments"}`, ""); err != nil { t.Fatalf("queue: %v", err) } @@ -529,7 +529,7 @@ func TestAdmitRun_DefaultCeiling_BlocksUnattendedRunWithZeroPolicies(t *testing. t.Fatalf("create agent: %v", err) } insertTestCostEvent(t, svc, agent.ID, "task-default-ceiling", int64(600_000)) - if err := svc.QueueRun(ctx, agent.ID, service.RunReasonRoutineTrigger, `{}`, ""); err != nil { + if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonRoutineTrigger, `{}`, ""); err != nil { t.Fatalf("queue: %v", err) } @@ -614,7 +614,7 @@ func TestAdmitRun_PricingDegradedBlock_UnmeasurableOutcome(t *testing.T) { if err := svc.CreateAgentInstance(ctx, agent); err != nil { t.Fatalf("create agent: %v", err) } - if err := svc.QueueRun(ctx, agent.ID, service.RunReasonRoutineTrigger, `{}`, ""); err != nil { + if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonRoutineTrigger, `{}`, ""); err != nil { t.Fatalf("queue: %v", err) } @@ -645,7 +645,7 @@ func TestAdmitRun_DefaultCeiling_ExemptsAttendedRun(t *testing.T) { } insertTestCostEvent(t, svc, agent.ID, "task-default-ceiling-attended", int64(600_000)) insertTestTask(t, svc, "task-default-ceiling-attended-run", "ws-1") - if err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, + if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, `{"task_id":"task-default-ceiling-attended-run"}`, ""); err != nil { t.Fatalf("queue: %v", err) } @@ -704,7 +704,7 @@ func TestBudgetAdmission_ActionsAreDistinguishable(t *testing.T) { if err := svc.CreateAgentInstance(ctx, agent); err != nil { t.Fatalf("create agent: %v", err) } - if err := svc.QueueRun(ctx, agent.ID, service.RunReasonRoutineTrigger, `{}`, ""); err != nil { + if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonRoutineTrigger, `{}`, ""); err != nil { t.Fatalf("queue: %v", err) } run, err := svc.ClaimNextRun(ctx) @@ -725,7 +725,7 @@ func TestBudgetAdmission_ActionsAreDistinguishable(t *testing.T) { if err := svc.CreateAgentInstance(ctx, agent); err != nil { t.Fatalf("create agent: %v", err) } - if err := svc.QueueRun(ctx, agent.ID, service.RunReasonRoutineTrigger, `{}`, ""); err != nil { + if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonRoutineTrigger, `{}`, ""); err != nil { t.Fatalf("queue: %v", err) } run, err := svc.ClaimNextRun(ctx) @@ -745,7 +745,7 @@ func TestBudgetAdmission_ActionsAreDistinguishable(t *testing.T) { if err := svc.CreateAgentInstance(ctx, agent); err != nil { t.Fatalf("create agent: %v", err) } - if err := svc.QueueRun(ctx, agent.ID, service.RunReasonRoutineTrigger, `{}`, ""); err != nil { + if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonRoutineTrigger, `{}`, ""); err != nil { t.Fatalf("queue: %v", err) } run, err := svc.ClaimNextRun(ctx) @@ -768,7 +768,7 @@ func TestBudgetAdmission_ActionsAreDistinguishable(t *testing.T) { if err := svc.CreateAgentInstance(ctx, agent); err != nil { t.Fatalf("create agent: %v", err) } - if err := svc.QueueRun(ctx, agent.ID, service.RunReasonRoutineTrigger, `{}`, ""); err != nil { + if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonRoutineTrigger, `{}`, ""); err != nil { t.Fatalf("queue: %v", err) } service.RunSchedulerTick(svc, ctx) @@ -787,7 +787,7 @@ func TestBudgetAdmission_ActionsAreDistinguishable(t *testing.T) { if err := svc.CreateAgentInstance(ctx, agent); err != nil { t.Fatalf("create agent: %v", err) } - if err := svc.QueueRun(ctx, agent.ID, service.RunReasonRoutineTrigger, `{}`, ""); err != nil { + if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonRoutineTrigger, `{}`, ""); err != nil { t.Fatalf("queue: %v", err) } run, err := svc.ClaimNextRun(ctx) @@ -818,7 +818,7 @@ func TestBudgetAdmission_ActionsAreDistinguishable(t *testing.T) { if err := svc.CreateAgentInstance(ctx, agent); err != nil { t.Fatalf("create agent: %v", err) } - if err := svc.QueueRun(ctx, agent.ID, service.RunReasonRoutineTrigger, `{}`, ""); err != nil { + if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonRoutineTrigger, `{}`, ""); err != nil { t.Fatalf("queue: %v", err) } run, err := svc.ClaimNextRun(ctx) @@ -840,7 +840,7 @@ func TestBudgetAdmission_ActionsAreDistinguishable(t *testing.T) { if err := svc.CreateAgentInstance(ctx, agent); err != nil { t.Fatalf("create agent: %v", err) } - if err := svc.QueueRun(ctx, agent.ID, service.RunReasonRoutineTrigger, `{}`, ""); err != nil { + if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonRoutineTrigger, `{}`, ""); err != nil { t.Fatalf("queue: %v", err) } run, err := svc.ClaimNextRun(ctx) @@ -863,7 +863,7 @@ func TestBudgetAdmission_ActionsAreDistinguishable(t *testing.T) { if err := svc.CreateAgentInstance(ctx, agent); err != nil { t.Fatalf("create agent: %v", err) } - if err := svc.QueueRun(ctx, agent.ID, service.RunReasonRoutineTrigger, `{}`, ""); err != nil { + if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonRoutineTrigger, `{}`, ""); err != nil { t.Fatalf("queue: %v", err) } run, err := svc.ClaimNextRun(ctx) @@ -884,7 +884,7 @@ func TestBudgetAdmission_ActionsAreDistinguishable(t *testing.T) { if err := svc.CreateAgentInstance(ctx, agent); err != nil { t.Fatalf("create agent: %v", err) } - if err := svc.QueueRun(ctx, agent.ID, service.RunReasonRoutineTrigger, `{}`, ""); err != nil { + if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonRoutineTrigger, `{}`, ""); err != nil { t.Fatalf("queue: %v", err) } run, err := svc.ClaimNextRun(ctx) @@ -911,7 +911,7 @@ func TestBudgetAdmission_ActionsAreDistinguishable(t *testing.T) { if err := svc.CreateAgentInstance(ctx, agent); err != nil { t.Fatalf("create agent: %v", err) } - if err := svc.QueueRun(ctx, agent.ID, service.RunReasonRoutineTrigger, `{}`, ""); err != nil { + if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonRoutineTrigger, `{}`, ""); err != nil { t.Fatalf("queue: %v", err) } service.RunSchedulerTick(svc, ctx) diff --git a/apps/backend/internal/office/service/budget_metrics_test.go b/apps/backend/internal/office/service/budget_metrics_test.go index 1afa2a62621..21fd4e8e1db 100644 --- a/apps/backend/internal/office/service/budget_metrics_test.go +++ b/apps/backend/internal/office/service/budget_metrics_test.go @@ -47,7 +47,7 @@ func TestAdmitRun_MetricBlockedByLimit(t *testing.T) { if err := svc.CreateAgentInstance(ctx, agent); err != nil { t.Fatalf("create agent: %v", err) } - if err := svc.QueueRun(ctx, agent.ID, service.RunReasonRoutineTrigger, `{}`, ""); err != nil { + if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonRoutineTrigger, `{}`, ""); err != nil { t.Fatalf("queue: %v", err) } @@ -78,7 +78,7 @@ func TestAdmitRun_MetricBlockedPricingDegraded(t *testing.T) { if err := svc.CreateAgentInstance(ctx, agent); err != nil { t.Fatalf("create agent: %v", err) } - if err := svc.QueueRun(ctx, agent.ID, service.RunReasonRoutineTrigger, `{}`, ""); err != nil { + if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonRoutineTrigger, `{}`, ""); err != nil { t.Fatalf("queue: %v", err) } @@ -104,7 +104,7 @@ func TestAdmitRun_MetricDeferredEvaluatorFault(t *testing.T) { t.Fatalf("create agent: %v", err) } insertTestTask(t, svc, "task-metric-deferred-fault", "ws-1") - if err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, + if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, `{"task_id":"task-metric-deferred-fault"}`, ""); err != nil { t.Fatalf("queue: %v", err) } @@ -126,7 +126,7 @@ func TestAdmitRun_MetricBlockedAbsentEvaluator(t *testing.T) { if err := svc.CreateAgentInstance(ctx, agent); err != nil { t.Fatalf("create agent: %v", err) } - if err := svc.QueueRun(ctx, agent.ID, service.RunReasonRoutineTrigger, `{}`, ""); err != nil { + if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonRoutineTrigger, `{}`, ""); err != nil { t.Fatalf("queue: %v", err) } @@ -156,7 +156,7 @@ func TestAdmitRun_MetricDeferredWorkspaceLookup(t *testing.T) { if err := svc.CreateAgentInstance(ctx, agent); err != nil { t.Fatalf("create agent: %v", err) } - if err := svc.QueueRun(ctx, agent.ID, service.RunReasonRoutineTrigger, `{}`, ""); err != nil { + if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonRoutineTrigger, `{}`, ""); err != nil { t.Fatalf("queue: %v", err) } run, err := svc.ClaimNextRun(ctx) @@ -192,7 +192,7 @@ func TestProcessRun_AgentRowDeleted_Cancels(t *testing.T) { if err := svc.CreateAgentInstance(ctx, agent); err != nil { t.Fatalf("create agent: %v", err) } - if err := svc.QueueRun(ctx, agent.ID, service.RunReasonRoutineTrigger, `{}`, ""); err != nil { + if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonRoutineTrigger, `{}`, ""); err != nil { t.Fatalf("queue: %v", err) } run, err := svc.ClaimNextRun(ctx) @@ -239,7 +239,7 @@ func TestAdmitRun_MetricCancelledNoWorkspace(t *testing.T) { if err := svc.CreateAgentInstance(ctx, agent); err != nil { t.Fatalf("create agent: %v", err) } - if err := svc.QueueRun(ctx, agent.ID, service.RunReasonRoutineTrigger, `{}`, ""); err != nil { + if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonRoutineTrigger, `{}`, ""); err != nil { t.Fatalf("queue: %v", err) } run, err := svc.ClaimNextRun(ctx) @@ -271,7 +271,7 @@ func TestAdmitRun_MetricCancelledStaleDeferral(t *testing.T) { if err := svc.CreateAgentInstance(ctx, agent); err != nil { t.Fatalf("create agent: %v", err) } - if err := svc.QueueRun(ctx, agent.ID, service.RunReasonRoutineTrigger, `{}`, ""); err != nil { + if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonRoutineTrigger, `{}`, ""); err != nil { t.Fatalf("queue: %v", err) } run, err := svc.ClaimNextRun(ctx) @@ -302,7 +302,7 @@ func TestAdmitRun_MetricAdmittedDefault(t *testing.T) { if err := svc.CreateAgentInstance(ctx, agent); err != nil { t.Fatalf("create agent: %v", err) } - if err := svc.QueueRun(ctx, agent.ID, service.RunReasonRoutineTrigger, `{}`, ""); err != nil { + if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonRoutineTrigger, `{}`, ""); err != nil { t.Fatalf("queue: %v", err) } @@ -335,7 +335,7 @@ func TestAdmitRun_MetricAdmittedDegradedWindow_ViaDefault(t *testing.T) { if err := svc.CreateAgentInstance(ctx, agent); err != nil { t.Fatalf("create agent: %v", err) } - if err := svc.QueueRun(ctx, agent.ID, service.RunReasonRoutineTrigger, `{}`, ""); err != nil { + if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonRoutineTrigger, `{}`, ""); err != nil { t.Fatalf("queue: %v", err) } @@ -369,7 +369,7 @@ func TestAdmitRun_MetricAdmittedDegradedWindow_ViaStoredPolicyBypassingGate5(t * t.Fatalf("create agent: %v", err) } insertTestTask(t, svc, "task-metric-degraded-attended", "ws-1") - if err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, + if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, `{"task_id":"task-metric-degraded-attended"}`, ""); err != nil { t.Fatalf("queue: %v", err) } diff --git a/apps/backend/internal/office/service/scheduler_integration_routing_test.go b/apps/backend/internal/office/service/scheduler_integration_routing_test.go index a5444ba1be3..94109842138 100644 --- a/apps/backend/internal/office/service/scheduler_integration_routing_test.go +++ b/apps/backend/internal/office/service/scheduler_integration_routing_test.go @@ -263,7 +263,7 @@ func TestSchedulerIntegration_NonSeatRunDoesNotReceiveDecisionSkill(t *testing.T svc.ExecSQL(t, `INSERT INTO tasks (id, workspace_id, workflow_step_id, title, description, created_at, updated_at) VALUES (?, ?, ?, ?, ?, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)`, "task-non-seat", "ws-1", "step-non-seat", "Non-seat task", "Review the change") - if err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, + if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, `{"task_id":"task-non-seat"}`, ""); err != nil { t.Fatalf("queue: %v", err) } From bb1a97f2fe12971706f0059e672cde5839e55073 Mon Sep 17 00:00:00 2001 From: Carlos Florencio Date: Sun, 13 Sep 2026 07:32:10 +0100 Subject: [PATCH 14/15] docs: add Office run dedup delivery package --- .../plans/office-run-dedup-generation/plan.md | 47 ++++++++++++ .../task-01-run-dedup-generation.md | 74 +++++++++++++++++++ 2 files changed, 121 insertions(+) create mode 100644 docs/plans/office-run-dedup-generation/plan.md create mode 100644 docs/plans/office-run-dedup-generation/task-01-run-dedup-generation.md diff --git a/docs/plans/office-run-dedup-generation/plan.md b/docs/plans/office-run-dedup-generation/plan.md new file mode 100644 index 00000000000..ff6da7af4e5 --- /dev/null +++ b/docs/plans/office-run-dedup-generation/plan.md @@ -0,0 +1,47 @@ +--- +created: 2026-09-13 +status: completed +requirements: + - REQ-OFFICE-RUN-DEDUP-001 + - REQ-OFFICE-RUN-DEDUP-002 + - REQ-OFFICE-RUN-DEDUP-003 + - REQ-OFFICE-RUN-DEDUP-004 +system_design: + - ../../specs/office/system-design/run-dedup-generation-01.md + - ../../specs/office/system-design/run-dedup-generation-02.md + - ../../specs/office/system-design/run-dedup-generation-03.md +--- + +# Implementation Plan: Office Run Deduplication Generation Identity + +## Overview + +Give Office wake requests an occurrence identity that survives retries and +allows later occurrences to run. Persist the task assignment generation, carry +it through task-created events, keep producer keys convergent, and report +deduplication outcomes with bounded metric labels. + +## Scope + +- Persist and increment `tasks.assignment_generation` for assignment writes. +- Include the assignee profile and generation in task-created event data. +- Use generation-aware keys for assignment and other durable wake occurrences. +- Use a keyless wake when a generation cannot be resolved. +- Keep agent-supplied metric reasons bounded to known labels or `custom`. +- Preserve the existing queue lookup, unique indexes, coalescing window, and run + lifecycle. + +## Implementation Wave + +- [x] [task-01-run-dedup-generation](task-01-run-dedup-generation.md) + +## Verification + +```bash +cd apps/backend && go test ./internal/runs/service ./internal/office/service \ + ./internal/office/dashboard ./internal/task/service +python3 scripts/lint-spec-files.py --all +``` + +The focused backend tests cover assignment event replay, same-agent +reassignment, bounded metric labels, and the existing deduplication paths. diff --git a/docs/plans/office-run-dedup-generation/task-01-run-dedup-generation.md b/docs/plans/office-run-dedup-generation/task-01-run-dedup-generation.md new file mode 100644 index 00000000000..15438035ae8 --- /dev/null +++ b/docs/plans/office-run-dedup-generation/task-01-run-dedup-generation.md @@ -0,0 +1,74 @@ +--- +id: "01-run-dedup-generation" +title: "Implement Office run deduplication generation identity" +status: done +wave: 1 +depends_on: [] +plan: "plan.md" +requirements: + - REQ-OFFICE-RUN-DEDUP-001 + - REQ-OFFICE-RUN-DEDUP-002 + - REQ-OFFICE-RUN-DEDUP-003 + - REQ-OFFICE-RUN-DEDUP-004 +acceptance_criteria: + - AC-OFFICE-RUN-DEDUP-001.1 + - AC-OFFICE-RUN-DEDUP-001.2 + - AC-OFFICE-RUN-DEDUP-001.3 + - AC-OFFICE-RUN-DEDUP-001.4 + - AC-OFFICE-RUN-DEDUP-001.5 + - AC-OFFICE-RUN-DEDUP-001.6 + - AC-OFFICE-RUN-DEDUP-001.7 + - AC-OFFICE-RUN-DEDUP-001.8 + - AC-OFFICE-RUN-DEDUP-001.9 + - AC-OFFICE-RUN-DEDUP-002.1 + - AC-OFFICE-RUN-DEDUP-002.2 + - AC-OFFICE-RUN-DEDUP-002.3 + - AC-OFFICE-RUN-DEDUP-002.4 + - AC-OFFICE-RUN-DEDUP-003.1 + - AC-OFFICE-RUN-DEDUP-003.2 + - AC-OFFICE-RUN-DEDUP-003.3 + - AC-OFFICE-RUN-DEDUP-003.4 + - AC-OFFICE-RUN-DEDUP-004.1 + - AC-OFFICE-RUN-DEDUP-004.2 + - AC-OFFICE-RUN-DEDUP-004.3 + - AC-OFFICE-RUN-DEDUP-004.4 + - AC-OFFICE-RUN-DEDUP-004.5 + - AC-OFFICE-RUN-DEDUP-004.6 +system_design: + - ../../specs/office/system-design/run-dedup-generation-01.md + - ../../specs/office/system-design/run-dedup-generation-02.md + - ../../specs/office/system-design/run-dedup-generation-03.md +--- + +# Task 01: Implement Office run deduplication generation identity + +## Scope + +- Add and preserve the task assignment generation across schema migrations. +- Carry the immutable assignee profile ID and generation in task-created events. +- Derive the same generation-aware assignment key in each producer. +- Keep unresolved generations keyless and observable. +- Bound metric label cardinality for agent-supplied reasons. +- Avoid changes to queue coalescing and other deferred deduplication work. + +## Acceptance + +An assignment replay uses one durable generation key. A later assignment to the +same agent gets a new generation and can enqueue work. A task-created event +replayed after its run is claimed remains deduplicated. An unknown metric reason +does not create a new permanent expvar label. + +## Verification + +```bash +cd apps/backend && go test ./internal/runs/service ./internal/office/service \ + ./internal/office/dashboard ./internal/task/service +python3 scripts/lint-spec-files.py --all +``` + +## Results + +Implemented in the contributor change plus fixup commit +`44a5ccc94d8d1fc212b9e839a240d1db2fe16e5f`. The fixup also carries the current +`main` branch and resolves the PR merge conflict. Focused backend tests and +specification lint pass. From d2648c8063f1d5a8473df2de31705f3058dfa823 Mon Sep 17 00:00:00 2001 From: nova28 <17953305+nova28@users.noreply.github.com> Date: Sun, 13 Sep 2026 15:16:22 +0800 Subject: [PATCH 15/15] test(office): match run-dedup metric assertions to the bounded reason allowlist A rebase onto main pulled in runs/service's metricReason() bounding: any reason outside its fixed allowlist collapses to "custom" on the office_run_dedup_total/office_run_dedup_keyless_total labels. The SpawnAgentRun and routine-wakeup tests used synthetic per-test reason strings and asserted on them verbatim, so they broke against the merged behavior. Assert the "custom" bucket instead. Co-Authored-By: Claude Sonnet 5 --- .../backendapp/adapters_office_wakeup_test.go | 6 ++++-- .../office/runtime/spawn_agent_run_key_test.go | 12 ++++++++---- 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/apps/backend/internal/backendapp/adapters_office_wakeup_test.go b/apps/backend/internal/backendapp/adapters_office_wakeup_test.go index f627b8e9157..cc58c77ba8a 100644 --- a/apps/backend/internal/backendapp/adapters_office_wakeup_test.go +++ b/apps/backend/internal/backendapp/adapters_office_wakeup_test.go @@ -59,8 +59,10 @@ func TestRoutineWakeupAdapter_CreateWakeupRequest_DurableConflictReportsCounter( t.Fatalf("second create error = %v, want ErrWakeupIdempotencyConflict", err) } - if !counterHasLabel(t, "office_run_dedup_total", "reason="+reason, "kind=durable", "queue=wakeup") { - t.Fatal("expected office_run_dedup_total to carry a durable/wakeup entry for this reason") + // reason isn't in runs/service's bounded metricReasons allowlist, so the + // label buckets it to "custom" rather than carrying it verbatim. + if !counterHasLabel(t, "office_run_dedup_total", "reason=custom", "kind=durable", "queue=wakeup") { + t.Fatal("expected office_run_dedup_total to carry a custom/durable/wakeup entry") } } diff --git a/apps/backend/internal/office/runtime/spawn_agent_run_key_test.go b/apps/backend/internal/office/runtime/spawn_agent_run_key_test.go index 2fce20e77d5..2e9e0e5eceb 100644 --- a/apps/backend/internal/office/runtime/spawn_agent_run_key_test.go +++ b/apps/backend/internal/office/runtime/spawn_agent_run_key_test.go @@ -138,8 +138,10 @@ func TestActionsSpawnAgentRun_EmptyKey_NotPrefixed_ByDesign(t *testing.T) { if got := runs.calls[0].IdempotencyKey; got != "" { t.Fatalf("idempotency key = %q, want empty (keyless by design)", got) } - if !spawnAgentRunKeylessCounterHasLabel(t, "test_spawn_empty_key", "by_design") { - t.Fatal("expected office_run_dedup_keyless_total to carry a by_design entry for this reason") + // Reason isn't in runs/service's bounded metricReasons allowlist, so it + // buckets to "custom" on the label rather than surviving verbatim. + if !spawnAgentRunKeylessCounterHasLabel(t, "custom", "by_design") { + t.Fatal("expected office_run_dedup_keyless_total to carry a custom/by_design entry") } } @@ -175,8 +177,10 @@ func TestActionsSpawnAgentRun_NoCallerRunID_EnqueuesKeylessUnresolved(t *testing if got := runs.calls[0].IdempotencyKey; got != "" { t.Fatalf("idempotency key = %q, want empty (keyless, no caller run to prefix with)", got) } - if !spawnAgentRunKeylessCounterHasLabel(t, "test_spawn_no_caller_run", "unresolved") { - t.Fatal("expected office_run_dedup_keyless_total to carry an unresolved entry") + // Reason isn't in runs/service's bounded metricReasons allowlist, so it + // buckets to "custom" on the label rather than surviving verbatim. + if !spawnAgentRunKeylessCounterHasLabel(t, "custom", "unresolved") { + t.Fatal("expected office_run_dedup_keyless_total to carry a custom/unresolved entry") } }