Name the parked and canceled resume outcomes on the route metric

ate.router.resume put budget exhaustion, lot rejection, client
cancellation and real failures all under unknown. Add parked_exhausted,
parked_rejected and canceled; unknown now means a failed resume or a
direction that resumes nothing.

Author: dims

cmd/atenet/internal/router/extproc/handler.go

@@ -61,7 +61,7 @@ type Result struct {
 
 	// Resume is the actor-resume outcome, as one of the ateattr.RouterResume*
 	// values. Empty means "unknown" — the direction never resumes an actor, or
-	// the request never got far enough to learn whether an activation ran.
+	// the request never got far enough to attempt one.
 	Resume string
 
 	// DynamicMetadata is attached to the ProcessingResponse alongside Response,

cmd/atenet/internal/router/extproc/metrics_test.go

@@ -195,6 +195,7 @@ func TestResultResume_DefaultsToUnknown(t *testing.T) {
 		{name: "none preserved", resume: ateattr.RouterResumeNone, want: ateattr.RouterResumeNone},
 		{name: "triggered preserved", resume: ateattr.RouterResumeTriggered, want: ateattr.RouterResumeTriggered},
 		{name: "joined preserved", resume: ateattr.RouterResumeJoined, want: ateattr.RouterResumeJoined},
+		{name: "parked_exhausted preserved", resume: ateattr.RouterResumeParkedExhausted, want: ateattr.RouterResumeParkedExhausted},
 		{name: "unknown preserved", resume: ateattr.RouterResumeUnknown, want: ateattr.RouterResumeUnknown},
 	}
 	for _, tt := range tests {

cmd/atenet/internal/router/ingress/flight.go

@@ -15,6 +15,7 @@
 package ingress
 
 import (
+	"errors"
 	"sync/atomic"
 
 	"go.ate.dev/substrate/pkg/proto/ateapipb"
@@ -71,17 +72,20 @@ func (f *resumeActorFlight) retryErr() error {
 // be called after f.done is closed.
 //
 // Only a resume that completed without an error says whether an activation
-// ran, so the three activation labels are reserved for that case. A failed
-// resume reports "unknown": the gRPC code alone does not carry the fact. A
-// canceled leader's flight outlives its request and keeps restoring the actor,
-// and a DeadlineExceeded can land in the middle of a restore.
+// ran, so the three activation labels are reserved for that case. A flight
+// that waited out its budget reports "parked_exhausted"; any other failure
+// reports "unknown", since the gRPC code alone does not carry the fact.
 func (f *resumeActorFlight) callerResult(reqID uint64) (*ateapipb.Actor, ResumeOutcome, error) {
 	res := f.result
 	if res == nil {
 		return nil, ResumeOutcomeUnknown, status.Error(codes.Internal, "resume call returned nil result")
 	}
 
 	if res.err != nil {
+		var budget *budgetExhaustedError
+		if errors.As(res.err, &budget) {
+			return nil, ResumeOutcomeParkedExhausted, res.err
+		}
 		return nil, ResumeOutcomeUnknown, res.err
 	}
 

cmd/atenet/internal/router/ingress/resumer.go

@@ -70,10 +70,13 @@ func (e *budgetExhaustedError) Unwrap() error { return e.lastErr }
 type ResumeOutcome string
 
 const (
-	ResumeOutcomeNone      ResumeOutcome = ateattr.RouterResumeNone
-	ResumeOutcomeTriggered ResumeOutcome = ateattr.RouterResumeTriggered
-	ResumeOutcomeJoined    ResumeOutcome = ateattr.RouterResumeJoined
-	ResumeOutcomeUnknown   ResumeOutcome = ateattr.RouterResumeUnknown
+	ResumeOutcomeNone            ResumeOutcome = ateattr.RouterResumeNone
+	ResumeOutcomeTriggered       ResumeOutcome = ateattr.RouterResumeTriggered
+	ResumeOutcomeJoined          ResumeOutcome = ateattr.RouterResumeJoined
+	ResumeOutcomeParkedExhausted ResumeOutcome = ateattr.RouterResumeParkedExhausted
+	ResumeOutcomeParkedRejected  ResumeOutcome = ateattr.RouterResumeParkedRejected
+	ResumeOutcomeCanceled        ResumeOutcome = ateattr.RouterResumeCanceled
+	ResumeOutcomeUnknown         ResumeOutcome = ateattr.RouterResumeUnknown
 )
 
 type resumeCallResult struct {
@@ -274,10 +277,10 @@ func flightResult(bgCtx context.Context, resumeResp *ateapipb.ResumeActorRespons
 func (r *ActorResumer) awaitFlight(ctx context.Context, f *resumeActorFlight, actorRef resources.ActorRef, reqID uint64) (*ateapipb.Actor, ResumeOutcome, error) {
 	select {
 	case <-ctx.Done():
-		// The caller's request context was canceled before the shared resume
+		// The caller's request context ended before the shared resume
 		// completed. The flight continues and may still activate the actor, so
-		// the outcome is "unknown", not "none".
-		return nil, ResumeOutcomeUnknown, ctx.Err()
+		// the outcome is "canceled", not "none".
+		return nil, ResumeOutcomeCanceled, ctx.Err()
 	case <-f.done:
 		// Fast path: the flight finished without ever retrying, or before this
 		// caller saw retrying. The lot is never touched.
@@ -298,15 +301,15 @@ func (r *ActorResumer) awaitFlight(ctx context.Context, f *resumeActorFlight, ac
 	retryErr := f.retryErr()
 	release, ok := r.enterLot(ctx, shedOutcome(retryErr))
 	if !ok {
-		return nil, ResumeOutcomeUnknown, parkingFullErr(actorRef.String(), retryErr)
+		return nil, ResumeOutcomeParkedRejected, parkingFullErr(actorRef.String(), retryErr)
 	}
 	var finalErr error
 	defer func() { release(parkOutcomeFor(finalErr)) }()
 
 	select {
 	case <-ctx.Done():
 		finalErr = ctx.Err()
-		return nil, ResumeOutcomeUnknown, finalErr
+		return nil, ResumeOutcomeCanceled, finalErr
 	case <-f.done:
 		actor, outcome, err := f.callerResult(reqID)
 		finalErr = err

cmd/atenet/internal/router/ingress/resumer_test.go

@@ -180,8 +180,8 @@ func TestActorResumer_ResumeActor(t *testing.T) {
 		if !errors.Is(err, context.Canceled) {
 			t.Errorf("expected context.Canceled, got %v", err)
 		}
-		if outcome != ResumeOutcomeUnknown {
-			t.Errorf("expected outcome %q on a canceled caller, got %q", ResumeOutcomeUnknown, outcome)
+		if outcome != ResumeOutcomeCanceled {
+			t.Errorf("expected outcome %q on a canceled caller, got %q", ResumeOutcomeCanceled, outcome)
 		}
 	})
 
@@ -368,7 +368,7 @@ func TestActorResumer_Parking(t *testing.T) {
 			// Budget large enough for a few ~100ms-spaced retries before it elapses;
 			// the pool never frees up.
 			resumer := NewActorResumer(mock, withParking(ParkedRequestConfig{Max: 1, Budget: 1500 * time.Millisecond}))
-			_, _, err := resumer.ResumeActor(context.Background(), testActorRef)
+			_, outcome, err := resumer.ResumeActor(context.Background(), testActorRef)
 			// The client must see the meaningful capacity error, not a generic
 			// timeout: status.Code must unwrap through the budget-exhaustion marker.
 			if got := status.Code(err); got != codes.FailedPrecondition {
@@ -378,6 +378,9 @@ func TestActorResumer_Parking(t *testing.T) {
 			if !errors.As(err, &budget) {
 				t.Errorf("expected the error to be marked as budget exhaustion, got %T (%v)", err, err)
 			}
+			if outcome != ResumeOutcomeParkedExhausted {
+				t.Errorf("expected outcome %q after park budget elapsed, got %q", ResumeOutcomeParkedExhausted, outcome)
+			}
 			mu.Lock()
 			defer mu.Unlock()
 			if calls < 2 {
@@ -564,14 +567,17 @@ func TestActorResumer_Parking(t *testing.T) {
 			}
 
 			resumer := NewActorResumer(mock, withParking(ParkedRequestConfig{Max: 1, Budget: budget}))
-			_, _, err := resumer.ResumeActor(context.Background(), testActorRef)
+			_, outcome, err := resumer.ResumeActor(context.Background(), testActorRef)
 			if got := status.Code(err); got != codes.ResourceExhausted {
 				t.Errorf("expected ResourceExhausted after a late retryable failure, got %v (err=%v)", got, err)
 			}
 			var budgetErr *budgetExhaustedError
 			if !errors.As(err, &budgetErr) {
 				t.Errorf("expected the error to be marked as budget exhaustion, got %T (%v)", err, err)
 			}
+			if outcome != ResumeOutcomeParkedExhausted {
+				t.Errorf("expected outcome %q after a late retryable failure, got %q", ResumeOutcomeParkedExhausted, outcome)
+			}
 			mu.Lock()
 			defer mu.Unlock()
 			if calls != 1 {
@@ -836,8 +842,8 @@ func TestActorResumer_LotAdmission(t *testing.T) {
 			if !errors.As(err, &reqErr) || reqErr.StatusCode != int(envoy_type.StatusCode_ServiceUnavailable) {
 				t.Fatalf("expected a 503 router-at-capacity denial, got %v", err)
 			}
-			if outcome != ResumeOutcomeUnknown {
-				t.Errorf("shed caller outcome = %q, want %q", outcome, ResumeOutcomeUnknown)
+			if outcome != ResumeOutcomeParkedRejected {
+				t.Errorf("shed caller outcome = %q, want %q", outcome, ResumeOutcomeParkedRejected)
 			}
 			// The caller was turned away at the transition: exactly one attempt
 			// had run.
@@ -901,8 +907,8 @@ func TestActorResumer_LotAdmission(t *testing.T) {
 			if !errors.As(err, &reqErr) || reqErr.StatusCode != int(envoy_type.StatusCode_ServiceUnavailable) {
 				t.Fatalf("joiner: expected a 503 router-at-capacity denial, got %v", err)
 			}
-			if outcome != ResumeOutcomeUnknown {
-				t.Errorf("joiner outcome = %q, want %q", outcome, ResumeOutcomeUnknown)
+			if outcome != ResumeOutcomeParkedRejected {
+				t.Errorf("joiner outcome = %q, want %q", outcome, ResumeOutcomeParkedRejected)
 			}
 
 			close(proceed)

docs/metrics/registry/metrics.yaml

@@ -467,15 +467,33 @@ groups:
                 and the resume activated the actor. This value is separate
                 because it is not a correct sample. One cold start with 50
                 requests is one slow activation and not 51.
+            - id: parked_exhausted
+              stability: development
+              value: parked_exhausted
+              brief: >
+                The parking budget ended while the resume was still blocked.
+                The request failed with the last capacity error. Every request
+                on that flight has this value. Not an activation time.
+            - id: parked_rejected
+              stability: development
+              value: parked_rejected
+              brief: >
+                The parking area was full, thus the router refused the request
+                at once. atenet.router.parking.rejected counts the same requests.
+            - id: canceled
+              stability: development
+              value: canceled
+              brief: >
+                The request ended before the resume gave an answer: the client
+                disconnected, or its deadline ended (see ate.router.outcome).
+                The flight continues, thus the actor can still start.
             - id: unknown
               stability: development
               value: unknown
               brief: >
-                The resume did not complete, thus the router cannot tell whether
-                an activation ran. The resume failed, or the request stopped
-                before the resume gave an answer, or the direction does not
-                resume an actor. Do not read the time in this series as an
-                activation time.
+                The resume failed for another reason (see ate.router.outcome),
+                or the direction does not resume an actor. Not an activation
+                time.
       - id: ate.router.outcome
         stability: development
         brief: The result of the route attempt.
@@ -1366,8 +1384,10 @@ groups:
       ate.router.resume before you read it. The triggered series is the
       activation time. The none series is the usual warm route. A sum across the
       two series puts milliseconds and tens of seconds in one distribution. The
-      unknown series holds the resumes that did not complete, thus it is neither
-      an activation time nor a warm route.
+      parked_exhausted, parked_rejected, canceled and unknown series hold the
+      resumes that did not complete, thus they are neither an activation time
+      nor a warm route. The count of parked_exhausted is the number of requests
+      that capacity, not a fault, turned away.
     annotations:
       substrate:
         emitted_by: [atenet-router]

docs/observability.md

@@ -306,7 +306,7 @@ For `ate.workerpool.desired_workers` and `ate.workerpool.ready_workers`:
 For `atenet.router.route.duration`:
 * `ate.router.outcome` categorizes the route attempt result: `ok`, `cancelled`, `timeout`, `no_capacity`, `failed_precondition`, `lock_conflict`, `not_found`, `unavailable`, `rate_limited`, or `resume_error`.
 * The router and the scheduler both report `no_capacity` for one condition, but they count different events. The scheduler records one sample on `ate.scheduler.assignment.duration` for each resume. The router records one sample for each request. Many requests can join one resume, and one parked request can cause more than one resume. Thus do not compare the two counts directly.
-* `ate.router.resume` indicates the singleflight execution state of actor resumption: `none` (the resume found the actor already running), `triggered` (this request completed a cold activation), `joined` (this request waited on another request's resume, which activated the actor), or `unknown` (the resume did not complete, so whether an activation ran is unknown). `ate.template.atespace` and `ate.template.name` hold `unknown` when the router has no template to name.
+* `ate.router.resume` indicates the singleflight execution state of actor resumption: `none` (the resume found the actor already running), `triggered` (this request completed a cold activation), `joined` (this request waited on another request's resume, which activated the actor), `parked_exhausted` (the request waited out the parking budget and failed with the last capacity error), `parked_rejected` (the parking lot was full, so the request was shed at once), `canceled` (the request ended, by a client disconnect or its deadline, before the resume answered), or `unknown` (the resume failed for another reason, so whether an activation ran is unknown). `ate.template.atespace` and `ate.template.name` hold `unknown` when the router has no template to name.
 
 For `ate.imagecache.requests`:
 * `ate.imagecache.outcome` is `hit` when the node holds a complete image record — every layer directory the record names is present — and `miss` when the lookup must pull. A failed lookup is neither: `error` is a failed lookup whatever the cause, and `cancelled` or `timeout` is the caller giving up, as on `ate.router.outcome`. So the hit ratio is `hit / (hit + miss)`, with failures and abandoned lookups out of the denominator.

docs/request-parking.md

@@ -174,8 +174,10 @@ bounds the wait.
   when its resume failed with `ResourceExhausted`, and `unavailable` otherwise.
   The `ate.router.outcome` label of this counter has the same value. Because
   `no_capacity` combines fast shed requests with requests that waited out the
-  parking budget, use `atenet.router.parking.rejected{ate_router_outcome="no_capacity"}`
-  to count the shed requests in it.
+  parking budget, read `ate.router.resume` on the route histogram to separate
+  them: a shed request reports `parked_rejected`, and a request that waited out
+  the budget reports `parked_exhausted`. A request whose client disconnected or
+  whose own deadline ended while parked reports `canceled`.
 
 **Status page** (`/statusz`): a "Request Parking" card shows whether parking is
 enabled, the current vs. maximum parked count, and the max wait.

internal/ateattr/ateattr.go

@@ -220,9 +220,17 @@ const (
 	// RouterResumeJoined indicates this request waited on another request's
 	// singleflight resume, which completed a cold activation.
 	RouterResumeJoined = "joined"
-	// RouterResumeUnknown indicates the resume did not complete, so the router
-	// cannot tell whether an activation ran. The resume failed, or the request
-	// stopped first, or the direction never resumes an actor.
+	// RouterResumeParkedExhausted: the parking budget elapsed while the resume
+	// was still blocked, so the request failed with the last capacity error.
+	RouterResumeParkedExhausted = "parked_exhausted"
+	// RouterResumeParkedRejected: the parking lot was full, so the request was
+	// shed at once.
+	RouterResumeParkedRejected = "parked_rejected"
+	// RouterResumeCanceled: the request ended (client disconnect or deadline)
+	// before the shared resume answered. The flight goes on.
+	RouterResumeCanceled = "canceled"
+	// RouterResumeUnknown: the resume failed for another reason, or the
+	// direction never resumes an actor.
 	RouterResumeUnknown = "unknown"
 )
 

internal/ateattr/ateattr_test.go

@@ -417,6 +417,9 @@ func TestMetricLabelValues(t *testing.T) {
 		{RouterResumeNone, "none"},
 		{RouterResumeTriggered, "triggered"},
 		{RouterResumeJoined, "joined"},
+		{RouterResumeParkedExhausted, "parked_exhausted"},
+		{RouterResumeParkedRejected, "parked_rejected"},
+		{RouterResumeCanceled, "canceled"},
 		{RouterResumeUnknown, "unknown"},
 	}
 	for _, tt := range tests {