Instances self-heal, restart, and report status correctly on the federation foundation

api/v1alpha/instance_types.go

@@ -404,8 +404,10 @@ const (
 	// InstanceReady indicates that the instance is ready
 	InstanceReady = "Ready"
 
-	// InstanceRunning indicates that the instance is running
-	InstanceRunning = "Running"
+	// InstanceAvailable indicates that the instance is available. It is True
+	// when the instance is serving and does not assert that a process is
+	// actively running at this instant.
+	InstanceAvailable = "Available"
 
 	// InstanceProgrammed indicates that the instance has been programmed
 	InstanceProgrammed = "Programmed"
@@ -458,20 +460,42 @@ const (
 	// InstanceReadyReasonSchedulingGatesPresent indicates that the instance is not ready because scheduling gates are present.
 	InstanceReadyReasonSchedulingGatesPresent = "SchedulingGatesPresent"
 
-	// InstanceReadyReasonRunning indicates that the instance is running
-	InstanceReadyReasonRunning = "Running"
+	// InstanceReadyReasonAvailable indicates that the instance is available
+	InstanceReadyReasonAvailable = "Available"
 
-	// InstanceRunningReasonStopped indicates that the instance is stopped
-	InstanceRunningReasonStopped = "Stopped"
+	// InstanceReadyReasonImageUnavailable indicates the provider could not pull
+	// the instance image (bad name, missing credentials, registry unreachable).
+	// This matches the reason written by translateWaitingReason in the unikraft
+	// provider when the container enters an image-pull waiting state.
+	InstanceReadyReasonImageUnavailable = "ImageUnavailable"
 
-	// InstanceRunningReasonStarting indicates that the instance is starting
-	InstanceRunningReasonStarting = "Starting"
+	// InstanceReadyReasonInstanceCrashing indicates the instance process started
+	// but is repeatedly exiting and being restarted (CrashLoopBackOff in the
+	// underlying runtime). This is user-actionable: the application itself is
+	// failing, not the platform.
+	InstanceReadyReasonInstanceCrashing = "InstanceCrashing"
 
-	// InstanceRunningReasonStopping indicates that the instance is stopping
-	InstanceRunningReasonStopping = "Stopping"
+	// InstanceReadyReasonConfigurationError indicates the runtime rejected the
+	// instance configuration before the process could start (e.g. invalid env
+	// variable injection, missing device). User must correct the workload spec.
+	InstanceReadyReasonConfigurationError = "ConfigurationError"
 
-	// InstanceRunningReasonRunning indicates that the instance is running
-	InstanceRunningReasonRunning = "Running"
+	// InstanceReadyReasonProvisioning indicates the instance runtime is still
+	// setting up the execution environment (container being created, image being
+	// unpacked). This is a transient, non-actionable state.
+	InstanceReadyReasonProvisioning = "Provisioning"
+
+	// InstanceAvailableReasonStopped indicates that the instance is stopped
+	InstanceAvailableReasonStopped = "Stopped"
+
+	// InstanceAvailableReasonStarting indicates that the instance is starting
+	InstanceAvailableReasonStarting = "Starting"
+
+	// InstanceAvailableReasonStopping indicates that the instance is stopping
+	InstanceAvailableReasonStopping = "Stopping"
+
+	// InstanceAvailableReasonAvailable indicates that the instance is available
+	InstanceAvailableReasonAvailable = "Available"
 
 	// InstanceProgrammedReasonPendingProgramming indicates that the instance has not been programmed
 	InstanceProgrammedReasonPendingProgramming = "PendingProgramming"
@@ -515,7 +539,7 @@ type Instance struct {
 
 	// Status defines the current state of an Instance.
 	//
-	// +kubebuilder:default={conditions:{{type:"Programmed",status:"Unknown",reason:"Pending", message:"Waiting for controller", lastTransitionTime: "1970-01-01T00:00:00Z"},{type:"Running",status:"Unknown",reason:"Pending", message:"Waiting for controller", lastTransitionTime: "1970-01-01T00:00:00Z"},{type:"Ready",status:"Unknown",reason:"Pending", message:"Waiting for controller", lastTransitionTime: "1970-01-01T00:00:00Z"},{type:"QuotaGranted",status:"Unknown",reason:"PendingEvaluation",message:"Waiting for quota evaluation",lastTransitionTime:"1970-01-01T00:00:00Z"}}}
+	// +kubebuilder:default={conditions:{{type:"Programmed",status:"Unknown",reason:"Pending", message:"Waiting for controller", lastTransitionTime: "1970-01-01T00:00:00Z"},{type:"Available",status:"Unknown",reason:"Pending", message:"Waiting for controller", lastTransitionTime: "1970-01-01T00:00:00Z"},{type:"Ready",status:"Unknown",reason:"Pending", message:"Waiting for controller", lastTransitionTime: "1970-01-01T00:00:00Z"},{type:"QuotaGranted",status:"Unknown",reason:"PendingEvaluation",message:"Waiting for quota evaluation",lastTransitionTime:"1970-01-01T00:00:00Z"}}}
 	Status InstanceStatus `json:"status,omitempty"`
 }
 

api/v1alpha/workload_types.go

@@ -58,15 +58,27 @@ type WorkloadStatus struct {
 	// The number of instances that currently exist
 	Replicas int32 `json:"replicas"`
 
-	// The number of instances which have the latest workload settings applied.
+	// The number of instances which have the latest workload settings applied
+	// and are programmed (a subset of UpdatedReplicas that are ready to serve).
 	CurrentReplicas int32 `json:"currentReplicas"`
 
+	// The number of instances updated to the latest template revision (their
+	// observed template hash matches the desired template), regardless of
+	// readiness. Lags Replicas during a rolling update or restart, then catches
+	// back up — making an in-progress roll observable.
+	UpdatedReplicas int32 `json:"updatedReplicas"`
+
 	// The desired number of instances
 	DesiredReplicas int32 `json:"desiredReplicas"`
 
 	// The number of instances which are ready.
 	ReadyReplicas int32 `json:"readyReplicas"`
 
+	// The most recent generation observed by the workload controller.
+	//
+	// +kubebuilder:validation:Optional
+	ObservedGeneration int64 `json:"observedGeneration,omitempty"`
+
 	// The current status of placemetns in a workload.
 	Placements []WorkloadPlacementStatus `json:"placements,omitempty"`
 
@@ -99,7 +111,7 @@ type WorkloadGatewayStatus struct {
 // +kubebuilder:printcolumn:name="Replicas",type=string,JSONPath=`.status.replicas`
 // +kubebuilder:printcolumn:name="Ready",type=string,JSONPath=`.status.readyReplicas`
 // +kubebuilder:printcolumn:name="Desired",type=string,JSONPath=`.status.desiredReplicas`
-// +kubebuilder:printcolumn:name="Up-to-date",type=string,JSONPath=`.status.currentReplicas`
+// +kubebuilder:printcolumn:name="Up-to-date",type=string,JSONPath=`.status.updatedReplicas`
 type Workload struct {
 	metav1.TypeMeta   `json:",inline"`
 	metav1.ObjectMeta `json:"metadata,omitempty"`
@@ -146,9 +158,14 @@ type WorkloadPlacementStatus struct {
 	// The number of instances that currently exist
 	Replicas int32 `json:"replicas"`
 
-	// The number of instances which have the latest workload settings applied.
+	// The number of instances which have the latest workload settings applied
+	// and are programmed (a subset of UpdatedReplicas that are ready to serve).
 	CurrentReplicas int32 `json:"currentReplicas"`
 
+	// The number of instances updated to the latest template revision, regardless
+	// of readiness. Lags Replicas during a rolling update or restart.
+	UpdatedReplicas int32 `json:"updatedReplicas"`
+
 	// The desired number of instances
 	DesiredReplicas int32 `json:"desiredReplicas"`
 

api/v1alpha/workloaddeployment_types.go

@@ -49,14 +49,28 @@ type WorkloadDeploymentStatus struct {
 	// The number of instances created
 	Replicas int32 `json:"replicas"`
 
-	// The number of instances which have the latest workload settings applied.
+	// The number of instances which have the latest workload settings applied
+	// and are programmed (a subset of UpdatedReplicas that are ready to serve).
 	CurrentReplicas int32 `json:"currentReplicas"`
 
+	// The number of instances updated to the latest template revision, i.e.
+	// whose observed template hash matches the desired template, regardless of
+	// readiness. Lags Replicas during a rolling update or restart, then catches
+	// back up — making an in-progress roll observable.
+	UpdatedReplicas int32 `json:"updatedReplicas"`
+
 	// The desired number of instances
 	DesiredReplicas int32 `json:"desiredReplicas"`
 
 	// The number of instances which are ready.
 	ReadyReplicas int32 `json:"readyReplicas"`
+
+	// The most recent generation observed by the deployment controller. When
+	// this matches metadata.generation, the controller has reconciled the
+	// latest spec (e.g. a restart request).
+	//
+	// +kubebuilder:validation:Optional
+	ObservedGeneration int64 `json:"observedGeneration,omitempty"`
 }
 
 const (
@@ -79,7 +93,7 @@ const (
 // +kubebuilder:printcolumn:name="Replicas",type=string,JSONPath=`.status.replicas`
 // +kubebuilder:printcolumn:name="Ready",type=string,JSONPath=`.status.readyReplicas`
 // +kubebuilder:printcolumn:name="Desired",type=string,JSONPath=`.status.desiredReplicas`
-// +kubebuilder:printcolumn:name="Up-to-date",type=string,JSONPath=`.status.currentReplicas`
+// +kubebuilder:printcolumn:name="Up-to-date",type=string,JSONPath=`.status.updatedReplicas`
 // +kubebuilder:printcolumn:name="Location Namespace",type=string,JSONPath=`.status.location.namespace`,priority=1
 // +kubebuilder:printcolumn:name="Location Name",type=string,JSONPath=`.status.location.name`,priority=1
 type WorkloadDeployment struct {

cmd/main.go

@@ -467,21 +467,34 @@ func ignoreCanceled(err error) error {
 // InstanceProjector). Called only when management controllers are enabled and
 // a federation REST config is available.
 func setupManagementControllers(mgr mcmanager.Manager, federationClient client.Client) ([]manager.Runnable, error) {
-	federator := &controller.WorkloadDeploymentFederator{FederationClient: federationClient}
+	// The federation manager provides a cached, watchable handle to the Karmada
+	// federation control plane. It backs the InstanceProjector's Instance watch
+	// and the WorkloadDeploymentFederator's downstream WorkloadDeployment status
+	// watch. A manager.Manager embeds a cluster.Cluster, so it can be passed
+	// directly anywhere a watchable federation cluster source is required.
+	federationMgr, err := manager.New(federationRestConfig, manager.Options{
+		Scheme:  scheme,
+		Metrics: metricsserver.Options{BindAddress: "0"},
+	})
+	if err != nil {
+		return nil, fmt.Errorf("federation manager: %w", err)
+	}
+
+	// The federator watches both the project WD (via the multicluster manager)
+	// and the downstream Karmada WD (via the federation cluster) so that status
+	// aggregated downstream by Karmada is mirrored back to the project WD
+	// immediately instead of on the next informer resync.
+	federator := &controller.WorkloadDeploymentFederator{
+		FederationClient:  federationClient,
+		FederationCluster: federationMgr,
+	}
 	if err := federator.SetupWithManager(mgr); err != nil {
 		return nil, fmt.Errorf("WorkloadDeploymentFederator: %w", err)
 	}
 
 	// InstanceProjector runs in the management plane, watches Instances written
 	// back by POP-cell operators to the Karmada federation control plane, and
 	// projects them into the corresponding project namespaces via the multicluster manager.
-	federationMgr, err := manager.New(federationRestConfig, manager.Options{
-		Scheme:  scheme,
-		Metrics: metricsserver.Options{BindAddress: "0"},
-	})
-	if err != nil {
-		return nil, fmt.Errorf("federation manager for InstanceProjector: %w", err)
-	}
 	if err = (&controller.InstanceProjector{
 		FederationClient: federationClient,
 		MCManager:        mgr,

config/base/crd/bases/compute.datumapis.com_instances.yaml

@@ -887,7 +887,7 @@ spec:
                 message: Waiting for controller
                 reason: Pending
                 status: Unknown
-                type: Running
+                type: Available
               - lastTransitionTime: "1970-01-01T00:00:00Z"
                 message: Waiting for controller
                 reason: Pending

config/base/crd/bases/compute.datumapis.com_workloaddeployments.yaml

@@ -34,7 +34,7 @@ spec:
     - jsonPath: .status.desiredReplicas
       name: Desired
       type: string
-    - jsonPath: .status.currentReplicas
+    - jsonPath: .status.updatedReplicas
       name: Up-to-date
       type: string
     - jsonPath: .status.location.namespace
@@ -1087,8 +1087,9 @@ spec:
                   type: object
                 type: array
               currentReplicas:
-                description: The number of instances which have the latest workload
-                  settings applied.
+                description: |-
+                  The number of instances which have the latest workload settings applied
+                  and are programmed (a subset of UpdatedReplicas that are ready to serve).
                 format: int32
                 type: integer
               desiredReplicas:
@@ -1109,6 +1110,13 @@ spec:
                 - name
                 - namespace
                 type: object
+              observedGeneration:
+                description: |-
+                  The most recent generation observed by the deployment controller. When
+                  this matches metadata.generation, the controller has reconciled the
+                  latest spec (e.g. a restart request).
+                format: int64
+                type: integer
               readyReplicas:
                 description: The number of instances which are ready.
                 format: int32
@@ -1117,11 +1125,20 @@ spec:
                 description: The number of instances created
                 format: int32
                 type: integer
+              updatedReplicas:
+                description: |-
+                  The number of instances updated to the latest template revision, i.e.
+                  whose observed template hash matches the desired template, regardless of
+                  readiness. Lags Replicas during a rolling update or restart, then catches
+                  back up — making an in-progress roll observable.
+                format: int32
+                type: integer
             required:
             - currentReplicas
             - desiredReplicas
             - readyReplicas
             - replicas
+            - updatedReplicas
             type: object
         type: object
     served: true

config/base/crd/bases/compute.datumapis.com_workloads.yaml

@@ -37,7 +37,7 @@ spec:
     - jsonPath: .status.desiredReplicas
       name: Desired
       type: string
-    - jsonPath: .status.currentReplicas
+    - jsonPath: .status.updatedReplicas
       name: Up-to-date
       type: string
     name: v1alpha
@@ -1081,8 +1081,9 @@ spec:
                   type: object
                 type: array
               currentReplicas:
-                description: The number of instances which have the latest workload
-                  settings applied.
+                description: |-
+                  The number of instances which have the latest workload settings applied
+                  and are programmed (a subset of UpdatedReplicas that are ready to serve).
                 format: int32
                 type: integer
               deployments:
@@ -1367,6 +1368,10 @@ spec:
                     - name
                     x-kubernetes-list-type: map
                 type: object
+              observedGeneration:
+                description: The most recent generation observed by the workload controller.
+                format: int64
+                type: integer
               placements:
                 description: The current status of placemetns in a workload.
                 items:
@@ -1432,8 +1437,9 @@ spec:
                         type: object
                       type: array
                     currentReplicas:
-                      description: The number of instances which have the latest workload
-                        settings applied.
+                      description: |-
+                        The number of instances which have the latest workload settings applied
+                        and are programmed (a subset of UpdatedReplicas that are ready to serve).
                       format: int32
                       type: integer
                     desiredReplicas:
@@ -1451,12 +1457,19 @@ spec:
                       description: The number of instances that currently exist
                       format: int32
                       type: integer
+                    updatedReplicas:
+                      description: |-
+                        The number of instances updated to the latest template revision, regardless
+                        of readiness. Lags Replicas during a rolling update or restart.
+                      format: int32
+                      type: integer
                   required:
                   - currentReplicas
                   - desiredReplicas
                   - name
                   - readyReplicas
                   - replicas
+                  - updatedReplicas
                   type: object
                 type: array
               readyReplicas:
@@ -1467,12 +1480,21 @@ spec:
                 description: The number of instances that currently exist
                 format: int32
                 type: integer
+              updatedReplicas:
+                description: |-
+                  The number of instances updated to the latest template revision (their
+                  observed template hash matches the desired template), regardless of
+                  readiness. Lags Replicas during a rolling update or restart, then catches
+                  back up — making an in-progress roll observable.
+                format: int32
+                type: integer
             required:
             - currentReplicas
             - deployments
             - desiredReplicas
             - readyReplicas
             - replicas
+            - updatedReplicas
             type: object
         required:
         - spec

config/components/controller_rbac/role.yaml

@@ -4,6 +4,13 @@ kind: ClusterRole
 metadata:
   name: compute
 rules:
+- apiGroups:
+  - ""
+  resources:
+  - events
+  verbs:
+  - create
+  - patch
 - apiGroups:
   - ""
   resources: