diff --git a/api/v1alpha1/olsconfig_types.go b/api/v1alpha1/olsconfig_types.go index c22bc6099..3f50fed98 100644 --- a/api/v1alpha1/olsconfig_types.go +++ b/api/v1alpha1/olsconfig_types.go @@ -254,6 +254,16 @@ type OLSSpec struct { // +kubebuilder:validation:Optional // +operator-sdk:csv:customresourcedefinitions:type=spec,displayName="Tools Approval Configuration",xDescriptors={"urn:alm:descriptor:com.tectonic.ui:advanced"} ToolsApprovalConfig *ToolsApprovalConfig `json:"toolsApprovalConfig,omitempty"` + // Enable in-process credential hot-reload for LLM provider secrets. + // When true, the operator will not restart the app-server when LLM credential + // secret data is rotated — the service re-reads credentials from disk on each request. + // IMPORTANT: Requires lightspeed-service with get_credentials() hot-reload support + // (service PR #2955 / RFE-9380). If enabled with an older service image, rotated + // credentials (including revoked keys) will remain stale until the pod is manually restarted. + // +kubebuilder:default=false + // +kubebuilder:validation:Optional + // +operator-sdk:csv:customresourcedefinitions:type=spec,displayName="Credential Hot Reload",xDescriptors={"urn:alm:descriptor:com.tectonic.ui:booleanSwitch"} + CredentialHotReload *bool `json:"credentialHotReload,omitempty"` } // Persistent Storage Configuration diff --git a/config/crd/bases/ols.openshift.io_olsconfigs.yaml b/config/crd/bases/ols.openshift.io_olsconfigs.yaml index 2e3a67eeb..e811350c1 100644 --- a/config/crd/bases/ols.openshift.io_olsconfigs.yaml +++ b/config/crd/bases/ols.openshift.io_olsconfigs.yaml @@ -468,6 +468,16 @@ spec: - postgres type: string type: object + credentialHotReload: + default: false + description: |- + Enable in-process credential hot-reload for LLM provider secrets. + When true, the operator will not restart the app-server when LLM credential + secret data is rotated — the service re-reads credentials from disk on each request. + IMPORTANT: Requires lightspeed-service with get_credentials() hot-reload support + (service PR #2955 / RFE-9380). If enabled with an older service image, rotated + credentials (including revoked keys) will remain stale until the pod is manually restarted. + type: boolean defaultModel: description: Default model for usage type: string diff --git a/docs/credential-hot-reload.md b/docs/credential-hot-reload.md new file mode 100644 index 000000000..c914403a2 --- /dev/null +++ b/docs/credential-hot-reload.md @@ -0,0 +1,52 @@ +# Credential Hot-Reload (RFE-9380) + +## Overview + +When `credentialHotReload` is enabled on the OLSConfig CR, the operator skips +rolling restarts of the app-server pod when LLM credential secret **data** +changes. The service re-reads credential files from disk on every LLM request, +so rotated tokens take effect without downtime. + +This feature is a companion to +[lightspeed-service PR #2955](https://github.com/openshift/lightspeed-service/pull/2955), +which adds the in-process credential reload on the service side. + +## Configuration + +```yaml +apiVersion: ols.openshift.io/v1alpha1 +kind: OLSConfig +metadata: + name: cluster +spec: + ols: + credentialHotReload: true # default: false +``` + +## Behavior + +| Event | `credentialHotReload: false` (default) | `credentialHotReload: true` | +|---|---|---| +| LLM secret **data** rotated (same secret name) | Rolling restart | **No restart** — service picks up new credentials on next request | +| LLM secret **ref** changed in CR (different secret name) | Rolling restart | Rolling restart (deployment volume spec changes) | +| TLS / MCP / Postgres secret changed | Rolling restart | Rolling restart (unchanged) | + +## Prerequisites + +- The lightspeed-service image must include the `get_credentials()` hot-reload + support (lightspeed-service >= the version containing PR #2955). If an older + service image is used with this flag enabled, rotated credentials will not be + picked up until the pod is manually restarted. + +## How It Works + +1. During reconciliation, the operator reads `spec.ols.credentialHotReload` from + the OLSConfig CR and stores it in the internal `WatcherConfig`, along with the + set of LLM provider secret names. + +2. When the secret watcher detects a `.data` change on an annotated secret, it + checks whether the secret is an LLM credential and the hot-reload flag is + enabled. If both conditions are true, the restart is skipped. + +3. Non-LLM secrets (TLS, MCP headers, Postgres) always trigger restarts + regardless of the flag. diff --git a/internal/controller/olsconfig_controller.go b/internal/controller/olsconfig_controller.go index fb1316bfd..b6ca905cb 100644 --- a/internal/controller/olsconfig_controller.go +++ b/internal/controller/olsconfig_controller.go @@ -553,6 +553,12 @@ func (r *OLSConfigReconciler) Reconcile(ctx context.Context, req ctrl.Request) ( return ctrl.Result{}, fmt.Errorf("failed to annotate external resources: %w", err) } + if olsconfig.Spec.OLSConfig.CredentialHotReload != nil && *olsconfig.Spec.OLSConfig.CredentialHotReload { + r.Logger.Info("credentialHotReload is enabled — LLM credential secret rotations will not "+ + "trigger app-server restarts. Requires lightspeed-service with get_credentials() "+ + "hot-reload support (service PR #2955 / RFE-9380)") + } + // 5. Phase 1: Reconcile independent resources if err := r.reconcileIndependentResources(ctx, olsconfig); err != nil { return ctrl.Result{}, err diff --git a/internal/controller/olsconfig_helpers.go b/internal/controller/olsconfig_helpers.go index 3474b8780..8386cac1d 100644 --- a/internal/controller/olsconfig_helpers.go +++ b/internal/controller/olsconfig_helpers.go @@ -384,6 +384,16 @@ func (r *OLSConfigReconciler) annotateExternalResources(ctx context.Context, if r.WatcherConfig != nil { r.WatcherConfig.AnnotatedConfigMapMapping = make(map[string][]string) r.WatcherConfig.AnnotatedSecretMapping = make(map[string][]string) + r.WatcherConfig.LLMSecretNames = make(map[string]bool) + + r.WatcherConfig.CredentialHotReload = cr.Spec.OLSConfig.CredentialHotReload != nil && + *cr.Spec.OLSConfig.CredentialHotReload + + for _, provider := range cr.Spec.LLMConfig.Providers { + if provider.CredentialsSecretRef.Name != "" { + r.WatcherConfig.LLMSecretNames[provider.CredentialsSecretRef.Name] = true + } + } } var errs []error diff --git a/internal/controller/utils/types.go b/internal/controller/utils/types.go index b2335e36f..9c966b34c 100644 --- a/internal/controller/utils/types.go +++ b/internal/controller/utils/types.go @@ -57,12 +57,19 @@ type ConfigMapWatcherConfig struct { SystemResources []SystemConfigMap } -// WatcherConfig contains all watcher configuration +// WatcherConfig contains all watcher configuration. +// NOTE: This struct is written by the reconciler and read by watcher event handlers +// (including predicate filters such as SecretWatcherFilter). This is safe because +// controller-runtime serializes reconcile calls and predicate evaluations on the same +// controller work queue. If MaxConcurrentReconciles is ever increased above 1, +// a sync.RWMutex must be added here. type WatcherConfig struct { Secrets SecretWatcherConfig ConfigMaps ConfigMapWatcherConfig AnnotatedSecretMapping map[string][]string AnnotatedConfigMapMapping map[string][]string + CredentialHotReload bool + LLMSecretNames map[string]bool } /*** controller internal ***/ diff --git a/internal/controller/watchers/watchers.go b/internal/controller/watchers/watchers.go index 80052d01e..3826dfe97 100644 --- a/internal/controller/watchers/watchers.go +++ b/internal/controller/watchers/watchers.go @@ -266,8 +266,19 @@ func SecretWatcherFilter(r reconciler.Reconciler, ctx context.Context, obj clien // Check 2: Look for watcher annotation (user-provided secrets) if _, exist := annotations[utils.WatcherAnnotationKey]; exist { - // For annotated secrets, determine affected deployments from mapping secretName := obj.GetName() + + // Skip restart for LLM credential secrets when hot-reload is enabled. + // The service re-reads credentials from disk on every request (RFE-9380). + if watcherConfig != nil && watcherConfig.CredentialHotReload { + if watcherConfig.LLMSecretNames[secretName] { + r.GetLogger().Info("Skipping restart for LLM credential secret (hot-reload enabled)", + "secret", secretName) + return + } + } + + // For annotated secrets, determine affected deployments from mapping var affectedDeployments []string var found bool if watcherConfig != nil { diff --git a/internal/controller/watchers/watchers_test.go b/internal/controller/watchers/watchers_test.go index cd940603d..f9cf2e01b 100644 --- a/internal/controller/watchers/watchers_test.go +++ b/internal/controller/watchers/watchers_test.go @@ -359,4 +359,114 @@ var _ = Describe("Watchers", func() { Expect(updated.Spec.Template.Annotations).To(HaveKey(utils.ForceReloadAnnotationKey)) }) }) + + Describe("Credential hot-reload", func() { + It("skips restart for LLM secret when CredentialHotReload is enabled", func() { + cr := utils.GetDefaultOLSConfigCR() + dep := &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: utils.OLSAppServerDeploymentName, + Namespace: utils.OLSNamespaceDefault, + }, + Spec: appsv1.DeploymentSpec{ + Selector: &metav1.LabelSelector{MatchLabels: map[string]string{"app": "ols"}}, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{Labels: map[string]string{"app": "ols"}}, + Spec: corev1.PodSpec{Containers: []corev1.Container{{Name: "c", Image: "img"}}}, + }, + }, + } + r := createTestReconciler(cr, dep) + + wc, _ := r.GetWatcherConfig().(*utils.WatcherConfig) + wc.CredentialHotReload = true + wc.LLMSecretNames = map[string]bool{"test-secret": true} + + sec := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: utils.OLSNamespaceDefault, + Name: "test-secret", + Annotations: map[string]string{utils.WatcherAnnotationKey: "1"}, + }, + Data: map[string][]byte{"apitoken": []byte("rotated-key")}, + } + SecretWatcherFilter(r, ctx, sec, true) + + updated := &appsv1.Deployment{} + Expect(r.Get(ctx, client.ObjectKeyFromObject(dep), updated)).To(Succeed()) + Expect(updated.Spec.Template.Annotations).NotTo(HaveKey(utils.ForceReloadAnnotationKey)) + }) + + It("restarts for LLM secret when CredentialHotReload is disabled", func() { + cr := utils.GetDefaultOLSConfigCR() + dep := &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: utils.OLSAppServerDeploymentName, + Namespace: utils.OLSNamespaceDefault, + }, + Spec: appsv1.DeploymentSpec{ + Selector: &metav1.LabelSelector{MatchLabels: map[string]string{"app": "ols"}}, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{Labels: map[string]string{"app": "ols"}}, + Spec: corev1.PodSpec{Containers: []corev1.Container{{Name: "c", Image: "img"}}}, + }, + }, + } + r := createTestReconciler(cr, dep) + + wc, _ := r.GetWatcherConfig().(*utils.WatcherConfig) + wc.CredentialHotReload = false + wc.LLMSecretNames = map[string]bool{"test-secret": true} + + sec := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: utils.OLSNamespaceDefault, + Name: "test-secret", + Annotations: map[string]string{utils.WatcherAnnotationKey: "1"}, + }, + Data: map[string][]byte{"apitoken": []byte("rotated-key")}, + } + SecretWatcherFilter(r, ctx, sec, true) + + updated := &appsv1.Deployment{} + Expect(r.Get(ctx, client.ObjectKeyFromObject(dep), updated)).To(Succeed()) + Expect(updated.Spec.Template.Annotations).To(HaveKey(utils.ForceReloadAnnotationKey)) + }) + + It("restarts for non-LLM secret even when CredentialHotReload is enabled", func() { + cr := utils.GetDefaultOLSConfigCR() + dep := &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: utils.OLSAppServerDeploymentName, + Namespace: utils.OLSNamespaceDefault, + }, + Spec: appsv1.DeploymentSpec{ + Selector: &metav1.LabelSelector{MatchLabels: map[string]string{"app": "ols"}}, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{Labels: map[string]string{"app": "ols"}}, + Spec: corev1.PodSpec{Containers: []corev1.Container{{Name: "c", Image: "img"}}}, + }, + }, + } + r := createTestReconciler(cr, dep) + + wc, _ := r.GetWatcherConfig().(*utils.WatcherConfig) + wc.CredentialHotReload = true + wc.LLMSecretNames = map[string]bool{"test-secret": true} + + sec := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: utils.OLSNamespaceDefault, + Name: "tls-secret", + Annotations: map[string]string{utils.WatcherAnnotationKey: "1"}, + }, + Data: map[string][]byte{"tls.crt": []byte("cert-data")}, + } + SecretWatcherFilter(r, ctx, sec, true) + + updated := &appsv1.Deployment{} + Expect(r.Get(ctx, client.ObjectKeyFromObject(dep), updated)).To(Succeed()) + Expect(updated.Spec.Template.Annotations).To(HaveKey(utils.ForceReloadAnnotationKey)) + }) + }) }) diff --git a/test/e2e/reconciliation_test.go b/test/e2e/reconciliation_test.go index 7c91d3da1..cec687851 100644 --- a/test/e2e/reconciliation_test.go +++ b/test/e2e/reconciliation_test.go @@ -389,4 +389,59 @@ var _ = Describe("Reconciliation From OLSConfig CR", Ordered, func() { }) + It("should skip app-server restart on LLM secret data rotation when credentialHotReload is enabled", func() { + deployment := &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: AppServerDeploymentName, + Namespace: OLSNameSpace, + }, + } + + By("enable credentialHotReload on the CR") + err = client.Update(cr, func(obj ctrlclient.Object) error { + config := obj.(*olsv1alpha1.OLSConfig) + hotReload := true + config.Spec.OLSConfig.CredentialHotReload = &hotReload + return nil + }) + Expect(err).NotTo(HaveOccurred()) + + // credentialHotReload only affects in-memory WatcherConfig, not the + // Deployment spec, so there is no Generation bump to wait for. + // Give the reconciler time to process the CR update. + time.Sleep(5 * time.Second) + + err = client.Get(deployment) + Expect(err).NotTo(HaveOccurred()) + generation := deployment.Generation + + By("rotate the LLM secret data (using second secret — CR was switched to it by an earlier test)") + llmSecret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: LLMTokenSecondSecretName, + Namespace: OLSNameSpace, + }, + } + err = client.Update(llmSecret, func(obj ctrlclient.Object) error { + s := obj.(*corev1.Secret) + s.Data[LLMApiTokenFileName] = []byte("rotated-key-value") + return nil + }) + Expect(err).NotTo(HaveOccurred()) + + By("verify the deployment generation did NOT change (no restart)") + Consistently(func() int64 { + Expect(client.Get(deployment)).To(Succeed()) + return deployment.Generation + }, 15*time.Second, 2*time.Second).Should(Equal(generation)) + + By("disable credentialHotReload to restore default behavior") + err = client.Update(cr, func(obj ctrlclient.Object) error { + config := obj.(*olsv1alpha1.OLSConfig) + config.Spec.OLSConfig.CredentialHotReload = nil + return nil + }) + Expect(err).NotTo(HaveOccurred()) + }) + })