OLS-3450: Add credentialHotReload flag to skip app-server restart on LLM secret rotation

api/v1alpha1/olsconfig_types.go

@@ -254,6 +254,16 @@ type OLSSpec struct {
 	// +kubebuilder:validation:Optional
 	// +operator-sdk:csv:customresourcedefinitions:type=spec,displayName="Tools Approval Configuration",xDescriptors={"urn:alm:descriptor:com.tectonic.ui:advanced"}
 	ToolsApprovalConfig *ToolsApprovalConfig `json:"toolsApprovalConfig,omitempty"`
+	// Enable in-process credential hot-reload for LLM provider secrets.
+	// When true, the operator will not restart the app-server when LLM credential
+	// secret data is rotated — the service re-reads credentials from disk on each request.
+	// IMPORTANT: Requires lightspeed-service with get_credentials() hot-reload support
+	// (service PR #2955 / RFE-9380). If enabled with an older service image, rotated
+	// credentials (including revoked keys) will remain stale until the pod is manually restarted.
+	// +kubebuilder:default=false
+	// +kubebuilder:validation:Optional
+	// +operator-sdk:csv:customresourcedefinitions:type=spec,displayName="Credential Hot Reload",xDescriptors={"urn:alm:descriptor:com.tectonic.ui:booleanSwitch"}
+	CredentialHotReload *bool `json:"credentialHotReload,omitempty"`
 }
 
 // Persistent Storage Configuration

config/crd/bases/ols.openshift.io_olsconfigs.yaml

@@ -468,6 +468,16 @@ spec:
                         - postgres
                         type: string
                     type: object
+                  credentialHotReload:
+                    default: false
+                    description: |-
+                      Enable in-process credential hot-reload for LLM provider secrets.
+                      When true, the operator will not restart the app-server when LLM credential
+                      secret data is rotated — the service re-reads credentials from disk on each request.
+                      IMPORTANT: Requires lightspeed-service with get_credentials() hot-reload support
+                      (service PR #2955 / RFE-9380). If enabled with an older service image, rotated
+                      credentials (including revoked keys) will remain stale until the pod is manually restarted.
+                    type: boolean
                   defaultModel:
                     description: Default model for usage
                     type: string

docs/credential-hot-reload.md

@@ -0,0 +1,52 @@
+# Credential Hot-Reload (RFE-9380)
+
+## Overview
+
+When `credentialHotReload` is enabled on the OLSConfig CR, the operator skips
+rolling restarts of the app-server pod when LLM credential secret **data**
+changes. The service re-reads credential files from disk on every LLM request,
+so rotated tokens take effect without downtime.
+
+This feature is a companion to
+[lightspeed-service PR #2955](https://github.com/openshift/lightspeed-service/pull/2955),
+which adds the in-process credential reload on the service side.
+
+## Configuration
+
+```yaml
+apiVersion: ols.openshift.io/v1alpha1
+kind: OLSConfig
+metadata:
+  name: cluster
+spec:
+  ols:
+    credentialHotReload: true   # default: false
+```
+
+## Behavior
+
+| Event | `credentialHotReload: false` (default) | `credentialHotReload: true` |
+|---|---|---|
+| LLM secret **data** rotated (same secret name) | Rolling restart | **No restart** — service picks up new credentials on next request |
+| LLM secret **ref** changed in CR (different secret name) | Rolling restart | Rolling restart (deployment volume spec changes) |
+| TLS / MCP / Postgres secret changed | Rolling restart | Rolling restart (unchanged) |
+
+## Prerequisites
+
+- The lightspeed-service image must include the `get_credentials()` hot-reload
+  support (lightspeed-service >= the version containing PR #2955). If an older
+  service image is used with this flag enabled, rotated credentials will not be
+  picked up until the pod is manually restarted.
+
+## How It Works
+
+1. During reconciliation, the operator reads `spec.ols.credentialHotReload` from
+   the OLSConfig CR and stores it in the internal `WatcherConfig`, along with the
+   set of LLM provider secret names.
+
+2. When the secret watcher detects a `.data` change on an annotated secret, it
+   checks whether the secret is an LLM credential and the hot-reload flag is
+   enabled. If both conditions are true, the restart is skipped.
+
+3. Non-LLM secrets (TLS, MCP headers, Postgres) always trigger restarts
+   regardless of the flag.

internal/controller/olsconfig_controller.go

@@ -553,6 +553,12 @@ func (r *OLSConfigReconciler) Reconcile(ctx context.Context, req ctrl.Request) (
 		return ctrl.Result{}, fmt.Errorf("failed to annotate external resources: %w", err)
 	}
 
+	if olsconfig.Spec.OLSConfig.CredentialHotReload != nil && *olsconfig.Spec.OLSConfig.CredentialHotReload {
+		r.Logger.Info("credentialHotReload is enabled — LLM credential secret rotations will not "+
+			"trigger app-server restarts. Requires lightspeed-service with get_credentials() "+
+			"hot-reload support (service PR #2955 / RFE-9380)")
+	}
+
 	// 5. Phase 1: Reconcile independent resources
 	if err := r.reconcileIndependentResources(ctx, olsconfig); err != nil {
 		return ctrl.Result{}, err

internal/controller/olsconfig_helpers.go

@@ -384,6 +384,16 @@ func (r *OLSConfigReconciler) annotateExternalResources(ctx context.Context,
 	if r.WatcherConfig != nil {
 		r.WatcherConfig.AnnotatedConfigMapMapping = make(map[string][]string)
 		r.WatcherConfig.AnnotatedSecretMapping = make(map[string][]string)
+		r.WatcherConfig.LLMSecretNames = make(map[string]bool)
+
+		r.WatcherConfig.CredentialHotReload = cr.Spec.OLSConfig.CredentialHotReload != nil &&
+			*cr.Spec.OLSConfig.CredentialHotReload
+
+		for _, provider := range cr.Spec.LLMConfig.Providers {
+			if provider.CredentialsSecretRef.Name != "" {
+				r.WatcherConfig.LLMSecretNames[provider.CredentialsSecretRef.Name] = true
+			}
+		}
 	}
 
 	var errs []error

internal/controller/utils/types.go

@@ -57,12 +57,19 @@ type ConfigMapWatcherConfig struct {
 	SystemResources []SystemConfigMap
 }
 
-// WatcherConfig contains all watcher configuration
+// WatcherConfig contains all watcher configuration.
+// NOTE: This struct is written by the reconciler and read by watcher event handlers
+// (including predicate filters such as SecretWatcherFilter). This is safe because
+// controller-runtime serializes reconcile calls and predicate evaluations on the same
+// controller work queue. If MaxConcurrentReconciles is ever increased above 1,
+// a sync.RWMutex must be added here.
 type WatcherConfig struct {
 	Secrets                   SecretWatcherConfig
 	ConfigMaps                ConfigMapWatcherConfig
 	AnnotatedSecretMapping    map[string][]string
 	AnnotatedConfigMapMapping map[string][]string
+	CredentialHotReload       bool
+	LLMSecretNames            map[string]bool
 }
 
 /*** controller internal ***/

internal/controller/watchers/watchers.go

@@ -266,8 +266,19 @@ func SecretWatcherFilter(r reconciler.Reconciler, ctx context.Context, obj clien
 
 	// Check 2: Look for watcher annotation (user-provided secrets)
 	if _, exist := annotations[utils.WatcherAnnotationKey]; exist {
-		// For annotated secrets, determine affected deployments from mapping
 		secretName := obj.GetName()
+
+		// Skip restart for LLM credential secrets when hot-reload is enabled.
+		// The service re-reads credentials from disk on every request (RFE-9380).
+		if watcherConfig != nil && watcherConfig.CredentialHotReload {
+			if watcherConfig.LLMSecretNames[secretName] {
+				r.GetLogger().Info("Skipping restart for LLM credential secret (hot-reload enabled)",
+					"secret", secretName)
+				return
+			}
+		}
+
+		// For annotated secrets, determine affected deployments from mapping
 		var affectedDeployments []string
 		var found bool
 		if watcherConfig != nil {

internal/controller/watchers/watchers_test.go

@@ -359,4 +359,114 @@ var _ = Describe("Watchers", func() {
 			Expect(updated.Spec.Template.Annotations).To(HaveKey(utils.ForceReloadAnnotationKey))
 		})
 	})
+
+	Describe("Credential hot-reload", func() {
+		It("skips restart for LLM secret when CredentialHotReload is enabled", func() {
+			cr := utils.GetDefaultOLSConfigCR()
+			dep := &appsv1.Deployment{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:      utils.OLSAppServerDeploymentName,
+					Namespace: utils.OLSNamespaceDefault,
+				},
+				Spec: appsv1.DeploymentSpec{
+					Selector: &metav1.LabelSelector{MatchLabels: map[string]string{"app": "ols"}},
+					Template: corev1.PodTemplateSpec{
+						ObjectMeta: metav1.ObjectMeta{Labels: map[string]string{"app": "ols"}},
+						Spec:       corev1.PodSpec{Containers: []corev1.Container{{Name: "c", Image: "img"}}},
+					},
+				},
+			}
+			r := createTestReconciler(cr, dep)
+
+			wc, _ := r.GetWatcherConfig().(*utils.WatcherConfig)
+			wc.CredentialHotReload = true
+			wc.LLMSecretNames = map[string]bool{"test-secret": true}
+
+			sec := &corev1.Secret{
+				ObjectMeta: metav1.ObjectMeta{
+					Namespace:   utils.OLSNamespaceDefault,
+					Name:        "test-secret",
+					Annotations: map[string]string{utils.WatcherAnnotationKey: "1"},
+				},
+				Data: map[string][]byte{"apitoken": []byte("rotated-key")},
+			}
+			SecretWatcherFilter(r, ctx, sec, true)
+
+			updated := &appsv1.Deployment{}
+			Expect(r.Get(ctx, client.ObjectKeyFromObject(dep), updated)).To(Succeed())
+			Expect(updated.Spec.Template.Annotations).NotTo(HaveKey(utils.ForceReloadAnnotationKey))
+		})
+
+		It("restarts for LLM secret when CredentialHotReload is disabled", func() {
+			cr := utils.GetDefaultOLSConfigCR()
+			dep := &appsv1.Deployment{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:      utils.OLSAppServerDeploymentName,
+					Namespace: utils.OLSNamespaceDefault,
+				},
+				Spec: appsv1.DeploymentSpec{
+					Selector: &metav1.LabelSelector{MatchLabels: map[string]string{"app": "ols"}},
+					Template: corev1.PodTemplateSpec{
+						ObjectMeta: metav1.ObjectMeta{Labels: map[string]string{"app": "ols"}},
+						Spec:       corev1.PodSpec{Containers: []corev1.Container{{Name: "c", Image: "img"}}},
+					},
+				},
+			}
+			r := createTestReconciler(cr, dep)
+
+			wc, _ := r.GetWatcherConfig().(*utils.WatcherConfig)
+			wc.CredentialHotReload = false
+			wc.LLMSecretNames = map[string]bool{"test-secret": true}
+
+			sec := &corev1.Secret{
+				ObjectMeta: metav1.ObjectMeta{
+					Namespace:   utils.OLSNamespaceDefault,
+					Name:        "test-secret",
+					Annotations: map[string]string{utils.WatcherAnnotationKey: "1"},
+				},
+				Data: map[string][]byte{"apitoken": []byte("rotated-key")},
+			}
+			SecretWatcherFilter(r, ctx, sec, true)
+
+			updated := &appsv1.Deployment{}
+			Expect(r.Get(ctx, client.ObjectKeyFromObject(dep), updated)).To(Succeed())
+			Expect(updated.Spec.Template.Annotations).To(HaveKey(utils.ForceReloadAnnotationKey))
+		})
+
+		It("restarts for non-LLM secret even when CredentialHotReload is enabled", func() {
+			cr := utils.GetDefaultOLSConfigCR()
+			dep := &appsv1.Deployment{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:      utils.OLSAppServerDeploymentName,
+					Namespace: utils.OLSNamespaceDefault,
+				},
+				Spec: appsv1.DeploymentSpec{
+					Selector: &metav1.LabelSelector{MatchLabels: map[string]string{"app": "ols"}},
+					Template: corev1.PodTemplateSpec{
+						ObjectMeta: metav1.ObjectMeta{Labels: map[string]string{"app": "ols"}},
+						Spec:       corev1.PodSpec{Containers: []corev1.Container{{Name: "c", Image: "img"}}},
+					},
+				},
+			}
+			r := createTestReconciler(cr, dep)
+
+			wc, _ := r.GetWatcherConfig().(*utils.WatcherConfig)
+			wc.CredentialHotReload = true
+			wc.LLMSecretNames = map[string]bool{"test-secret": true}
+
+			sec := &corev1.Secret{
+				ObjectMeta: metav1.ObjectMeta{
+					Namespace:   utils.OLSNamespaceDefault,
+					Name:        "tls-secret",
+					Annotations: map[string]string{utils.WatcherAnnotationKey: "1"},
+				},
+				Data: map[string][]byte{"tls.crt": []byte("cert-data")},
+			}
+			SecretWatcherFilter(r, ctx, sec, true)
+
+			updated := &appsv1.Deployment{}
+			Expect(r.Get(ctx, client.ObjectKeyFromObject(dep), updated)).To(Succeed())
+			Expect(updated.Spec.Template.Annotations).To(HaveKey(utils.ForceReloadAnnotationKey))
+		})
+	})
 })

test/e2e/reconciliation_test.go

@@ -389,4 +389,59 @@ var _ = Describe("Reconciliation From OLSConfig CR", Ordered, func() {
 
 	})
 
+	It("should skip app-server restart on LLM secret data rotation when credentialHotReload is enabled", func() {
+		deployment := &appsv1.Deployment{
+			ObjectMeta: metav1.ObjectMeta{
+				Name:      AppServerDeploymentName,
+				Namespace: OLSNameSpace,
+			},
+		}
+
+		By("enable credentialHotReload on the CR")
+		err = client.Update(cr, func(obj ctrlclient.Object) error {
+			config := obj.(*olsv1alpha1.OLSConfig)
+			hotReload := true
+			config.Spec.OLSConfig.CredentialHotReload = &hotReload
+			return nil
+		})
+		Expect(err).NotTo(HaveOccurred())
+
+		// credentialHotReload only affects in-memory WatcherConfig, not the
+		// Deployment spec, so there is no Generation bump to wait for.
+		// Give the reconciler time to process the CR update.
+		time.Sleep(5 * time.Second)
+
+		err = client.Get(deployment)
+		Expect(err).NotTo(HaveOccurred())
+		generation := deployment.Generation
+
+		By("rotate the LLM secret data (using second secret — CR was switched to it by an earlier test)")
+		llmSecret := &corev1.Secret{
+			ObjectMeta: metav1.ObjectMeta{
+				Name:      LLMTokenSecondSecretName,
+				Namespace: OLSNameSpace,
+			},
+		}
+		err = client.Update(llmSecret, func(obj ctrlclient.Object) error {
+			s := obj.(*corev1.Secret)
+			s.Data[LLMApiTokenFileName] = []byte("rotated-key-value")
+			return nil
+		})
+		Expect(err).NotTo(HaveOccurred())
+
+		By("verify the deployment generation did NOT change (no restart)")
+		Consistently(func() int64 {
+			Expect(client.Get(deployment)).To(Succeed())
+			return deployment.Generation
+		}, 15*time.Second, 2*time.Second).Should(Equal(generation))
+
+		By("disable credentialHotReload to restore default behavior")
+		err = client.Update(cr, func(obj ctrlclient.Object) error {
+			config := obj.(*olsv1alpha1.OLSConfig)
+			config.Spec.OLSConfig.CredentialHotReload = nil
+			return nil
+		})
+		Expect(err).NotTo(HaveOccurred())
+	})
+
 })