feat: resolve embedding credentials natively from the OS keyring

CHANGELOG.md

@@ -2,6 +2,7 @@
 
 ## Unreleased
 
+- Allow explicit OS keyring credentials for embeddings, with safe initialization retries during capture and unchanged environment-based defaults.
 - Use CrawlKit v0.16.1 to bound live-embedding shutdown when cancellation interrupts retry or completion persistence and cleanup storage is unavailable.
 - Generate deterministic, aggregate-only Discord field notes alongside daily published activity reports, with paired Markdown/JSON artifacts, separate markers preserving legacy narrative notes, explicit timestamp and coverage limits, and filtered-publication cleanup.
 

docs/configuration.md

@@ -89,6 +89,10 @@ enabled = false
 provider = "openai"
 model = "text-embedding-3-small"
 api_key_env = "OPENAI_API_KEY"
+# Optional OS keyring selection (no environment fallback):
+# api_key_source = "keyring"
+# api_key_keyring_service = "discrawl/embeddings"
+# api_key_keyring_account = "api-key"
 dimensions = 512 # optional OpenAI projection; omit for provider default
 batch_size = 64
 max_input_chars = 12000

docs/guides/embeddings.md

@@ -41,6 +41,30 @@ discrawl embed --rebuild --limit 1000
 
 For OpenAI `text-embedding-3-small`, `dimensions` can project vectors to a smaller size. Leave it unset for the provider default, or use a positive value such as `512` to reduce local vector storage. Run `embed --rebuild` after changing it.
 
+## Credentials
+
+By default, credentials come from `api_key_env`; no OS keyring is queried.
+Credential-free local providers keep working with an empty `api_key_env`.
+To use an existing OS keyring item instead, configure:
+
+```toml
+[search.embeddings]
+api_key_source = "keyring"
+api_key_keyring_service = "discrawl/embeddings"
+api_key_keyring_account = "api-key"
+```
+
+Keep your other embedding settings. The service and account identify an existing
+item; the API key itself is never written to the configuration or exported to the
+environment. Keyring selection is explicit and does not fall back to an
+environment variable. `api_key_source = "env"` restores the default behavior.
+
+`tail --embed-live` keeps capturing while a credential is missing, empty, or
+locked. Native background work reports `embedding_provider_configuration` and
+retries initialization after one minute. A pending keyring prompt does not block
+worker cancellation, and only one lookup is outstanding. Once initialized, the
+provider retains its credential until the process restarts.
+
 ## Local provider example
 
 ```toml

internal/cli/admin_commands.go

@@ -13,7 +13,6 @@ import (
 	"syscall"
 	"time"
 
-	"github.com/openclaw/crawlkit/embed"
 	"github.com/openclaw/discrawl/internal/config"
 	"github.com/openclaw/discrawl/internal/discord"
 	"github.com/openclaw/discrawl/internal/discorddesktop"
@@ -539,9 +538,7 @@ func (r *runtime) runEmbed(args []string) error {
 	}
 	providerFactory := r.newEmbed
 	if providerFactory == nil {
-		providerFactory = func(cfg config.EmbeddingsConfig) (embed.Provider, error) {
-			return embed.NewProvider(crawlkitEmbeddingConfig(cfg))
-		}
+		providerFactory = newEmbeddingProvider
 	}
 	provider, err := providerFactory(r.cfg.Search.Embeddings)
 	if err != nil {
@@ -601,7 +598,7 @@ func (r *runtime) runDoctor(args []string) error {
 		report["share_stale_after"] = cfg.Share.StaleAfter
 	}
 	if cfg.Search.Embeddings.Enabled {
-		check := embed.CheckProvider(r.ctx, crawlkitEmbeddingConfig(cfg.Search.Embeddings))
+		check := checkEmbeddingProvider(r.ctx, cfg.Search.Embeddings)
 		report["embeddings"] = check.Status
 		report["embeddings_provider"] = check.Provider
 		report["embeddings_model"] = check.Model

internal/cli/embedding_provider.go

@@ -0,0 +1,60 @@
+package cli
+
+import (
+	"context"
+	"errors"
+	"net"
+	"net/url"
+	"strings"
+
+	"github.com/openclaw/crawlkit/embed"
+	"github.com/openclaw/discrawl/internal/config"
+)
+
+func newEmbeddingProvider(cfg config.EmbeddingsConfig) (embed.Provider, error) {
+	switch strings.ToLower(strings.TrimSpace(cfg.APIKeySource)) {
+	case "", "env":
+		// Preserve Crawlkit's provider-specific optional/required env behavior.
+		return embed.NewProvider(crawlkitEmbeddingConfig(cfg))
+	case "keyring":
+		key, err := config.ResolveEmbeddingKeyringAPIKey(cfg)
+		if err != nil {
+			return nil, err
+		}
+		return embed.NewProvider(crawlkitEmbeddingConfig(cfg), embed.WithAPIKey(key))
+	default:
+		return nil, errors.New("unsupported embedding api_key_source; use env or keyring")
+	}
+}
+
+func checkEmbeddingProvider(ctx context.Context, cfg config.EmbeddingsConfig) embed.CheckResult {
+	if cfg.APIKeySource == "" || cfg.APIKeySource == "env" {
+		return embed.CheckProvider(ctx, crawlkitEmbeddingConfig(cfg))
+	}
+	result := embed.CheckResult{Provider: cfg.Provider, Model: cfg.Model, BaseURL: cfg.BaseURL, Status: "ok"}
+	provider, err := newEmbeddingProvider(cfg)
+	if err != nil {
+		result.Status, result.Warning = "warning", err.Error()
+		return result
+	}
+	// CheckProvider has no per-call credential option. Keep its local-only
+	// probe boundary without exporting a keyring credential into the environment.
+	probe := cfg.Provider == embed.ProviderOllama || cfg.Provider == embed.ProviderLlamaCpp
+	if cfg.Provider == embed.ProviderOpenAICompatible {
+		u, parseErr := url.Parse(cfg.BaseURL)
+		if parseErr == nil {
+			host := u.Hostname()
+			probe = host == "localhost" || net.ParseIP(host).IsLoopback()
+		}
+	}
+	if probe {
+		probeCtx, cancel := context.WithTimeout(ctx, embed.DefaultProbeTimeout)
+		defer cancel()
+		if _, err := provider.Embed(probeCtx, []string{"discrawl probe"}); err != nil {
+			result.Status, result.Warning = "warning", "embedding provider probe failed"
+		} else {
+			result.Probed = true
+		}
+	}
+	return result
+}

internal/cli/embedding_provider_test.go

@@ -0,0 +1,178 @@
+package cli
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"log/slog"
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"path/filepath"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"github.com/openclaw/crawlkit/embed"
+	"github.com/openclaw/crawlkit/worker"
+	"github.com/openclaw/discrawl/internal/config"
+	"github.com/openclaw/discrawl/internal/store"
+	"github.com/openclaw/discrawl/internal/syncer"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"github.com/zalando/go-keyring"
+)
+
+func TestNativeEmbeddingCredentialsReachProviderWithoutEnvironmentExport(t *testing.T) {
+	for _, source := range []string{"", "env", "keyring"} {
+		t.Run(source, func(t *testing.T) {
+			keyring.MockInit()
+			t.Setenv("DISCRAWL_TEST_EMBED_KEY", "environment-value")
+			want := "environment-value"
+			if source == "keyring" {
+				want = "Bot example"
+				require.NoError(t, keyring.Set("test-embeddings", "test-account", want))
+			} else {
+				keyring.MockInitWithError(errors.New("keyring must not be accessed"))
+			}
+			server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+				assert.Equal(t, "Bearer "+want, r.Header.Get("Authorization"))
+				w.Header().Set("Content-Type", "application/json")
+				_, _ = fmt.Fprint(w, `{"model":"fixture","data":[{"index":0,"embedding":[1,2]}]}`)
+			}))
+			defer server.Close()
+			cfg := config.EmbeddingsConfig{Provider: "openai_compatible", Model: "fixture", BaseURL: server.URL, APIKeyEnv: "DISCRAWL_TEST_EMBED_KEY", APIKeySource: source, APIKeyKeyringService: "test-embeddings", APIKeyKeyringAccount: "test-account"}
+			p, err := newEmbeddingProvider(cfg)
+			require.NoError(t, err)
+			_, err = p.Embed(t.Context(), []string{"hello"})
+			require.NoError(t, err)
+			check := checkEmbeddingProvider(t.Context(), cfg)
+			require.Equal(t, "ok", check.Status)
+			require.True(t, check.Probed)
+			require.NotContains(t, fmt.Sprint(check), want)
+			require.Equal(t, "environment-value", os.Getenv("DISCRAWL_TEST_EMBED_KEY"))
+		})
+	}
+}
+
+func TestNativeEmbeddingCredentialFailuresAndRecovery(t *testing.T) {
+	keyring.MockInit()
+	t.Setenv("DISCRAWL_TEST_EMBED_KEY", "environment-must-not-be-fallback")
+	cfg := config.EmbeddingsConfig{Provider: "openai", APIKeySource: "keyring", APIKeyEnv: "DISCRAWL_TEST_EMBED_KEY", APIKeyKeyringService: "test-embeddings", APIKeyKeyringAccount: "test-account"}
+	_, err := newEmbeddingProvider(cfg)
+	require.EqualError(t, err, "embedding keyring credential is unavailable")
+	check := checkEmbeddingProvider(t.Context(), cfg)
+	require.Equal(t, "warning", check.Status)
+	require.NoError(t, keyring.Set("test-embeddings", "test-account", " "))
+	_, err = newEmbeddingProvider(cfg)
+	require.EqualError(t, err, "embedding keyring credential is empty")
+	keyring.MockInitWithError(errors.New("locked: sensitive-material"))
+	_, err = newEmbeddingProvider(cfg)
+	require.EqualError(t, err, "embedding keyring credential is unavailable")
+	keyring.MockInit()
+	require.NoError(t, keyring.Set("test-embeddings", "test-account", "example"))
+	check = checkEmbeddingProvider(t.Context(), cfg)
+	require.Equal(t, "ok", check.Status)
+	require.False(t, check.Probed) // Remote OpenAI checks do not send a paid probe.
+	cfg.APIKeySource = "unsupported"
+	_, err = newEmbeddingProvider(cfg)
+	require.ErrorContains(t, err, "unsupported embedding api_key_source")
+}
+
+func TestNativeEmbeddingCredentialFreeProvider(t *testing.T) {
+	keyring.MockInitWithError(errors.New("keyring must not be accessed"))
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		assert.Empty(t, r.Header.Get("Authorization"))
+		_, _ = fmt.Fprint(w, `{"model":"fixture","data":[{"index":0,"embedding":[1,2]}]}`)
+	}))
+	defer server.Close()
+	p, err := newEmbeddingProvider(config.EmbeddingsConfig{Provider: "openai_compatible", BaseURL: server.URL, Model: "fixture"})
+	require.NoError(t, err)
+	_, err = p.Embed(t.Context(), []string{"hello"})
+	require.NoError(t, err)
+}
+
+func TestDeferredKeyringLookupDoesNotBlockCancellationOrMultiplyLookups(t *testing.T) {
+	entered, release := make(chan struct{}), make(chan struct{})
+	var lookups atomic.Int32
+	p := &deferredEmbeddingProvider{create: func() (embed.Provider, error) {
+		lookups.Add(1)
+		close(entered)
+		<-release
+		return liveCLIProvider(func(context.Context, []string) (embed.EmbeddingBatch, error) {
+			return embed.EmbeddingBatch{Vectors: [][]float32{{1, 2}}}, nil
+		}), nil
+	}}
+	ctx, cancel := context.WithCancel(t.Context())
+	done := make(chan error, 1)
+	go func() { _, err := p.Embed(ctx, []string{"one"}); done <- err }()
+	<-entered
+	cancel()
+	require.ErrorIs(t, <-done, context.Canceled)
+	for range 3 {
+		_, err := p.Embed(ctx, []string{"two"})
+		require.ErrorIs(t, err, context.Canceled)
+	}
+	require.EqualValues(t, 1, lookups.Load())
+	timed, stop := context.WithTimeout(t.Context(), time.Millisecond)
+	defer stop()
+	_, timeoutErr := p.Embed(timed, []string{"timed out"})
+	var failure *worker.Failure
+	require.ErrorAs(t, timeoutErr, &failure)
+	require.True(t, failure.Pause) // A locked prompt must not exhaust job attempts.
+	require.Equal(t, "embedding_provider_configuration", failure.Code)
+	close(release)
+	_, err := p.Embed(t.Context(), []string{"three"})
+	require.NoError(t, err)
+	require.EqualValues(t, 1, lookups.Load())
+}
+
+func TestDeferredCredentialLookupPanicIsSafe(t *testing.T) {
+	p := &deferredEmbeddingProvider{create: func() (embed.Provider, error) { panic("sensitive-material") }}
+	_, err := p.Embed(t.Context(), []string{"hello"})
+	require.ErrorContains(t, err, "embedding_provider_configuration")
+	require.NotContains(t, err.Error(), "sensitive")
+}
+
+func TestTailNativeKeyringRecoveryKeepsCaptureRunning(t *testing.T) {
+	keyring.MockInit()
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		assert.Equal(t, "Bearer example", r.Header.Get("Authorization"))
+		_, _ = fmt.Fprint(w, `{"model":"fixture","data":[{"index":0,"embedding":[1,2]}]}`)
+	}))
+	defer server.Close()
+	ctx, cancel := context.WithTimeout(t.Context(), 80*time.Second)
+	defer cancel()
+	dir := t.TempDir()
+	cfg := config.Default()
+	cfg.DBPath, cfg.CacheDir, cfg.LogDir = filepath.Join(dir, "archive.db"), filepath.Join(dir, "cache"), filepath.Join(dir, "logs")
+	cfg.Share.RepoPath, cfg.Share.AutoUpdate = filepath.Join(dir, "share"), false
+	cfg.Search.Embeddings = config.EmbeddingsConfig{Enabled: true, Provider: "openai_compatible", Model: "fixture", BaseURL: server.URL, APIKeySource: "keyring", APIKeyKeyringService: "recovery-test", APIKeyKeyringAccount: "test-account", BatchSize: 1}
+	path := filepath.Join(dir, "config.toml")
+	require.NoError(t, config.Write(path, cfg))
+	fake := &fakeSyncService{callTailReady: true}
+	rt := tailTestRuntime(ctx, path, fake)
+	rt.newSyncer = func(_ syncer.Client, s *store.Store, _ *slog.Logger) syncService {
+		return &liveCLISync{fakeSyncService: fake, live: func(ctx context.Context) error {
+			require.NoError(t, s.UpsertMessageWithOptions(ctx, store.MessageRecord{ID: "100", GuildID: "g", ChannelID: "c", Content: "first", NormalizedContent: "first"}, store.WriteOptions{EnqueueEmbedding: true}))
+			require.Eventually(t, func() bool {
+				status, err := s.ReadEmbeddingWorkerStatus(ctx)
+				return err == nil && status != nil && status.State == "paused"
+			}, 5*time.Second, 20*time.Millisecond)
+			// The lookup has finished. Restore the stub credential while the
+			// production worker is paused, then allow its normal retry timer.
+			require.NoError(t, keyring.Set("recovery-test", "test-account", "example"))
+			require.NoError(t, s.UpsertMessageWithOptions(ctx, store.MessageRecord{ID: "101", GuildID: "g", ChannelID: "c", Content: "second", NormalizedContent: "second"}, store.WriteOptions{EnqueueEmbedding: true}))
+			var count int
+			require.NoError(t, s.DB().QueryRowContext(ctx, `select count(*) from messages`).Scan(&count))
+			require.Equal(t, 2, count)
+			require.Eventually(t, func() bool {
+				_ = s.DB().QueryRowContext(ctx, `select count(*) from message_embeddings`).Scan(&count)
+				return count == 2
+			}, 65*time.Second, 50*time.Millisecond)
+			return nil
+		}}
+	}
+	require.NoError(t, rt.dispatch([]string{"tail", "--embed-live", "--guild", "g"}))
+	require.Equal(t, 1, fake.tailCalls)
+}

internal/cli/embedding_worker.go

@@ -8,7 +8,6 @@ import (
 
 	"github.com/openclaw/crawlkit/embed"
 	"github.com/openclaw/crawlkit/worker"
-	"github.com/openclaw/discrawl/internal/config"
 	"github.com/openclaw/discrawl/internal/store"
 )
 
@@ -18,17 +17,52 @@ type deferredEmbeddingProvider struct {
 	mu       sync.Mutex
 	provider embed.Provider
 	create   func() (embed.Provider, error)
+	pending  *embeddingProviderAttempt
+}
+
+type embeddingProviderAttempt struct {
+	done     chan struct{}
+	provider embed.Provider
+	err      error
 }
 
 func (p *deferredEmbeddingProvider) Embed(ctx context.Context, inputs []string) (embed.EmbeddingBatch, error) {
 	p.mu.Lock()
 	if p.provider == nil {
-		v, err := p.create()
-		if err != nil || v == nil {
+		if p.pending == nil {
+			attempt := &embeddingProviderAttempt{done: make(chan struct{})}
+			p.pending = attempt
+			// OS keyrings can wait for an interactive unlock. Share one lookup
+			// across workers/retries; cancellation must not wait on the prompt.
+			go func() {
+				defer func() {
+					if recover() != nil {
+						attempt.err = errors.New("embedding provider initialization failed")
+					}
+					close(attempt.done)
+				}()
+				attempt.provider, attempt.err = p.create()
+			}()
+		}
+		attempt := p.pending
+		p.mu.Unlock()
+		select {
+		case <-ctx.Done():
+			if errors.Is(ctx.Err(), context.DeadlineExceeded) {
+				return embed.EmbeddingBatch{}, &worker.Failure{Code: "embedding_provider_configuration", Pause: true, RetryAfter: time.Minute}
+			}
+			return embed.EmbeddingBatch{}, ctx.Err()
+		case <-attempt.done:
+		}
+		p.mu.Lock()
+		if p.pending == attempt {
+			p.pending = nil
+		}
+		if attempt.err != nil || attempt.provider == nil {
 			p.mu.Unlock()
 			return embed.EmbeddingBatch{}, &worker.Failure{Code: "embedding_provider_configuration", Pause: true, RetryAfter: time.Minute}
 		}
-		p.provider = v
+		p.provider = attempt.provider
 	}
 	provider := p.provider
 	p.mu.Unlock()
@@ -38,9 +72,7 @@ func (p *deferredEmbeddingProvider) Embed(ctx context.Context, inputs []string)
 func (r *runtime) runTailWithEmbeddingWorker(ctx context.Context, guilds []string, repair time.Duration) error {
 	create := r.newEmbed
 	if create == nil {
-		create = func(c config.EmbeddingsConfig) (embed.Provider, error) {
-			return embed.NewProvider(crawlkitEmbeddingConfig(c))
-		}
+		create = newEmbeddingProvider
 	}
 	provider := &deferredEmbeddingProvider{create: func() (embed.Provider, error) { return create(r.cfg.Search.Embeddings) }}
 	w, err := r.store.NewEmbeddingWorker(ctx, provider, store.EmbeddingDrainOptions{Provider: r.cfg.Search.Embeddings.Provider, Model: r.cfg.Search.Embeddings.Model, InputVersion: store.EmbeddingInputVersion, MaxInputChars: r.cfg.Search.Embeddings.MaxInputChars, BatchSize: r.cfg.Search.Embeddings.BatchSize, RequestTimeout: mustDuration(r.cfg.Search.Embeddings.RequestTimeout)})