diff --git a/DOCS.md b/DOCS.md index 97286abd4..bc932af6b 100644 --- a/DOCS.md +++ b/DOCS.md @@ -955,7 +955,21 @@ The optional project filter is enforced: an observation owned by another project ### mem_get_observation -Get full untruncated content of a specific observation by ID. +Get the content of a specific observation by ID. + +With no extra parameters this returns the full untruncated body (unchanged). Optional partial-read parameters reduce token cost on large observations. Offsets and lengths are counted in **runes**, not bytes. + +**Mechanical paging — `offset` + `limit`** + +`mem_get_observation(id: 475, offset: 12000, limit: 2000)` returns runes `[offset, offset+limit)`. `limit` without `offset` starts at `0`. `offset` past the end returns an empty window (no error). Default `limit` is 2000. + +Optional partial-read numeric values must be safe integers; negative values are rejected. + +**Match-scoped read — `find` + `context`** + +`mem_get_observation(id: 475, find: "test 7", context: 600)` returns merged windows of `context` runes on each side of every literal match, prefixed with each window's rune offset. Overlapping or adjacent contexts are merged; the header still reports literal occurrence count. `find` with no matches returns zero windows (no error). `context` without `find` is an error. Default `context` is 600. + +`offset`/`limit` and `find`/`context` are mutually exclusive. ### mem_session_summary diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index d6ed5eaa4..6a2792c59 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -61,7 +61,7 @@ Next session starts → Previous session context is injected automatically | `mem_session_summary` | Save end-of-session summary | | `mem_context` | Get recent context from previous sessions | | `mem_timeline` | Chronological context around a specific observation | -| `mem_get_observation` | Get full content of a specific memory | +| `mem_get_observation` | Get observation content by ID (full body, or optional rune-ranged / find windows) | | `mem_save_prompt` | Save a user prompt for future context | | `mem_stats` | Memory system statistics | | `mem_session_start` | Register a session start | @@ -85,7 +85,7 @@ Token-efficient memory retrieval — don't dump everything, drill in: ``` 1. mem_search "auth middleware" → compact results with IDs (~100 tokens each) 2. mem_timeline observation_id=42 → what happened before/after in that session -3. mem_get_observation id=42 → full untruncated content +3. mem_get_observation id=42 → full body, or offset/limit / find/context windows ``` --- diff --git a/internal/mcp/mcp.go b/internal/mcp/mcp.go index 29bc15dc6..d713f5189 100644 --- a/internal/mcp/mcp.go +++ b/internal/mcp/mcp.go @@ -18,8 +18,10 @@ import ( "encoding/json" "errors" "fmt" + "math" "os" "path/filepath" + "strconv" "strings" "time" @@ -660,7 +662,7 @@ Examples: if shouldRegister("mem_get_observation", allowlist) { srv.AddTool( mcp.NewTool("mem_get_observation", - mcp.WithDescription("Get the full content of a specific observation by ID. Use when you need the complete, untruncated content of an observation found via mem_search or mem_timeline."), + mcp.WithDescription("Get the content of a specific observation by ID. Omit partial-read params for the full untruncated body. Use offset/limit for a rune-ranged slice, or find/context for merged windows around literal occurrences. The find header reports occurrences, not window count. Optional numeric inputs must be safe integers; negative values are rejected. The two groups are mutually exclusive."), mcp.WithTitleAnnotation("Get Observation"), mcp.WithReadOnlyHintAnnotation(true), mcp.WithDestructiveHintAnnotation(false), @@ -670,6 +672,18 @@ Examples: mcp.Required(), mcp.Description("The observation ID to retrieve"), ), + mcp.WithNumber("offset", + mcp.Description("Safe-integer rune offset for a ranged read. Negative values are rejected. Mutually exclusive with find/context. Omit with no other partial-read params to return the full body."), + ), + mcp.WithNumber("limit", + mcp.Description("Safe-integer rune length for a ranged read (default 2000). Negative values are rejected. Limit without offset starts at offset 0. Mutually exclusive with find/context."), + ), + mcp.WithString("find", + mcp.Description("Literal substring to locate inside the observation. Returns merged windows around overlapping or adjacent contexts; the header reports literal occurrences. Mutually exclusive with offset/limit."), + ), + mcp.WithNumber("context", + mcp.Description("Safe-integer rune padding on each side of every find match (default 600). Negative values are rejected. Overlapping or adjacent contexts merge; the header reports literal occurrences. Requires find."), + ), ), handleGetObservation(s, cfg, activity), ) @@ -1978,12 +1992,21 @@ func handleGetObservation(s *store.Store, cfg MCPConfig, activities ...*SessionA if id == 0 { return mcp.NewToolResultError("id is required"), nil } + readReq, requestErr := observationReadRequest(req) + if requestErr != nil { + return mcp.NewToolResultError(requestErr.Error()), nil + } obs, err := s.GetObservation(id) if err != nil { return mcp.NewToolResultError(fmt.Sprintf("Observation #%d not found", id)), nil } + read, readErr := store.ResolveObservationRead(obs.Content, readReq) + if readErr != nil { + return mcp.NewToolResultError(readErr.Error()), nil + } + // Resolve project from process override/cwd (REQ-310, REQ-314). No per-call // override is possible for get-by-ID. detRes, detErr := resolveReadProjectWithProcessOverride(s, "", cfg.DefaultProject) @@ -2003,14 +2026,13 @@ func handleGetObservation(s *store.Store, cfg MCPConfig, activities ...*SessionA } duplicateMeta := fmt.Sprintf("\nDuplicates: %d", obs.DuplicateCount) revisionMeta := fmt.Sprintf("\nRevisions: %d", obs.RevisionCount) - - result := fmt.Sprintf("#%d [%s] %s\n%s\nSession: %s%s%s\nCreated: %s", - obs.ID, obs.Type, obs.Title, - obs.Content, + meta := fmt.Sprintf("Session: %s%s%s\nCreated: %s", obs.SessionID, obsProject+scope+topic, toolName+duplicateMeta+revisionMeta, timeutil.FormatLocal(obs.CreatedAt), ) + result := formatGetObservationResult(obs, read, meta) + if detErr != nil { return readProjectErrorResult(activity, detRes, detErr), nil } @@ -2018,6 +2040,91 @@ func handleGetObservation(s *store.Store, cfg MCPConfig, activities ...*SessionA } } +func observationReadRequest(req mcp.CallToolRequest) (store.ObservationReadRequest, error) { + args := req.GetArguments() + out := store.ObservationReadRequest{} + var err error + if out.Offset, err = optionalSafeIntArgument(args, "offset"); err != nil { + return store.ObservationReadRequest{}, err + } + if out.Limit, err = optionalSafeIntArgument(args, "limit"); err != nil { + return store.ObservationReadRequest{}, err + } + if v, ok := args["find"].(string); ok { + out.Find = &v + } + if out.Context, err = optionalSafeIntArgument(args, "context"); err != nil { + return store.ObservationReadRequest{}, err + } + return out, nil +} + +func optionalSafeIntArgument(args map[string]any, name string) (*int, error) { + raw, present := args[name] + if !present { + return nil, nil + } + v, ok := raw.(float64) + if !ok { + return nil, fmt.Errorf("%s must be a safe integer", name) + } + const maxSafeMCPInteger = (1 << 53) - 1 + maxInt := int(^uint(0) >> 1) + minInt := -maxInt - 1 + if math.IsNaN(v) || math.IsInf(v, 0) || math.Trunc(v) != v || v < -maxSafeMCPInteger || v > maxSafeMCPInteger || v < float64(minInt) || v > float64(maxInt) { + return nil, fmt.Errorf("%s must be a safe integer", name) + } + n := int(v) + return &n, nil +} + +func formatGetObservationResult(obs *store.Observation, read store.ObservationReadResult, meta string) string { + if read.Mode == store.ObservationReadFull { + return fmt.Sprintf("#%d [%s] %s\n%s\n%s", + obs.ID, obs.Type, obs.Title, read.Content, meta) + } + + header := fmt.Sprintf("#%d %q — %s", obs.ID, obs.Title, formatRuneCount(read.TotalRunes)) + var b strings.Builder + if read.Mode == store.ObservationReadFind { + fmt.Fprintf(&b, "%s, %d matches for %q\n", header, read.MatchCount, read.Find) + for _, w := range read.Windows { + fmt.Fprintf(&b, "\n[offset %s]\n%s\n", formatIntWithComma(w.Offset), w.Content) + } + } else { + fmt.Fprintf(&b, "%s\n\n[offset %s, limit %s]\n%s\n", + header, formatIntWithComma(read.Offset), formatIntWithComma(read.Limit), read.Content) + } + b.WriteString("\n") + b.WriteString(meta) + return b.String() +} + +func formatRuneCount(n int) string { + return formatIntWithComma(n) + " runes total" +} + +func formatIntWithComma(n int) string { + if n < 0 { + return "-" + formatIntWithComma(-n) + } + s := strconv.Itoa(n) + if len(s) <= 3 { + return s + } + pre := len(s) % 3 + if pre == 0 { + pre = 3 + } + var b strings.Builder + b.WriteString(s[:pre]) + for i := pre; i < len(s); i += 3 { + b.WriteByte(',') + b.WriteString(s[i : i+3]) + } + return b.String() +} + // handleSessionSummary returns a tool handler function that saves a comprehensive // end-of-session summary memory. It supports explicit project override matching // the precedence of mem_save. diff --git a/internal/mcp/observation_partial_test.go b/internal/mcp/observation_partial_test.go new file mode 100644 index 000000000..34a1575b0 --- /dev/null +++ b/internal/mcp/observation_partial_test.go @@ -0,0 +1,294 @@ +package mcp + +import ( + "context" + "encoding/json" + "math" + "strconv" + "strings" + "testing" + + mcppkg "github.com/mark3labs/mcp-go/mcp" + + "github.com/Gentleman-Programming/engram/v2/internal/store" +) + +func seedObservation(t *testing.T, s *store.Store, title, content string) int64 { + t.Helper() + if err := s.CreateSession("s-partial", "engram", "/tmp/engram"); err != nil { + t.Fatalf("create session: %v", err) + } + id, err := s.AddObservation(store.AddObservationParams{ + SessionID: "s-partial", + Type: "architecture", + Title: title, + Content: content, + Project: "engram", + }) + if err != nil { + t.Fatalf("add observation: %v", err) + } + return id +} + +func callGetObservation(t *testing.T, s *store.Store, args map[string]any) string { + t.Helper() + res, err := handleGetObservation(s, MCPConfig{})(context.Background(), mcppkg.CallToolRequest{ + Params: mcppkg.CallToolParams{Arguments: args}, + }) + if err != nil { + t.Fatalf("handler error: %v", err) + } + if res.IsError { + t.Fatalf("tool error: %s", callResultText(t, res)) + } + return observationResultText(t, callResultText(t, res)) +} + +func observationResultText(t *testing.T, raw string) string { + t.Helper() + var env map[string]any + if err := json.Unmarshal([]byte(raw), &env); err != nil { + return raw + } + result, _ := env["result"].(string) + if result == "" { + return raw + } + return result +} + +func TestHandleGetObservationNoParamsUnchanged(t *testing.T) { + s := newMCPTestStore(t) + id := seedObservation(t, s, "Full body", "complete text") + text := callGetObservation(t, s, map[string]any{"id": float64(id)}) + if !strings.Contains(text, "#"+strconv.FormatInt(id, 10)+" [architecture] Full body") { + t.Fatalf("expected full header, got %q", text) + } + if !strings.Contains(text, "complete text") { + t.Fatalf("expected full content, got %q", text) + } +} + +func TestHandleGetObservationLimitWithoutOffset(t *testing.T) { + s := newMCPTestStore(t) + id := seedObservation(t, s, "design Part A", "abcdefghij") + text := callGetObservation(t, s, map[string]any{ + "id": float64(id), + "limit": float64(4), + }) + if !strings.Contains(text, "[offset 0, limit 4]") { + t.Fatalf("expected offset 0 for limit-only, got %q", text) + } + if !strings.Contains(text, "abcd") { + t.Fatalf("expected leading slice, got %q", text) + } +} + +func TestHandleGetObservationRange(t *testing.T) { + s := newMCPTestStore(t) + id := seedObservation(t, s, "design Part A", "abcdefghij") + text := callGetObservation(t, s, map[string]any{ + "id": float64(id), + "offset": float64(2), + "limit": float64(3), + }) + if !strings.Contains(text, ` — 10 runes total`) { + t.Fatalf("expected rune total, got %q", text) + } + if !strings.Contains(text, "[offset 2, limit 3]") { + t.Fatalf("expected range marker, got %q", text) + } + if !strings.Contains(text, "cde") { + t.Fatalf("expected ranged slice, got %q", text) + } +} + +func TestHandleGetObservationFindWindows(t *testing.T) { + s := newMCPTestStore(t) + id := seedObservation(t, s, "design Part A", "aaTESTyyyyyTESTcc") + text := callGetObservation(t, s, map[string]any{ + "id": float64(id), + "find": "TEST", + "context": float64(2), + }) + if !strings.Contains(text, "2 matches for") { + t.Fatalf("expected match count, got %q", text) + } + if !strings.Contains(text, "[offset 0]") || !strings.Contains(text, "[offset 9]") { + t.Fatalf("expected window offsets, got %q", text) + } + if strings.Contains(text, "yyyyTEST") { + t.Fatalf("expected distant text to remain outside both windows, got %q", text) + } +} + +func TestHandleGetObservationFindMergesOverlappingOccurrences(t *testing.T) { + s := newMCPTestStore(t) + id := seedObservation(t, s, "overlap", "aaaaa") + text := callGetObservation(t, s, map[string]any{ + "id": float64(id), + "find": "aa", + "context": float64(0), + }) + if !strings.Contains(text, "4 matches for") { + t.Fatalf("expected overlapping occurrence count, got %q", text) + } + if strings.Count(text, "[offset ") != 1 || strings.Count(text, "aaaaa") != 1 { + t.Fatalf("expected one merged window with one body, got %q", text) + } +} + +func TestHandleGetObservationFindDefaultContextDoesNotDuplicateContent(t *testing.T) { + s := newMCPTestStore(t) + find := "needle" + content := strings.Repeat("a", store.DefaultFindContext+100) + find + strings.Repeat("b", 100) + find + strings.Repeat("c", store.DefaultFindContext+100) + id := seedObservation(t, s, "repeated", content) + text := callGetObservation(t, s, map[string]any{ + "id": float64(id), + "find": find, + }) + if !strings.Contains(text, "2 matches for") { + t.Fatalf("expected occurrence count, got %q", text) + } + if strings.Count(text, "[offset ") != 1 || strings.Count(text, find) != 3 { + t.Fatalf("expected one merged body containing each match once, got %q", text) + } +} + +func TestHandleGetObservationFindNotFound(t *testing.T) { + s := newMCPTestStore(t) + id := seedObservation(t, s, "design Part A", "nope") + text := callGetObservation(t, s, map[string]any{ + "id": float64(id), + "find": "TEST", + }) + if !strings.Contains(text, "0 matches for") { + t.Fatalf("expected zero matches, got %q", text) + } +} + +func TestHandleGetObservationOffsetPastEnd(t *testing.T) { + s := newMCPTestStore(t) + id := seedObservation(t, s, "short", "abc") + text := callGetObservation(t, s, map[string]any{ + "id": float64(id), + "offset": float64(50), + }) + if !strings.Contains(text, "[offset 50, limit 2,000]") { + t.Fatalf("expected empty past-end range, got %q", text) + } +} + +func TestHandleGetObservationMutuallyExclusive(t *testing.T) { + s := newMCPTestStore(t) + id := seedObservation(t, s, "x", "body") + res, err := handleGetObservation(s, MCPConfig{})(context.Background(), mcppkg.CallToolRequest{ + Params: mcppkg.CallToolParams{Arguments: map[string]any{ + "id": float64(id), + "offset": float64(0), + "find": "body", + }}, + }) + if err != nil { + t.Fatalf("handler error: %v", err) + } + if !res.IsError { + t.Fatal("expected mutually exclusive error") + } + if !strings.Contains(callResultText(t, res), "mutually exclusive") { + t.Fatalf("error = %q", callResultText(t, res)) + } +} + +func TestHandleGetObservationContextRequiresFind(t *testing.T) { + s := newMCPTestStore(t) + id := seedObservation(t, s, "x", "body") + res, err := handleGetObservation(s, MCPConfig{})(context.Background(), mcppkg.CallToolRequest{ + Params: mcppkg.CallToolParams{Arguments: map[string]any{ + "id": float64(id), + "context": float64(10), + }}, + }) + if err != nil { + t.Fatalf("handler error: %v", err) + } + if !res.IsError || !strings.Contains(callResultText(t, res), "context requires find") { + t.Fatalf("error = %q", callResultText(t, res)) + } +} + +func TestHandleGetObservationRangeUsesRunesNotBytes(t *testing.T) { + s := newMCPTestStore(t) + id := seedObservation(t, s, "utf8", "aé😊z") + text := callGetObservation(t, s, map[string]any{ + "id": float64(id), + "offset": float64(1), + "limit": float64(2), + }) + if !strings.Contains(text, "é😊") { + t.Fatalf("expected rune slice, got %q", text) + } +} + +func TestHandleGetObservationRejectsUnsafeOptionalNumbers(t *testing.T) { + s := newMCPTestStore(t) + id := seedObservation(t, s, "numbers", "abcdef") + cases := []struct { + name string + args map[string]any + want string + }{ + {"fractional offset", map[string]any{"id": float64(id), "offset": 1.5}, "offset must be a safe integer"}, + {"fractional limit", map[string]any{"id": float64(id), "limit": 1.5}, "limit must be a safe integer"}, + {"fractional context", map[string]any{"id": float64(id), "find": "cd", "context": 1.5}, "context must be a safe integer"}, + {"out of range offset", map[string]any{"id": float64(id), "offset": math.MaxFloat64}, "offset must be a safe integer"}, + {"out of range limit", map[string]any{"id": float64(id), "limit": math.MaxFloat64}, "limit must be a safe integer"}, + {"out of range context", map[string]any{"id": float64(id), "find": "cd", "context": math.MaxFloat64}, "context must be a safe integer"}, + {"nan offset", map[string]any{"id": float64(id), "offset": math.NaN()}, "offset must be a safe integer"}, + {"infinite limit", map[string]any{"id": float64(id), "limit": math.Inf(1)}, "limit must be a safe integer"}, + {"wrong type offset", map[string]any{"id": float64(id), "offset": "1"}, "offset must be a safe integer"}, + {"wrong type limit", map[string]any{"id": float64(id), "limit": true}, "limit must be a safe integer"}, + {"wrong type context", map[string]any{"id": float64(id), "find": "cd", "context": "1"}, "context must be a safe integer"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + res, err := handleGetObservation(s, MCPConfig{})(context.Background(), mcppkg.CallToolRequest{ + Params: mcppkg.CallToolParams{Arguments: tc.args}, + }) + if err != nil { + t.Fatalf("handler error: %v", err) + } + if !res.IsError || !strings.Contains(callResultText(t, res), tc.want) { + t.Fatalf("result = %q, want tool error containing %q", callResultText(t, res), tc.want) + } + }) + } +} + +func TestHandleGetObservationNegativeOptionalNumbersUseStoreValidation(t *testing.T) { + s := newMCPTestStore(t) + id := seedObservation(t, s, "numbers", "abcdef") + cases := []struct { + name string + args map[string]any + want string + }{ + {"offset", map[string]any{"id": float64(id), "offset": float64(-1)}, "offset must be >= 0"}, + {"limit", map[string]any{"id": float64(id), "limit": float64(-1)}, "limit must be >= 0"}, + {"context", map[string]any{"id": float64(id), "find": "cd", "context": float64(-1)}, "context must be >= 0"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + res, err := handleGetObservation(s, MCPConfig{})(context.Background(), mcppkg.CallToolRequest{ + Params: mcppkg.CallToolParams{Arguments: tc.args}, + }) + if err != nil { + t.Fatalf("handler error: %v", err) + } + if !res.IsError || !strings.Contains(callResultText(t, res), tc.want) { + t.Fatalf("result = %q, want store validation %q", callResultText(t, res), tc.want) + } + }) + } +} diff --git a/internal/mcp/testdata/tool-contract-v1.json b/internal/mcp/testdata/tool-contract-v1.json index edd9aeaed..13142dc65 100644 --- a/internal/mcp/testdata/tool-contract-v1.json +++ b/internal/mcp/testdata/tool-contract-v1.json @@ -48,7 +48,11 @@ "mem_get_observation": { "type": ["object"], "properties": { - "id": {"type":["number"],"additionalProperties":true} + "context": {"type":["number"],"additionalProperties":true}, + "find": {"type":["string"],"additionalProperties":true}, + "id": {"type":["number"],"additionalProperties":true}, + "limit": {"type":["number"],"additionalProperties":true}, + "offset": {"type":["number"],"additionalProperties":true} }, "required": ["id"], "additionalProperties": true}, diff --git a/internal/store/observation_partial.go b/internal/store/observation_partial.go new file mode 100644 index 000000000..f4dcbc7e8 --- /dev/null +++ b/internal/store/observation_partial.go @@ -0,0 +1,201 @@ +package store + +const ( + // DefaultPartialReadLimit is the rune window used when limit is omitted + // from a ranged mem_get_observation read. + DefaultPartialReadLimit = 2000 + // DefaultFindContext is the rune padding on each side of a find match + // when context is omitted. + DefaultFindContext = 600 +) + +// ObservationReadMode is how mem_get_observation selects content. +type ObservationReadMode int + +const ( + ObservationReadFull ObservationReadMode = iota + ObservationReadRange + ObservationReadFind +) + +// ObservationReadRequest is the optional partial-read contract for GetObservation. +// Pointers distinguish omitted values from explicit zeros. +type ObservationReadRequest struct { + Offset *int + Limit *int + Find *string + Context *int +} + +// ObservationWindow is one rune-indexed slice of observation content. +type ObservationWindow struct { + Offset int + Content string +} + +// ObservationReadResult is the resolved partial or full read of one body. +type ObservationReadResult struct { + Mode ObservationReadMode + TotalRunes int + Offset int + Limit int + Find string + MatchCount int + Windows []ObservationWindow + Content string +} + +func (r ObservationReadRequest) rangeRequested() bool { + return r.Offset != nil || r.Limit != nil +} + +func (r ObservationReadRequest) findRequested() bool { + return r.Find != nil || r.Context != nil +} + +// ResolveObservationRead applies the issue #812 validation and rune-indexed +// slice rules to a stored body. Offsets and limits are counted in runes. +func ResolveObservationRead(content string, req ObservationReadRequest) (ObservationReadResult, error) { + runes := []rune(content) + result := ObservationReadResult{TotalRunes: len(runes)} + + if req.rangeRequested() && req.findRequested() { + return ObservationReadResult{}, ErrPartialReadModesExclusive + } + if req.Context != nil && req.Find == nil { + return ObservationReadResult{}, ErrContextRequiresFind + } + if req.Offset != nil && *req.Offset < 0 { + return ObservationReadResult{}, ErrPartialReadOffsetNegative + } + if req.Limit != nil && *req.Limit < 0 { + return ObservationReadResult{}, ErrPartialReadLimitNegative + } + if req.Context != nil && *req.Context < 0 { + return ObservationReadResult{}, ErrPartialReadContextNegative + } + + if !req.rangeRequested() && !req.findRequested() { + result.Mode = ObservationReadFull + result.Content = content + return result, nil + } + + if req.findRequested() { + result.Mode = ObservationReadFind + find := "" + if req.Find != nil { + find = *req.Find + } + contextRunes := DefaultFindContext + if req.Context != nil { + contextRunes = *req.Context + } + result.Find = find + result.Windows, result.MatchCount = findContentWindows(runes, find, contextRunes) + return result, nil + } + + offset := 0 + if req.Offset != nil { + offset = *req.Offset + } + limit := DefaultPartialReadLimit + if req.Limit != nil { + limit = *req.Limit + } + result.Mode = ObservationReadRange + result.Offset = offset + result.Limit = limit + result.Content = sliceRunes(runes, offset, limit) + result.Windows = []ObservationWindow{{Offset: offset, Content: result.Content}} + return result, nil +} + +func sliceRunes(runes []rune, offset, limit int) string { + if offset >= len(runes) || limit == 0 { + return "" + } + if limit >= len(runes)-offset { + return string(runes[offset:]) + } + return string(runes[offset : offset+limit]) +} + +type observationInterval struct { + start int + end int +} + +func findContentWindows(runes []rune, find string, contextRunes int) ([]ObservationWindow, int) { + needle := []rune(find) + if len(needle) == 0 { + return nil, 0 + } + prefix := runePrefixTable(needle) + merged := make([]observationInterval, 0) + matchCount := 0 + matched := 0 + for i, r := range runes { + for matched > 0 && r != needle[matched] { + matched = prefix[matched-1] + } + if r == needle[matched] { + matched++ + } + if matched != len(needle) { + continue + } + + matchCount++ + matchStart := i - len(needle) + 1 + matchEnd := i + 1 + start := 0 + if contextRunes < matchStart { + start = matchStart - contextRunes + } + end := matchEnd + if contextRunes >= len(runes)-end { + end = len(runes) + } else { + end += contextRunes + } + appendMergedObservationInterval(&merged, observationInterval{start: start, end: end}) + matched = prefix[matched-1] + } + + windows := make([]ObservationWindow, 0, len(merged)) + for _, interval := range merged { + windows = append(windows, ObservationWindow{ + Offset: interval.start, + Content: string(runes[interval.start:interval.end]), + }) + } + return windows, matchCount +} + +func appendMergedObservationInterval(merged *[]observationInterval, interval observationInterval) { + last := len(*merged) - 1 + if last < 0 || interval.start > (*merged)[last].end { + *merged = append(*merged, interval) + return + } + if interval.end > (*merged)[last].end { + (*merged)[last].end = interval.end + } +} + +func runePrefixTable(needle []rune) []int { + prefix := make([]int, len(needle)) + matched := 0 + for i := 1; i < len(needle); i++ { + for matched > 0 && needle[i] != needle[matched] { + matched = prefix[matched-1] + } + if needle[i] == needle[matched] { + matched++ + } + prefix[i] = matched + } + return prefix +} diff --git a/internal/store/observation_partial_test.go b/internal/store/observation_partial_test.go new file mode 100644 index 000000000..cee4679aa --- /dev/null +++ b/internal/store/observation_partial_test.go @@ -0,0 +1,269 @@ +package store + +import ( + "errors" + "strings" + "testing" + "unicode/utf8" +) + +func TestResolveObservationReadFullUnchanged(t *testing.T) { + content := "plain body" + got, err := ResolveObservationRead(content, ObservationReadRequest{}) + if err != nil { + t.Fatalf("ResolveObservationRead: %v", err) + } + if got.Mode != ObservationReadFull || got.Content != content { + t.Fatalf("full read = %+v", got) + } + if got.TotalRunes != len([]rune(content)) { + t.Fatalf("total runes = %d", got.TotalRunes) + } +} + +func TestResolveObservationReadRangeRuneSlice(t *testing.T) { + content := "aé😊z" // runes: a é 😊 z + offset := 1 + limit := 2 + got, err := ResolveObservationRead(content, ObservationReadRequest{Offset: &offset, Limit: &limit}) + if err != nil { + t.Fatalf("ResolveObservationRead: %v", err) + } + if got.Content != "é😊" { + t.Fatalf("range content = %q", got.Content) + } + if !utf8.ValidString(got.Content) { + t.Fatal("range slice split a multi-byte rune") + } +} + +func TestResolveObservationReadLimitWithoutOffsetStartsAtZero(t *testing.T) { + content := "abcdefghij" + limit := 3 + got, err := ResolveObservationRead(content, ObservationReadRequest{Limit: &limit}) + if err != nil { + t.Fatalf("ResolveObservationRead: %v", err) + } + if got.Offset != 0 || got.Content != "abc" { + t.Fatalf("limit-only read = %+v", got) + } +} + +func TestResolveObservationReadOffsetPastEndIsEmpty(t *testing.T) { + content := "abc" + offset := 10 + got, err := ResolveObservationRead(content, ObservationReadRequest{Offset: &offset}) + if err != nil { + t.Fatalf("ResolveObservationRead: %v", err) + } + if got.Content != "" { + t.Fatalf("past-end content = %q", got.Content) + } + if got.Limit != DefaultPartialReadLimit { + t.Fatalf("default limit = %d", got.Limit) + } +} + +func TestResolveObservationReadFindWindows(t *testing.T) { + content := "xxTESTyyyyyTESTzz" + find := "TEST" + contextRunes := 2 + got, err := ResolveObservationRead(content, ObservationReadRequest{Find: &find, Context: &contextRunes}) + if err != nil { + t.Fatalf("ResolveObservationRead: %v", err) + } + if len(got.Windows) != 2 { + t.Fatalf("windows = %d", len(got.Windows)) + } + if got.Windows[0].Offset != 0 || got.Windows[0].Content != "xxTESTyy" { + t.Fatalf("first window = %+v", got.Windows[0]) + } + if got.Windows[1].Offset != 9 || got.Windows[1].Content != "yyTESTzz" { + t.Fatalf("second window = %+v", got.Windows[1]) + } +} + +func TestResolveObservationReadFindMergesOverlappingOccurrences(t *testing.T) { + content := "aaaaa" + find := "aa" + contextRunes := 0 + got, err := ResolveObservationRead(content, ObservationReadRequest{Find: &find, Context: &contextRunes}) + if err != nil { + t.Fatalf("ResolveObservationRead: %v", err) + } + if got.MatchCount != 4 { + t.Fatalf("match count = %d, want 4", got.MatchCount) + } + if len(got.Windows) != 1 { + t.Fatalf("windows = %d, want 1", len(got.Windows)) + } + if got.Windows[0].Offset != 0 || got.Windows[0].Content != content { + t.Fatalf("merged window = %+v", got.Windows[0]) + } +} + +func TestResolveObservationReadFindMergesAdjacentContexts(t *testing.T) { + find := "a" + contextRunes := 1 + got, err := ResolveObservationRead("abca", ObservationReadRequest{Find: &find, Context: &contextRunes}) + if err != nil { + t.Fatalf("ResolveObservationRead: %v", err) + } + if got.MatchCount != 2 || len(got.Windows) != 1 { + t.Fatalf("matches/windows = %d/%d, want 2/1", got.MatchCount, len(got.Windows)) + } + if got.Windows[0].Offset != 0 || got.Windows[0].Content != "abca" { + t.Fatalf("merged adjacent window = %+v", got.Windows[0]) + } +} + +func TestResolveObservationReadFindDefaultContextDoesNotDuplicateContent(t *testing.T) { + find := "needle" + content := strings.Repeat("a", DefaultFindContext+100) + find + strings.Repeat("b", 100) + find + strings.Repeat("c", DefaultFindContext+100) + got, err := ResolveObservationRead(content, ObservationReadRequest{Find: &find}) + if err != nil { + t.Fatalf("ResolveObservationRead: %v", err) + } + if got.MatchCount != 2 || len(got.Windows) != 1 { + t.Fatalf("matches/windows = %d/%d, want 2/1", got.MatchCount, len(got.Windows)) + } + windowRunes := 0 + for _, window := range got.Windows { + windowRunes += len([]rune(window.Content)) + } + if windowRunes > len([]rune(content)) { + t.Fatalf("window runes = %d, source runes = %d", windowRunes, len([]rune(content))) + } +} + +func TestResolveObservationReadFindWindowRunesStayBoundedAcrossIndependentWindows(t *testing.T) { + find := "needle" + contextRunes := 10 + content := strings.Repeat("a", 20) + find + strings.Repeat("b", 2_000) + find + strings.Repeat("c", 20) + got, err := ResolveObservationRead(content, ObservationReadRequest{Find: &find, Context: &contextRunes}) + if err != nil { + t.Fatalf("ResolveObservationRead: %v", err) + } + if got.MatchCount != 2 || len(got.Windows) != 2 { + t.Fatalf("matches/windows = %d/%d, want 2/2", got.MatchCount, len(got.Windows)) + } + windowRunes := 0 + for _, window := range got.Windows { + windowRunes += len([]rune(window.Content)) + } + if windowRunes > len([]rune(content)) { + t.Fatalf("window runes = %d, source runes = %d", windowRunes, len([]rune(content))) + } +} + +func TestResolveObservationReadFindLargeNearMatch(t *testing.T) { + find := strings.Repeat("a", 5_000) + "b" + content := strings.Repeat("a", 10_000) + "b" + contextRunes := 0 + got, err := ResolveObservationRead(content, ObservationReadRequest{Find: &find, Context: &contextRunes}) + if err != nil { + t.Fatalf("ResolveObservationRead: %v", err) + } + if got.MatchCount != 1 || len(got.Windows) != 1 { + t.Fatalf("matches/windows = %d/%d, want 1/1", got.MatchCount, len(got.Windows)) + } + if got.Windows[0].Offset != 5_000 || got.Windows[0].Content != find { + t.Fatalf("near-match window = %+v", got.Windows[0]) + } +} + +func TestResolveObservationReadHugeLimitsClampToContent(t *testing.T) { + huge := int(^uint(0) >> 1) + cases := []struct { + name string + req ObservationReadRequest + }{ + {"range limit", ObservationReadRequest{Limit: &huge}}, + {"find context", func() ObservationReadRequest { + find := "cd" + return ObservationReadRequest{Find: &find, Context: &huge} + }()}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + got, err := ResolveObservationRead("abcdef", tc.req) + if err != nil { + t.Fatalf("ResolveObservationRead: %v", err) + } + if tc.req.Find == nil && got.Content != "abcdef" { + t.Fatalf("range content = %q", got.Content) + } + if tc.req.Find != nil && (len(got.Windows) != 1 || got.Windows[0].Content != "abcdef") { + t.Fatalf("find windows = %+v", got.Windows) + } + }) + } +} + +func TestResolveObservationReadFindNotFound(t *testing.T) { + find := "missing" + got, err := ResolveObservationRead("hello", ObservationReadRequest{Find: &find}) + if err != nil { + t.Fatalf("ResolveObservationRead: %v", err) + } + if len(got.Windows) != 0 { + t.Fatalf("expected zero windows, got %d", len(got.Windows)) + } +} + +func TestResolveObservationReadFindRuneOffset(t *testing.T) { + content := "áéTEST" + find := "TEST" + contextRunes := 0 + got, err := ResolveObservationRead(content, ObservationReadRequest{Find: &find, Context: &contextRunes}) + if err != nil { + t.Fatalf("ResolveObservationRead: %v", err) + } + if len(got.Windows) != 1 || got.Windows[0].Offset != 2 { + t.Fatalf("rune offset window = %+v", got.Windows) + } +} + +func TestResolveObservationReadValidation(t *testing.T) { + offset := 0 + find := "x" + contextRunes := 1 + neg := -1 + + cases := []struct { + name string + req ObservationReadRequest + want error + }{ + {"both modes", ObservationReadRequest{Offset: &offset, Find: &find}, ErrPartialReadModesExclusive}, + {"context without find", ObservationReadRequest{Context: &contextRunes}, ErrContextRequiresFind}, + {"negative offset", ObservationReadRequest{Offset: &neg}, ErrPartialReadOffsetNegative}, + {"negative limit", ObservationReadRequest{Limit: &neg}, ErrPartialReadLimitNegative}, + {"negative context", ObservationReadRequest{Find: &find, Context: &neg}, ErrPartialReadContextNegative}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + _, err := ResolveObservationRead("body", tc.req) + if !errors.Is(err, tc.want) { + t.Fatalf("err = %v, want %v", err, tc.want) + } + }) + } +} + +func TestResolveObservationReadDefaultContext(t *testing.T) { + find := "needle" + pad := strings.Repeat("a", DefaultFindContext+10) + content := pad + find + pad + got, err := ResolveObservationRead(content, ObservationReadRequest{Find: &find}) + if err != nil { + t.Fatalf("ResolveObservationRead: %v", err) + } + if len(got.Windows) != 1 { + t.Fatalf("windows = %d", len(got.Windows)) + } + wantLen := DefaultFindContext + len([]rune(find)) + DefaultFindContext + if len([]rune(got.Windows[0].Content)) != wantLen { + t.Fatalf("default context window runes = %d, want %d", len([]rune(got.Windows[0].Content)), wantLen) + } +} diff --git a/internal/store/store.go b/internal/store/store.go index 2437f89e9..76ca25e11 100644 --- a/internal/store/store.go +++ b/internal/store/store.go @@ -66,6 +66,11 @@ var ( ErrObservationProjectImmutable = errors.New("observation project cannot be reassigned") ErrObservationTitleRequired = errors.New("observation title is required") ErrObservationContentRequired = errors.New("observation content is required") + ErrPartialReadModesExclusive = errors.New("offset/limit and find/context are mutually exclusive") + ErrContextRequiresFind = errors.New("context requires find") + ErrPartialReadOffsetNegative = errors.New("offset must be >= 0") + ErrPartialReadLimitNegative = errors.New("limit must be >= 0") + ErrPartialReadContextNegative = errors.New("context must be >= 0") ErrPromptContentRequired = errors.New("prompt content is required") )