Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 8 additions & 0 deletions internal/server/server.go
Original file line number Diff line number Diff line change
Expand Up @@ -491,6 +491,10 @@ func (s *Server) handleSearch(w http.ResponseWriter, r *http.Request) {
MatchMode: matchMode,
})
if err != nil {
if errors.Is(err, store.ErrFTSQueryTooLarge) {
jsonError(w, http.StatusBadRequest, err.Error())
return
}
jsonError(w, http.StatusInternalServerError, err.Error())
return
}
Expand Down Expand Up @@ -787,6 +791,10 @@ func (s *Server) handleSearchPrompts(w http.ResponseWriter, r *http.Request) {
queryInt(r, "limit", 10),
)
if err != nil {
if errors.Is(err, store.ErrFTSQueryTooLarge) {
jsonError(w, http.StatusBadRequest, err.Error())
return
}
jsonError(w, http.StatusInternalServerError, err.Error())
return
}
Expand Down
76 changes: 76 additions & 0 deletions internal/server/server_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -837,6 +837,82 @@ func TestHandleSearchForwardsMatchModeAndAllProjects(t *testing.T) {
}
}

func TestSearchEndpointsRejectOversizedFTSQueries(t *testing.T) {
srv := New(newServerTestStore(t), 0)
query := strings.Repeat("a", 65_537)
want := "fts query too large: got 65537 bytes; maximum is 65536 bytes; shorten the query and retry"

for _, path := range []string{"/search", "/prompts/search"} {
t.Run(path, func(t *testing.T) {
req := httptest.NewRequest(http.MethodGet, path+"?q="+url.QueryEscape(query), nil)
rec := httptest.NewRecorder()
srv.Handler().ServeHTTP(rec, req)

if rec.Code != http.StatusBadRequest {
t.Fatalf("status = %d, want 400: %s", rec.Code, rec.Body.String())
}
var response map[string]string
if err := json.Unmarshal(rec.Body.Bytes(), &response); err != nil {
t.Fatalf("decode response: %v", err)
}
if got := response["error"]; got != want {
t.Fatalf("error = %q, want %q", got, want)
}
})
}
}

func TestSearchEndpointsReturnInternalErrorForStoreFailure(t *testing.T) {
st := newServerTestStore(t)
if err := st.Close(); err != nil {
t.Fatalf("close store: %v", err)
}
srv := New(st, 0)

for _, path := range []string{"/search?q=x", "/prompts/search?q=x"} {
t.Run(path, func(t *testing.T) {
rec := httptest.NewRecorder()
srv.Handler().ServeHTTP(rec, httptest.NewRequest(http.MethodGet, path, nil))

if rec.Code != http.StatusInternalServerError {
t.Fatalf("status = %d, want 500: %s", rec.Code, rec.Body.String())
}
var response map[string]string
if err := json.Unmarshal(rec.Body.Bytes(), &response); err != nil {
t.Fatalf("decode response: %v", err)
}
if response["error"] == "" {
t.Fatalf("internal store error response missing error: %s", rec.Body.String())
}
})
}
}

func TestSearchEndpointsRejectTooManyShortFTSTerms(t *testing.T) {
srv := New(newServerTestStore(t), 0)
query := strings.TrimSpace(strings.Repeat("x ", 769))
want := "fts query too large: got 769 terms; maximum is 768 terms for short-term search; shorten the query and retry"

for _, path := range []string{"/search", "/prompts/search"} {
t.Run(path, func(t *testing.T) {
req := httptest.NewRequest(http.MethodGet, path+"?q="+url.QueryEscape(query), nil)
rec := httptest.NewRecorder()
srv.Handler().ServeHTTP(rec, req)

if rec.Code != http.StatusBadRequest {
t.Fatalf("status = %d, want 400: %s", rec.Code, rec.Body.String())
}
var response map[string]string
if err := json.Unmarshal(rec.Body.Bytes(), &response); err != nil {
t.Fatalf("decode response: %v", err)
}
if got := response["error"]; got != want {
t.Fatalf("error = %q, want %q", got, want)
}
})
}
}

func TestHandleSearchAllowsEmptyMatchMode(t *testing.T) {
st := newServerTestStore(t)
srv := New(st, 0)
Expand Down
3 changes: 3 additions & 0 deletions internal/store/relations.go
Original file line number Diff line number Diff line change
Expand Up @@ -383,6 +383,9 @@ func (s *Store) FindCandidates(savedID int64, opts CandidateOptions) ([]Candidat
if strings.TrimSpace(queryText) == "" {
queryText = title
}
if err := validateFTSQueryLength(queryText); err != nil {
return nil, err
}
ftsQuery := sanitizeFTSCandidates(queryText)
if ftsQuery == "" {
return nil, nil
Expand Down
57 changes: 57 additions & 0 deletions internal/store/relations_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -110,6 +110,63 @@ func TestFindCandidates_HappyPath(t *testing.T) {
}
}

func TestFindCandidatesValidatesStoredTitleLength(t *testing.T) {
s := setupRelationsStore(t)
_, _ = addTestObs(t, s, "candidate title", "decision", "testproject", "project")

atLimitID, _ := addTestObs(t, s, strings.Repeat("Γ©", 32_768), "decision", "testproject", "project")
if _, err := s.FindCandidates(atLimitID, CandidateOptions{
Project: "testproject",
Scope: "project",
Query: "candidate",
SkipInsert: true,
}); err != nil {
t.Fatalf("FindCandidates with a 65536-byte stored title: %v", err)
}

title := strings.Repeat("a", 65_537)
savedID, _ := addTestObs(t, s, title, "decision", "testproject", "project")
_, err := s.FindCandidates(savedID, CandidateOptions{
Project: "testproject",
Scope: "project",
SkipInsert: true,
})
if err == nil {
t.Fatal("FindCandidates with a 65537-byte stored title returned nil error")
}
if !errors.Is(err, ErrFTSQueryTooLarge) {
t.Fatalf("FindCandidates error = %v, want ErrFTSQueryTooLarge", err)
}
if got, want := err.Error(), "fts query too large: got 65537 bytes; maximum is 65536 bytes; shorten the query and retry"; got != want {
t.Fatalf("FindCandidates error = %q, want %q", got, want)
}
}

func TestFindCandidatesUsesEffectiveQueryForLengthValidation(t *testing.T) {
s := setupRelationsStore(t)
_, _ = addTestObs(t, s, "candidate title", "decision", "testproject", "project")

oversizedTitleID, _ := addTestObs(t, s, strings.Repeat("a", 65_537), "decision", "testproject", "project")
if _, err := s.FindCandidates(oversizedTitleID, CandidateOptions{
Project: "testproject",
Scope: "project",
Query: "candidate",
SkipInsert: true,
}); err != nil {
t.Fatalf("FindCandidates with valid override query: %v", err)
}

_, err := s.FindCandidates(oversizedTitleID, CandidateOptions{
Project: "testproject",
Scope: "project",
Query: strings.Repeat("b", 65_537),
SkipInsert: true,
})
if !errors.Is(err, ErrFTSQueryTooLarge) {
t.Fatalf("FindCandidates error = %v, want ErrFTSQueryTooLarge", err)
}
}

func TestFindCandidates_EscapesInteriorQuotes(t *testing.T) {
s := setupRelationsStore(t)
_, _ = addTestObs(t, s, `hello"world candidate`, "decision", "testproject", "project")
Expand Down
38 changes: 38 additions & 0 deletions internal/store/store.go
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,16 @@ import (
// See https://www.sqlite.org/rescode.html#constraint_foreignkey
const sqliteConstraintForeignKey = 787

const (
maxFTSQueryBytes = 65_536

// maxFTSShortTermQueryTerms keeps LIKE fallbacks below empirically observed
// SQLite failures. Filtered SearchContext first failed at 990 terms, while
// SearchPrompts first failed at 997 terms. The cap of 768 leaves at least
// 221 terms of headroom below the observed worst path.
maxFTSShortTermQueryTerms = 768
)

const (
sqlitePrimaryBusy = 5
sqlitePrimaryLocked = 6
Expand Down Expand Up @@ -67,6 +77,7 @@ var (
ErrObservationTitleRequired = errors.New("observation title is required")
ErrObservationContentRequired = errors.New("observation content is required")
ErrPromptContentRequired = errors.New("prompt content is required")
ErrFTSQueryTooLarge = errors.New("fts query too large")
)

// Sentinel errors for relation sync apply path (Phase 2).
Expand Down Expand Up @@ -3252,13 +3263,19 @@ func (s *Store) compactionPrompts(sessionID, project string, limit int) ([]Promp
}

func (s *Store) SearchPrompts(query string, project string, limit int) ([]Prompt, error) {
if err := validateFTSQueryLength(query); err != nil {
return nil, err
}
if limit <= 0 {
limit = 10
}

var sql string
var args []any
if hasShortFTSTerm(query) {
if err := validateShortTermFTSQuery(query); err != nil {
return nil, err
}
sql, args = buildPromptLIKEQuery(query, project, limit)
} else {
ftsQuery := sanitizeFTS(query)
Expand Down Expand Up @@ -3714,6 +3731,10 @@ func (s *Store) SearchContext(ctx context.Context, query string, opts SearchOpti
return nil, fmt.Errorf("invalid match_mode %q: must be \"all\" or \"any\"", opts.MatchMode)
}

if err := validateFTSQueryLength(query); err != nil {
return nil, err
}

// Normalize project filter so "Engram" finds records stored as "engram"
opts.Project, _ = NormalizeProject(opts.Project)

Expand Down Expand Up @@ -3785,6 +3806,9 @@ func (s *Store) SearchContext(ctx context.Context, query string, opts SearchOpti
var sqlQ string
var args []any
if hasShortFTSTerm(query) {
if err := validateShortTermFTSQuery(query); err != nil {
return nil, err
}
sqlQ, args = buildSearchLIKEQuery(query, opts, limit)
} else {
// Build FTS5 query: "all" (default) uses AND semantics; "any" uses OR for broader recall.
Expand Down Expand Up @@ -8859,6 +8883,20 @@ func stripPrivateTags(s string) string {
return result
}

func validateFTSQueryLength(query string) error {
if len(query) > maxFTSQueryBytes {
return fmt.Errorf("%w: got %d bytes; maximum is %d bytes; shorten the query and retry", ErrFTSQueryTooLarge, len(query), maxFTSQueryBytes)
}
return nil
}

func validateShortTermFTSQuery(query string) error {
if terms := len(searchTerms(query)); terms > maxFTSShortTermQueryTerms {
return fmt.Errorf("%w: got %d terms; maximum is %d terms for short-term search; shorten the query and retry", ErrFTSQueryTooLarge, terms, maxFTSShortTermQueryTerms)
}
return nil
}

// sanitizeFTS wraps each word in quotes so FTS5 doesn't choke on special chars.
// "fix auth bug" β†’ `"fix" "auth" "bug"`
func sanitizeFTS(query string) string {
Expand Down
Loading