fix: clean duplicate_* index tables on retract; test transit read-cache wrap

- refresh_indexes_with_tx_data removed a retracted fact from the three
  PSet indexes but left its copy in duplicate_datoms and the
  duplicate_*_by_attr/entity tables (populated by full rebuilds via
  set_indexes_from_datoms/restore). find_avet_exact consults
  duplicate_avet_by_attr, so a restored+retracted unique-attr fact still
  resolved lookup refs instead of raising 'Nothing found for entity id'.
  Now the fact is filtered from all duplicate tables on retract, matching
  upstream where a retracted datom is gone entirely.
- test_melange_transit_backend: regression test for the transit read
  cache wrapping at 1936 entries (the melange-transit 0.1.0 misdecode
  fixed in 0.1.2) — a ^XX cache code past the wrap must resolve to the
  overwritten slot, not the stale one.
- test_query_engine_fixes: retract_cleans_duplicate_avet_tables covers
  the stale-lookup-ref repro.

Author: tiensonqin

impl/db.ml

@@ -222,6 +222,30 @@ let add_datom_to_indexes db datom =
   ; max_datom_e = max db.max_datom_e datom.e
   }
 
+let remove_fact_from_duplicate_tables db active =
+  (* Retraction removes the fact entirely, so every stored copy — including
+     the ones kept out of the PSet indexes in the duplicate tables — must go.
+     The tables are shared with prior db values, so rebuild them rather than
+     mutate in place. *)
+  let duplicate_datoms = List.filter (fun d -> not (same_fact d active)) db.duplicate_datoms in
+  if duplicate_datoms == db.duplicate_datoms then
+    db
+  else
+    let duplicate_aevt_datoms = List.sort (Util.compare_datom Aevt) duplicate_datoms in
+    let duplicate_avet_datoms =
+      duplicate_datoms
+      |> List.filter (fun datom -> Schema.schema_attr_is_avet_accessible db.schema datom.a)
+      |> List.sort (Util.compare_datom Avet)
+    in
+    { db with
+      duplicate_datoms
+    ; duplicate_aevt_datoms
+    ; duplicate_avet_datoms
+    ; duplicate_eavt_by_entity = duplicate_eavt_by_entity duplicate_datoms
+    ; duplicate_aevt_by_attr = duplicate_datoms_by_attr duplicate_aevt_datoms
+    ; duplicate_avet_by_attr = duplicate_datoms_by_attr duplicate_avet_datoms
+    }
+
 let refresh_indexes_with_tx_data db tx_data =
   let db =
     List.fold_left
@@ -232,6 +256,7 @@ let refresh_indexes_with_tx_data db tx_data =
           match find_active_datom_by_fact db datom with
           | None -> db
           | Some active ->
+            let db = remove_fact_from_duplicate_tables db active in
             { db with
               eavt_index = PSet.remove active db.eavt_index
             ; aevt_index = PSet.remove active db.aevt_index

test/test_melange_transit_backend.ml

@@ -70,8 +70,41 @@ let test_native_backend_converts_edn_values () =
       ] -> ()
   | _ -> failwith "native melange-transit backend should convert EDN maps to Transit maps"
 
+let test_native_backend_read_cache_wraps_at_max_entries () =
+  (* transit-js caches both writes and reads; both caches wrap at
+     44*44 = 1936 entries, so ^XX cache codes reuse slots. Encoding
+     1936+64 distinct cacheable keywords then repeating the last one
+     emits ^1C — index 1999 wrapped to slot 63, which the read cache
+     must resolve to the overwritten entry, not the original.
+     melange-transit 0.1.0 indexed the read cache by Hashtbl.length
+     without wrapping and decoded it as the stale number-63. *)
+  let max_cache_entries = 44 * 44 in
+  let count = max_cache_entries + 64 in
+  let last = count - 1 in
+  let kw i = Json.Keyword (Printf.sprintf "long.namespace.attr/number-%d" i) in
+  let payload = Json.Array (List.init count kw @ [ kw last ]) in
+  match Json.of_string (Json.to_string payload) with
+  | Json.Array values ->
+    let decode i =
+      match List.nth values i with
+      | Json.Keyword s -> s
+      | _ -> failf "entry %d should decode to a keyword" i
+    in
+    List.iteri
+      (fun i _ ->
+        let expected = Printf.sprintf "long.namespace.attr/number-%d" i in
+        let actual = decode i in
+        if not (String.equal expected actual) then
+          failf "entry %d: expected %S, got %S" i expected actual)
+      (List.init count Fun.id);
+    expect_equal "wrapped cache reference"
+      (Printf.sprintf "long.namespace.attr/number-%d" last)
+      (decode count)
+  | _ -> failwith "expected a transit array"
+
 let () =
   test_native_backend_decodes_logseq_storage_shape ();
   test_native_backend_writes_transit_json ();
   test_native_backend_roundtrips_verbose_storage_keys ();
-  test_native_backend_converts_edn_values ()
+  test_native_backend_converts_edn_values ();
+  test_native_backend_read_cache_wraps_at_max_entries ()

test/test_query_engine_fixes.ml

@@ -543,6 +543,53 @@ let test_entity_map_lookup_ref_never_resolves_raises () =
      then
        failf "unexpected error message: %s" msg)
 
+let test_retract_cleans_duplicate_avet_tables () =
+  (* Bug 5: datoms that compare equal under eavt (same fact stored twice, as
+     produced by a full index rebuild over restore) are kept out of the PSet
+     indexes in the duplicate_* tables — and find_avet_exact consults
+     duplicate_avet_by_attr. Retraction removed the fact from the three PSets
+     but left the duplicate copy behind, so [:block/uuid u] still resolved
+     after the entity was retracted. Upstream retracts the fact entirely and
+     the lookup ref raises "Nothing found for entity id". *)
+  let u1 = "11111111-1111-1111-1111-111111111111" in
+  let uuid_datom =
+    { e = 1; a = "block/uuid"; v = Uuid u1; tx = 1; added = true }
+  in
+  let db =
+    init_db ~schema:(block_schema ())
+      [ uuid_datom
+      ; { e = 1; a = "block/title"; v = String "Page"; tx = 1; added = true }
+      ; uuid_datom ]
+  in
+  (if List.length db.duplicate_datoms = 0 then
+     failf "precondition: init_db should route the repeated fact to duplicate_datoms");
+  let db = db |> db_with [ RetractEntity (Entity_id 1) ] in
+  (match datoms db Eavt ~a:"block/uuid" () |> List.of_seq with
+   | [] -> ()
+   | datoms -> failf "retracted uuid datom still visible in eavt: %d" (List.length datoms));
+  (match datoms db Avet ~a:"block/uuid" () |> List.of_seq with
+   | [] -> ()
+   | datoms -> failf "retracted uuid datom still visible in avet: %d" (List.length datoms));
+  (try
+     ignore
+       (db
+        |> db_with
+             [ Entity
+                 { db_id = None
+                 ; attrs =
+                     [ "block/title", One_value (String "Child")
+                     ; "block/parent", One_value (lookup_ref "block/uuid" (Uuid u1))
+                     ]
+                 }
+             ]);
+     failf "lookup ref to a retracted entity should raise"
+   with
+   | Invalid_argument msg ->
+     if not
+          (String.starts_with ~prefix:"Nothing found for entity id" msg)
+     then
+       failf "unexpected error message: %s" msg)
+
 (* Bug D: upstream maybe-wrap-multival — inside a multival attr a 2-element
    collection is a lookup ref only when its head names a unique-identity attr;
    otherwise it expands into individual values *)
@@ -892,4 +939,5 @@ let () =
     ; "rule_attr_var_property", test_rule_attr_var_property
     ; "rule_tags_query_with_eid_input", test_rule_tags_query_with_eid_input
     ; "edn_symbol_special_chars", test_edn_symbol_special_chars
+    ; "retract_cleans_duplicate_avet_tables", test_retract_cleans_duplicate_avet_tables
     ]