Skip to content

Commit 1696226

Browse files
tiensonqintiensonqin
committed
Align benchmark query suite with upstream
Rename and extend the OCaml and upstream JS benchmark outputs to use upstream DataScript names for add-1/add-5 and q1/q2/q3/q4/q5/qpred cases. Keep query values aligned with upstream keyword semantics and add a fast bench output regression under dune runtest. Verified: dune runtest bench; dune build @check; dune build @bench; outliner and memory benchmark scripts; dune runtest.
1 parent bac503d commit 1696226

4 files changed

Lines changed: 101 additions & 8 deletions

File tree

‎bench/bench_ocaml.ml‎

Lines changed: 42 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -150,7 +150,7 @@ let person size i =
150150
; "last-name", One_value (String last_names.((i - 1) mod Array.length last_names))
151151
; "age", One_value (Int ((i * 37) mod 100))
152152
; "salary", One_value (Int ((i * 7919) mod 100_000))
153-
; "sex", One_value (String (if i mod 2 = 0 then "male" else "female"))
153+
; "sex", One_value (Keyword (if i mod 2 = 0 then "male" else "female"))
154154
; "friend", One_value (Ref friend)
155155
; "alias", Many_values [ String ("alias-" ^ string_of_int (i mod 10)); String ("tag-" ^ string_of_int (i mod 17)) ]
156156
]
@@ -176,6 +176,25 @@ let add_one_by_one size =
176176
(empty_db ~schema ())
177177
(people size)
178178

179+
let add_one_datom_per_tx size =
180+
let add_entity db entity =
181+
match entity with
182+
| Entity { db_id = Some entity_ref; attrs; _ } ->
183+
List.fold_left
184+
(fun db (attr, value) ->
185+
let tx =
186+
match value with
187+
| One_value value -> [ Add (entity_ref, attr, value) ]
188+
| Many_values values -> List.map (fun value -> Add (entity_ref, attr, value)) values
189+
| One_entity _ | Many_entities _ -> [ entity ]
190+
in
191+
db_with tx db)
192+
db
193+
attrs
194+
| _ -> db_with [ entity ] db
195+
in
196+
List.fold_left add_entity (empty_db ~schema ()) (people size)
197+
179198
let main () =
180199
let config = parse_args () in
181200
let runtime_label =
@@ -186,15 +205,34 @@ let main () =
186205
Printf.printf "runtime\t%s\n" runtime_label;
187206
Printf.printf "size\t%d\n" config.size;
188207
let db = lazy (build_db config.size) in
208+
bench config "add-1" (fun () -> consume_db (add_one_datom_per_tx config.size));
209+
bench config "add-5" (fun () -> consume_db (add_one_by_one config.size));
189210
bench config "add-all" (fun () -> consume_db (build_db config.size));
190-
bench config "add-one-by-one" (fun () -> consume_db (add_one_by_one config.size));
191211
bench config "storage-roundtrip" (fun () ->
192212
consume_db (build_storage_db config.size));
193213
bench config "datoms-name" (fun () -> consume_int (seq_length (datoms (Lazy.force db) Aevt ~a:"name" ())));
194-
bench config "query-name-age" (fun () ->
214+
bench config "q1" (fun () ->
215+
consume_rows (q_string (Lazy.force db) "[:find ?e :where [?e :name \"Ivan\"]]"));
216+
bench config "q2" (fun () ->
195217
consume_rows (q_string (Lazy.force db) "[:find ?e ?a :where [?e :name \"Ivan\"] [?e :age ?a]]"));
196-
bench config "query-salary-pred" (fun () ->
218+
bench config "q3" (fun () ->
219+
consume_rows (q_string (Lazy.force db) "[:find ?e ?a :where [?e :name \"Ivan\"] [?e :age ?a] [?e :sex :male]]"));
220+
bench config "q4" (fun () ->
221+
consume_rows (q_string (Lazy.force db) "[:find ?e ?l ?a :where [?e :name \"Ivan\"] [?e :last-name ?l] [?e :age ?a] [?e :sex :male]]"));
222+
bench config "q5-shortcircuit" (fun () ->
223+
consume_rows
224+
(q_string
225+
~inputs:[ Arg_scalar (Result_value (String "Anastasia")); Arg_scalar (Result_value (Int 35)) ]
226+
(Lazy.force db)
227+
"[:find ?e ?n ?l ?a ?s ?al :in $ ?n ?a :where [?e :name ?n] [?e :age ?a] [?e :last-name ?l] [?e :sex ?s] [?e :alias ?al]]"));
228+
bench config "qpred1" (fun () ->
197229
consume_rows (q_string (Lazy.force db) "[:find ?e ?s :where [?e :salary ?s] [(> ?s 50000)]]"));
230+
bench config "qpred2" (fun () ->
231+
consume_rows
232+
(q_string
233+
~inputs:[ Arg_scalar (Result_value (Int 50000)) ]
234+
(Lazy.force db)
235+
"[:find ?e ?s :in $ ?min-s :where [?e :salary ?s] [(> ?s ?min-s)]]"));
198236
bench config "pull-one" (fun () ->
199237
consume_pull (pull (Lazy.force db) [ Pull_attr "name"; Pull_attr "age"; Pull_ref ("friend", [ Pull_attr "name"; Pull_attr "age" ]) ] (Entity_id 1)));
200238
Printf.eprintf "blackhole=%d\n%!" !blackhole

‎bench/bench_output_test.sh‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
#!/usr/bin/env bash
2+
set -euo pipefail
3+
4+
bench_exe="$1"
5+
if [[ "$bench_exe" != */* ]]; then
6+
bench_exe="./$bench_exe"
7+
fi
8+
output="$("$bench_exe" --size 20 --warmup-ms 1 --sample-ms 1 --samples 1 2>/dev/null)"
9+
10+
for name in add-1 add-5 add-all q1 q2 q3 q4 q5-shortcircuit qpred1 qpred2 pull-one; do
11+
if ! grep -q "^${name}[[:space:]]" <<<"$output"; then
12+
echo "missing benchmark: $name" >&2
13+
echo "$output" >&2
14+
exit 1
15+
fi
16+
done

‎bench/bench_upstream.js‎

Lines changed: 35 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -111,7 +111,7 @@ function person(size, i) {
111111
"last-name": lastNames[(i - 1) % lastNames.length],
112112
age: (i * 37) % 100,
113113
salary: (i * 7919) % 100000,
114-
sex: i % 2 === 0 ? "male" : "female",
114+
sex: i % 2 === 0 ? ":male" : ":female",
115115
friend,
116116
alias: [`alias-${i % 10}`, `tag-${i % 17}`],
117117
};
@@ -137,6 +137,21 @@ function addOneByOne(size) {
137137
return db;
138138
}
139139

140+
function addOneDatomPerTx(size) {
141+
let db = d.empty_db(schema);
142+
for (const entity of people(size)) {
143+
const id = entity[":db/id"];
144+
for (const [attr, value] of Object.entries(entity)) {
145+
if (attr === ":db/id") continue;
146+
const values = Array.isArray(value) ? value : [value];
147+
for (const item of values) {
148+
db = d.db_with(db, [[":db/add", id, attr, item]]);
149+
}
150+
}
151+
}
152+
return db;
153+
}
154+
140155
function main() {
141156
const config = parseArgs(process.argv.slice(2));
142157
console.log("runtime\tupstream-cljs-js");
@@ -147,15 +162,31 @@ function main() {
147162
return cachedDb;
148163
};
149164

165+
bench(config, "add-1", () => consumeDb(addOneDatomPerTx(config.size)));
166+
bench(config, "add-5", () => consumeDb(addOneByOne(config.size)));
150167
bench(config, "add-all", () => consumeDb(buildDb(config.size)));
151-
bench(config, "add-one-by-one", () => consumeDb(addOneByOne(config.size)));
152168
bench(config, "datoms-name", () => consumeInt(d.datoms(db(), ":aevt", "name").length));
153-
bench(config, "query-name-age", () =>
169+
bench(config, "q1", () =>
170+
consumeRows(d.q('[:find ?e :where [?e "name" "Ivan"]]', db()))
171+
);
172+
bench(config, "q2", () =>
154173
consumeRows(d.q('[:find ?e ?a :where [?e "name" "Ivan"] [?e "age" ?a]]', db()))
155174
);
156-
bench(config, "query-salary-pred", () =>
175+
bench(config, "q3", () =>
176+
consumeRows(d.q('[:find ?e ?a :where [?e "name" "Ivan"] [?e "age" ?a] [?e "sex" :male]]', db()))
177+
);
178+
bench(config, "q4", () =>
179+
consumeRows(d.q('[:find ?e ?l ?a :where [?e "name" "Ivan"] [?e "last-name" ?l] [?e "age" ?a] [?e "sex" :male]]', db()))
180+
);
181+
bench(config, "q5-shortcircuit", () =>
182+
consumeRows(d.q('[:find ?e ?n ?l ?a ?s ?al :in $ ?n ?a :where [?e "name" ?n] [?e "age" ?a] [?e "last-name" ?l] [?e "sex" ?s] [?e "alias" ?al]]', db(), "Anastasia", 35))
183+
);
184+
bench(config, "qpred1", () =>
157185
consumeRows(d.q('[:find ?e ?s :where [?e "salary" ?s] [(> ?s 50000)]]', db()))
158186
);
187+
bench(config, "qpred2", () =>
188+
consumeRows(d.q('[:find ?e ?s :in $ ?min_s :where [?e "salary" ?s] [(> ?s ?min_s)]]', db(), 50000))
189+
);
159190
bench(config, "pull-one", () =>
160191
consumePull(d.pull(db(), '["name" "age" {"friend" ["name" "age"]}]', 1))
161192
);

‎bench/dune‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -48,3 +48,11 @@
4848
BENCH_SKIP_BUILD
4949
1
5050
(run bash %{dep:../script/benchmark_vs_cljs.sh}))))
51+
52+
(rule
53+
(alias runtest)
54+
(deps
55+
bench_ocaml.exe
56+
bench_output_test.sh)
57+
(action
58+
(run bash %{dep:bench_output_test.sh} %{dep:bench_ocaml.exe})))

0 commit comments

Comments
 (0)