Skip to content

Commit d888e54

Browse files
authored
Optimise min/max computation for VarBinView arrays (#9740)
Leverage view values to perform fast min/max for VarBinView arrays. Unify the view comparison logic across multiple places in the project under ResolvedViews --------- Signed-off-by: Robert Kruszewski <github@robertk.io>
1 parent 61a4bb3 commit d888e54

13 files changed

Lines changed: 957 additions & 343 deletions

File tree

‎vortex-array/benches/aggregate_max.rs‎

Lines changed: 125 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,8 +8,13 @@ use mimalloc::MiMalloc;
88
use rand::prelude::*;
99
use vortex_array::IntoArray;
1010
use vortex_array::VortexSessionExecute;
11+
use vortex_array::aggregate_fn::NumericalAggregateOpts;
12+
use vortex_array::aggregate_fn::fns::min_max::min_max;
1113
use vortex_array::array_session;
1214
use vortex_array::arrays::PrimitiveArray;
15+
use vortex_array::arrays::VarBinViewArray;
16+
use vortex_array::dtype::DType;
17+
use vortex_array::dtype::Nullability;
1318
use vortex_session::VortexSession;
1419

1520
#[global_allocator]
@@ -106,3 +111,123 @@ fn max_i32_nulls_scattered(bencher: Bencher) {
106111
})
107112
.bench_refs(|(a, ctx)| a.statistics().compute_max::<i32>(ctx));
108113
}
114+
115+
// String extrema. Every view carries its value's first four bytes inline, so the shape that
116+
// matters is how far into a value its order is settled: `distinct_prefixes` and
117+
// `low_cardinality` never have to look past the view, while the log-shaped columns (every value
118+
// starting `thread-`/`req-`/`This is `) tie on those bytes and have to resolve the value.
119+
//
120+
// Like the primitive benches, each iteration gets a fresh array: `min_max` caches `Stat::Min`
121+
// and `Stat::Max` on the array it scanned, so a reused array would only be scanned once.
122+
123+
// Resolving and comparing string values costs far more per row than a primitive, so these scan a
124+
// smaller column to keep each benchmark under 1ms in CodSpeed simulation.
125+
const STRING_N: usize = 4_096;
126+
127+
fn bench_min_max(bencher: Bencher, values: Vec<Option<String>>) {
128+
bencher
129+
.with_inputs(|| {
130+
(
131+
VarBinViewArray::from_iter(
132+
values.iter().map(|v| v.as_deref()),
133+
DType::Utf8(Nullability::Nullable),
134+
)
135+
.into_array(),
136+
SESSION.create_execution_ctx(),
137+
)
138+
})
139+
.bench_refs(|(a, ctx)| min_max(a, ctx, NumericalAggregateOpts::default()));
140+
}
141+
142+
/// Long values with distinct leading bytes: the prefix rejects almost every element.
143+
#[divan::bench]
144+
fn max_utf8_view_distinct_prefixes(bencher: Bencher) {
145+
let mut rng = StdRng::seed_from_u64(6);
146+
bench_min_max(
147+
bencher,
148+
(0..STRING_N)
149+
.map(|_| {
150+
Some(format!(
151+
"{:016x}-{:016x}",
152+
rng.random::<u64>(),
153+
rng.random::<u64>()
154+
))
155+
})
156+
.collect(),
157+
);
158+
}
159+
160+
/// Log levels: short inlined values over a five-value dictionary, with distinct leading bytes.
161+
#[divan::bench]
162+
fn max_utf8_view_low_cardinality(bencher: Bencher) {
163+
bench_min_max(
164+
bencher,
165+
(0..STRING_N)
166+
.map(|i| {
167+
Some(
168+
match i % 5 {
169+
0 => "ERROR",
170+
1 => "WARN",
171+
2 => "INFO",
172+
3 => "DEBUG",
173+
_ => "TRACE",
174+
}
175+
.to_string(),
176+
)
177+
})
178+
.collect(),
179+
);
180+
}
181+
182+
/// Thread names: short inlined values that tie on the inline prefix and are settled a few
183+
/// bytes later.
184+
#[divan::bench]
185+
fn max_utf8_view_tied_prefix_inlined(bencher: Bencher) {
186+
bench_min_max(
187+
bencher,
188+
(0..STRING_N)
189+
.map(|i| Some(format!("thread-{}", i % 8)))
190+
.collect(),
191+
);
192+
}
193+
194+
/// Request ids: 20-byte buffer-backed values that tie on the inline prefix `req-`.
195+
#[divan::bench]
196+
fn max_utf8_view_tied_prefix_outlined(bencher: Bencher) {
197+
bench_min_max(
198+
bencher,
199+
(0..STRING_N)
200+
.map(|i| Some(format!("req-{i:016x}")))
201+
.collect(),
202+
);
203+
}
204+
205+
/// Log lines: long, buffer-backed values that agree for their first twenty-six bytes.
206+
#[divan::bench]
207+
fn max_utf8_view_shared_prefix(bencher: Bencher) {
208+
bench_min_max(
209+
bencher,
210+
(0..STRING_N)
211+
.map(|i| {
212+
Some(format!(
213+
"This is log message number {i} with some additional content to simulate real logs"
214+
))
215+
})
216+
.collect(),
217+
);
218+
}
219+
220+
/// Distinct prefixes with ~50% random nulls, producing many short valid runs.
221+
#[divan::bench]
222+
fn max_utf8_view_nulls_scattered(bencher: Bencher) {
223+
let mut rng = StdRng::seed_from_u64(7);
224+
bench_min_max(
225+
bencher,
226+
(0..STRING_N)
227+
.map(|_| {
228+
rng.random_bool(0.5)
229+
.then(|| format!("{:016x}-{:016x}", rng.random::<u64>(), rng.random::<u64>()))
230+
})
231+
.collect(),
232+
);
233+
}

‎vortex-array/src/aggregate_fn/fns/min_max/mod.rs‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@ mod bool;
55
mod decimal;
66
mod extension;
77
mod primitive;
8-
mod varbin;
8+
mod varbinview;
99

1010
use std::sync::LazyLock;
1111

@@ -19,7 +19,7 @@ use self::bool::accumulate_bool;
1919
use self::decimal::accumulate_decimal;
2020
use self::extension::accumulate_extension;
2121
use self::primitive::accumulate_primitive;
22-
use self::varbin::accumulate_varbinview;
22+
use self::varbinview::accumulate_varbinview;
2323
use crate::ArrayRef;
2424
use crate::Canonical;
2525
use crate::Columnar;

‎vortex-array/src/aggregate_fn/fns/min_max/varbin.rs‎

Lines changed: 0 additions & 82 deletions
This file was deleted.

0 commit comments

Comments
 (0)