Skip to content

Commit 7d23aa5

Browse files
joseph-isaacsjoseph-isaacs
committed
Merge develop after NEON small-table take landed
Signed-off-by: Joe Isaacs <joe.isaacs@live.co.uk>
2 parents d138ae0 + 750b36d commit 7d23aa5

17 files changed

Lines changed: 1018 additions & 351 deletions

File tree

‎encodings/runend/src/array.rs‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -481,21 +481,21 @@ pub(super) fn run_end_canonicalize(
481481
}
482482
DType::Primitive(..) => {
483483
let pvalues = array.values().clone().execute_as("values", ctx)?;
484-
runend_decode_primitive(pends, pvalues, array.offset(), array.len(), ctx)?.into_array()
484+
runend_decode_primitive(pends, pvalues, array.offset(), array.len(), ctx)?
485485
}
486486
DType::Decimal(..) => {
487487
let values = array
488488
.values()
489489
.clone()
490490
.execute_as::<DecimalArray>("values", ctx)?;
491-
runend_decode_decimal(pends, values, array.offset(), array.len(), ctx)?.into_array()
491+
runend_decode_decimal(pends, values, array.offset(), array.len(), ctx)?
492492
}
493493
DType::Utf8(_) | DType::Binary(_) => {
494494
let values = array
495495
.values()
496496
.clone()
497497
.execute_as::<VarBinViewArray>("values", ctx)?;
498-
runend_decode_varbinview(pends, values, array.offset(), array.len(), ctx)?.into_array()
498+
runend_decode_varbinview(pends, values, array.offset(), array.len(), ctx)?
499499
}
500500
DType::List(..) => {
501501
let values = array

‎encodings/runend/src/compress.rs‎

Lines changed: 22 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -190,11 +190,16 @@ pub fn runend_decode_primitive(
190190
offset: usize,
191191
length: usize,
192192
ctx: &mut ExecutionCtx,
193-
) -> VortexResult<PrimitiveArray> {
193+
) -> VortexResult<ArrayRef> {
194194
let validity_mask = values
195195
.as_ref()
196196
.validity()?
197197
.execute_mask(values.as_ref().len(), ctx)?;
198+
if matches!(validity_mask, Mask::AllFalse(_)) {
199+
return Ok(
200+
ConstantArray::new(Scalar::null(values.dtype().as_nullable()), length).into_array(),
201+
);
202+
}
198203
Ok(match_each_native_ptype!(values.ptype(), |P| {
199204
match_each_unsigned_integer_ptype!(ends.ptype(), |E| {
200205
runend_decode_typed_primitive(
@@ -204,6 +209,7 @@ pub fn runend_decode_primitive(
204209
values.dtype().nullability(),
205210
length,
206211
)
212+
.into_array()
207213
})
208214
}))
209215
}
@@ -294,11 +300,16 @@ pub fn runend_decode_decimal(
294300
offset: usize,
295301
length: usize,
296302
ctx: &mut ExecutionCtx,
297-
) -> VortexResult<DecimalArray> {
303+
) -> VortexResult<ArrayRef> {
298304
let validity_mask = values
299305
.as_ref()
300306
.validity()?
301307
.execute_mask(values.as_ref().len(), ctx)?;
308+
if matches!(validity_mask, Mask::AllFalse(_)) {
309+
return Ok(
310+
ConstantArray::new(Scalar::null(values.dtype().as_nullable()), length).into_array(),
311+
);
312+
}
302313

303314
Ok(match_each_decimal_value_type!(values.values_type(), |D| {
304315
let (decoded, validity) = match_each_unsigned_integer_ptype!(ends.ptype(), |E| {
@@ -310,7 +321,7 @@ pub fn runend_decode_decimal(
310321
length,
311322
)
312323
});
313-
DecimalArray::new(decoded, values.decimal_dtype(), validity)
324+
DecimalArray::new(decoded, values.decimal_dtype(), validity).into_array()
314325
}))
315326
}
316327

@@ -321,11 +332,16 @@ pub fn runend_decode_varbinview(
321332
offset: usize,
322333
length: usize,
323334
ctx: &mut ExecutionCtx,
324-
) -> VortexResult<VarBinViewArray> {
335+
) -> VortexResult<ArrayRef> {
325336
let validity_mask = values
326337
.as_ref()
327338
.validity()?
328339
.execute_mask(values.as_ref().len(), ctx)?;
340+
if matches!(validity_mask, Mask::AllFalse(_)) {
341+
return Ok(
342+
ConstantArray::new(Scalar::null(values.dtype().as_nullable()), length).into_array(),
343+
);
344+
}
329345
let views = values.views();
330346

331347
let (decoded_views, validity) = match_each_unsigned_integer_ptype!(ends.ptype(), |E| {
@@ -345,7 +361,8 @@ pub fn runend_decode_varbinview(
345361
// buffers, so all buffer indices and offsets remain valid.
346362
Ok(unsafe {
347363
VarBinViewArray::new_handle_unchecked(view_handle, parts.buffers, parts.dtype, validity)
348-
})
364+
}
365+
.into_array())
349366
}
350367

351368
#[cfg(test)]

‎vortex-array/benches/aggregate_max.rs‎

Lines changed: 125 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,8 +8,13 @@ use mimalloc::MiMalloc;
88
use rand::prelude::*;
99
use vortex_array::IntoArray;
1010
use vortex_array::VortexSessionExecute;
11+
use vortex_array::aggregate_fn::NumericalAggregateOpts;
12+
use vortex_array::aggregate_fn::fns::min_max::min_max;
1113
use vortex_array::array_session;
1214
use vortex_array::arrays::PrimitiveArray;
15+
use vortex_array::arrays::VarBinViewArray;
16+
use vortex_array::dtype::DType;
17+
use vortex_array::dtype::Nullability;
1318
use vortex_session::VortexSession;
1419

1520
#[global_allocator]
@@ -106,3 +111,123 @@ fn max_i32_nulls_scattered(bencher: Bencher) {
106111
})
107112
.bench_refs(|(a, ctx)| a.statistics().compute_max::<i32>(ctx));
108113
}
114+
115+
// String extrema. Every view carries its value's first four bytes inline, so the shape that
116+
// matters is how far into a value its order is settled: `distinct_prefixes` and
117+
// `low_cardinality` never have to look past the view, while the log-shaped columns (every value
118+
// starting `thread-`/`req-`/`This is `) tie on those bytes and have to resolve the value.
119+
//
120+
// Like the primitive benches, each iteration gets a fresh array: `min_max` caches `Stat::Min`
121+
// and `Stat::Max` on the array it scanned, so a reused array would only be scanned once.
122+
123+
// Resolving and comparing string values costs far more per row than a primitive, so these scan a
124+
// smaller column to keep each benchmark under 1ms in CodSpeed simulation.
125+
const STRING_N: usize = 4_096;
126+
127+
fn bench_min_max(bencher: Bencher, values: Vec<Option<String>>) {
128+
bencher
129+
.with_inputs(|| {
130+
(
131+
VarBinViewArray::from_iter(
132+
values.iter().map(|v| v.as_deref()),
133+
DType::Utf8(Nullability::Nullable),
134+
)
135+
.into_array(),
136+
SESSION.create_execution_ctx(),
137+
)
138+
})
139+
.bench_refs(|(a, ctx)| min_max(a, ctx, NumericalAggregateOpts::default()));
140+
}
141+
142+
/// Long values with distinct leading bytes: the prefix rejects almost every element.
143+
#[divan::bench]
144+
fn max_utf8_view_distinct_prefixes(bencher: Bencher) {
145+
let mut rng = StdRng::seed_from_u64(6);
146+
bench_min_max(
147+
bencher,
148+
(0..STRING_N)
149+
.map(|_| {
150+
Some(format!(
151+
"{:016x}-{:016x}",
152+
rng.random::<u64>(),
153+
rng.random::<u64>()
154+
))
155+
})
156+
.collect(),
157+
);
158+
}
159+
160+
/// Log levels: short inlined values over a five-value dictionary, with distinct leading bytes.
161+
#[divan::bench]
162+
fn max_utf8_view_low_cardinality(bencher: Bencher) {
163+
bench_min_max(
164+
bencher,
165+
(0..STRING_N)
166+
.map(|i| {
167+
Some(
168+
match i % 5 {
169+
0 => "ERROR",
170+
1 => "WARN",
171+
2 => "INFO",
172+
3 => "DEBUG",
173+
_ => "TRACE",
174+
}
175+
.to_string(),
176+
)
177+
})
178+
.collect(),
179+
);
180+
}
181+
182+
/// Thread names: short inlined values that tie on the inline prefix and are settled a few
183+
/// bytes later.
184+
#[divan::bench]
185+
fn max_utf8_view_tied_prefix_inlined(bencher: Bencher) {
186+
bench_min_max(
187+
bencher,
188+
(0..STRING_N)
189+
.map(|i| Some(format!("thread-{}", i % 8)))
190+
.collect(),
191+
);
192+
}
193+
194+
/// Request ids: 20-byte buffer-backed values that tie on the inline prefix `req-`.
195+
#[divan::bench]
196+
fn max_utf8_view_tied_prefix_outlined(bencher: Bencher) {
197+
bench_min_max(
198+
bencher,
199+
(0..STRING_N)
200+
.map(|i| Some(format!("req-{i:016x}")))
201+
.collect(),
202+
);
203+
}
204+
205+
/// Log lines: long, buffer-backed values that agree for their first twenty-six bytes.
206+
#[divan::bench]
207+
fn max_utf8_view_shared_prefix(bencher: Bencher) {
208+
bench_min_max(
209+
bencher,
210+
(0..STRING_N)
211+
.map(|i| {
212+
Some(format!(
213+
"This is log message number {i} with some additional content to simulate real logs"
214+
))
215+
})
216+
.collect(),
217+
);
218+
}
219+
220+
/// Distinct prefixes with ~50% random nulls, producing many short valid runs.
221+
#[divan::bench]
222+
fn max_utf8_view_nulls_scattered(bencher: Bencher) {
223+
let mut rng = StdRng::seed_from_u64(7);
224+
bench_min_max(
225+
bencher,
226+
(0..STRING_N)
227+
.map(|_| {
228+
rng.random_bool(0.5)
229+
.then(|| format!("{:016x}-{:016x}", rng.random::<u64>(), rng.random::<u64>()))
230+
})
231+
.collect(),
232+
);
233+
}

‎vortex-array/src/aggregate_fn/fns/min_max/mod.rs‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@ mod bool;
55
mod decimal;
66
mod extension;
77
mod primitive;
8-
mod varbin;
8+
mod varbinview;
99

1010
use std::sync::LazyLock;
1111

@@ -19,7 +19,7 @@ use self::bool::accumulate_bool;
1919
use self::decimal::accumulate_decimal;
2020
use self::extension::accumulate_extension;
2121
use self::primitive::accumulate_primitive;
22-
use self::varbin::accumulate_varbinview;
22+
use self::varbinview::accumulate_varbinview;
2323
use crate::ArrayRef;
2424
use crate::Canonical;
2525
use crate::Columnar;

‎vortex-array/src/aggregate_fn/fns/min_max/varbin.rs‎

Lines changed: 0 additions & 82 deletions
This file was deleted.

0 commit comments

Comments
 (0)