Skip to content

Commit ccb1a6f

Browse files
authored
Move users of map_each_with_validity and map_each_in_place to lane kernels and remove those methods (#10139)
Signed-off-by: Robert Kruszewski <github@robertk.io>
1 parent c1b204c commit ccb1a6f

12 files changed

Lines changed: 247 additions & 167 deletions

File tree

‎Cargo.lock‎

Lines changed: 2 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎encodings/fastlanes/Cargo.toml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@ prost = { workspace = true }
2828
rand = { workspace = true, optional = true }
2929
vortex-array = { workspace = true }
3030
vortex-buffer = { workspace = true }
31+
vortex-compute = { workspace = true }
3132
vortex-error = { workspace = true }
3233
vortex-mask = { workspace = true }
3334
vortex-session = { workspace = true }

‎encodings/fastlanes/src/for_/array/for_compress.rs‎

Lines changed: 45 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@ use vortex_array::ExecutionCtx;
1111
use vortex_array::IntoArray;
1212
use vortex_array::arrays::ConstantArray;
1313
use vortex_array::arrays::PrimitiveArray;
14+
use vortex_array::arrays::primitive::PrimitiveArrayExt;
1415
use vortex_array::dtype::NativePType;
1516
use vortex_array::expr::stats::Stat;
1617
use vortex_array::match_each_integer_ptype;
@@ -19,9 +20,11 @@ use vortex_array::validity::Validity;
1920
use vortex_buffer::BitBuffer;
2021
use vortex_buffer::Buffer;
2122
use vortex_buffer::BufferMut;
23+
use vortex_compute::lane_kernels::IndexedSourceExt;
2224
use vortex_error::VortexResult;
2325
use vortex_error::vortex_err;
2426
use vortex_mask::AllOr;
27+
use vortex_mask::Mask;
2528

2629
use crate::FL_CHUNK_SIZE;
2730
use crate::FoR;
@@ -53,21 +56,29 @@ impl FoRData {
5356
}
5457
}
5558

59+
/// Subtracts `min` from every valid value. Null slots are written as zero so they cost no bits in
60+
/// a downstream bit-packing, and the validity bit selects the zero without a branch per value.
5661
fn encode_primitive<T: NativePType + WrappingSub + PrimInt>(
5762
parray: PrimitiveArray,
5863
min: T,
5964
ctx: &mut ExecutionCtx,
6065
) -> VortexResult<PrimitiveArray> {
61-
// Set null values to the min value, ensuring that decompress into a value in the primitive
62-
// range (and stop them wrapping around).
63-
let encoded = parray.map_each_with_validity::<T, _, _>(ctx, |(v, bool)| {
64-
if bool {
65-
v.wrapping_sub(&min)
66-
} else {
67-
T::zero()
68-
}
69-
})?;
70-
Ok(encoded)
66+
let validity = parray.validity()?;
67+
let len = parray.len();
68+
let values = parray.as_slice::<T>();
69+
let subtract = |v: T| v.wrapping_sub(&min);
70+
71+
let mut encoded = BufferMut::<T>::with_capacity_in(len, ctx.allocator().clone());
72+
let out = &mut encoded.spare_capacity_mut()[..len];
73+
match validity.execute_mask(len, ctx)? {
74+
Mask::AllTrue(_) => values.map_into(out, subtract),
75+
Mask::AllFalse(_) => out.fill(MaybeUninit::new(T::zero())),
76+
Mask::Values(mask) => values.map_masked_into(mask.bit_buffer(), out, subtract),
77+
}
78+
// SAFETY: each branch writes every lane of `out`, which spans exactly `len` items.
79+
unsafe { encoded.set_len(len) };
80+
81+
Ok(PrimitiveArray::new(encoded.freeze(), validity))
7182
}
7283

7384
fn encode_chunked_typed<T: NativePType + WrappingSub + PrimInt>(
@@ -331,6 +342,30 @@ mod test {
331342
assert_eq!(encoded, Scalar::from(0i32));
332343
}
333344

345+
#[test]
346+
fn test_compress_nullable_zeroes_null_slots() -> VortexResult<()> {
347+
let mut ctx = SESSION.create_execution_ctx();
348+
let array = PrimitiveArray::from_option_iter(
349+
(0..200i32).map(|i| (i % 3 != 0).then_some(1_000 + i)),
350+
);
351+
let compressed = FoRData::encode(array.clone(), &mut ctx)?;
352+
let reference = compressed
353+
.constant_reference()
354+
.ok_or_else(|| vortex_err!("expected a constant reference"))?;
355+
assert_eq!(i32::try_from(&reference)?, 1_001);
356+
357+
let encoded = compressed
358+
.encoded()
359+
.clone()
360+
.execute::<PrimitiveArray>(&mut ctx)?;
361+
let expected: Vec<i32> = (0..200i32)
362+
.map(|i| if i % 3 != 0 { i - 1 } else { 0 })
363+
.collect();
364+
assert_eq!(encoded.as_slice::<i32>(), expected.as_slice());
365+
assert_arrays_eq!(compressed, array, &mut ctx);
366+
Ok(())
367+
}
368+
334369
#[test]
335370
fn test_overflow() -> VortexResult<()> {
336371
let mut ctx = SESSION.create_execution_ctx();

‎encodings/fastlanes/src/for_/array/for_decompress.rs‎

Lines changed: 35 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,8 @@ use vortex_array::scalar::Scalar;
2626
use vortex_buffer::Buffer;
2727
use vortex_buffer::BufferAllocatorRef;
2828
use vortex_buffer::BufferMut;
29+
use vortex_compute::lane_kernels::IndexedSinkExt;
30+
use vortex_compute::lane_kernels::IndexedSourceExt;
2931
use vortex_error::VortexExpect;
3032
use vortex_error::VortexResult;
3133
use vortex_error::vortex_err;
@@ -56,7 +58,7 @@ fn decompress_one_ref(
5658
return fused_decompress(array, bp, ctx);
5759
}
5860

59-
add_reference(array, reference)
61+
add_reference(array, reference, ctx)
6062
}
6163

6264
/// Unpack a BitPacked child and add the constant reference in one pass.
@@ -88,15 +90,20 @@ fn fused_decompress_typed<
8890
}
8991

9092
/// Decode `encoded`, then add `reference` to every value.
91-
fn add_reference(array: &FoRArray, reference: &Scalar) -> VortexResult<PrimitiveArray> {
93+
fn add_reference(
94+
array: &FoRArray,
95+
reference: &Scalar,
96+
ctx: &mut ExecutionCtx,
97+
) -> VortexResult<PrimitiveArray> {
9298
match_each_integer_ptype!(array.ptype(), |T| {
93-
add_reference_typed::<T>(array, reference)
99+
add_reference_typed::<T>(array, reference, ctx)
94100
})
95101
}
96102

97103
fn add_reference_typed<T: NativePType + WrappingAdd + PrimInt>(
98104
array: &FoRArray,
99105
reference: &Scalar,
106+
ctx: &mut ExecutionCtx,
100107
) -> VortexResult<PrimitiveArray> {
101108
let encoded = array.encoded().as_::<Primitive>().into_owned();
102109
let min = reference
@@ -108,18 +115,38 @@ fn add_reference_typed<T: NativePType + WrappingAdd + PrimInt>(
108115
}
109116
let validity = encoded.validity()?;
110117
Ok(PrimitiveArray::new(
111-
decompress_primitive(encoded.into_buffer::<T>(), min),
118+
decompress_primitive(encoded.try_into_buffer_mut::<T>(), min, ctx),
112119
validity,
113120
))
114121
}
115122

123+
/// Adds `min` to every value. A uniquely owned buffer is mapped in place, a shared one is mapped
124+
/// into a new allocation.
116125
fn decompress_primitive<T: NativePType + WrappingAdd + PrimInt>(
117-
values: Buffer<T>,
126+
values: Result<BufferMut<T>, Buffer<T>>,
118127
min: T,
128+
ctx: &mut ExecutionCtx,
119129
) -> Buffer<T> {
120-
values
121-
.map_each_in_place(move |v| v.wrapping_add(&min))
122-
.freeze()
130+
let add = |v: T| v.wrapping_add(&min);
131+
132+
match values {
133+
Ok(mut values) => {
134+
values.as_mut_slice().map_into_in_place(add);
135+
values.freeze()
136+
}
137+
Err(values) => {
138+
let len = values.len();
139+
let mut decoded = BufferMut::<T>::with_capacity_in(len, ctx.allocator().clone());
140+
values
141+
.as_slice()
142+
.map_into(&mut decoded.spare_capacity_mut()[..len], add);
143+
144+
// SAFETY: `map_into` writes every lane of the `len` items.
145+
unsafe { decoded.set_len(len) };
146+
147+
decoded.freeze()
148+
}
149+
}
123150
}
124151

125152
/// Decompress an array whose chunks have different references.

‎encodings/zigzag/Cargo.toml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@ version = { workspace = true }
1616
[dependencies]
1717
vortex-array = { workspace = true }
1818
vortex-buffer = { workspace = true }
19+
vortex-compute = { workspace = true }
1920
vortex-error = { workspace = true }
2021
vortex-mask = { workspace = true }
2122
vortex-session = { workspace = true }

‎encodings/zigzag/src/compress.rs‎

Lines changed: 18 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,8 @@ use vortex_array::dtype::NativePType;
99
use vortex_array::dtype::PType;
1010
use vortex_array::validity::Validity;
1111
use vortex_buffer::BufferMut;
12+
use vortex_compute::lane_kernels::IndexedSinkExt;
13+
use vortex_compute::lane_kernels::ReinterpretSink;
1214
use vortex_error::VortexExpect;
1315
use vortex_error::VortexResult;
1416
use vortex_error::vortex_bail;
@@ -34,16 +36,19 @@ pub fn zigzag_encode(parray: ArrayView<'_, Primitive>) -> VortexResult<ZigZagArr
3436
}
3537

3638
fn zigzag_encode_primitive<T: ExternalZigZag + NativePType>(
37-
values: BufferMut<T>,
39+
mut values: BufferMut<T>,
3840
validity: Validity,
3941
) -> PrimitiveArray
4042
where
4143
<T as ExternalZigZag>::UInt: NativePType,
4244
{
43-
PrimitiveArray::new(
44-
values.map_each_in_place(|v| T::encode(v)).freeze(),
45-
validity,
46-
)
45+
ReinterpretSink::<T, T::UInt>::new(values.as_mut_slice()).map_into_in_place(T::encode);
46+
47+
// SAFETY: `ReinterpretSink::new` checks that both types have the same size and alignment,
48+
// and `map_into_in_place` writes a valid value into every lane.
49+
let encoded = unsafe { values.transmute::<T::UInt>() };
50+
51+
PrimitiveArray::new(encoded.freeze(), validity)
4752
}
4853

4954
pub fn zigzag_decode(parray: PrimitiveArray) -> PrimitiveArray {
@@ -63,16 +68,19 @@ pub fn zigzag_decode(parray: PrimitiveArray) -> PrimitiveArray {
6368
}
6469

6570
fn zigzag_decode_primitive<T: ExternalZigZag + NativePType>(
66-
values: BufferMut<T::UInt>,
71+
mut values: BufferMut<T::UInt>,
6772
validity: Validity,
6873
) -> PrimitiveArray
6974
where
7075
<T as ExternalZigZag>::UInt: NativePType,
7176
{
72-
PrimitiveArray::new(
73-
values.map_each_in_place(|v| T::decode(v)).freeze(),
74-
validity,
75-
)
77+
ReinterpretSink::<T::UInt, T>::new(values.as_mut_slice()).map_into_in_place(T::decode);
78+
79+
// SAFETY: `ReinterpretSink::new` checks that both types have the same size and alignment,
80+
// and `map_into_in_place` writes a valid value into every lane.
81+
let decoded = unsafe { values.transmute::<T>() };
82+
83+
PrimitiveArray::new(decoded.freeze(), validity)
7684
}
7785

7886
#[cfg(test)]

‎vortex-array/src/arrays/primitive/array/mod.rs‎

Lines changed: 0 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,6 @@
33

44
use std::fmt::Display;
55
use std::fmt::Formatter;
6-
use std::iter::repeat;
76

87
use smallvec::smallvec;
98
use vortex_buffer::Alignment;
@@ -486,31 +485,6 @@ impl Array<Primitive> {
486485
validity,
487486
}
488487
}
489-
490-
pub fn map_each_with_validity<T, R, F>(self, ctx: &mut ExecutionCtx, f: F) -> VortexResult<Self>
491-
where
492-
T: NativePType,
493-
R: NativePType,
494-
F: FnMut((T, bool)) -> R,
495-
{
496-
let validity = PrimitiveArrayExt::validity(&self);
497-
let data = self.into_data();
498-
let buf_iter = data.to_buffer::<T>().into_iter();
499-
500-
let buffer = match &validity {
501-
Validity::NonNullable | Validity::AllValid => {
502-
Buffer::<R>::from_trusted_len_iter(buf_iter.zip(repeat(true)).map(f))
503-
}
504-
Validity::AllInvalid => {
505-
Buffer::<R>::from_trusted_len_iter(buf_iter.zip(repeat(false)).map(f))
506-
}
507-
Validity::Array(val) => {
508-
let val = val.clone().execute::<BoolArray>(ctx)?.into_bit_buffer();
509-
Buffer::<R>::from_trusted_len_iter(buf_iter.zip(val.iter()).map(f))
510-
}
511-
};
512-
Ok(PrimitiveArray::new(buffer, validity))
513-
}
514488
}
515489

516490
impl PrimitiveData {

‎vortex-buffer/benches/vortex_buffer.rs‎

Lines changed: 0 additions & 61 deletions
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,6 @@ use num_traits::PrimInt;
1212
use vortex_buffer::Buffer;
1313
use vortex_buffer::BufferMut;
1414
use vortex_error::VortexExpect;
15-
use vortex_error::vortex_err;
1615

1716
#[global_allocator]
1817
static GLOBAL: MiMalloc = MiMalloc;
@@ -47,66 +46,6 @@ fn from_iter<B: FromIterator<i32>>(n: i32) {
4746
B::from_iter((0..n).map(|i| i % i32::MAX));
4847
}
4948

50-
trait MapEach<T, R> {
51-
type Output;
52-
53-
fn map_each<F>(self, f: F) -> Self::Output
54-
where
55-
F: FnMut(T) -> R;
56-
}
57-
58-
impl<T: ArrowNativeType, R: ArrowNativeType> MapEach<T, R> for Arrow<ScalarBuffer<T>> {
59-
type Output = Arrow<ScalarBuffer<R>>;
60-
61-
fn map_each<F>(self, f: F) -> Self::Output
62-
where
63-
F: FnMut(T) -> R,
64-
{
65-
Arrow(ScalarBuffer::from(
66-
self.0
67-
.into_inner()
68-
.into_vec::<T>()
69-
.map_err(|_| vortex_err!("Failed to convert Arrow buffer into a mut vec"))
70-
.vortex_expect("Failed to convert Arrow buffer into a mut vec")
71-
.into_iter()
72-
.map(f)
73-
.collect::<Vec<R>>(),
74-
))
75-
}
76-
}
77-
78-
impl<T: Copy, R> MapEach<T, R> for Buffer<T> {
79-
type Output = BufferMut<R>;
80-
81-
fn map_each<F>(self, f: F) -> Self::Output
82-
where
83-
F: FnMut(T) -> R,
84-
{
85-
Buffer::<T>::map_each_in_place(self, f)
86-
}
87-
}
88-
89-
impl<T: Copy, R> MapEach<T, R> for BufferMut<T> {
90-
type Output = BufferMut<R>;
91-
92-
fn map_each<F>(self, f: F) -> Self::Output
93-
where
94-
F: FnMut(T) -> R,
95-
{
96-
BufferMut::<T>::map_each_in_place(self, f)
97-
}
98-
}
99-
100-
#[divan::bench(
101-
types = [Arrow<ScalarBuffer<i32>>, Buffer<i32>, BufferMut<i32>],
102-
args = INPUT_SIZE,
103-
)]
104-
fn map_each<B: MapEach<i32, u32> + FromIterator<i32>>(bencher: Bencher, n: i32) {
105-
bencher
106-
.with_inputs(|| B::from_iter((0..n).map(|i| i % i32::MAX)))
107-
.bench_values(|buffer| B::map_each(buffer, |i| (i as u32) + 1));
108-
}
109-
11049
/// Element-at-a-time pushes cost tens of nanoseconds each, so cap the top size to keep the
11150
/// benchmark runtime bounded.
11251
const PUSH_INPUT_SIZE: &[i32] = &[128, 1024, 2048, 16_384, 32_768];

0 commit comments

Comments
 (0)