@@ -11,6 +11,7 @@ use vortex_array::ExecutionCtx;
1111use vortex_array:: IntoArray ;
1212use vortex_array:: arrays:: ConstantArray ;
1313use vortex_array:: arrays:: PrimitiveArray ;
14+ use vortex_array:: arrays:: primitive:: PrimitiveArrayExt ;
1415use vortex_array:: dtype:: NativePType ;
1516use vortex_array:: expr:: stats:: Stat ;
1617use vortex_array:: match_each_integer_ptype;
@@ -19,9 +20,11 @@ use vortex_array::validity::Validity;
1920use vortex_buffer:: BitBuffer ;
2021use vortex_buffer:: Buffer ;
2122use vortex_buffer:: BufferMut ;
23+ use vortex_compute:: lane_kernels:: IndexedSourceExt ;
2224use vortex_error:: VortexResult ;
2325use vortex_error:: vortex_err;
2426use vortex_mask:: AllOr ;
27+ use vortex_mask:: Mask ;
2528
2629use crate :: FL_CHUNK_SIZE ;
2730use crate :: FoR ;
@@ -53,21 +56,29 @@ impl FoRData {
5356 }
5457}
5558
59+ /// Subtracts `min` from every valid value. Null slots are written as zero so they cost no bits in
60+ /// a downstream bit-packing, and the validity bit selects the zero without a branch per value.
5661fn encode_primitive < T : NativePType + WrappingSub + PrimInt > (
5762 parray : PrimitiveArray ,
5863 min : T ,
5964 ctx : & mut ExecutionCtx ,
6065) -> VortexResult < PrimitiveArray > {
61- // Set null values to the min value, ensuring that decompress into a value in the primitive
62- // range (and stop them wrapping around).
63- let encoded = parray. map_each_with_validity :: < T , _ , _ > ( ctx, |( v, bool) | {
64- if bool {
65- v. wrapping_sub ( & min)
66- } else {
67- T :: zero ( )
68- }
69- } ) ?;
70- Ok ( encoded)
66+ let validity = parray. validity ( ) ?;
67+ let len = parray. len ( ) ;
68+ let values = parray. as_slice :: < T > ( ) ;
69+ let subtract = |v : T | v. wrapping_sub ( & min) ;
70+
71+ let mut encoded = BufferMut :: < T > :: with_capacity_in ( len, ctx. allocator ( ) . clone ( ) ) ;
72+ let out = & mut encoded. spare_capacity_mut ( ) [ ..len] ;
73+ match validity. execute_mask ( len, ctx) ? {
74+ Mask :: AllTrue ( _) => values. map_into ( out, subtract) ,
75+ Mask :: AllFalse ( _) => out. fill ( MaybeUninit :: new ( T :: zero ( ) ) ) ,
76+ Mask :: Values ( mask) => values. map_masked_into ( mask. bit_buffer ( ) , out, subtract) ,
77+ }
78+ // SAFETY: each branch writes every lane of `out`, which spans exactly `len` items.
79+ unsafe { encoded. set_len ( len) } ;
80+
81+ Ok ( PrimitiveArray :: new ( encoded. freeze ( ) , validity) )
7182}
7283
7384fn encode_chunked_typed < T : NativePType + WrappingSub + PrimInt > (
@@ -331,6 +342,30 @@ mod test {
331342 assert_eq ! ( encoded, Scalar :: from( 0i32 ) ) ;
332343 }
333344
345+ #[ test]
346+ fn test_compress_nullable_zeroes_null_slots ( ) -> VortexResult < ( ) > {
347+ let mut ctx = SESSION . create_execution_ctx ( ) ;
348+ let array = PrimitiveArray :: from_option_iter (
349+ ( 0 ..200i32 ) . map ( |i| ( i % 3 != 0 ) . then_some ( 1_000 + i) ) ,
350+ ) ;
351+ let compressed = FoRData :: encode ( array. clone ( ) , & mut ctx) ?;
352+ let reference = compressed
353+ . constant_reference ( )
354+ . ok_or_else ( || vortex_err ! ( "expected a constant reference" ) ) ?;
355+ assert_eq ! ( i32 :: try_from( & reference) ?, 1_001 ) ;
356+
357+ let encoded = compressed
358+ . encoded ( )
359+ . clone ( )
360+ . execute :: < PrimitiveArray > ( & mut ctx) ?;
361+ let expected: Vec < i32 > = ( 0 ..200i32 )
362+ . map ( |i| if i % 3 != 0 { i - 1 } else { 0 } )
363+ . collect ( ) ;
364+ assert_eq ! ( encoded. as_slice:: <i32 >( ) , expected. as_slice( ) ) ;
365+ assert_arrays_eq ! ( compressed, array, & mut ctx) ;
366+ Ok ( ( ) )
367+ }
368+
334369 #[ test]
335370 fn test_overflow ( ) -> VortexResult < ( ) > {
336371 let mut ctx = SESSION . create_execution_ctx ( ) ;
0 commit comments