1414//! | at most 16 per byte of value width | decode only the selected values |
1515//! | otherwise | decode to scratch, then compact the scratch |
1616//!
17- //! The scratch buffer stays in cache. It is compacted by copying the slices, or with the same
18- //! SIMD, byte compress and scalar kernels that `filter_buffer` chooses for the chunk's density.
17+ //! The scratch buffer stays in cache. It is compacted by copying the slices, by copying the runs
18+ //! of set bits when they average at least [`MIN_RUN_BYTES`], and otherwise with the same SIMD,
19+ //! byte compress and scalar kernels that `filter_buffer` chooses for the chunk's density.
1920
2021use std:: mem:: MaybeUninit ;
2122use std:: ptr;
@@ -31,6 +32,7 @@ use super::simd_compress;
3132use super :: simd_compress:: SLACK_BYTES ;
3233use super :: slice:: MaskBits ;
3334use super :: slice:: compact_by_bitmap;
35+ use super :: slice:: compact_runs_by_bitmap;
3436use super :: slice:: low_bits_mask;
3537use crate :: dtype:: NativePType ;
3638
@@ -48,6 +50,10 @@ const SPARSE_VALUES_PER_BYTE: usize = 16;
4850/// The largest number of selected values for which a chunk decodes only those values.
4951const MAX_SPARSE_VALUES : usize = SPARSE_VALUES_PER_BYTE * 8 ;
5052
53+ /// Chunks whose runs of selected values average at least this many bytes copy each run. Shorter
54+ /// runs are faster to compact with the SIMD and byte compress kernels.
55+ const MIN_RUN_BYTES : usize = 96 ;
56+
5157/// Number of mask bytes read to assemble the words of a chunk at an arbitrary bit offset.
5258const CHUNK_WORD_BYTES : usize = FILTER_CHUNK_LEN / 8 + 8 ;
5359
@@ -192,10 +198,9 @@ impl<'a, T: NativePType, D: ChunkDecoder<T>> ChunkedFilter<'a, T, D> {
192198 } else {
193199 let dst = self . dst ( ) . cast :: < T > ( ) ;
194200 let src = & self . decode_scratch ( chunk_idx) [ chunk_offset..] [ ..chunk_len] ;
195- let density = selected as f64 / chunk_len as f64 ;
196201 // SAFETY: the words select only values of `src` and clear any bits past
197202 // `chunk_len`, and the output has room for every selected value plus the slack.
198- let written = unsafe { compact_words ( src, words, density , dst) } ;
203+ let written = unsafe { compact_words ( src, words, selected , dst) } ;
199204 debug_assert_eq ! ( written, selected) ;
200205 self . written += selected;
201206 }
@@ -271,22 +276,33 @@ impl<'a, T: NativePType, D: ChunkDecoder<T>> ChunkedFilter<'a, T, D> {
271276 }
272277}
273278
274- /// Copies the values of `src` selected by `words` to `dst` with the kernel that
275- /// [`filter_buffer`](super::buffer::filter_buffer) chooses for `density`, and returns the number
279+ /// Copies the `selected` values of `src` selected by `words` to `dst` and returns the number
276280/// copied.
277281///
282+ /// Runs of set bits that average at least [`MIN_RUN_BYTES`] are copied run by run. Otherwise the kernel is the one that
283+ /// [`filter_buffer`](super::buffer::filter_buffer) chooses for the density.
284+ ///
278285/// # Safety
279286///
280287/// `words` must hold `src.len()` bits with any bits past `src.len()` cleared, and `dst` must be
281288/// valid for writes of every selected value plus [`SLACK_BYTES`].
282- unsafe fn compact_words < T : Copy > ( src : & [ T ] , words : & [ u64 ] , density : f64 , dst : * mut T ) -> usize {
289+ unsafe fn compact_words < T : Copy > ( src : & [ T ] , words : & [ u64 ] , selected : usize , dst : * mut T ) -> usize {
283290 let bits = MaskBits :: Words {
284291 words,
285292 len : src. len ( ) ,
286293 } ;
294+ // A run that crosses a word boundary counts once in each word, as the run walk copies it.
295+ let runs = words
296+ . iter ( )
297+ . map ( |& word| ( word & !( word << 1 ) ) . count_ones ( ) as usize )
298+ . sum :: < usize > ( ) ;
299+ let density = selected as f64 / src. len ( ) as f64 ;
300+
287301 // SAFETY: forwarded from the caller contract.
288302 unsafe {
289- if let Some ( written) = simd_compress:: compress_bits ( src, bits, density, dst) {
303+ if selected * size_of :: < T > ( ) >= MIN_RUN_BYTES * runs {
304+ compact_runs_by_bitmap ( src, bits, dst)
305+ } else if let Some ( written) = simd_compress:: compress_bits ( src, bits, density, dst) {
290306 written
291307 } else if density >= byte_compress_density_threshold :: < T > ( ) {
292308 byte_compress:: compress_words ( src, words, dst)
@@ -422,19 +438,20 @@ mod tests {
422438
423439 /// Runs long enough that the cached slices are used rather than the bitmap.
424440 #[ rstest]
425- fn filter_chunked_by_slices_matches_expected (
441+ fn filter_chunked_runs_matches_expected (
426442 #[ values( 9 , 12 , 200 , 3000 ) ] run_len : usize ,
427- #[ values( 16 , 64 , 1500 ) ] gap : usize ,
443+ #[ values( 1 , 16 , 64 , 1500 ) ] gap : usize ,
428444 #[ values( 0 , 3 , 1000 ) ] offset : usize ,
445+ #[ values( false , true ) ] slices : bool ,
429446 ) {
430447 let bits = ( 0 ..5000 )
431448 . map ( |i| i % ( run_len + gap) < run_len)
432449 . collect :: < BitBuffer > ( ) ;
433450
434- check :: < u8 > ( & bits, offset, true ) ;
435- check :: < u16 > ( & bits, offset, true ) ;
436- check :: < u32 > ( & bits, offset, true ) ;
437- check :: < u64 > ( & bits, offset, true ) ;
451+ check :: < u8 > ( & bits, offset, slices ) ;
452+ check :: < u16 > ( & bits, offset, slices ) ;
453+ check :: < u32 > ( & bits, offset, slices ) ;
454+ check :: < u64 > ( & bits, offset, slices ) ;
438455 }
439456
440457 #[ rstest]
0 commit comments