1111//! Import this module's [`ScanBuilder`] to select V2 explicitly. It accepts a [`ScanFile`]
1212//! directly and supports projection, filters, row ranges, and selections. Unsupported layouts return an
1313//! error; limits are not part of this builder's API. Work is partitioned into fixed
14- //! chunks of up to 65,536 rows, or 1,048,576 rows for filtered scans, independently of the stored
15- //! layout's chunk boundaries.
14+ //! chunks of up to 65,536 rows, or up to 1,048,576 rows for filtered scans, independently of the
15+ //! stored layout's chunk boundaries. Filtered scans use smaller chunks when the projection reads
16+ //! many bytes per row or the row range would give fewer chunks than threads.
1617//! Full-file projections use smaller 8,192-row multiples when serialized bytes per row are high.
1718//!
1819//! A [`ScanFile`] carrying a [`ScanCache`] shares its lowered plan, segment index, and dictionary
@@ -45,6 +46,7 @@ use vortex_error::VortexExpect;
4546use vortex_error:: VortexResult ;
4647use vortex_io:: request:: IoService ;
4748use vortex_io:: session:: RuntimeSessionExt ;
49+ use vortex_mask:: Mask ;
4850use vortex_scan:: planning:: planner:: WorkScope ;
4951use vortex_scan:: selection:: Selection ;
5052use vortex_scan:: strict_sorted_buffer:: StrictSortedBuffer ;
@@ -55,6 +57,7 @@ pub use self::cache::ScanCache;
5557use crate :: LayoutRef ;
5658use crate :: plan:: physical:: lower;
5759use crate :: plan:: physical:: optimize;
60+ use crate :: plan:: physical:: pipeline:: Scan ;
5861use crate :: plan:: physical:: plan_row_idx_expression;
5962use crate :: scan:: arrow:: to_record_batch;
6063use crate :: scan:: execution:: FilterPlans ;
@@ -76,13 +79,40 @@ fn rows_per_chunk(row_count: u64, locations: &[SegmentLocation]) -> usize {
7679 if row_count <= MAX_ROWS_PER_CHUNK as u64 {
7780 return MAX_ROWS_PER_CHUNK ;
7881 }
79- let bytes: u64 = locations
82+ let blocks = blocks_per_target_bytes ( total_bytes ( locations) , row_count) ;
83+ blocks. clamp ( 1 , 8 ) as usize * ROW_BLOCK_SIZE
84+ }
85+
86+ /// Rows per task of a filtered scan over `rows` rows of the file.
87+ ///
88+ /// Large chunks amortize filter evaluation over cheap projections. A projection that reads many
89+ /// bytes per row, or a row range that would otherwise give fewer chunks than threads, gets smaller
90+ /// chunks so the surviving rows still spread across the runtime's threads.
91+ fn filtered_rows_per_chunk (
92+ rows : u64 ,
93+ row_count : u64 ,
94+ projected_bytes : u64 ,
95+ parallelism : usize ,
96+ ) -> usize {
97+ let max_blocks = ( FILTERED_ROWS_PER_CHUNK / ROW_BLOCK_SIZE ) as u64 ;
98+ let min_blocks = ( MAX_ROWS_PER_CHUNK / ROW_BLOCK_SIZE ) as u64 ;
99+ let by_bytes = blocks_per_target_bytes ( projected_bytes, row_count) . max ( 1 ) ;
100+ let by_threads = ( rows / parallelism. max ( 1 ) as u64 / ROW_BLOCK_SIZE as u64 ) . max ( min_blocks) ;
101+ let blocks = by_bytes. min ( by_threads) . min ( max_blocks) ;
102+ usize:: try_from ( blocks) . vortex_expect ( "at most FILTERED_ROWS_PER_CHUNK blocks" ) * ROW_BLOCK_SIZE
103+ }
104+
105+ /// The row blocks that hold [`TARGET_BYTES_PER_CHUNK`] when `bytes` spread over `row_count` rows.
106+ fn blocks_per_target_bytes ( bytes : u64 , row_count : u64 ) -> u64 {
107+ let bytes_per_row = bytes. div_ceil ( row_count. max ( 1 ) ) . max ( 1 ) ;
108+ TARGET_BYTES_PER_CHUNK / bytes_per_row / ROW_BLOCK_SIZE as u64
109+ }
110+
111+ fn total_bytes ( locations : & [ SegmentLocation ] ) -> u64 {
112+ locations
80113 . iter ( )
81114 . map ( |location| u64:: from ( location. length ) )
82- . sum ( ) ;
83- let bytes_per_row = bytes. div_ceil ( row_count) . max ( 1 ) ;
84- let blocks = TARGET_BYTES_PER_CHUNK / bytes_per_row / ROW_BLOCK_SIZE as u64 ;
85- blocks. clamp ( 1 , 8 ) as usize * ROW_BLOCK_SIZE
115+ . sum ( )
86116}
87117
88118/// Fixed boundaries in the file's row domain, clipped to the requested range.
@@ -309,18 +339,40 @@ impl<A: 'static + Send> PreparedScan<A> {
309339 plan_row_idx_expression ( builder. projection . clone ( ) , root) ?,
310340 & builder. session ,
311341 ) ?;
312- cache. insert_projection ( builder. projection , plan)
342+ cache. insert_projection ( builder. projection . clone ( ) , plan)
313343 }
314344 }
315345 } ;
346+ let rows = builder. row_range . unwrap_or ( 0 ..row_count) ;
316347 let chunk_rows = if filter. is_some ( ) {
317- FILTERED_ROWS_PER_CHUNK
348+ let projected_bytes = if identity {
349+ total_bytes ( & file. locations )
350+ } else {
351+ cache. projection_bytes ( & builder. projection , || {
352+ let segments = Scan :: segments (
353+ & builder. session ,
354+ & projection,
355+ 0 ..row_count,
356+ & Mask :: new_true ( usize:: try_from ( row_count) ?) ,
357+ ) ?;
358+ Ok ( segments
359+ . iter ( )
360+ . map ( |segment| u64:: from ( file. locations [ * * segment as usize ] . length ) )
361+ . sum ( ) )
362+ } ) ?
363+ } ;
364+ filtered_rows_per_chunk (
365+ rows. end - rows. start ,
366+ row_count,
367+ projected_bytes,
368+ get_available_parallelism ( ) . unwrap_or ( 1 ) ,
369+ )
318370 } else if identity {
319371 rows_per_chunk ( row_count, & file. locations )
320372 } else {
321373 MAX_ROWS_PER_CHUNK
322374 } ;
323- let splits = row_chunks ( builder . row_range . unwrap_or ( 0 ..row_count ) , chunk_rows) ;
375+ let splits = row_chunks ( rows , chunk_rows) ;
324376 let ranges = match cache. ranges ( ) {
325377 Some ( ranges) => ranges,
326378 None => cache. insert_ranges ( segment_ranges ( & file. locations ) ) ,
0 commit comments