Skip to content

Commit c05ea8e

Browse files
committed
Compress fixed-size lists without their null rows
A fixed-size list stores `list_size` elements for every row, null or not, and the compressor only ever compressed those elements, so it had to encode whatever sits under the null rows. A UUID column that is mostly null was compressed as dense bytes unless the elements happened to be 90% zeros, and once mixed with a dense chunk not at all. The compressor now offers fixed-size lists to schemes before compressing their elements, after canonicalizing the elements so a scheme's output is compared against the list's real size. The new `FixedSizeListSparseScheme` estimates the saving of storing only the valid rows plus their bit-packed positions, and when it is at least 10% encodes the list as `Sparse` with a null fill, its values being the valid rows compressed recursively. On the data from #10417, the mostly-null file shrinks from 25,296 to 9,056 bytes and the rewrite of both files from 292,808 to 190,768 bytes. Signed-off-by: Claude <noreply@anthropic.com> Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_015XzXtnsrnEsBNVa9R2VqKQ
1 parent c3ad942 commit c05ea8e

8 files changed

Lines changed: 242 additions & 0 deletions

File tree

‎Cargo.lock‎

Lines changed: 1 addition & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎vortex-btrblocks/Cargo.toml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@ vortex-edition = { workspace = true }
2828
vortex-error = { workspace = true }
2929
vortex-fastlanes = { workspace = true }
3030
vortex-fsst = { workspace = true }
31+
vortex-mask = { workspace = true }
3132
vortex-onpair = { workspace = true }
3233
vortex-pco = { workspace = true, optional = true }
3334
vortex-runend = { workspace = true }

‎vortex-btrblocks/src/builder.rs‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@ use crate::SchemeExt;
1717
use crate::SchemeId;
1818
use crate::allowed_ids::AllowedIds;
1919
use crate::schemes::binary;
20+
use crate::schemes::fixed_size_list;
2021
use crate::schemes::float;
2122
use crate::schemes::integer;
2223
use crate::schemes::string;
@@ -73,6 +74,7 @@ impl CompressionMode {
7374
float::FloatRLEScheme.id(),
7475
float::NullDominatedSparseScheme.id(),
7576
string::NullDominatedSparseScheme.id(),
77+
fixed_size_list::FixedSizeListSparseScheme.id(),
7678
string::StringDictScheme.id(),
7779
binary::BinaryDictScheme.id(),
7880
]);
Lines changed: 163 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,163 @@
1+
// SPDX-License-Identifier: Apache-2.0
2+
// SPDX-FileCopyrightText: Copyright the Vortex contributors
3+
4+
//! Sparse encoding for fixed-size lists with many null rows.
5+
6+
use vortex_array::ArrayId;
7+
use vortex_array::ArrayRef;
8+
use vortex_array::Canonical;
9+
use vortex_array::ExecutionCtx;
10+
use vortex_array::IntoArray;
11+
use vortex_array::VTable;
12+
use vortex_array::arrays::PrimitiveArray;
13+
use vortex_array::arrays::primitive::PrimitiveArrayExt;
14+
use vortex_array::scalar::Scalar;
15+
use vortex_array::validity::Validity;
16+
use vortex_buffer::Buffer;
17+
use vortex_compressor::scheme::ChildSelection;
18+
use vortex_compressor::scheme::CompressionEstimate;
19+
use vortex_compressor::scheme::DescendantExclusion;
20+
use vortex_compressor::scheme::EstimateVerdict;
21+
use vortex_error::VortexResult;
22+
use vortex_mask::AllOr;
23+
use vortex_mask::Mask;
24+
use vortex_sparse::Sparse;
25+
26+
use crate::ArrayAndStats;
27+
use crate::CascadingCompressor;
28+
use crate::CompressorContext;
29+
use crate::Scheme;
30+
use crate::SchemeExt;
31+
use crate::schemes::integer::IntDictScheme;
32+
use crate::schemes::integer::IntRLEScheme;
33+
use crate::schemes::integer::RunEndScheme;
34+
use crate::schemes::integer::SparseScheme as IntSparseScheme;
35+
36+
/// The smallest estimated ratio of the dense list's size to the sparse one's worth encoding for.
37+
const MIN_ESTIMATED_RATIO: f64 = 1.1;
38+
39+
/// Sparse encoding of a fixed-size list that stores only its valid rows.
40+
///
41+
/// A fixed-size list keeps `list_size` elements for every row, null or not, so compressing its
42+
/// elements has to encode the arbitrary values under null rows too. This scheme instead stores the
43+
/// positions of the valid rows and a list of only those rows, with null as the fill value.
44+
#[derive(Debug, Copy, Clone, PartialEq, Eq)]
45+
pub struct FixedSizeListSparseScheme;
46+
47+
impl Scheme for FixedSizeListSparseScheme {
48+
fn scheme_name(&self) -> &'static str {
49+
"vortex.fixed_size_list.sparse"
50+
}
51+
52+
fn matches(&self, canonical: &Canonical) -> bool {
53+
matches!(canonical, Canonical::FixedSizeList(_)) && canonical.dtype().is_nullable()
54+
}
55+
56+
fn produced_encodings(&self) -> Vec<ArrayId> {
57+
vec![Sparse.id()]
58+
}
59+
60+
/// Children: values=0, indices=1.
61+
fn num_children(&self) -> usize {
62+
2
63+
}
64+
65+
/// Sparse indices (child 1) are monotonically increasing positions with all unique values.
66+
fn descendant_exclusions(&self) -> Vec<DescendantExclusion> {
67+
vec![
68+
DescendantExclusion {
69+
excluded: IntDictScheme.id(),
70+
children: ChildSelection::One(1),
71+
},
72+
DescendantExclusion {
73+
excluded: RunEndScheme.id(),
74+
children: ChildSelection::One(1),
75+
},
76+
DescendantExclusion {
77+
excluded: IntRLEScheme.id(),
78+
children: ChildSelection::One(1),
79+
},
80+
DescendantExclusion {
81+
excluded: IntSparseScheme.id(),
82+
children: ChildSelection::One(1),
83+
},
84+
]
85+
}
86+
87+
/// Compares the list's size with the valid rows plus their bit-packed positions.
88+
fn expected_compression_ratio(
89+
&self,
90+
data: &ArrayAndStats,
91+
_compress_ctx: CompressorContext,
92+
exec_ctx: &mut ExecutionCtx,
93+
) -> CompressionEstimate {
94+
let array = data.array();
95+
let len = array.len();
96+
let Ok(mask) = validity_mask(array, exec_ctx) else {
97+
return CompressionEstimate::Verdict(EstimateVerdict::Skip);
98+
};
99+
let valid = mask.true_count();
100+
if valid == 0 || valid == len {
101+
return CompressionEstimate::Verdict(EstimateVerdict::Skip);
102+
}
103+
104+
let dense_bytes = array.nbytes() as f64;
105+
let row_bytes = dense_bytes / len as f64;
106+
let index_bytes = f64::from(usize::BITS - (len - 1).leading_zeros()) / 8.0;
107+
let sparse_bytes = valid as f64 * (row_bytes + index_bytes);
108+
109+
let ratio = dense_bytes / sparse_bytes;
110+
if ratio < MIN_ESTIMATED_RATIO {
111+
return CompressionEstimate::Verdict(EstimateVerdict::Skip);
112+
}
113+
CompressionEstimate::Verdict(EstimateVerdict::Ratio(ratio))
114+
}
115+
116+
fn compress(
117+
&self,
118+
compressor: &CascadingCompressor,
119+
data: &ArrayAndStats,
120+
compress_ctx: CompressorContext,
121+
exec_ctx: &mut ExecutionCtx,
122+
) -> VortexResult<ArrayRef> {
123+
let array = data.array();
124+
let mask = validity_mask(array, exec_ctx)?;
125+
let AllOr::Some(valid_indices) = mask.indices() else {
126+
// The estimate skips lists without both valid and null rows.
127+
return Ok(array.clone());
128+
};
129+
130+
let values = array.filter(mask.clone())?;
131+
let compressed_values =
132+
compressor.compress_child(&values, &compress_ctx, self.id(), 0, exec_ctx)?;
133+
134+
let indices = PrimitiveArray::new(
135+
valid_indices
136+
.iter()
137+
.map(|&idx| idx as u64)
138+
.collect::<Buffer<u64>>(),
139+
Validity::NonNullable,
140+
)
141+
.narrow(exec_ctx)?;
142+
let compressed_indices = compressor.compress_child(
143+
&indices.into_array(),
144+
&compress_ctx,
145+
self.id(),
146+
1,
147+
exec_ctx,
148+
)?;
149+
150+
Ok(Sparse::try_new(
151+
compressed_indices,
152+
compressed_values,
153+
array.len(),
154+
Scalar::null(array.dtype().clone()),
155+
)?
156+
.into_array())
157+
}
158+
}
159+
160+
/// Returns which rows of `array` are valid.
161+
fn validity_mask(array: &ArrayRef, exec_ctx: &mut ExecutionCtx) -> VortexResult<Mask> {
162+
array.validity()?.execute_mask(array.len(), exec_ctx)
163+
}

‎vortex-btrblocks/src/schemes/mod.rs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@ pub mod integer;
99
pub mod string;
1010

1111
pub mod decimal;
12+
pub mod fixed_size_list;
1213
pub mod temporal;
1314

1415
pub(crate) mod patches;

‎vortex-btrblocks/src/session.rs‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@ use crate::Scheme;
1313
use crate::SchemeExt;
1414
use crate::schemes::binary;
1515
use crate::schemes::decimal;
16+
use crate::schemes::fixed_size_list;
1617
use crate::schemes::float;
1718
use crate::schemes::integer;
1819
use crate::schemes::string;
@@ -91,6 +92,8 @@ impl Default for CompressionSession {
9192
&decimal::DECIMAL_V1,
9293
// Temporal schemes.
9394
&temporal::TemporalScheme,
95+
// Fixed-size list schemes.
96+
&fixed_size_list::FixedSizeListSparseScheme,
9497
],
9598
}
9699
}

‎vortex-btrblocks/src/tests.rs‎

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@ use vortex_array::IntoArray;
2121
use vortex_array::VortexSessionExecute;
2222
use vortex_array::aggregate_fn::fns::sum::sum;
2323
use vortex_array::arrays::DecimalArray;
24+
use vortex_array::arrays::FixedSizeListArray;
2425
use vortex_array::arrays::ListArray;
2526
use vortex_array::arrays::PrimitiveArray;
2627
use vortex_array::arrays::StructArray;
@@ -32,6 +33,7 @@ use vortex_array::dtype::DecimalDType;
3233
use vortex_array::dtype::Nullability;
3334
use vortex_array::extension::datetime::TimeUnit;
3435
use vortex_array::validity::Validity;
36+
use vortex_buffer::Buffer;
3537
#[cfg(feature = "zstd")]
3638
use vortex_edition::EDITION_DECLARATIONS;
3739
#[cfg(feature = "zstd")]
@@ -44,6 +46,7 @@ use vortex_edition::EditionSessionExt;
4446
use vortex_edition::declarations::core::CORE_2026_08_3;
4547
use vortex_error::VortexResult;
4648
use vortex_fastlanes::Delta;
49+
use vortex_sparse::Sparse;
4750

4851
use crate::BtrBlocksCompressor;
4952
use crate::BtrBlocksCompressorBuilder;
@@ -135,6 +138,48 @@ fn test_delta_unaligned_roundtrip(#[case] input: PrimitiveArray) -> VortexResult
135138
Ok(())
136139
}
137140

141+
/// Fixed-size lists of random bytes, like UUIDs, with the rows `is_valid` picks valid. The null
142+
/// rows hold zeros, as Arrow's fixed-size binary arrays do.
143+
fn random_byte_lists(rows: usize, is_valid: impl Fn(usize) -> bool) -> ArrayRef {
144+
let mut rng = StdRng::seed_from_u64(42);
145+
let elements = (0..rows * 16)
146+
.map(|i| {
147+
if is_valid(i / 16) {
148+
rng.next_u32().to_le_bytes()[0]
149+
} else {
150+
0
151+
}
152+
})
153+
.collect::<Buffer<u8>>();
154+
let validity = Validity::from_iter((0..rows).map(&is_valid));
155+
FixedSizeListArray::new(elements.into_array(), 16, validity, rows).into_array()
156+
}
157+
158+
#[rstest]
159+
#[case::mostly_null(|row| row % 20 == 0, true)]
160+
#[case::half_null(|row| row % 2 == 0, true)]
161+
// Dropping one row in twenty saves less than storing the valid rows' positions costs.
162+
#[case::few_nulls(|row| row % 20 != 0, false)]
163+
#[case::all_valid(|_| true, false)]
164+
fn test_fixed_size_list_drops_null_rows(
165+
#[case] is_valid: fn(usize) -> bool,
166+
#[case] sparse: bool,
167+
) -> VortexResult<()> {
168+
let compressor = BtrBlocksCompressorBuilder::from_session(&SESSION)
169+
.unrestricted()
170+
.build();
171+
let input = random_byte_lists(8_000, is_valid);
172+
let compressed = assert_roundtrip(&compressor, &input)?;
173+
assert_eq!(
174+
compressed.is::<Sparse>(),
175+
sparse,
176+
"{}",
177+
compressed.display_tree()
178+
);
179+
180+
Ok(())
181+
}
182+
138183
#[cfg(feature = "zstd")]
139184
#[rstest]
140185
#[case::array_level(CORE_2026_08_3, vortex_zstd::Zstd.id())]

‎vortex-compressor/src/compressor/cascade.rs‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -172,6 +172,32 @@ impl CascadingCompressor {
172172
}
173173
Canonical::Map(map_array) => self.compress_map_array(map_array, compress_ctx, exec_ctx),
174174
Canonical::FixedSizeList(fsl_array) => {
175+
// Canonicalize the elements first, so that schemes see the list's real size and
176+
// their output is accepted only if it beats it.
177+
let elements = fsl_array
178+
.elements()
179+
.clone()
180+
.execute::<CanonicalValidity>(exec_ctx)?
181+
.0
182+
.compact(exec_ctx)?
183+
.into_array();
184+
let fsl_array = FixedSizeListArray::try_new(
185+
elements,
186+
fsl_array.list_size(),
187+
fsl_array.validity()?,
188+
fsl_array.len(),
189+
)?;
190+
191+
// Schemes over the whole list can drop the elements of null rows, which
192+
// compressing the elements alone has to keep.
193+
if let Selection::Compressed(compressed) = self.choose_and_compress(
194+
Canonical::FixedSizeList(fsl_array.clone()),
195+
compress_ctx,
196+
exec_ctx,
197+
)? {
198+
return Ok(compressed);
199+
}
200+
175201
let compressed_elems = self.compress(fsl_array.elements(), exec_ctx)?;
176202

177203
Ok(FixedSizeListArray::try_new(

0 commit comments

Comments
 (0)