Make protobuf metadata fields crate-private

Signed-off-by: Connor Tsui <connor.tsui20@gmail.com>

Author: connortsui20

encodings/datetime-parts/src/array.rs

@@ -64,11 +64,11 @@ pub struct DateTimePartsMetadata {
     // Validity lives in the days array
     // TODO(ngates): we should actually model this with a Tuple array when we have one.
     #[prost(enumeration = "PType", tag = "1")]
-    pub days_ptype: i32,
+    pub(crate) days_ptype: i32,
     #[prost(enumeration = "PType", tag = "2")]
-    pub seconds_ptype: i32,
+    pub(crate) seconds_ptype: i32,
     #[prost(enumeration = "PType", tag = "3")]
-    pub subseconds_ptype: i32,
+    pub(crate) subseconds_ptype: i32,
 }
 
 impl DateTimePartsMetadata {

encodings/fastlanes/src/rle/vtable/mod.rs

@@ -49,17 +49,17 @@ pub type RLEArray = Array<RLE>;
 #[derive(Clone, prost::Message)]
 pub struct RLEMetadata {
     #[prost(uint64, tag = "1")]
-    pub values_len: u64,
+    pub(crate) values_len: u64,
     #[prost(uint64, tag = "2")]
-    pub indices_len: u64,
+    pub(crate) indices_len: u64,
     #[prost(int32, tag = "3")]
-    pub indices_ptype: i32,
+    pub(crate) indices_ptype: i32,
     #[prost(uint64, tag = "4")]
-    pub values_idx_offsets_len: u64,
+    pub(crate) values_idx_offsets_len: u64,
     #[prost(int32, tag = "5")]
-    pub values_idx_offsets_ptype: i32,
+    pub(crate) values_idx_offsets_ptype: i32,
     #[prost(uint64, tag = "6", default = "0")]
-    pub offset: u64,
+    pub(crate) offset: u64,
 }
 
 impl RLEMetadata {

encodings/onpair/src/array.rs

@@ -81,24 +81,24 @@ pub type OnPairArray = Array<OnPair>;
 pub struct OnPairMetadata {
     /// Width of the per-row primitive `uncompressed_lengths` child.
     #[prost(enumeration = "PType", tag = "1")]
-    pub uncompressed_lengths_ptype: i32,
+    pub(crate) uncompressed_lengths_ptype: i32,
     /// Number of dictionary tokens. `dict_offsets` has length `dict_size + 1`.
     #[prost(uint32, tag = "3")]
-    pub dict_size: u32,
+    pub(crate) dict_size: u32,
     /// Length of the `codes` slot child. A sliced array may retain codes that
     /// fall outside its visible row range.
     #[prost(uint64, tag = "4")]
-    pub codes_len: u64,
+    pub(crate) codes_len: u64,
     /// PType of the `dict_offsets` slot child (defaults to U32, may be
     /// narrowed to U16/U8 by the cascading compressor when values fit).
     #[prost(enumeration = "PType", tag = "5")]
-    pub dict_offsets_ptype: i32,
+    pub(crate) dict_offsets_ptype: i32,
     /// PType of the `codes` slot child.
     #[prost(enumeration = "PType", tag = "6")]
-    pub codes_ptype: i32,
+    pub(crate) codes_ptype: i32,
     /// PType of the `codes_offsets` slot child.
     #[prost(enumeration = "PType", tag = "7")]
-    pub codes_offsets_ptype: i32,
+    pub(crate) codes_offsets_ptype: i32,
 }
 
 impl OnPairMetadata {

encodings/parquet-variant/src/vtable.rs

@@ -67,13 +67,13 @@ pub struct ParquetVariant;
 struct ParquetVariantMetadataProto {
     /// Whether the un-shredded `value` child is present.
     #[prost(bool, tag = "1")]
-    pub has_value: bool,
+    pub(crate) has_value: bool,
     /// DType of the shredded `typed_value`, if present.
     #[prost(message, optional, tag = "2")]
-    pub typed_value_dtype: Option<pb::DType>,
+    pub(crate) typed_value_dtype: Option<pb::DType>,
     /// Whether the `value` child is nullable.
     #[prost(bool, tag = "3")]
-    pub value_nullable: bool,
+    pub(crate) value_nullable: bool,
 }
 
 /// A [`ParquetVariant`]-encoded Vortex array.

encodings/pco/src/lib.rs

@@ -34,7 +34,7 @@ pub struct PcoPageInfo {
     // count of values.
     /// Number of valid primitive values stored in this page.
     #[prost(uint32, tag = "1")]
-    pub n_values: u32,
+    pub(crate) n_values: u32,
 }
 
 // We're calling this Info instead of Metadata because ChunkMeta refers to a specific
@@ -44,7 +44,7 @@ pub struct PcoPageInfo {
 pub struct PcoChunkInfo {
     /// Pages contained in this chunk.
     #[prost(message, repeated, tag = "1")]
-    pub pages: Vec<PcoPageInfo>,
+    pub(crate) pages: Vec<PcoPageInfo>,
 }
 
 #[derive(Clone, prost::Message)]
@@ -54,10 +54,10 @@ pub struct PcoMetadata {
     // no issue duplicating it here per PcoArray
     /// Pco file header bytes.
     #[prost(bytes, tag = "1")]
-    pub header: Vec<u8>,
+    pub(crate) header: Vec<u8>,
     /// Metadata for each compressed chunk.
     #[prost(message, repeated, tag = "2")]
-    pub chunks: Vec<PcoChunkInfo>,
+    pub(crate) chunks: Vec<PcoChunkInfo>,
 }
 
 #[cfg(test)]

encodings/runend/src/array.rs

@@ -63,11 +63,11 @@ pub type RunEndArray = Array<RunEnd>;
 #[derive(Clone, prost::Message)]
 pub struct RunEndMetadata {
     #[prost(int32, tag = "1")]
-    pub ends_ptype: i32,
+    pub(crate) ends_ptype: i32,
     #[prost(uint64, tag = "2")]
-    pub num_runs: u64,
+    pub(crate) num_runs: u64,
     #[prost(uint64, tag = "3")]
-    pub offset: u64,
+    pub(crate) offset: u64,
 }
 
 impl RunEndMetadata {

encodings/zstd/src/lib.rs

@@ -81,10 +81,24 @@ pub(crate) fn validate_frame_content_size(
 pub struct ZstdFrameMetadata {
     /// Uncompressed byte size of this frame.
     #[prost(uint64, tag = "1")]
-    pub uncompressed_size: u64,
+    pub(crate) uncompressed_size: u64,
     /// Number of valid values stored in this frame.
     #[prost(uint64, tag = "2")]
-    pub n_values: u64,
+    pub(crate) n_values: u64,
+}
+
+impl ZstdFrameMetadata {
+    /// Returns the uncompressed byte size of this frame.
+    ///
+    /// Returns an error if the stored size does not fit in [`usize`].
+    pub fn get_uncompressed_size(&self) -> VortexResult<usize> {
+        usize::try_from(self.uncompressed_size).map_err(|_| {
+            vortex_err!(
+                "ZstdFrameMetadata uncompressed_size must fit in usize, got {}",
+                self.uncompressed_size
+            )
+        })
+    }
 }
 
 #[derive(Clone, prost::Message)]
@@ -93,40 +107,47 @@ pub struct ZstdMetadata {
     // optional, will be 0 if there's no dictionary
     /// Dictionary size in bytes, or `0` when no dictionary is present.
     #[prost(uint32, tag = "1")]
-    pub dictionary_size: u32,
+    pub(crate) dictionary_size: u32,
     /// Metadata for each compressed frame.
     #[prost(message, repeated, tag = "2")]
-    pub frames: Vec<ZstdFrameMetadata>,
+    pub(crate) frames: Vec<ZstdFrameMetadata>,
+}
+
+impl ZstdMetadata {
+    /// Returns the metadata for the compressed frames in storage order.
+    pub fn frames(&self) -> &[ZstdFrameMetadata] {
+        &self.frames
+    }
 }
 
 #[derive(Clone, prost::Message)]
 /// Serialized metadata for the unstable `ZstdBuffers` encoding.
 pub struct ZstdBuffersMetadata {
     /// Encoding id of the inner array whose buffers were compressed.
     #[prost(string, tag = "1")]
-    pub inner_encoding_id: String,
+    pub(crate) inner_encoding_id: String,
     /// Serialized metadata of the inner array.
     #[prost(bytes = "vec", tag = "2")]
-    pub inner_metadata: Vec<u8>,
+    pub(crate) inner_metadata: Vec<u8>,
     /// Uncompressed byte size of each compressed buffer.
     #[prost(uint64, repeated, tag = "3")]
-    pub uncompressed_sizes: Vec<u64>,
+    pub(crate) uncompressed_sizes: Vec<u64>,
     /// Alignment of each buffer in bytes (must be a power of two).
     #[prost(uint32, repeated, tag = "4")]
-    pub buffer_alignments: Vec<u32>,
+    pub(crate) buffer_alignments: Vec<u32>,
     /// DType of child arrays. Children belong to inner encodings, and their
     /// dtypes don't persist after serialization, so we need to retrieve them
     /// from metadata.
     #[prost(message, repeated, tag = "5")]
-    pub child_dtypes: Vec<pb::DType>,
+    pub(crate) child_dtypes: Vec<pb::DType>,
     /// Length of each child array, ordered as "child_dtypes"
     #[prost(uint64, repeated, tag = "6")]
-    pub child_lens: Vec<u64>,
+    pub(crate) child_lens: Vec<u64>,
     /// Number of leading children that belong to the inner array. Any child after them holds
     /// the validity. Absent in arrays written before the validity was stored separately.
     #[prost(uint32, optional, tag = "7")]
-    pub inner_child_count: Option<u32>,
+    pub(crate) inner_child_count: Option<u32>,
     /// Index of the child holding the validity, if it is not implied by the dtype.
     #[prost(uint32, optional, tag = "8")]
-    pub validity_child: Option<u32>,
+    pub(crate) validity_child: Option<u32>,
 }

vortex-array/src/arrays/bool/vtable/mod.rs

@@ -52,7 +52,7 @@ pub(crate) fn initialize(session: &VortexSession) {
 pub struct BoolMetadata {
     // The offset in bits must be <8
     #[prost(uint32, tag = "1")]
-    pub offset: u32,
+    pub(crate) offset: u32,
 }
 
 impl ArrayHash for BoolData {

vortex-array/src/arrays/variant/vtable/mod.rs

@@ -51,7 +51,7 @@ pub struct Variant;
 #[derive(Clone, prost::Message)]
 struct VariantMetadataProto {
     #[prost(message, optional, tag = "1")]
-    pub shredded_dtype: Option<pb::DType>,
+    pub(crate) shredded_dtype: Option<pb::DType>,
 }
 
 impl VTable for Variant {

vortex-bench/src/conversions.rs

@@ -471,10 +471,7 @@ fn geoparquet_columns(metadata: &ParquetMetaData) -> HashSet<String> {
 
 /// The erased `vortex.st.wkb` extension dtype over a binary `storage` dtype.
 fn wkb_ext_dtype(storage: &DType) -> VortexResult<ExtDTypeRef> {
-    Ok(
-        ExtDType::<WellKnownBinary>::try_new(SpatialMetadata { crs: None }, storage.clone())?
-            .erased(),
-    )
+    Ok(ExtDType::<WellKnownBinary>::try_new(SpatialMetadata::default(), storage.clone())?.erased())
 }
 
 /// Re-type the named binary columns of a struct `dtype` as `vortex.st.wkb`, so the column