Updated duckdb client to read the nested stats.

Author: thorfour

vortex-duckdb/cpp/copy_function.cpp

@@ -11,7 +11,6 @@
 #include "duckdb/main/capi/capi_internal.hpp"
 #include "duckdb/main/client_context.hpp"
 #include "duckdb/main/connection.hpp"
-#include "duckdb/parser/keyword_helper.hpp"
 #include "duckdb/parser/parsed_data/create_copy_function_info.hpp"
 
 unique_ptr<FunctionData> copy_to_bind(ClientContext &,
@@ -40,7 +39,7 @@ unique_ptr<FunctionData> copy_to_bind(ClientContext &,
         throw BinderException(IntoErrString(error_out));
     }
     auto cdata = unique_ptr<CData>(reinterpret_cast<CData *>(ffi_bind_data));
-    return make_uniq<VortexCopyBindData>(std::move(cdata), names);
+    return make_uniq<VortexCopyBindData>(std::move(cdata));
 }
 
 unique_ptr<GlobalFunctionData>
@@ -86,7 +85,7 @@ void copy_to_get_written_statistics(ClientContext &,
     gstate.Cast<VortexCopyGlobalState>().written_stats = &statistics;
 }
 
-void copy_to_finalize(ClientContext &, FunctionData &bind_data, GlobalFunctionData &gstate) {
+void copy_to_finalize(ClientContext &, FunctionData &, GlobalFunctionData &gstate) {
     auto &global = gstate.Cast<VortexCopyGlobalState>();
     void *const ffi_global = global.ffi_global->DataPtr();
     duckdb_vx_error error_out = nullptr;
@@ -98,25 +97,18 @@ void copy_to_finalize(ClientContext &, FunctionData &bind_data, GlobalFunctionDa
     if (!global.written_stats) {
         return;
     }
-    auto &names = bind_data.Cast<VortexCopyBindData>().column_names;
     duckdb_vx_written_file_statistics file_stats;
     if (!duckdb_copy_function_get_written_file_statistics(ffi_global, &file_stats)) {
         // Statistics were requested (written_stats is set) but the finished write produced none;
         // that is an internal inconsistency, not a silently empty result.
         throw InternalException("vortex COPY: written statistics were requested but not produced");
     }
-    if (file_stats.num_columns != names.size()) {
-        throw InternalException("vortex COPY: %llu statistics columns for %llu written columns",
-                                file_stats.num_columns,
-                                names.size());
-    }
     D_ASSERT(global.written_stats != nullptr);
     global.written_stats->row_count = file_stats.row_count;
     global.written_stats->file_size_bytes = file_stats.file_size_bytes;
     global.written_stats->footer_size_bytes = Value::UBIGINT(file_stats.footer_size_bytes);
-    // Keyed by top-level column name only. The vortex footer reports one statistics set per
-    // top-level field, so nested struct/list leaf columns get no statistics here (unlike parquet,
-    // which recurses to leaf paths). Flat tables are fully covered.
+    // One entry per leaf column: nested struct fields are reported at their full path and structs
+    // get no entry of their own, as in parquet.
     for (idx_t i = 0; i < file_stats.num_columns; i++) {
         duckdb_vx_written_column_statistics col_stats {};
         duckdb_vx_error col_error = nullptr;
@@ -145,10 +137,9 @@ void copy_to_finalize(ClientContext &, FunctionData &bind_data, GlobalFunctionDa
         if (col_stats.has_nan_stat) {
             column["has_nan"] = Value::BOOLEAN(col_stats.contains_nan);
         }
-        // DuckLake keys column statistics by a quoted, dot-separated path (see
-        // DuckLakeUtil::ParseQuotedList); match the parquet writer, which quotes each name.
-        global.written_stats->column_statistics.emplace(KeywordHelper::WriteQuoted(names[i], '"'),
-                                                        std::move(column));
+        auto column_key = reinterpret_cast<Value *>(col_stats.column_key)->ToString();
+        duckdb_destroy_value(&col_stats.column_key);
+        global.written_stats->column_statistics.emplace(std::move(column_key), std::move(column));
     }
 }
 

vortex-duckdb/cpp/include/copy_function.hpp

@@ -8,12 +8,9 @@
 using namespace duckdb;
 
 struct VortexCopyBindData final : TableFunctionData {
-    VortexCopyBindData(unique_ptr<CData> ffi_bind, vector<string> column_names)
-        : ffi_bind(std::move(ffi_bind)), column_names(std::move(column_names)) {
+    VortexCopyBindData(unique_ptr<CData> ffi_bind) : ffi_bind(std::move(ffi_bind)) {
     }
     unique_ptr<CData> ffi_bind;
-    // Column names in write order, used to key WRITTEN_FILE_STATISTICS.
-    vector<string> column_names;
 };
 
 struct VortexCopyGlobalState final : GlobalFunctionData {

vortex-duckdb/cpp/include/table_function.h

@@ -83,6 +83,9 @@ typedef struct {
 
 // Per-column statistics of a written Vortex file.
 typedef struct {
+    // Owned VARCHAR value holding the column's quoted, dot-separated path (e.g. `"s"."b"`), the key
+    // format DuckLake expects; the caller must destroy it.
+    duckdb_value column_key;
     // Owned values, null if absent; the caller must destroy them.
     duckdb_value min;
     duckdb_value max;

vortex-duckdb/src/copy.rs

@@ -26,6 +26,8 @@ use vortex::error::vortex_bail;
 use vortex::error::vortex_err;
 use vortex::expr::stats::Precision;
 use vortex::expr::stats::Stat;
+use vortex::file::CompressedFieldSizes;
+use vortex::file::FileStatistics;
 use vortex::file::WriteOptionsSessionExt;
 use vortex::file::WriteSummary;
 use vortex::file::multi::parse_uri_or_path;
@@ -55,11 +57,13 @@ pub struct CopyFunctionBind {
 }
 assert_impl_all!(CopyFunctionBind: Send, Clone);
 
-/// The per-column compressed sizes are computed once here rather than per column, since DuckDB
-/// queries statistics one column at a time.
+/// The compressed field sizes and the statistics columns are computed once here rather than per
+/// column, since DuckDB queries statistics one column at a time.
 struct FinishedWrite {
     summary: WriteSummary,
-    column_sizes: Vec<u64>,
+    column_sizes: Option<CompressedFieldSizes>,
+    /// Indices into the footer's [`FileStatistics`] of the entries reported as statistics columns.
+    stats_columns: Vec<usize>,
 }
 
 /// Write to a file has two phases, writing data chunks and then closing the file.
@@ -160,12 +164,18 @@ pub fn copy_to_finalize(init_global: &mut CopyFunctionGlobal) -> VortexResult<()
             .take()
             .vortex_expect("no file to close");
         // Keep the write summary (footer + size) so DuckLake can read per-file statistics back
-        // without re-opening the file. Compute the per-column compressed sizes once, up front.
+        // without re-opening the file. Compute the compressed field sizes once, up front.
         let summary = task.await?;
-        let column_sizes = summary.compressed_column_sizes().unwrap_or_default();
+        let column_sizes = summary.footer().compressed_field_sizes().ok();
+        let stats_columns = summary
+            .footer()
+            .statistics()
+            .map(leaf_stats_columns)
+            .unwrap_or_default();
         *init_global.finished.lock() = Some(FinishedWrite {
             summary,
             column_sizes,
+            stats_columns,
         });
         Ok(())
     })
@@ -182,6 +192,8 @@ pub(crate) struct WrittenFileStats {
 /// Per-column statistics of the written Vortex file. `min`/`max` are DuckDB values converted from
 /// the Vortex scalar; every field is optional and omitted when the statistic is not available.
 pub(crate) struct WrittenColumnStats {
+    /// The column's path in DuckLake's key format (see [`ducklake_column_key`]).
+    pub column_key: String,
     pub min: Option<Value>,
     pub max: Option<Value>,
     pub null_count: Option<u64>,
@@ -193,7 +205,11 @@ pub(crate) struct WrittenColumnStats {
 /// Read file-level statistics back from the finished write. `None` before finalize.
 pub(crate) fn written_file_stats(global: &CopyFunctionGlobal) -> Option<WrittenFileStats> {
     let guard = global.finished.lock();
-    Some(file_stats_from_summary(&guard.as_ref()?.summary))
+    let finished = guard.as_ref()?;
+    Some(file_stats_from_summary(
+        &finished.summary,
+        finished.stats_columns.len(),
+    ))
 }
 
 /// Read per-column statistics for `column_index` from the finished write. `Ok(None)` if the file is
@@ -206,15 +222,45 @@ pub(crate) fn written_column_stats(
     let Some(finished) = guard.as_ref() else {
         return Ok(None);
     };
-    column_stats_from_summary(&finished.summary, column_index, &finished.column_sizes).map(Some)
+    column_stats_from_summary(
+        &finished.summary,
+        &finished.stats_columns,
+        column_index,
+        finished.column_sizes.as_ref(),
+    )
+    .map(Some)
 }
 
-fn file_stats_from_summary(summary: &WriteSummary) -> WrittenFileStats {
-    let num_columns = summary
-        .footer()
-        .dtype()
-        .as_struct_fields_opt()
-        .map_or(1, StructFields::nfields);
+/// The indices of the leaf entries of `file_stats`, which are the columns statistics are reported
+/// for. Like parquet, which reports one entry per leaf column writer, struct columns get no entry of
+/// their own; nested fields are reported at their full path. Lists and maps are opaque leaves of the
+/// statistics layout, so they are reported at their own path rather than per element.
+fn leaf_stats_columns(file_stats: &FileStatistics) -> Vec<usize> {
+    file_stats
+        .dtypes()
+        .iter()
+        .enumerate()
+        .filter(|(_, dtype)| !dtype.is_struct())
+        .map(|(index, _)| index)
+        .collect()
+}
+
+/// DuckLake keys column statistics by a quoted, dot-separated path (see
+/// `DuckLakeUtil::ParseQuotedList`). This matches the parquet writer, which quotes each name with
+/// `KeywordHelper::WriteQuoted(name, '"')`, escaping embedded quotes by doubling them. A list
+/// element is named `element`, as in parquet's list encoding.
+fn ducklake_column_key(path: &FieldPath) -> String {
+    path.parts()
+        .iter()
+        .map(|part| {
+            let name = part.as_name().unwrap_or("element");
+            format!("\"{}\"", name.replace('"', "\"\""))
+        })
+        .collect::<Vec<_>>()
+        .join(".")
+}
+
+fn file_stats_from_summary(summary: &WriteSummary, num_columns: usize) -> WrittenFileStats {
     WrittenFileStats {
         row_count: summary.row_count(),
         file_size_bytes: summary.size(),
@@ -224,45 +270,38 @@ fn file_stats_from_summary(summary: &WriteSummary) -> WrittenFileStats {
     }
 }
 
-/// Per-column statistics from a finished write's summary and its precomputed compressed sizes
-/// (`column_sizes`, indexed by top-level field position).
-///
-/// Only top-level columns are covered: nested struct/list leaf columns are not reported (parquet,
-/// by contrast, recurses to leaf paths). Flat tables - the common DuckLake case - are fully
-/// covered.
+/// Statistics for the `column_index`-th statistics column of a finished write, where
+/// `stats_columns` are the [leaf entries][leaf_stats_columns] of its footer statistics and
+/// `column_sizes` its compressed field sizes.
 fn column_stats_from_summary(
     summary: &WriteSummary,
+    stats_columns: &[usize],
     column_index: usize,
-    column_sizes: &[u64],
+    column_sizes: Option<&CompressedFieldSizes>,
 ) -> VortexResult<WrittenColumnStats> {
     let file_stats = summary
         .footer()
         .statistics()
         .ok_or_else(|| vortex_err!("written file has no statistics"))?;
-    let struct_fields = summary
-        .footer()
-        .dtype()
-        .as_struct_fields_opt()
-        .ok_or_else(|| vortex_err!("written file dtype is not a struct"))?;
-    let name = struct_fields.names().get(column_index).ok_or_else(|| {
+    let stats_index = *stats_columns.get(column_index).ok_or_else(|| {
         vortex_err!(
-            "column index {column_index} out of range for {} top-level fields",
-            struct_fields.nfields()
+            "column index {column_index} out of range for {} statistics columns",
+            stats_columns.len()
         )
     })?;
-    let (stats, dtype) = file_stats
-        .get_by_path(&FieldPath::from_name(name.clone()))
-        .ok_or_else(|| vortex_err!("no statistics for column {name}"))?;
+    let (stats, dtype) = file_stats.get(stats_index);
+    let path = &file_stats.paths()[stats_index];
 
     Ok(WrittenColumnStats {
+        column_key: ducklake_column_key(path),
         min: exact_scalar_to_duckdb(stats.get(Stat::Min), dtype)?,
         max: exact_scalar_to_duckdb(stats.get(Stat::Max), dtype)?,
         null_count: exact_u64(stats.get(Stat::NullCount)),
         // NaNCount is exact only for float columns, so this is emitted just for them (as in parquet).
         has_nan: exact_u64(stats.get(Stat::NaNCount)).map(|count| count > 0),
         num_values: summary.row_count(),
         // On-disk compressed size; excludes bytes not attributable to a column (e.g. struct validity).
-        column_size_bytes: column_sizes.get(column_index).copied(),
+        column_size_bytes: column_sizes.and_then(|sizes| sizes.get(path)),
     })
 }
 
@@ -376,44 +415,71 @@ mod tests {
         })
     }
 
+    /// Statistics for the `column_index`-th statistics column of `summary`, resolved the same way
+    /// `copy_to_finalize` and [`written_column_stats`] do.
+    fn column_stats(
+        summary: &WriteSummary,
+        column_index: usize,
+    ) -> VortexResult<WrittenColumnStats> {
+        let stats_columns = summary
+            .footer()
+            .statistics()
+            .map(leaf_stats_columns)
+            .unwrap_or_default();
+        let column_sizes = summary.footer().compressed_field_sizes()?;
+        column_stats_from_summary(summary, &stats_columns, column_index, Some(&column_sizes))
+    }
+
     #[test]
     fn column_stats_out_of_range_is_an_error() {
         let summary = write_summary(pruning_aggregate_fns());
-        assert!(column_stats_from_summary(&summary, 0, &[]).is_ok());
-        assert!(column_stats_from_summary(&summary, 1, &[]).is_err());
+        assert!(column_stats(&summary, 0).is_ok());
+        assert!(column_stats(&summary, 1).is_err());
     }
 
     #[test]
     fn column_stats_without_file_statistics_is_an_error() {
         let summary = write_summary(vec![]);
-        assert!(column_stats_from_summary(&summary, 0, &[]).is_err());
+        assert!(column_stats(&summary, 0).is_err());
     }
 
     #[test]
-    fn column_stats_resolves_by_name_past_a_nullable_struct_column() {
-        // Regression test: a nullable top-level struct field ("s") contributes both a `s.b` entry
-        // and a trailing entry for its own null count to the nested (post-order) stats layout, so
-        // that layout has one more entry than there are top-level fields. Resolving a later
-        // top-level column ("c", index 1) by flat position into that layout would silently return
-        // `s`'s own null-count-only entry (which has no Min) instead of `c`'s stats.
+    fn column_stats_reports_nested_leaves_by_quoted_path() -> VortexResult<()> {
+        // A nullable struct `s` contributes `s.b` and a trailing own entry to the post-order stats
+        // layout; only the leaves are statistics columns, keyed by their full quoted path.
         let summary = RUNTIME.block_on(async {
             let b = buffer![10i32, 20, 30].into_array();
             let inner = StructArray::new(["b"].into(), [b], 3, Validity::AllValid).into_array();
             let c = buffer![7u32, 8, 9].into_array();
-            let outer = StructArray::new(["s", "c"].into(), [inner, c], 3, Validity::NonNullable)
-                .into_array();
+            let outer =
+                StructArray::new(["s", "c\"q"].into(), [inner, c], 3, Validity::NonNullable)
+                    .into_array();
 
             let mut buf = ByteBufferMut::empty();
             let mut writer = SESSION
                 .write_options()
                 .with_file_statistics(pruning_aggregate_fns())
                 .writer(&mut buf, outer.dtype().clone());
-            writer.push(outer).await.unwrap();
-            writer.finish().await.unwrap()
-        });
-
-        let stats = column_stats_from_summary(&summary, 1, &[]).unwrap();
-        assert!(stats.min.is_some());
-        assert!(stats.max.is_some());
+            writer.push(outer).await?;
+            writer.finish().await
+        })?;
+
+        let file_stats = summary
+            .footer()
+            .statistics()
+            .ok_or_else(|| vortex_err!("no statistics"))?;
+        assert_eq!(leaf_stats_columns(file_stats).len(), 2);
+        let nested = column_stats(&summary, 0)?;
+        assert_eq!(nested.column_key, "\"s\".\"b\"");
+        assert!(nested.min.is_some());
+        assert!(nested.column_size_bytes.is_some_and(|size| size > 0));
+
+        let top_level = column_stats(&summary, 1)?;
+        assert_eq!(top_level.column_key, "\"c\"\"q\"");
+        assert!(top_level.min.is_some());
+        assert!(top_level.max.is_some());
+
+        assert!(column_stats(&summary, 2).is_err());
+        Ok(())
     }
 }

vortex-duckdb/src/ffi.rs

@@ -33,6 +33,7 @@ use crate::duckdb::Expression;
 use crate::duckdb::LogicalType;
 use crate::duckdb::LogicalTypeRef;
 use crate::duckdb::TableInitInput;
+use crate::duckdb::Value;
 use crate::duckdb::try_or;
 use crate::duckdb::try_or_null;
 use crate::file_reader::OpenFileReader;
@@ -495,6 +496,7 @@ pub unsafe extern "C-unwind" fn duckdb_copy_function_get_written_column_statisti
             return Ok(false);
         };
         let out = unsafe { &mut *out };
+        out.column_key = Value::from(stats.column_key.as_str()).into_ptr();
         out.min = stats.min.map_or(ptr::null_mut(), |v| v.into_ptr());
         out.max = stats.max.map_or(ptr::null_mut(), |v| v.into_ptr());
         out.has_null_count = stats.null_count.is_some();

vortex-sqllogictest/slt/duckdb/copy_return_stats.slt

@@ -11,10 +11,10 @@ TO '${WORK_DIR}/copy_return_stats.vortex' (FORMAT vortex, RETURN_STATS);
 ----
 <REGEX>:\$\{WORK_DIR\}/copy_return_stats\.vortex 2 \d+ 0 \{'"d"'=\{column_size_bytes=\d+, max=1\.5, min=1\.5, null_count=1, num_values=2\}, '"i"'=\{column_size_bytes=\d+, max=2, min=1, null_count=0, num_values=2\}, '"s"'=\{column_size_bytes=\d+, max=b, min=a, null_count=0, num_values=2\}\} NULL
 
-# Vortex reports statistics per top-level field, so nested columns are covered by the file-level
-# counts only; the COPY must still succeed.
+# As in parquet, statistics are reported per leaf column: struct fields are keyed by their full
+# quoted path and the struct itself gets no entry. Lists are reported as a single column.
 query TIII??
 COPY (SELECT {'a': 1, 'b': 2} AS st, [1, 2, 3] AS lst, 7 AS i)
 TO '${WORK_DIR}/copy_return_stats_nested.vortex' (FORMAT vortex, RETURN_STATS);
 ----
-<REGEX>:\$\{WORK_DIR\}/copy_return_stats_nested\.vortex 1 \d+ 0 \{'"i"'=\{column_size_bytes=\d+, max=7, min=7, null_count=0, num_values=1\}, '"lst"'=\{column_size_bytes=\d+, null_count=0, num_values=1\}, '"st"'=\{column_size_bytes=\d+, null_count=0, num_values=1\}\} NULL
+<REGEX>:\$\{WORK_DIR\}/copy_return_stats_nested\.vortex 1 \d+ 0 \{'"i"'=\{column_size_bytes=\d+, max=7, min=7, null_count=0, num_values=1\}, '"lst"'=\{column_size_bytes=\d+, null_count=0, num_values=1\}, '"st"\."a"'=\{column_size_bytes=\d+, max=1, min=1, null_count=0, num_values=1\}, '"st"\."b"'=\{column_size_bytes=\d+, max=2, min=2, null_count=0, num_values=1\}\} NULL