Skip to content

Commit d6e1c86

Browse files
authored
fix(datafusion): preserve list element field metadata in physical schema (#10285)
## What `calculate_physical_schema` reattaches each field's reference metadata for struct children and map key/value entries, but the List, FixedSizeList, and ListView branches built their element field without it. Any element metadata — an `ARROW:extension:name`, a `PARQUET:field_id` — was dropped, so the physical schema disagreed with the reference schema on list elements. ## Fix Carry `logical_elem.metadata()` onto the element field in all three branches, as the struct and map branches already do. Added a regression test covering list, fixed-size list, and list-view element metadata. ## AI assistance Prepared with AI assistance; I have reviewed and verified the change. Signed-off-by: jackylee-ch <qcsd2011@gmail.com>
1 parent 9a08b82 commit d6e1c86

1 file changed

Lines changed: 51 additions & 3 deletions

File tree

‎vortex-datafusion/src/convert/schema.rs‎

Lines changed: 51 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -148,7 +148,8 @@ fn calculate_physical_field_type(
148148
logical_elem.name(),
149149
physical_elem_type,
150150
logical_elem.is_nullable(),
151-
);
151+
)
152+
.with_metadata(logical_elem.metadata().clone());
152153
match logical_type {
153154
DataType::List(_) => DataType::List(physical_field.into()),
154155
DataType::LargeList(_) => DataType::LargeList(physical_field.into()),
@@ -173,7 +174,8 @@ fn calculate_physical_field_type(
173174
logical_elem.name(),
174175
physical_elem_type,
175176
logical_elem.is_nullable(),
176-
);
177+
)
178+
.with_metadata(logical_elem.metadata().clone());
177179
DataType::FixedSizeList(physical_field.into(), *size)
178180
} else {
179181
return Err(exec_datafusion_err!(
@@ -240,7 +242,8 @@ fn calculate_physical_field_type(
240242
logical_elem.name(),
241243
physical_elem_type,
242244
logical_elem.is_nullable(),
243-
);
245+
)
246+
.with_metadata(logical_elem.metadata().clone());
244247
match logical_type {
245248
DataType::ListView(_) => DataType::ListView(physical_field.into()),
246249
DataType::LargeListView(_) => DataType::LargeListView(physical_field.into()),
@@ -646,4 +649,49 @@ mod tests {
646649
let names: Vec<&str> = labels.iter().map(|f| f.name().as_str()).collect();
647650
assert_eq!(names, label_names);
648651
}
652+
653+
#[test]
654+
fn test_list_family_conversion_preserves_element_metadata() -> DFResult<()> {
655+
let elem = || {
656+
Field::new("item", DataType::Int32, true)
657+
.with_metadata([("elem_meta".to_owned(), "v".to_owned())].into())
658+
};
659+
let logical_schema = Schema::new(vec![
660+
Field::new("list_col", DataType::List(Arc::new(elem())), true),
661+
Field::new(
662+
"fsl_col",
663+
DataType::FixedSizeList(Arc::new(elem()), 3),
664+
true,
665+
),
666+
Field::new("listview_col", DataType::ListView(Arc::new(elem())), true),
667+
]);
668+
let i32n = || Arc::new(DType::Primitive(PType::I32, Nullability::Nullable));
669+
let dtype = DType::Struct(
670+
StructFields::from_iter([
671+
("list_col", DType::List(i32n(), Nullability::Nullable)),
672+
(
673+
"fsl_col",
674+
DType::FixedSizeList(i32n(), 3, Nullability::Nullable),
675+
),
676+
("listview_col", DType::List(i32n(), Nullability::Nullable)),
677+
]),
678+
Nullability::NonNullable,
679+
);
680+
681+
let physical_schema =
682+
calculate_physical_schema(&dtype, &logical_schema, &ArrowSession::default())?;
683+
684+
for name in ["list_col", "fsl_col", "listview_col"] {
685+
let elem_field = match physical_schema.field_with_name(name)?.data_type() {
686+
DataType::List(f) | DataType::FixedSizeList(f, _) | DataType::ListView(f) => f,
687+
other => panic!("expected list-family type for {name}, got {other:?}"),
688+
};
689+
assert_eq!(
690+
elem_field.metadata().get("elem_meta"),
691+
Some(&"v".to_owned()),
692+
"element metadata dropped for {name}"
693+
);
694+
}
695+
Ok(())
696+
}
649697
}

0 commit comments

Comments
 (0)