support multiple globs in DuckDB and MultiFileDataSource (#7390)

## Summary

Adds support for multiple globs in DuckDB, similar to Parquet,

E.g. instead of `read_vortex('*.vortex')` now you can also do
`read_vortex(['s3://bucket1/path/*.vortex', '/a/b/*.vortex'])`

Note that multiple file systems are supported in a single scan.

This is implemented and wired through into the underlying Rust
MultiFileDataSource.

## Testing

We add new DuckDB e2e tests for multi-glob

Couldn't figure out a great way to test multiple file systems, just
multiple globs

---------

Signed-off-by: Andrew Duffy <andrew@a10y.dev>

Author: a10y

benchmarks/datafusion-bench/src/main.rs

@@ -317,8 +317,7 @@ async fn register_v2_tables<B: Benchmark + ?Sized>(
         };
 
         let multi_ds = MultiFileDataSource::new(SESSION.clone())
-            .with_filesystem(fs)
-            .with_glob(glob_pattern)
+            .with_glob(glob_pattern, Some(fs))
             .build()
             .await?;
 

vortex-duckdb/cpp/table_function.cpp

@@ -383,6 +383,8 @@ extern "C" duckdb_state duckdb_vx_tfunc_register(duckdb_database ffi_db, const d
         auto &system_catalog = Catalog::GetSystemCatalog(*db);
         auto data = CatalogTransaction::GetSystemTransaction(*db);
         CreateTableFunctionInfo tf_info(tf);
+        // Allow registering multiple overloads with the same name but different parameter types.
+        tf_info.on_conflict = OnCreateConflict::ALTER_ON_CONFLICT;
         system_catalog.CreateFunction(data, tf_info);
     } catch (...) {
         return DuckDBError;

vortex-duckdb/src/e2e_test/vortex_scan_test.rs

@@ -350,6 +350,40 @@ fn test_vortex_scan_multiple_files() {
     assert_eq!(total_sum, 21);
 }
 
+#[test]
+fn test_vortex_scan_multiple_globs() {
+    // Test scanning multiple directories using a list of glob patterns.
+    let (tempdir1, tempdir2, _file1, _file2, _file3) = RUNTIME.block_on(async {
+        let tempdir1 = tempfile::tempdir().unwrap();
+        let tempdir2 = tempfile::tempdir().unwrap();
+
+        let file1 = write_vortex_file_to_dir(tempdir1.path(), "numbers", buffer![1i32, 2, 3]).await;
+        let file2 = write_vortex_file_to_dir(tempdir1.path(), "numbers", buffer![4i32, 5, 6]).await;
+        let file3 =
+            write_vortex_file_to_dir(tempdir2.path(), "numbers", buffer![7i32, 8, 9, 10]).await;
+
+        (tempdir1, tempdir2, file1, file2, file3)
+    });
+
+    // Create glob patterns for each directory.
+    let glob_pattern1 = format!("{}/*.vortex", tempdir1.path().display());
+    let glob_pattern2 = format!("{}/*.vortex", tempdir2.path().display());
+
+    // Scan files from both directories using a list of globs.
+    let conn = database_connection();
+    let result = conn
+        .query(&format!(
+            "SELECT SUM(numbers) FROM read_vortex(['{glob_pattern1}', '{glob_pattern2}'])"
+        ))
+        .unwrap();
+    let chunk = result.into_iter().next().unwrap();
+    let vec = chunk.get_vector(0);
+    let total_sum = vec.as_slice_with_len::<i64>(chunk.len().as_())[0];
+
+    // 1+2+3 + 4+5+6 + 7+8+9+10 = 55
+    assert_eq!(total_sum, 55);
+}
+
 #[test]
 fn test_vortex_scan_over_http() {
     let file = RUNTIME.block_on(async {

vortex-duckdb/src/lib.rs

@@ -21,6 +21,7 @@ use crate::duckdb::DatabaseRef;
 use crate::duckdb::LogicalType;
 use crate::duckdb::Value;
 use crate::multi_file::VortexMultiFileScan;
+use crate::multi_file::VortexMultiFileScanList;
 
 mod convert;
 mod datasource;
@@ -56,6 +57,9 @@ pub fn initialize(db: &DatabaseRef) -> VortexResult<()> {
     )?;
     db.register_table_function::<VortexMultiFileScan>(c"vortex_scan")?;
     db.register_table_function::<VortexMultiFileScan>(c"read_vortex")?;
+    // Register list overloads for multi-glob scanning (e.g., read_vortex(['a.vortex', 'b.vortex']))
+    db.register_table_function::<VortexMultiFileScanList>(c"vortex_scan")?;
+    db.register_table_function::<VortexMultiFileScanList>(c"read_vortex")?;
     db.register_copy_function::<VortexCopyFunction>(c"vortex", c"vortex")
 }
 

vortex-duckdb/src/multi_file.rs

@@ -5,18 +5,23 @@ use std::path::Path;
 use std::path::absolute;
 use std::sync::Arc;
 
+use itertools::Itertools;
 use url::Url;
 use vortex::error::VortexResult;
+use vortex::error::vortex_bail;
 use vortex::error::vortex_err;
 use vortex::file::multi::MultiFileDataSource;
+use vortex::io::filesystem::FileSystemRef;
 use vortex::io::runtime::BlockingRuntime;
 use vortex::scan::DataSourceRef;
+use vortex_utils::aliases::hash_map::HashMap;
 
 use crate::RUNTIME;
 use crate::SESSION;
 use crate::datasource::DataSourceTableFunction;
 use crate::duckdb::BindInputRef;
 use crate::duckdb::ClientContextRef;
+use crate::duckdb::ExtractedValue;
 use crate::duckdb::LogicalType;
 use crate::filesystem::resolve_filesystem;
 
@@ -60,34 +65,95 @@ fn normalize_path(path: std::path::PathBuf) -> std::path::PathBuf {
 #[derive(Debug)]
 pub struct VortexMultiFileScan;
 
+/// Variant of [`VortexMultiFileScan`] that accepts a list of globs.
+///
+/// This is registered as a separate overload to handle `read_vortex(['a.vortex', 'b.vortex'])`.
+#[derive(Debug)]
+pub struct VortexMultiFileScanList;
+
 impl DataSourceTableFunction for VortexMultiFileScan {
     fn parameters() -> Vec<LogicalType> {
         vec![LogicalType::varchar()]
     }
 
     fn bind(ctx: &ClientContextRef, input: &BindInputRef) -> VortexResult<DataSourceRef> {
-        let glob_url_parameter = input
-            .get_parameter(0)
-            .ok_or_else(|| vortex_err!("Missing file glob parameter"))?;
+        bind_multi_file_scan(ctx, input)
+    }
+}
+
+impl DataSourceTableFunction for VortexMultiFileScanList {
+    fn parameters() -> Vec<LogicalType> {
+        vec![
+            LogicalType::list_type(LogicalType::varchar())
+                .unwrap_or_else(|_| unreachable!("creating list<varchar> type should not fail")),
+        ]
+    }
+
+    fn bind(ctx: &ClientContextRef, input: &BindInputRef) -> VortexResult<DataSourceRef> {
+        bind_multi_file_scan(ctx, input)
+    }
+}
 
-        // Parse the URL and separate the base URL (keep scheme, host, etc.) from the path.
-        let glob_url_string = glob_url_parameter.as_string();
-        let glob_url_str = glob_url_string.as_str();
-        let glob_url = parse_glob_url(glob_url_str)?;
+/// Shared bind logic for both single-glob and multi-glob variants.
+fn bind_multi_file_scan(
+    ctx: &ClientContextRef,
+    input: &BindInputRef,
+) -> VortexResult<DataSourceRef> {
+    let glob_url_parameter = input
+        .get_parameter(0)
+        .ok_or_else(|| vortex_err!("Missing file glob parameter"))?;
 
+    // The input to the table function can either be a single glob, or a List of glob patterns.
+    let glob_strings: Vec<String> = match glob_url_parameter.extract() {
+        ExtractedValue::Varchar(glob) => {
+            vec![glob.to_string()]
+        }
+        ExtractedValue::List(globs) => globs
+            .into_iter()
+            .map(|glob| {
+                let ExtractedValue::Varchar(string) = glob.extract() else {
+                    vortex_bail!("list element must be Varchar type")
+                };
+
+                Ok(string.to_string())
+            })
+            .try_collect()?,
+        _ => vortex_bail!("Invalid argument to read_vortex table function"),
+    };
+
+    // Parse each glob URL and resolve its filesystem.
+    let mut glob_urls: Vec<Url> = Vec::with_capacity(glob_strings.len());
+    for glob_str in &glob_strings {
+        glob_urls.push(parse_glob_url(glob_str)?);
+    }
+
+    // Cache filesystems by base URL to avoid resolving the same filesystem multiple times.
+    let mut fs_cache: HashMap<Url, FileSystemRef> = HashMap::new();
+    for glob_url in &glob_urls {
         let mut base_url = glob_url.clone();
         base_url.set_path("");
+        if !fs_cache.contains_key(&base_url) {
+            let fs = resolve_filesystem(&base_url, ctx)?;
+            fs_cache.insert(base_url, fs);
+        }
+    }
 
-        let fs = resolve_filesystem(&base_url, ctx)?;
+    RUNTIME.block_on(async {
+        let mut builder = MultiFileDataSource::new(SESSION.clone());
 
-        RUNTIME.block_on(async {
-            let builder = MultiFileDataSource::new(SESSION.clone())
-                .with_filesystem(fs)
-                .with_glob(glob_url.path());
-            let ds = builder.build().await?;
-            VortexResult::Ok(Arc::new(ds) as DataSourceRef)
-        })
-    }
+        for glob_url in &glob_urls {
+            let mut base_url = glob_url.clone();
+            base_url.set_path("");
+            let fs = fs_cache
+                .get(&base_url)
+                .map(Arc::clone)
+                .unwrap_or_else(|| unreachable!("fs should be cached for all base URLs"));
+            builder = builder.with_glob(glob_url.path(), Some(fs));
+        }
+
+        let ds = builder.build().await?;
+        Ok(Arc::new(ds) as DataSourceRef)
+    })
 }
 
 #[cfg(test)]

vortex-file/public-api.lock

@@ -10,9 +10,7 @@ pub async fn vortex_file::multi::MultiFileDataSource::build(self) -> vortex_erro
 
 pub fn vortex_file::multi::MultiFileDataSource::new(session: vortex_session::VortexSession) -> Self
 
-pub fn vortex_file::multi::MultiFileDataSource::with_filesystem(self, fs: vortex_io::filesystem::FileSystemRef) -> Self
-
-pub fn vortex_file::multi::MultiFileDataSource::with_glob(self, glob: impl core::convert::Into<alloc::string::String>) -> Self
+pub fn vortex_file::multi::MultiFileDataSource::with_glob(self, glob: impl core::convert::Into<alloc::string::String>, fs: core::option::Option<vortex_io::filesystem::FileSystemRef>) -> Self
 
 pub fn vortex_file::multi::MultiFileDataSource::with_open_options(self, f: impl core::ops::function::Fn(vortex_file::VortexOpenOptions) -> vortex_file::VortexOpenOptions + core::marker::Send + core::marker::Sync + 'static) -> Self