feat(cuda): support column projection and dictionary decoding in file scans (#9934)

## Summary
- Add column projection to CUDA scans, avoiding reads and decoding for
unselected columns.
- Decode dictionaries for consistent Arrow stream schemas, preserving
defaults for other exports.
- Cache schemas without reading a batch.
- Preserve row-block boundaries, register the CUDA layout edition, and
generate the C header.

---------

Signed-off-by: Alexander Droste <alexander.droste@protonmail.com>

Author: 0ax1

.github/workflows/cuda.yaml

@@ -45,6 +45,8 @@ jobs:
               - "vortex-test/**"
               - "pyproject.toml"
               - "uv.lock"
+              - "Cargo.toml"
+              - "Cargo.lock"
               - ".github/workflows/**"
 
   cuda-build-lint:
@@ -71,6 +73,8 @@ jobs:
             cargo build --profile ci --locked --all-features --all-targets
             -p vortex-cuda -p vortex-cuda-ffi -p vortex-cub -p vortex-nvcomp
             -p gpu-scan-cli -p vortex-test-e2e-cuda -p vortex-python-cuda
+      - name: Verify generated CUDA FFI header is up to date
+        run: git --no-pager diff --exit-code -- vortex-cuda/ffi/cinclude/vortex_cuda.h
       - name: Clippy CUDA crates
         run: |
           cargo clippy --profile ci --locked --all-features --all-targets \

Cargo.lock

@@ -22,8 +22,13 @@ vortex-cuda = { path = ".." }
 vortex-ffi = { path = "../../vortex-ffi" }
 
 [dev-dependencies]
+cudarc = { workspace = true }
+tempfile = { workspace = true }
 vortex-cuda-macros = { workspace = true }
 
+[build-dependencies]
+cbindgen = { workspace = true }
+
 [lib]
 name = "vortex_cuda_ffi"
 crate-type = ["rlib", "staticlib", "cdylib"]

vortex-cuda/ffi/Cargo.toml

@@ -27,3 +27,8 @@ pool and CUDA state are reused as well.
 On Linux, use `vx_cuda_scan_path_arrow_device_stream_with_options` with
 `vx_cuda_scan_options.flags = VX_CUDA_SCAN_FLAG_DIRECT_IO` to bypass the operating system page
 cache for pooled data-plane reads. Footer and zone-map reads remain buffered on the host.
+
+## Generated header
+
+`build.rs` generates `cinclude/vortex_cuda.h` with cbindgen. Edit `src/lib.rs` or `cbindgen.toml`
+and commit the regenerated header. It skips clang-format; CUDA CI checks for header drift.

vortex-cuda/ffi/README.md

@@ -0,0 +1,60 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright the Vortex contributors
+
+use std::error::Error;
+use std::fs;
+use std::fs::OpenOptions;
+use std::io;
+use std::io::Write;
+use std::process;
+
+fn main() -> Result<(), Box<dyn Error>> {
+    println!("cargo:rerun-if-changed=src");
+    println!("cargo:rerun-if-changed=cbindgen.toml");
+    println!("cargo:rerun-if-changed=build.rs");
+
+    let header = "cinclude/vortex_cuda.h";
+    let mut generated = Vec::new();
+    // Parse only the FFI source to avoid macro expansion and recursive CUDA builds.
+    cbindgen::Builder::new()
+        .with_src("src/lib.rs")
+        .with_config(cbindgen::Config::from_file("cbindgen.toml")?)
+        .generate()?
+        .write(&mut generated);
+    match fs::read(header) {
+        Ok(existing) if existing == generated => return Ok(()),
+        Ok(_) => {}
+        Err(error) if error.kind() == io::ErrorKind::NotFound => {}
+        Err(error) => return Err(format!("failed to read {header}: {error}").into()),
+    }
+    publish_header(header, &generated)
+        .map_err(|error| format!("failed to publish {header}: {error}"))?;
+    Ok(())
+}
+
+fn publish_header(header: &str, generated: &[u8]) -> io::Result<()> {
+    let mut attempt = 0_u64;
+    let (temporary, mut file) = loop {
+        let temporary = format!("{header}.{}.{attempt}.tmp", process::id());
+        match OpenOptions::new()
+            .write(true)
+            .create_new(true)
+            .open(&temporary)
+        {
+            Ok(file) => break (temporary, file),
+            Err(error) if error.kind() == io::ErrorKind::AlreadyExists => attempt += 1,
+            Err(error) => return Err(error),
+        }
+    };
+
+    // Publish atomically so concurrent builds never see a truncated header.
+    let written = file.write_all(generated);
+    drop(file);
+    let result = written.and_then(|()| fs::rename(&temporary, header));
+    if result.is_err()
+        && let Err(error) = fs::remove_file(&temporary)
+    {
+        eprintln!("failed to remove temporary header {temporary}: {error}");
+    }
+    result
+}

vortex-cuda/ffi/build.rs

@@ -0,0 +1,77 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright the Vortex contributors
+
+language = "C"
+braces = "SameLine"
+cpp_compat = true
+usize_is_size_t = true
+style = "both"
+no_includes = true
+
+header = """
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright the Vortex contributors
+
+// clang-format off
+
+#pragma once
+
+// THIS FILE IS AUTO-GENERATED, DO NOT MAKE EDITS DIRECTLY
+
+#include <stddef.h>
+#include <stdint.h>
+
+#include "vortex.h"
+
+/* Link against the CUDA-enabled FFI library that provides both the base Vortex FFI and these CUDA
+ * entry points. Do not pass Vortex handles between independently linked Rust FFI libraries. */
+
+/* Definitions from the Arrow C Device data interface. Define USE_OWN_ARROW_DEVICE to skip them.
+ * https://arrow.apache.org/docs/format/CDeviceDataInterface.html */
+#if !defined(ARROW_C_DEVICE_DATA_INTERFACE) && !defined(USE_OWN_ARROW_DEVICE)
+#define ARROW_C_DEVICE_DATA_INTERFACE
+
+typedef int32_t ArrowDeviceType;
+#define ARROW_DEVICE_CPU          1
+#define ARROW_DEVICE_CUDA         2
+#define ARROW_DEVICE_CUDA_HOST    3
+#define ARROW_DEVICE_OPENCL       4
+#define ARROW_DEVICE_VULKAN       7
+#define ARROW_DEVICE_METAL        8
+#define ARROW_DEVICE_VPI          9
+#define ARROW_DEVICE_ROCM         10
+#define ARROW_DEVICE_ROCM_HOST    11
+#define ARROW_DEVICE_EXT_DEV      12
+#define ARROW_DEVICE_CUDA_MANAGED 13
+#define ARROW_DEVICE_ONEAPI       14
+#define ARROW_DEVICE_WEBGPU       15
+#define ARROW_DEVICE_HEXAGON      16
+
+struct ArrowDeviceArray {
+    struct ArrowArray array;
+    int64_t device_id;
+    ArrowDeviceType device_type;
+    void *sync_event;
+    int64_t reserved[3];
+};
+#endif
+
+#if !defined(ARROW_C_DEVICE_STREAM_INTERFACE) && !defined(USE_OWN_ARROW_DEVICE)
+#define ARROW_C_DEVICE_STREAM_INTERFACE
+struct ArrowDeviceArrayStream {
+    ArrowDeviceType device_type;
+    int (*get_schema)(struct ArrowDeviceArrayStream *, struct ArrowSchema *out);
+    int (*get_next)(struct ArrowDeviceArrayStream *, struct ArrowDeviceArray *out);
+    const char *(*get_last_error)(struct ArrowDeviceArrayStream *);
+    void (*release)(struct ArrowDeviceArrayStream *);
+    void *private_data;
+};
+#endif
+"""
+
+trailer = "// clang-format on"
+
+# These externally defined Arrow ABI types use struct tags, not typedef names.
+[export.rename]
+"ArrowDeviceArray" = "struct ArrowDeviceArray"
+"ArrowDeviceArrayStream" = "struct ArrowDeviceArrayStream"