Merge develop into ngates/id

Signed-off-by: Nicholas Gates <nick@nickgates.com>

Author: joseph-isaacs

.github/AGENTS.md

@@ -5,7 +5,6 @@ Nightly is required for:
 - `-Z` flags: sanitizers (`-Zsanitizer=address`), miri (`-Zmiri-*`), publish (`-Zpublish-timeout`)
 - `cargo-fuzz` (requires nightly)
 - `public-api` xtask (nightly rustdoc JSON)
-- `--cfg vortex_nightly` (enables `portable_simd` feature gate)
 
 Everything else (build, clippy, tests, docs, benchmarks, packaging) should use stable.
 

.github/actions/setup-prebuild/action.yml

@@ -30,12 +30,12 @@ runs:
       shell: bash
       run: |
         mkdir -p ~/.config/sccache
-        echo 'server_startup_timeout_ms = 15000' > ~/.config/sccache/config
+        echo 'server_startup_timeout_ms = 60000' > ~/.config/sccache/config
 
     - name: Pre-start sccache server
       if: github.repository == 'vortex-data/vortex' && inputs.enable-sccache == 'true'
       shell: bash
-      run: sccache --start-server &
+      run: sccache --start-server
 
     # Fallback path: full setup for forks
     - name: Full Rust setup

.github/actions/setup-rust/action.yml

@@ -69,7 +69,7 @@ runs:
       shell: bash
       run: |
         mkdir -p ~/.config/sccache
-        echo 'server_startup_timeout_ms = 15000' > ~/.config/sccache/config
+        echo 'server_startup_timeout_ms = 60000' > ~/.config/sccache/config
 
     - name: Rust Compile Cache
       if: inputs.enable-sccache == 'true'
@@ -78,15 +78,12 @@ runs:
     - name: Pre-start sccache server
       if: inputs.enable-sccache == 'true'
       shell: bash
-      run: sccache --start-server &
+      run: sccache --start-server
 
     - name: Install Protoc (for lance-encoding build step)
       if: runner.os != 'Windows'
       uses: ./.github/actions/setup-protoc
 
-    - name: Install Ninja (for DuckDB build system)
-      uses: seanmiddleditch/gha-setup-ninja@v6
-
     - name: Install Sweep
       shell: bash
       if: ${{ inputs.timestamp == 'true' && github.ref_name == 'develop' }}

.github/scripts/close_fixed_fuzzer_issues.py

@@ -162,7 +162,6 @@ def build_fuzz_target(target: str) -> bool:
     """Build the fuzz target once. Returns True on success."""
     print(f"\nBuilding fuzz target: {target}")
     env = os.environ.copy()
-    env["RUSTFLAGS"] = "--cfg vortex_nightly"
     result = run(
         ["cargo", "+nightly", "fuzz", "build", "--dev", "--sanitizer=none", target],
         env=env,
@@ -173,7 +172,6 @@ def build_fuzz_target(target: str) -> bool:
 def retest_crash(target: str, crash_path: str, timeout_secs: int = 120) -> str:
     """Run the fuzz target with the crash file. Returns 'fixed', 'reproduces', or 'timeout'."""
     env = os.environ.copy()
-    env["RUSTFLAGS"] = "--cfg vortex_nightly"
     try:
         result = run(
             [

.github/scripts/fuzz_report/cli.py

@@ -363,8 +363,9 @@ def cmd_report(args: argparse.Namespace) -> int:
         print(f"Commented on #{existing_issue}", file=sys.stderr)
         _write_github_output("issue_number", str(existing_issue))
     else:
-        fuzz_target = variables.get("FUZZ_TARGET", "unknown")
-        title = f"Fuzzing Crash: {crash_info.error_variant} in {fuzz_target}"
+        # Use FUZZ_NAME for the title (descriptive name), fall back to FUZZ_TARGET
+        fuzz_name = variables.get("FUZZ_NAME") or variables.get("FUZZ_TARGET", "unknown")
+        title = f"Fuzzing Crash: {crash_info.error_variant} in {fuzz_name}"
 
         body = render_template(str(TEMPLATES_DIR / "new_issue.md"), variables, use_env=False)
         body_file = Path("issue_body.md")

.github/workflows/approvals.yml

@@ -0,0 +1,46 @@
+name: PR Approval Check
+
+on:
+  pull_request_review:
+    types: [submitted, dismissed]
+
+jobs:
+  check-approvals:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check required approvals
+        uses: actions/github-script@450193c5abd4cdb17ba9f3ffcfe8f635c4bb6c2a
+        with:
+          script: |
+            const pr = context.payload.pull_request;
+            const reviews = await github.rest.pulls.listReviews({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              pull_number: pr.number,
+            });
+
+            // Count unique human approvals (latest review per user)
+            const latestByUser = {};
+            for (const review of reviews.data) {
+              if (review.user.type === 'Bot') continue;
+              latestByUser[review.user.login] = review.state;
+            }
+            const approvalCount = Object.values(latestByUser)
+              .filter(state => state === 'APPROVED').length;
+
+            // Determine if PR author is a bot/GitHub Actions
+            const authorType = pr.user.type;        // 'Bot' vs 'User'
+            const authorLogin = pr.user.login;       // e.g. 'github-actions[bot]'
+            const isBot = authorType === 'Bot' || authorLogin.endsWith('[bot]');
+
+            const required = isBot ? 2 : 1;
+
+            console.log(`PR author: ${authorLogin} (${authorType}), isBot: ${isBot}`);
+            console.log(`Approvals: ${approvalCount} / ${required} required`);
+
+            if (approvalCount < required) {
+              core.setFailed(
+                `This PR needs ${required} human approval(s) but has ${approvalCount}. ` +
+                `(Author is ${isBot ? 'a bot' : 'human'})`
+              );
+            }

.github/workflows/bench-dispatch.yml

@@ -0,0 +1,49 @@
+# Dispatches benchmark workflows when benchmark labels are added to pull requests.
+# This is a separate workflow so that non-benchmark label events don't create
+# phantom check suites that obscure in-progress benchmark runs on the PR.
+
+name: Benchmark Dispatch
+
+on:
+  pull_request:
+    types: [labeled]
+    branches: ["develop"]
+
+permissions:
+  actions: write
+  contents: read
+  pull-requests: write  # for label removal and PR comments
+  id-token: write  # enables AWS-GitHub OIDC
+
+jobs:
+  remove-bench-label:
+    runs-on: ubuntu-latest
+    timeout-minutes: 2
+    if: github.event.label.name == 'action/benchmark'
+    steps:
+      - uses: actions-ecosystem/action-remove-labels@v1
+        if: github.event.pull_request.head.repo.full_name == 'vortex-data/vortex'
+        with:
+          labels: action/benchmark
+          fail_on_error: true
+
+  bench:
+    needs: remove-bench-label
+    uses: ./.github/workflows/bench-pr.yml
+    secrets: inherit
+
+  remove-sql-label:
+    runs-on: ubuntu-latest
+    timeout-minutes: 2
+    if: github.event.label.name == 'action/benchmark-sql'
+    steps:
+      - uses: actions-ecosystem/action-remove-labels@v1
+        if: github.event.pull_request.head.repo.full_name == 'vortex-data/vortex'
+        with:
+          labels: action/benchmark-sql
+          fail_on_error: true
+
+  sql-bench:
+    needs: remove-sql-label
+    uses: ./.github/workflows/sql-pr.yml
+    secrets: inherit

.github/workflows/bench-pr.yml

@@ -1,4 +1,5 @@
-# Runs all benchmarks once when we add the `action/benchmark` tag to a pull request.
+# Runs all benchmarks once for a pull request.
+# Called from bench-dispatch.yml when the `action/benchmark` label is added.
 
 name: PR Benchmarks
 
@@ -9,32 +10,16 @@ concurrency:
   cancel-in-progress: false
 
 on:
-  pull_request:
-    types: [labeled, synchronize]
-    branches: ["develop"]
+  workflow_call: { }
   workflow_dispatch: { }
 
 permissions:
-  actions: write  # for removing labels
   contents: read
   pull-requests: write  # for commenting on PRs
   id-token: write  # enables AWS-GitHub OIDC
 
 jobs:
-  label_trigger:
-    runs-on: ubuntu-latest
-    timeout-minutes: 120
-    if: ${{ contains(github.event.head_commit.message, '[benchmark]') || github.event.label.name == 'action/benchmark' && github.event_name == 'pull_request' }}
-    steps:
-      # We remove the benchmark label first so that the workflow can be re-triggered.
-      - uses: actions-ecosystem/action-remove-labels@v1
-        if: ${{ github.event.pull_request.head.repo.full_name == 'vortex-data/vortex' }}
-        with:
-          labels: action/benchmark
-          fail_on_error: true
-
   bench:
-    needs: label_trigger
     timeout-minutes: 120
     runs-on: >-
       ${{ github.repository == 'vortex-data/vortex'
@@ -48,7 +33,6 @@ jobs:
             build_args: "--features lance"
           - id: compress-bench
             name: Compression
-    if: ${{ contains(github.event.head_commit.message, '[benchmark]') || github.event.label.name == 'action/benchmark' && github.event_name == 'pull_request' }}
     steps:
       - uses: runs-on/action@v2
         if: github.event.pull_request.head.repo.fork == false
@@ -74,7 +58,7 @@ jobs:
       - name: Build binary
         shell: bash
         env:
-          RUSTFLAGS: "-C target-cpu=native -C force-frame-pointers=yes"
+          RUSTFLAGS: "-C target-cpu=native"
         run: |
           cargo build --package ${{ matrix.benchmark.id }} --profile release_debug ${{ matrix.benchmark.build_args }}
 
@@ -137,17 +121,16 @@ jobs:
           comment-tag: bench-pr-comment-${{ matrix.benchmark.id }}
 
       - name: Comment PR on failure
-        if: failure() && inputs.mode == 'pr' && github.event.pull_request.head.repo.fork == false
+        if: failure() && github.event.pull_request.head.repo.fork == false
         uses: thollander/actions-comment-pull-request@v3
         with:
           message: |
-            # 🚨🚨🚨❌❌❌ BENCHMARK FAILED ❌❌❌🚨🚨🚨
+            # BENCHMARK FAILED
 
             Benchmark `${{ matrix.benchmark.name }}` failed! Check the [workflow run](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}) for details.
           comment-tag: bench-pr-comment-${{ matrix.benchmark.id }}
 
   sql:
-    needs: label_trigger
     uses: ./.github/workflows/sql-benchmarks.yml
     secrets: inherit
     with:
@@ -158,7 +141,8 @@ jobs:
             "id": "clickbench-nvme",
             "subcommand": "clickbench",
             "name": "Clickbench on NVME",
-            "targets": "datafusion:parquet,datafusion:vortex,duckdb:parquet,duckdb:vortex,duckdb:duckdb"
+            "targets": "datafusion:parquet,datafusion:vortex,duckdb:parquet,duckdb:vortex,duckdb:duckdb",
+            "extra_data_formats": "vortex-compact"
           },
           {
             "id": "tpch-nvme",

.github/workflows/bench.yml

@@ -71,7 +71,7 @@ jobs:
       - name: Build binary
         shell: bash
         env:
-          RUSTFLAGS: "-C target-cpu=native -C force-frame-pointers=yes"
+          RUSTFLAGS: "-C target-cpu=native"
         run: |
           cargo build --bin ${{ matrix.benchmark.id }} --profile release_debug ${{ matrix.benchmark.build_args }}