ci(python): run Python benchmarks with CodSpeed

Run the existing vortex-python benchmark suite under pytest-codspeed in
simulation mode, so CodSpeed tracks read and Arrow export speed through
the Python bindings.

pytest-codspeed rejects the `disable_gc` marker option. The benchmarks
no longer set it, and the pytest-benchmark runs pass
`--benchmark-disable-gc` instead.

Signed-off-by: Robert Kruszewski <github@robertk.io>

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01Suq4jdU4jCnGsJeRRtGMF5

Author: robert3005

.github/workflows/ci.yml

@@ -149,7 +149,7 @@ jobs:
 
       - name: Pytest Benchmarks - Vortex
         run: |
-          bash ../scripts/bench-taskset.sh uv run --all-packages pytest --benchmark-only benchmark/
+          bash ../scripts/bench-taskset.sh uv run --all-packages pytest --benchmark-only --benchmark-disable-gc benchmark/
         working-directory: vortex-python/
 
   python-windows:

.github/workflows/codspeed.yml

@@ -126,6 +126,38 @@ jobs:
           # Cached package restoration fails on dangling libc6-dbg documentation symlinks.
           cache-instruments: "false"
 
+  bench-codspeed-python:
+    name: "Benchmark with Codspeed (Python)"
+    timeout-minutes: 45
+    runs-on: >-
+      ${{ github.repository == 'vortex-data/vortex'
+          && format('runs-on={0}/runner=amd64-medium/image=ubuntu24-full-x64-pre-v2/extras=s3-cache/tag=bench-codspeed-python', github.run_id)
+          || 'ubuntu-latest' }}
+    env:
+      # The bench profile keeps the debug info that CodSpeed uses to attribute native frames.
+      MATURIN_PEP517_ARGS: "--profile bench"
+    steps:
+      - uses: runs-on/action@v2
+        if: github.repository == 'vortex-data/vortex'
+        with:
+          sccache: s3
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7
+      - uses: ./.github/actions/setup-prebuild
+        with:
+          enable-sccache: ${{ github.repository == 'vortex-data/vortex' && 'true' || 'false' }}
+      - uses: ./.github/actions/system-info
+      # Build the extension before measurement, so the benchmark step does not compile it.
+      - name: Build Python extension
+        run: uv sync --all-packages
+      - name: Run benchmarks
+        uses: CodSpeedHQ/action@f842b9a3ffe213dbb81768fd7f7eeb4dfcf1c369  # v5
+        with:
+          run: cd vortex-python && uv run --all-packages --no-sync pytest benchmark/ --codspeed
+          token: ${{ secrets.CODSPEED_TOKEN }}
+          mode: "simulation"
+          # Cached package restoration fails on dangling libc6-dbg documentation symlinks.
+          cache-instruments: "false"
+
   # Compile on smaller VMs from the same CPU family as the measurement hosts. Globally
   # enabled features also affect build scripts, so the build host must support them.
   bench-codspeed-cpu-features-build:

uv.lock

@@ -16,27 +16,27 @@ def _has_mean(t: pa.DataType) -> bool:
     return is_integer(t) or is_floating(t)
 
 
-@pytest.mark.benchmark(group="aggregation", disable_gc=True)
+@pytest.mark.benchmark(group="aggregation")
 def test_arrow_table_aggregation(benchmark: BenchmarkFixture, vxf: vx.VortexFile) -> None:
     aggregations: list[tuple[str, Literal["mean"]]] = [
         (field.name, "mean") for field in vxf.dtype.to_arrow_schema() if _has_mean(field.type)
     ]
     benchmark(lambda: pa.concat_tables(x.to_arrow_table() for x in vxf.scan()).group_by([]).aggregate(aggregations))
 
 
-@pytest.mark.benchmark(group="aggregation", disable_gc=True)
+@pytest.mark.benchmark(group="aggregation")
 def test_polars_aggregation(benchmark: BenchmarkFixture, vxf: vx.VortexFile) -> None:
     lf = vxf.to_polars()
     benchmark(lambda: lf.mean().collect().to_arrow())
 
 
-@pytest.mark.benchmark(group="aggregation", disable_gc=True)
+@pytest.mark.benchmark(group="aggregation")
 def test_polars_streaming_aggregation(benchmark: BenchmarkFixture, vxf: vx.VortexFile) -> None:
     lf = vxf.to_polars()
     benchmark(lambda: lf.mean().collect(engine="streaming").to_arrow())
 
 
-@pytest.mark.benchmark(group="aggregation", disable_gc=True)
+@pytest.mark.benchmark(group="aggregation")
 def test_duckdb_aggregation(benchmark: BenchmarkFixture, vxf: vx.VortexFile) -> None:
     conn = duckdb.connect(database=":memory:")
     ds = vxf.to_dataset()

vortex-python/benchmark/test_aggregation.py

@@ -11,30 +11,30 @@
 from vortex.expr import column
 
 
-@pytest.mark.benchmark(group="filter", disable_gc=True)
+@pytest.mark.benchmark(group="filter")
 def test_scan_filter(benchmark: BenchmarkFixture, vxf: vx.VortexFile) -> None:
     benchmark(lambda: pa.concat_tables(x.to_arrow_table() for x in vxf.scan(expr=column("x") >= 50_000)))
 
 
-@pytest.mark.benchmark(group="filter", disable_gc=True)
+@pytest.mark.benchmark(group="filter")
 def test_repeated_scan_filter(benchmark: BenchmarkFixture, vxf: vx.VortexFile) -> None:
     rscan = vxf.to_repeated_scan(expr=column("x") > 50_000)
     benchmark(lambda: pa.concat_tables(x.to_arrow_table() for x in rscan.execute()))
 
 
-@pytest.mark.benchmark(group="filter", disable_gc=True)
+@pytest.mark.benchmark(group="filter")
 def test_polars_filter(benchmark: BenchmarkFixture, vxf: vx.VortexFile) -> None:
     lf = vxf.to_polars()
     benchmark(lambda: lf.filter(pl.col("x") >= pl.lit(50_000).cast(pl.Int64)).collect().to_arrow())
 
 
-@pytest.mark.benchmark(group="filter", disable_gc=True)
+@pytest.mark.benchmark(group="filter")
 def test_polars_streaming_filter(benchmark: BenchmarkFixture, vxf: vx.VortexFile) -> None:
     lf = vxf.to_polars()
     benchmark(lambda: lf.filter(pl.col("x") >= pl.lit(50_000).cast(pl.Int64)).collect(engine="streaming").to_arrow())
 
 
-@pytest.mark.benchmark(group="filter", disable_gc=True)
+@pytest.mark.benchmark(group="filter")
 def test_duckdb_filter(benchmark: BenchmarkFixture, vxf: vx.VortexFile) -> None:
     conn = duckdb.connect(database=":memory:")
     ds = vxf.to_dataset()

vortex-python/benchmark/test_filter.py

@@ -9,30 +9,30 @@
 import vortex as vx
 
 
-@pytest.mark.benchmark(group="scalar_at", disable_gc=True)
+@pytest.mark.benchmark(group="scalar_at")
 def test_scan_scalar_at(benchmark: BenchmarkFixture, vxf: vx.VortexFile) -> None:
     benchmark(lambda: pa.concat_tables(x.to_arrow_table() for x in vxf.scan(indices=vx.array([50_000]))))
 
 
-@pytest.mark.benchmark(group="scalar_at", disable_gc=True)
+@pytest.mark.benchmark(group="scalar_at")
 def test_repeated_scan_scalar_at(benchmark: BenchmarkFixture, vxf: vx.VortexFile) -> None:
     rscan = vxf.to_repeated_scan()
     benchmark(lambda: rscan.scalar_at(50_000))
 
 
-@pytest.mark.benchmark(group="scalar_at", disable_gc=True)
+@pytest.mark.benchmark(group="scalar_at")
 def test_polars_scalar_at(benchmark: BenchmarkFixture, vxf: vx.VortexFile) -> None:
     lf = vxf.to_polars()
     benchmark(lambda: lf.slice(50_000, 50_001).collect().to_arrow())
 
 
-@pytest.mark.benchmark(group="scalar_at", disable_gc=True)
+@pytest.mark.benchmark(group="scalar_at")
 def test_polars_streaming_scalar_at(benchmark: BenchmarkFixture, vxf: vx.VortexFile) -> None:
     lf = vxf.to_polars()
     benchmark(lambda: lf.slice(50_000, 50_001).collect(engine="streaming").to_arrow())
 
 
-@pytest.mark.benchmark(group="scalar_at", disable_gc=True)
+@pytest.mark.benchmark(group="scalar_at")
 def test_duckdb_scalar_at(benchmark: BenchmarkFixture, vxf: vx.VortexFile) -> None:
     conn = duckdb.connect(database=":memory:")
     ds = vxf.to_dataset()

vortex-python/benchmark/test_scalar_at.py

@@ -9,30 +9,30 @@
 import vortex as vx
 
 
-@pytest.mark.benchmark(group="scan", disable_gc=True)
+@pytest.mark.benchmark(group="scan")
 def test_scan(benchmark: BenchmarkFixture, vxf: vx.VortexFile) -> None:
     benchmark(lambda: pa.concat_tables(x.to_arrow_table() for x in vxf.scan()))
 
 
-@pytest.mark.benchmark(group="scan", disable_gc=True)
+@pytest.mark.benchmark(group="scan")
 def test_repeated_scan(benchmark: BenchmarkFixture, vxf: vx.VortexFile) -> None:
     rscan = vxf.to_repeated_scan()
     benchmark(lambda: pa.concat_tables(x.to_arrow_table() for x in rscan.execute()))
 
 
-@pytest.mark.benchmark(group="scan", disable_gc=True)
+@pytest.mark.benchmark(group="scan")
 def test_polars(benchmark: BenchmarkFixture, vxf: vx.VortexFile) -> None:
     lf = vxf.to_polars()
     benchmark(lambda: lf.collect().to_arrow())
 
 
-@pytest.mark.benchmark(group="scan", disable_gc=True)
+@pytest.mark.benchmark(group="scan")
 def test_polars_streaming(benchmark: BenchmarkFixture, vxf: vx.VortexFile) -> None:
     lf = vxf.to_polars()
     benchmark(lambda: lf.collect(engine="streaming").to_arrow())
 
 
-@pytest.mark.benchmark(group="scan", disable_gc=True)
+@pytest.mark.benchmark(group="scan")
 def test_duckdb(benchmark: BenchmarkFixture, vxf: vx.VortexFile) -> None:
     conn = duckdb.connect(database=":memory:")
     ds = vxf.to_dataset()

vortex-python/benchmark/test_scan.py

@@ -12,7 +12,6 @@
 
 @pytest.mark.parametrize("protocol", [4, 5], ids=lambda p: f"p{p}")
 @pytest.mark.parametrize("operation", ["dumps", "loads", "roundtrip"])
-@pytest.mark.benchmark(disable_gc=True)
 def test_pickle(
     benchmark: BenchmarkFixture,
     array_fixture: vx.Array,

vortex-python/benchmark/test_serialization.py

@@ -85,6 +85,7 @@ dev = [
     "polars>=1.32.0",
     "pyarrow-stubs>=17.16",
     "pytest-benchmark>=5.1.0",
+    "pytest-codspeed>=5.0.3",
     "pytest-xdist>=3.5.0",
     "ray>=2.48",
     # forced transitive bumps

vortex-python/pyproject.toml

@@ -25,4 +25,4 @@ source "$ROOT/.venv/bin/activate"
 maturin develop --release --manifest-path "$PYTHON_DIR/Cargo.toml"
 
 # Run benchmarks. Extra args are forwarded to pytest.
-pytest "$PYTHON_DIR/benchmark" "$@"
+pytest --benchmark-disable-gc "$PYTHON_DIR/benchmark" "$@"