ci(bench): run TPC-DS SF=10 and SF=100 nightly on NVMe and S3 (#10508)

## Summary

TPC-DS only ran at SF=1 (NVMe). At ~1 GB the data is cache-resident and
joins are small, so it says little about scan, pruning, spill or memory
behaviour at realistic sizes, and TPC-DS's 99 queries cover far more of
that than TPC-H's 22. SF=100 is too slow for PR or per-merge runs, so
this adds SF=10 and SF=100 to the nightly preset, next to the existing
TPC-H SF=100 runs.

Author: joseph-isaacs

bench-orchestrator/bench_orchestrator/ci_matrix/catalog.py

@@ -20,7 +20,7 @@
     "pr-compact": "Pull-request SQL benchmarks for Vortex Compact plus Parquet controls.",
     "pr-all": "The union of the pr, pr-compact and pr-full matrices (action/bench-all).",
     "pr-full": "Every regular SQL benchmark at full PR target coverage (action/bench-sql-extended).",
-    "nightly": "Large-scale SF=100 TPC-H on NVMe and S3 at default targets.",
+    "nightly": "Large-scale SF=100 TPC-H on NVMe and S3, and SF=10/100 TPC-DS on NVMe and S3, at default targets.",
 }
 
 # Reusable target coverage
@@ -186,6 +186,43 @@
             "develop": STANDARD_WITH_DUCKDB,
         },
     ),
+    BenchmarkCase(
+        id="tpcds-nvme-10",
+        benchmark=Benchmark.TPCDS,
+        name="TPC-DS SF=10 on NVME",
+        scale_factor=10.0,
+        runs={"nightly": DEFAULT},
+    ),
+    BenchmarkCase(
+        id="tpcds-nvme-100",
+        benchmark=Benchmark.TPCDS,
+        name="TPC-DS SF=100 on NVME",
+        scale_factor=100.0,
+        # 99 queries across four targets: fewer passes keep the job inside its timeout.
+        iterations=3,
+        runs={"nightly": DEFAULT},
+    ),
+    BenchmarkCase(
+        id="tpcds-s3-10",
+        benchmark=Benchmark.TPCDS,
+        name="TPC-DS SF=10 on S3",
+        storage=Storage.S3,
+        scale_factor=10.0,
+        local_dir="vortex-bench/data/tpcds/10.0",
+        remote_key="tpcds/10.0",
+        runs={"nightly": DEFAULT},
+    ),
+    BenchmarkCase(
+        id="tpcds-s3-100",
+        benchmark=Benchmark.TPCDS,
+        name="TPC-DS SF=100 on S3",
+        storage=Storage.S3,
+        scale_factor=100.0,
+        iterations=3,
+        local_dir="vortex-bench/data/tpcds/100.0",
+        remote_key="tpcds/100.0",
+        runs={"nightly": DEFAULT},
+    ),
     BenchmarkCase(
         id="statpopgen",
         benchmark=Benchmark.STATPOPGEN,

bench-orchestrator/tests/test_matrix.py

@@ -55,7 +55,14 @@
     "pr-compact": COMPACT_IDS,
     "pr-all": PR_FULL_IDS,
     "pr-full": PR_FULL_IDS,
-    "nightly": ("tpch-nvme", "tpch-s3"),
+    "nightly": (
+        "tpch-nvme",
+        "tpch-s3",
+        "tpcds-nvme-10",
+        "tpcds-nvme-100",
+        "tpcds-s3-10",
+        "tpcds-s3-100",
+    ),
 }