feat: add Hugging Face Hub as test data source support for SDK- #169

src/evalhub/cli/main.py

@@ -26,6 +26,7 @@
     EvaluationExportsOCI,
     ExperimentConfig,
     GitTestDataRef,
+    HFTestDataRef,
     JobStatus,
     JobSubmissionRequest,
     ModelAuth,
@@ -354,6 +355,26 @@ def _build_request_from_flags(
     default=None,
     help="Kubernetes Secret name with username/password keys for private git repos (inline flags only).",
 )
+@click.option(
+    "--test-data-hf-repo-id",
+    default=None,
+    help="Hugging Face Hub repository ID for custom test data (inline flags only).",
+)
+@click.option(
+    "--test-data-hf-revision",
+    default=None,
+    help="HF branch, tag, or commit SHA to download (inline flags only).",
+)
+@click.option(
+    "--test-data-hf-sub-path",
+    default=None,
+    help="Path within the HF repository to mount at /test_data (inline flags only).",
+)
+@click.option(
+    "--test-data-hf-secret",
+    default=None,
+    help="Kubernetes Secret name with a token key for gated HF repositories (inline flags only).",
+)
 @click.option(
     "--wait", "wait_for", is_flag=True, default=False, help="Block until job completes."
 )
@@ -407,6 +428,10 @@ def eval_run(
     test_data_git_ref: str | None,
     test_data_git_sub_path: str | None,
     test_data_git_secret: str | None,
+    test_data_hf_repo_id: str | None,
+    test_data_hf_revision: str | None,
+    test_data_hf_sub_path: str | None,
+    test_data_hf_secret: str | None,
     wait_for: bool,
     watch_for: bool,
     timeout: float | None,
@@ -455,6 +480,13 @@ def eval_run(
           --model-name llama3 --provider lm_evaluation_harness -b your_benchmark_id \\
           --test-data-git-url https://github.com/org/benchmarks.git --test-data-git-ref v1.0 \\
           --test-data-git-sub-path arc_easy --test-data-git-secret my-git-credentials
+      evalhub eval run --name my-eval --model-url http://vllm:8000/v1 \\
+          --model-name llama3 --provider lm_evaluation_harness -b your_benchmark_id \\
+          --test-data-hf-repo-id eval-hub-test/evalhub-offline-testdata --test-data-hf-revision main
+      evalhub eval run --name my-eval --model-url http://vllm:8000/v1 \\
+          --model-name llama3 --provider lm_evaluation_harness -b your_benchmark_id \\
+          --test-data-hf-repo-id cais/mmlu --test-data-hf-sub-path data/train \\
+          --test-data-hf-secret my-hf-credentials
     """
     if wait_for and watch_for:
         raise click.UsageError("--wait and --watch are mutually exclusive.")
@@ -528,16 +560,24 @@ def eval_run(
             raise click.ClickException(
                 "--test-data-git-sub-path and --test-data-git-secret require --test-data-git-url."
             )
+        if (
+            test_data_hf_revision or test_data_hf_sub_path or test_data_hf_secret
+        ) and not test_data_hf_repo_id:
+            raise click.ClickException(
+                "--test-data-hf-revision, --test-data-hf-sub-path, and "
+                "--test-data-hf-secret require --test-data-hf-repo-id."
+            )
         active_sources = sum(
             [
                 bool(all(s3_flags)),
                 bool(test_data_pvc_claim_name),
                 bool(test_data_git_url),
+                bool(test_data_hf_repo_id),
             ]
         )
         if active_sources > 1:
             raise click.ClickException(
-                "Cannot specify more than one test data source (s3, pvc, git). Use only one."
+                "Cannot specify more than one test data source (s3, pvc, git, hf). Use only one."
             )
         test_data_ref: TestDataRef | None = None
         if all(s3_flags):
@@ -564,6 +604,15 @@ def eval_run(
                     secret_ref=test_data_git_secret,
                 )
             )
+        elif test_data_hf_repo_id:
+            test_data_ref = TestDataRef(
+                hf=HFTestDataRef(
+                    repo_id=test_data_hf_repo_id,
+                    revision=test_data_hf_revision,
+                    sub_path=test_data_hf_sub_path,
+                    secret_ref=test_data_hf_secret,
+                )
+            )
         request = _build_request_from_flags(
             name=cast(str, name),
             model_url=cast(str, model_url),

src/evalhub/models/__init__.py

@@ -31,6 +31,7 @@
     FrameworkInfo,
     GitTestDataRef,
     HealthResponse,
+    HFTestDataRef,
     # Status and metadata
     JobPhase,
     JobsList,
@@ -121,6 +122,7 @@
     "BenchmarkConfig",
     "BenchmarkInfo",
     "GitTestDataRef",
+    "HFTestDataRef",
     "PVCTestDataRef",
     "S3TestDataRef",
     "TestDataRef",

src/evalhub/models/api.py

@@ -382,8 +382,40 @@ def check_url_scheme(self) -> GitTestDataRef:
         return self
 
 
+class HFTestDataRef(BaseModel):
+    """Hugging Face Hub repository source for custom test data.
+
+    The repository is downloaded at ``revision`` (or the default branch when omitted)
+    into ``/test_data`` before the adapter runs. When ``sub_path`` is set, only that
+    path within the repository is exposed.
+    """
+
+    repo_id: str = Field(
+        ...,
+        description="Hugging Face Hub repository ID (for example cais/mmlu or org/dataset)",
+    )
+    revision: str | None = Field(
+        default=None,
+        description="Branch, tag, or commit SHA to download (default branch when omitted)",
+    )
+    sub_path: str | None = Field(
+        default=None,
+        description="Optional path within the repository to mount at /test_data",
+    )
+    secret_ref: str | None = Field(
+        default=None,
+        description="Kubernetes Secret name containing a token key for gated repositories",
+    )
+
+    @model_validator(mode="after")
+    def check_repo_id(self) -> HFTestDataRef:
+        if not self.repo_id.strip():
+            raise ValueError("repo_id must not be blank")
+        return self
+
+
 class TestDataRef(BaseModel):
-    """Reference to an external test data source. Exactly one of s3, pvc, or git must be set."""
+    """Reference to an external test data source. Exactly one of s3, pvc, git, or hf must be set."""
 
     s3: S3TestDataRef | None = Field(default=None, description="S3 data source")
     pvc: PVCTestDataRef | None = Field(
@@ -392,21 +424,24 @@ class TestDataRef(BaseModel):
     git: GitTestDataRef | None = Field(
         default=None, description="Git repository data source"
     )
+    hf: HFTestDataRef | None = Field(
+        default=None, description="Hugging Face Hub repository data source"
+    )
     resolved_sha: str | None = Field(
         default=None,
         description="Resolved content identity (e.g. git commit SHA). Server-populated; stripped from submission payloads.",
     )
 
     @model_validator(mode="after")
     def check_exactly_one_source(self) -> TestDataRef:
-        sources = [s for s in (self.s3, self.pvc, self.git) if s is not None]
+        sources = [s for s in (self.s3, self.pvc, self.git, self.hf) if s is not None]
         if len(sources) > 1:
             raise ValueError(
-                "Cannot specify more than one test data source (s3, pvc, git)"
+                "Cannot specify more than one test data source (s3, pvc, git, hf)"
             )
         if len(sources) == 0:
             raise ValueError(
-                "Must specify exactly one test data source (s3, pvc, or git)"
+                "Must specify exactly one test data source (s3, pvc, git, or hf)"
             )
         return self
 

tests/unit/test_cli_eval.py

@@ -899,6 +899,142 @@ def test_eval_run_help_pvc_flags(self, runner: CliRunner) -> None:
         assert "--test-data-pvc-claim-name" in result.output
         assert "--test-data-pvc-sub-path" in result.output
 
+    def test_run_with_hf_repo_id(
+        self, runner: CliRunner, config_file: Path, mock_client: MagicMock
+    ) -> None:
+        mock_client.jobs.submit.return_value = _make_job()
+        with patch("evalhub.cli.main.get_client", return_value=mock_client):
+            result = runner.invoke(
+                main,
+                [
+                    "eval",
+                    "run",
+                    "--name",
+                    "hf-eval",
+                    "--model-url",
+                    "http://vllm:8000/v1",
+                    "--model-name",
+                    "llama3",
+                    "--provider",
+                    "lm_eval",
+                    "-b",
+                    "mmlu",
+                    "--test-data-hf-repo-id",
+                    "eval-hub-test/evalhub-offline-testdata",
+                ],
+            )
+        assert result.exit_code == 0
+        req = mock_client.jobs.submit.call_args[0][0]
+        assert req.benchmarks[0].test_data_ref is not None
+        hf = req.benchmarks[0].test_data_ref.hf
+        assert hf is not None
+        assert hf.repo_id == "eval-hub-test/evalhub-offline-testdata"
+        assert hf.revision is None
+        assert hf.sub_path is None
+        assert hf.secret_ref is None
+
+    def test_run_with_hf_all_fields(
+        self, runner: CliRunner, config_file: Path, mock_client: MagicMock
+    ) -> None:
+        mock_client.jobs.submit.return_value = _make_job()
+        with patch("evalhub.cli.main.get_client", return_value=mock_client):
+            result = runner.invoke(
+                main,
+                [
+                    "eval",
+                    "run",
+                    "--name",
+                    "hf-eval",
+                    "--model-url",
+                    "http://vllm:8000/v1",
+                    "--model-name",
+                    "llama3",
+                    "--provider",
+                    "lm_eval",
+                    "-b",
+                    "mmlu",
+                    "--test-data-hf-repo-id",
+                    "cais/mmlu",
+                    "--test-data-hf-revision",
+                    "main",
+                    "--test-data-hf-sub-path",
+                    "data/train",
+                    "--test-data-hf-secret",
+                    "my-hf-credentials",
+                ],
+            )
+        assert result.exit_code == 0
+        hf = mock_client.jobs.submit.call_args[0][0].benchmarks[0].test_data_ref.hf
+        assert hf.repo_id == "cais/mmlu"
+        assert hf.revision == "main"
+        assert hf.sub_path == "data/train"
+        assert hf.secret_ref == "my-hf-credentials"
+
+    def test_run_hf_and_pvc_mutually_exclusive(
+        self, runner: CliRunner, config_file: Path, mock_client: MagicMock
+    ) -> None:
+        with patch("evalhub.cli.main.get_client", return_value=mock_client):
+            result = runner.invoke(
+                main,
+                [
+                    "eval",
+                    "run",
+                    "--name",
+                    "hf-eval",
+                    "--model-url",
+                    "http://vllm:8000/v1",
+                    "--model-name",
+                    "llama3",
+                    "--provider",
+                    "lm_eval",
+                    "-b",
+                    "mmlu",
+                    "--test-data-hf-repo-id",
+                    "org/dataset",
+                    "--test-data-pvc-claim-name",
+                    "my-datasets-pvc",
+                ],
+            )
+        assert result.exit_code != 0
+        assert "Cannot specify more than one test data source" in result.output
+        mock_client.jobs.submit.assert_not_called()
+
+    def test_run_hf_sub_path_requires_repo_id(
+        self, runner: CliRunner, config_file: Path, mock_client: MagicMock
+    ) -> None:
+        with patch("evalhub.cli.main.get_client", return_value=mock_client):
+            result = runner.invoke(
+                main,
+                [
+                    "eval",
+                    "run",
+                    "--name",
+                    "hf-eval",
+                    "--model-url",
+                    "http://vllm:8000/v1",
+                    "--model-name",
+                    "llama3",
+                    "--provider",
+                    "lm_eval",
+                    "-b",
+                    "mmlu",
+                    "--test-data-hf-sub-path",
+                    "data/train",
+                ],
+            )
+        assert result.exit_code != 0
+        assert "--test-data-hf-sub-path" in result.output
+        assert "--test-data-hf-repo-id" in result.output
+        mock_client.jobs.submit.assert_not_called()
+
+    def test_eval_run_help_hf_flags(self, runner: CliRunner) -> None:
+        result = runner.invoke(main, ["eval", "run", "--help"])
+        assert result.exit_code == 0
+        assert "--test-data-hf-repo-id" in result.output
+        assert "--test-data-hf-revision" in result.output
+        assert "--test-data-hf-sub-path" in result.output
+        assert "--test-data-hf-secret" in result.output
+
 
 # --- eval status ---