rework base task

README.md

@@ -189,11 +189,13 @@ from eval_framework.main import main
 from eval_framework.tasks.eval_config import EvalConfig
 from template_formatting.formatter import HFFormatter
 
+
 # Define your model
 class MyHuggingFaceModel(HFLLM):
     LLM_NAME = "microsoft/DialoGPT-medium"
     DEFAULT_FORMATTER = partial(HFFormatter, "microsoft/DialoGPT-medium")
 
+
 if __name__ == "__main__":
     # Initialize your model
     llm = MyHuggingFaceModel()

docs/add_new_benchmark_guide.md

@@ -23,16 +23,16 @@ All benchmarks inherit from `BaseTask[SubjectType]` and must implement several r
 ```python
 class YourBenchmark(BaseTask[str]):  # or BaseTask[Enum] for multiple subjects
     # === CORE CONFIGURATION ===
-    NAME: str                           # Display name for the benchmark
-    DATASET_PATH: str                   # HuggingFace dataset path or local path
-    SAMPLE_SPLIT: str                   # Dataset split for evaluation samples
-    FEWSHOT_SPLIT: str                  # Dataset split for few-shot examples
-    RESPONSE_TYPE: ResponseType         # COMPLETION or LOGLIKELIHOODS
-    METRICS: list[type[BaseMetric]]     # List of metric classes to compute
-    SUBJECTS: list[SubjectType]         # List of subjects/categories to evaluate
+    NAME: str  # Display name for the benchmark
+    DATASET_PATH: str  # HuggingFace dataset path or local path
+    SAMPLE_SPLIT: str  # Dataset split for evaluation samples
+    FEWSHOT_SPLIT: str  # Dataset split for few-shot examples
+    RESPONSE_TYPE: ResponseType  # COMPLETION or LOGLIKELIHOODS
+    METRICS: list[type[BaseMetric]]  # List of metric classes to compute
+    SUBJECTS: list[SubjectType]  # List of subjects/categories to evaluate
 
     # === OPTIONAL CONFIGURATION ===
-    HF_REVISION: str | None = None      # Git revision for reproducibility
+    HF_REVISION: str | None = None  # Git revision for reproducibility
     PERTURBATION_UNMODIFIABLE_WORDS: list[str] | None = None  # Words to protect from perturbation
     LANGUAGE: Language | dict[str, Language] | dict[str, tuple[Language, Language]] | None = None  # Language(s) tested
 ```
@@ -44,6 +44,7 @@ def _get_instruction_text(self, item: dict[str, Any]) -> str:
     """Generate the instruction/question text for a sample."""
     pass
 
+
 def _get_ground_truth(self, item: dict[str, Any]) -> str | None | list[str]:
     """Extract the correct answer(s) from a dataset item."""
     pass
@@ -56,38 +57,46 @@ def _get_initial_prompt_text(self, item: dict[str, Any]) -> str:
     """Text to prepend to the first message."""
     return ""
 
+
 def _get_system_prompt_text(self, item: dict[str, Any]) -> str | None:
     """System message content."""
     return None
 
+
 def _get_cue_text(self, item: dict[str, Any]) -> str:
     """Text to append as assistant cue (e.g., 'Answer:')."""
     return ""
 
+
 def _get_possible_completions(self, item: dict[str, Any]) -> list[str] | None:
     """For loglikelihood tasks: list of answer choices."""
     return None
 
+
 def _get_fewshot_target_text(self, item: dict[str, Any]) -> str:
     """Target text for few-shot examples."""
     target = self._get_ground_truth(item)
     assert target is not None and isinstance(target, str)
     return target
 
+
 def _get_context(self, item: dict[str, Any]) -> BaseMetricContext | list[BaseMetricContext] | None:
     """Additional parameters for evaluation metrics."""
     return None
 
+
 def _sample_fewshot_examples(self, item: dict[str, Any]) -> list[dict]:
     """Custom few-shot sampling logic."""
     # Default implementation samples randomly from FEWSHOT_SPLIT
     pass
 
+
 def _create_samples(self, item: dict[str, Any], index: int, subject: str) -> list[Sample]:
     """Create one or more samples from a dataset item."""
     # Default creates single sample - override for multi-sample items
     pass
 
+
 def post_process_generated_completion(self, completion_text: str, sample: Sample | None = None) -> str:
     """Post-process model completions (e.g., extract final answer)."""
     return completion_text
@@ -151,7 +160,6 @@ from eval_framework.metrics.completion.niah_accuracy import NIAHAccuracy
 from eval_framework.metrics.completion.text_counter import WordCounter
 from eval_framework.metrics.completion.text_counter import ParagraphCounter
 from eval_framework.metrics.completion.text_counter import ResponseToOriginalLengthRatio
-
 ```
 
 #### Loglikelihood Metrics
@@ -214,7 +222,6 @@ from eval_framework.metrics.llm.llm_judge_sql import LLMJudgeSql
 
 from eval_framework.metrics.llm.llm_judge_world_knowledge import LLMJudgeWorldKnowledge
 # Evaluates whether a summary contains information that goes beyond the reference text (also known as "world knowledge"), returning a boolean classification with detailed reasoning for the assessment. (English, French and German)
-
 ```
 
 ## Implementation Examples and Patterns
@@ -231,6 +238,7 @@ from eval_framework.tasks.base import BaseTask
 from eval_framework.models.sample import ResponseType
 from eval_framework.metrics.completion.accuracy_completion import AccuracyCompletion
 
+
 class GeographyQATask(BaseTask[str]):
     # Required class attributes
     NAME = "GeographyQA"

docs/completion_task_guide.md

@@ -27,7 +27,7 @@ class YourCompletionTask(BaseTask[str]):
 
     def _get_ground_truth(self, item: dict[str, Any]) -> str:
         """Extract the correct answer from the dataset."""
-        return item['answer']
+        return item["answer"]
 ```
 
 ## Step-by-Step Implementation
@@ -41,6 +41,7 @@ from eval_framework.tasks.base import BaseTask
 from eval_framework.models.sample import ResponseType
 from eval_framework.metrics.completion.accuracy_completion import AccuracyCompletion
 
+
 class MathQATask(BaseTask[str]):
     NAME = "MathQA"
     DATASET_PATH = "math_qa_dataset"
@@ -76,7 +77,7 @@ This method extracts the correct answer:
 def _get_ground_truth(self, item: dict[str, Any]) -> str:
     """Extract the correct answer from the dataset item."""
     # Simple case - direct answer
-    return item['answer']
+    return item["answer"]
 
     # For numeric answers, you might want to normalize
     # return str(float(item['answer']))
@@ -102,7 +103,7 @@ class QATask(BaseTask[str]):
         return f"Question: {item['question']}\nAnswer:"
 
     def _get_ground_truth(self, item: dict[str, Any]) -> str:
-        return item['answer']
+        return item["answer"]
 
     def _get_cue_text(self, item: dict[str, Any]) -> str:
         return "Answer:"  # Helps model start response correctly
@@ -123,13 +124,14 @@ class MathTask(BaseTask[str]):
         return f"Problem: {item['problem']}\nSolution:"
 
     def _get_ground_truth(self, item: dict[str, Any]) -> str:
-        return item['solution']
+        return item["solution"]
 
     def post_process_generated_completion(self, completion_text: str, sample: Sample | None = None) -> str:
         """Extract final numerical answer from solution."""
         import re
+
         # Look for "The answer is X" pattern
-        match = re.search(r'The answer is (\d+(?:\.\d+)?)', completion_text)
+        match = re.search(r"The answer is (\d+(?:\.\d+)?)", completion_text)
         if match:
             return match.group(1)
         return completion_text.strip()
@@ -140,11 +142,14 @@ class MathTask(BaseTask[str]):
 from eval_framework.metrics.completion.code_execution_pass_at_one import CodeExecutionPassAtOne
 from eval_framework.shared.types import BaseMetricContext
 
+
 class CodeTaskMetricContext(BaseMetricContext):
     """Will be passed to the metric for this task."""
+
     test_cases: list
     entry_point: str
 
+
 class CodeTask(BaseTask[str]):
     NAME = "Code Generation"
     DATASET_PATH = "code_dataset"
@@ -158,13 +163,13 @@ class CodeTask(BaseTask[str]):
         return f"Complete this function:\n{item['prompt']}"
 
     def _get_ground_truth(self, item: dict[str, Any]) -> str:
-        return item['canonical_solution']
+        return item["canonical_solution"]
 
     def _get_context(self, item: dict[str, Any]) -> CodeTaskMetricContext:
         """Provide test cases for code execution."""
         return CodeTaskMetricContext(
-            test_cases=item['text_cases'],
-            entry_point=item['entry_point'],
+            test_cases=item["text_cases"],
+            entry_point=item["entry_point"],
         )
 ```
 
@@ -231,6 +236,7 @@ from eval_framework.metrics.completion.csv_format import CSVFormat
 # Custom metrics using LLM judges
 from eval_framework.metrics.llm.llm_judge_score import LLMJudgeScore
 
+
 class YourTask(BaseTask[str]):
     # Choose metrics appropriate for your task
     METRICS = [AccuracyCompletion, Rouge1, MathReasoningCompletion]
@@ -244,6 +250,7 @@ from eval_framework.tasks.base import BaseTask
 from eval_framework.models.sample import ResponseType
 from eval_framework.metrics.completion.accuracy_completion import AccuracyCompletion
 
+
 class GeographyQuizTask(BaseTask[str]):
     NAME = "Geography Quiz"
     DATASET_PATH = "geography_quiz"
@@ -259,7 +266,7 @@ class GeographyQuizTask(BaseTask[str]):
 
     def _get_ground_truth(self, item: dict[str, Any]) -> str:
         """Extract the correct capital city."""
-        return item['capital']
+        return item["capital"]
 
     def _get_system_prompt_text(self, item: dict[str, Any]) -> str:
         """Provide context about the task."""

docs/evaluate_huggingface_model.md

@@ -15,11 +15,13 @@ from eval_framework.main import main
 from eval_framework.tasks.eval_config import EvalConfig
 from template_formatting.formatter import HFFormatter
 
+
 # Define your model
 class MyHuggingFaceModel(HFLLM):
     LLM_NAME = "context-labs/meta-llama-Llama-3.2-3B-Instruct-FP16"
     DEFAULT_FORMATTER = partial(HFFormatter, "context-labs/meta-llama-Llama-3.2-3B-Instruct-FP16")
 
+
 if __name__ == "__main__":
     # Initialize your model
     llm = MyHuggingFaceModel()
@@ -65,6 +67,7 @@ The formatter determines how prompts are structured for your model. Choose based
 ```python
 from template_formatting.formatter import ConcatFormatter
 
+
 class BaseModel(HFLLM):
     LLM_NAME = "meta-llama/Llama-3.2-3B"
     DEFAULT_FORMATTER = ConcatFormatter
@@ -75,6 +78,7 @@ class BaseModel(HFLLM):
 ```python
 from template_formatting.formatter import Llama3Formatter
 
+
 class Llama3Model(HFLLM):
     LLM_NAME = "meta-llama/Meta-Llama-3-8B-Instruct"
     DEFAULT_FORMATTER = Llama3Formatter
@@ -85,6 +89,7 @@ class Llama3Model(HFLLM):
 ```python
 from template_formatting.mistral_formatter import MistralFormatter
 
+
 class MistralModel(HFLLM):
     LLM_NAME = "mistralai/Mistral-7B-Instruct-v0.1"
     DEFAULT_FORMATTER = MistralFormatter
@@ -95,6 +100,7 @@ class MistralModel(HFLLM):
 from template_formatting.formatter import HFFormatter
 from functools import partial
 
+
 class ChatModel(HFLLM):
     LLM_NAME = "meta-llama/Llama-3.2-3B-Instruct"
     DEFAULT_FORMATTER = partial(HFFormatter, "meta-llama/Llama-3.2-3B-Instruct")
@@ -113,10 +119,12 @@ class Llama3_8B(HFLLM):
     LLM_NAME = "meta-llama/Meta-Llama-3-8B-Instruct"
     DEFAULT_FORMATTER = Llama3Formatter
 
+
 class Mistral7B(HFLLM):
     LLM_NAME = "mistralai/Mistral-7B-Instruct-v0.1"
     DEFAULT_FORMATTER = MistralFormatter
 
+
 class Qwen2_7B(HFLLM):
     LLM_NAME = "Qwen/Qwen2-7B-Instruct"
     DEFAULT_FORMATTER = partial(HFFormatter, "Qwen/Qwen2-7B-Instruct")
@@ -128,6 +136,7 @@ class SmolLM(HFLLM):
     LLM_NAME = "HuggingFaceTB/SmolLM-1.7B-Instruct"
     DEFAULT_FORMATTER = partial(HFFormatter, "HuggingFaceTB/SmolLM-1.7B-Instruct")
 
+
 class TinyLlama(HFLLM):
     LLM_NAME = "TinyLlama/TinyLlama-1.1B-Chat-v1.0"
     DEFAULT_FORMATTER = partial(HFFormatter, "TinyLlama/TinyLlama-1.1B-Chat-v1.0")
@@ -144,15 +153,14 @@ from eval_framework.tasks.eval_config import EvalConfig
 
 config = EvalConfig(
     # Core settings
-    task_name="MMLU",                    # Benchmark to run
-    num_fewshot=5,                       # Number of examples in prompt
-    num_samples=100,                     # How many questions to evaluate
-    output_dir=Path("./eval_results"),        # Where to save results
-    llm_class=YourModelClass,            # Your model class
-
+    task_name="MMLU",  # Benchmark to run
+    num_fewshot=5,  # Number of examples in prompt
+    num_samples=100,  # How many questions to evaluate
+    output_dir=Path("./eval_results"),  # Where to save results
+    llm_class=YourModelClass,  # Your model class
     # Optional settings
-    task_subjects=["astronomy"],             # Specific subjects (if applicable)
-    batch_size=8,                        # Batch processing size
+    task_subjects=["astronomy"],  # Specific subjects (if applicable)
+    batch_size=8,  # Batch processing size
 )
 ```