diff --git a/src/evaluate/__init__.py b/src/evaluate/__init__.py index a8c25bd9..96745b6f 100644 --- a/src/evaluate/__init__.py +++ b/src/evaluate/__init__.py @@ -45,7 +45,7 @@ from .info import ComparisonInfo, EvaluationModuleInfo, MeasurementInfo, MetricInfo from .inspect import inspect_evaluation_module, list_evaluation_modules from .loading import load -from .module import CombinedEvaluations, Comparison, EvaluationModule, Measurement, Metric, combine +from .module import CombinedEvaluations, Comparison, EvaluationModule, EvaluationModuleError, Measurement, Metric, combine from .saving import save from .utils import * from .utils import gradio, logging diff --git a/src/evaluate/module.py b/src/evaluate/module.py index ca38b9b1..46dbe9f8 100644 --- a/src/evaluate/module.py +++ b/src/evaluate/module.py @@ -80,6 +80,23 @@ def format_chunk(chunk): return f"[{format_chunk(obj[:3])}, ..., {format_chunk(obj[-3:])}]" +class EvaluationModuleError(Exception): + """Raised when an evaluation module's compute step fails. + + Wraps internal errors (e.g. from sklearn or numpy) so callers can catch + evaluate-specific failures without catching broad ``Exception``. + + Example:: + + import evaluate + acc = evaluate.load("accuracy") + try: + acc.compute(predictions=[], references=[]) + except evaluate.EvaluationModuleError as e: + print(f"Metric failed: {e}") + """ + + class EvaluationModuleInfoMixin: """This base class exposes some attributes of EvaluationModuleInfo at the base level of the EvaluationModule for easy access. @@ -464,7 +481,14 @@ def compute(self, *, predictions=None, references=None, **kwargs) -> Optional[di inputs = {input_name: self.data[input_name][:] for input_name in self._feature_names()} with temp_seed(self.seed): - output = self._compute(**inputs, **compute_kwargs) + try: + output = self._compute(**inputs, **compute_kwargs) + except EvaluationModuleError: + raise + except Exception as e: + raise EvaluationModuleError( + f"Metric '{self.name}' failed during compute: {e}" + ) from e if self.buf_writer is not None: self.buf_writer = None diff --git a/src/evaluate/naming.py b/src/evaluate/naming.py index 6335cf1b..f5411e85 100644 --- a/src/evaluate/naming.py +++ b/src/evaluate/naming.py @@ -44,12 +44,52 @@ def snakecase_to_camelcase(name): def filename_prefix_for_name(name): + """Return the snake_case filename prefix for a given dataset name. + + Args: + name (`str`): Dataset name (must not contain path separators). + + Returns: + `str`: The snake_case version of `name`, used as the base prefix for data files. + + Raises: + `ValueError`: If `name` looks like a file path rather than a plain dataset name. + + Example: + ```py + >>> filename_prefix_for_name("MyDataset") + 'my_dataset' + ``` + """ if os.path.basename(name) != name: raise ValueError(f"Should be a dataset name, not a path: {name}") return camelcase_to_snakecase(name) def filename_prefix_for_split(name, split): + """Return the filename prefix for a specific dataset split. + + Combines the snake_case dataset name with the split name, following + the pattern ``-``. + + Args: + name (`str`): Dataset name (must not contain path separators). + split (`str`): Split identifier, e.g. ``"train"``, ``"test"``, or + a dotted sub-split such as ``"train.clean"``. + + Returns: + `str`: The filename prefix in the form ``-``. + + Raises: + `ValueError`: If `name` looks like a path, or `split` does not match + the expected pattern ``^\\w+(\\.\\w+)*$``. + + Example: + ```py + >>> filename_prefix_for_split("MyDataset", "train") + 'my_dataset-train' + ``` + """ if os.path.basename(name) != name: raise ValueError(f"Should be a dataset name, not a path: {name}") if not re.match(_split_re, split): @@ -58,6 +98,25 @@ def filename_prefix_for_split(name, split): def filepattern_for_dataset_split(dataset_name, split, data_dir, filetype_suffix=None): + """Return a glob pattern matching all shard files for a dataset split. + + Args: + dataset_name (`str`): Dataset name (must not contain path separators). + split (`str`): Split identifier, e.g. ``"train"`` or ``"validation"``. + data_dir (`str`): Directory where the dataset files are stored. + filetype_suffix (`str`, *optional*): File extension to append before the + glob wildcard, e.g. ``"parquet"`` → ``"my_dataset-train.parquet*"``. + When ``None`` the pattern has no extension suffix. + + Returns: + `str`: A glob pattern of the form ``/[.]*``. + + Example: + ```py + >>> filepattern_for_dataset_split("MyDataset", "train", "/data", "parquet") + '/data/my_dataset-train.parquet*' + ``` + """ prefix = filename_prefix_for_split(dataset_name, split) if filetype_suffix: prefix += f".{filetype_suffix}" @@ -66,6 +125,24 @@ def filepattern_for_dataset_split(dataset_name, split, data_dir, filetype_suffix def filename_for_dataset_split(dataset_name, split, filetype_suffix=None): + """Return the filename (without directory) for a dataset split. + + Args: + dataset_name (`str`): Dataset name (must not contain path separators). + split (`str`): Split identifier, e.g. ``"train"`` or ``"test"``. + filetype_suffix (`str`, *optional*): File extension to append, e.g. + ``"arrow"`` → ``"my_dataset-train.arrow"``. When ``None`` no + extension is added. + + Returns: + `str`: The filename string, e.g. ``"my_dataset-train.arrow"``. + + Example: + ```py + >>> filename_for_dataset_split("MyDataset", "train", "arrow") + 'my_dataset-train.arrow' + ``` + """ prefix = filename_prefix_for_split(dataset_name, split) if filetype_suffix: prefix += f".{filetype_suffix}" @@ -73,6 +150,27 @@ def filename_for_dataset_split(dataset_name, split, filetype_suffix=None): def filepath_for_dataset_split(dataset_name, split, data_dir, filetype_suffix=None): + """Return the full file path for a dataset split file. + + Combines `data_dir` and the result of :func:`filename_for_dataset_split` + to produce an absolute-or-relative path suitable for reading or writing. + + Args: + dataset_name (`str`): Dataset name (must not contain path separators). + split (`str`): Split identifier, e.g. ``"train"`` or ``"validation"``. + data_dir (`str`): Directory where the dataset files are stored. + filetype_suffix (`str`, *optional*): File extension without the leading + dot, e.g. ``"arrow"``. When ``None`` no extension is added. + + Returns: + `str`: Full path of the form ``/``. + + Example: + ```py + >>> filepath_for_dataset_split("MyDataset", "train", "/data", "arrow") + '/data/my_dataset-train.arrow' + ``` + """ filename = filename_for_dataset_split( dataset_name=dataset_name, split=split,