diff --git a/tests/conftest.py b/tests/conftest.py index 932f0324..a21eea67 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -17,12 +17,22 @@ import pandas as pd import pytest - _ENV_VARS = ( - "SRC_PATH", "LABEL_FILE", "TABLE_NAME", "TITLE", - "BACKEND_TOKEN", "CLIENT_ID", "CLIENT_PASSWORD", - "CLIENT_ENV", "LOG_LEVEL", "BATCH_SIZE", - "MYSQL_HOST", "MYSQL_PORT", "DB_USER", "DB_PASSWORD", "DB_NAME", + "SRC_PATH", + "LABEL_FILE", + "TABLE_NAME", + "TITLE", + "BACKEND_TOKEN", + "CLIENT_ID", + "CLIENT_PASSWORD", + "CLIENT_ENV", + "LOG_LEVEL", + "BATCH_SIZE", + "MYSQL_HOST", + "MYSQL_PORT", + "DB_USER", + "DB_PASSWORD", + "DB_NAME", ) @@ -101,39 +111,6 @@ def _make( return _make -@pytest.fixture -def make_tokenizer(tmp_path): - """Write a HuggingFace-style tokenizer.json into a dir, return that dir. - - ``vocab`` is the model.vocab token list; ``added`` is the added_tokens - content list. Pass ``raw=`` to write invalid JSON. - """ - - def _make( - vocab: Optional[Sequence[str]] = ("hello", "world", "[MASK]", "[PAD]"), - added: Optional[Sequence[str]] = None, - *, - subdir: str = "src", - raw: Optional[str] = None, - ) -> Path: - src = tmp_path / subdir - src.mkdir(parents=True, exist_ok=True) - path = src / "tokenizer.json" - if raw is not None: - path.write_text(raw, encoding="utf-8") - return src - - body: Dict[str, Any] = {"model": {}} - if vocab is not None: - body["model"]["vocab"] = {tok: i for i, tok in enumerate(vocab)} - if added is not None: - body["added_tokens"] = [{"content": t} for t in added] - path.write_text(json.dumps(body), encoding="utf-8") - return src - - return _make - - @pytest.fixture def make_image(tmp_path): """Write a solid-color image of a given size and return its path.""" diff --git a/tests/test_coverage_gaps.py b/tests/test_coverage_gaps.py index e9e96010..3374eed0 100644 --- a/tests/test_coverage_gaps.py +++ b/tests/test_coverage_gaps.py @@ -20,11 +20,11 @@ from tracebloc_ingestor.utils.logging import setup_logging from tracebloc_ingestor.config import Config - # =========================================================================== # utils: constants, logging # =========================================================================== + def test_dataformat_helpers(): formats = DataFormat.get_all_formats() assert DataFormat.IMAGE in formats @@ -45,8 +45,10 @@ def test_config_log_level_string_override(): # cli: conventions, run # =========================================================================== + def test_data_format_for_unknown_raises(): from tracebloc_ingestor.cli.conventions import _data_format_for + with pytest.raises(ValueError): _data_format_for("not_a_category") @@ -54,9 +56,14 @@ def test_data_format_for_unknown_raises(): def test_build_ingestor_unknown_source_type_raises(): from tracebloc_ingestor.cli import run as run_mod from tracebloc_ingestor.cli.conventions import ResolvedConfig + resolved = ResolvedConfig( - category=None, table_name="t", intent="train", - data_format="image", source_type="parquet", source_path="/x", + category=None, + table_name="t", + intent="train", + data_format="image", + source_type="parquet", + source_path="/x", ) with pytest.raises(ValueError, match="Unknown source_type"): run_mod._build_ingestor(MagicMock(), MagicMock(), resolved) @@ -120,9 +127,11 @@ def test_base_validator_str_and_repr(): # validators_mapping: schema branches # =========================================================================== + def test_text_classification_with_schema_adds_data_validator(): from tracebloc_ingestor.utils.validators_mapping import map_validators from tracebloc_ingestor.validators.data_validator import DataValidator + v = map_validators(TaskCategory.TEXT_CLASSIFICATION, {"schema": {"a": "INT"}}) assert any(isinstance(x, DataValidator) for x in v) @@ -130,6 +139,7 @@ def test_text_classification_with_schema_adds_data_validator(): def test_mlm_with_schema_adds_data_validator(): from tracebloc_ingestor.utils.validators_mapping import map_validators from tracebloc_ingestor.validators.data_validator import DataValidator + v = map_validators(TaskCategory.MASKED_LANGUAGE_MODELING, {"schema": {"a": "INT"}}) assert any(isinstance(x, DataValidator) for x in v) @@ -168,8 +178,10 @@ def test_data_validator_boolean_other_dtype(): def test_data_validator_date_exception(monkeypatch): v = DataValidator(schema={"d": "DATE"}) - with patch("tracebloc_ingestor.validators.data_validator.pd.to_datetime", - side_effect=RuntimeError("bad")): + with patch( + "tracebloc_ingestor.validators.data_validator.pd.to_datetime", + side_effect=RuntimeError("bad"), + ): res = v._validate_date(pd.Series(["2024-01-01"]), "d", "DATE") assert not res["is_valid"] @@ -191,44 +203,41 @@ def test_duplicate_validate_exception(monkeypatch): def test_duplicate_check_directory_exists_exception(): v = DuplicateValidator(dest_path="/x") - with patch("tracebloc_ingestor.validators.duplicate_validator.Path", - side_effect=RuntimeError("boom")): + with patch( + "tracebloc_ingestor.validators.duplicate_validator.Path", + side_effect=RuntimeError("boom"), + ): assert v._check_directory_exists() is False def test_duplicate_create_directory_exception(): v = DuplicateValidator(dest_path="/x") - with patch("tracebloc_ingestor.validators.duplicate_validator.Path", - side_effect=RuntimeError("boom")): + with patch( + "tracebloc_ingestor.validators.duplicate_validator.Path", + side_effect=RuntimeError("boom"), + ): assert v._create_directory_if_needed() is False # =========================================================================== -# table_name / tokenizer / keypoint / numeric: outer except +# table_name / keypoint / numeric: outer except # =========================================================================== + def test_table_name_validate_exception(): from tracebloc_ingestor.validators.table_name_validator import TableNameValidator + v = TableNameValidator() with patch.object(v, "_validate_table_names", side_effect=RuntimeError("boom")): res = v.validate(None) assert not res.is_valid -def test_tokenizer_validate_generic_exception(clean_env, make_tokenizer): - from tracebloc_ingestor.validators.tokenizer_validator import TokenizerValidator - src = make_tokenizer(vocab=["[MASK]", "[PAD]"]) - clean_env.setenv("SRC_PATH", str(src)) - v = TokenizerValidator() - with patch.object(v, "_extract_vocab", side_effect=RuntimeError("boom")): - res = v.validate(None) - assert not res.is_valid - - def test_keypoint_annotation_validate_exception(): from tracebloc_ingestor.validators.keypoint_annotation_validator import ( KeypointAnnotationValidator, ) + v = KeypointAnnotationValidator() with patch.object(v, "_load_data", side_effect=RuntimeError("boom")): res = v.validate("x") @@ -239,6 +248,7 @@ def test_keypoint_visibility_validate_exception(): from tracebloc_ingestor.validators.keypoint_visibility_validator import ( KeypointVisibilityValidator, ) + v = KeypointVisibilityValidator() with patch.object(v, "_load_data", side_effect=RuntimeError("boom")): res = v.validate("x") @@ -249,6 +259,7 @@ def test_numeric_columns_validate_exception(): from tracebloc_ingestor.validators.numeric_columns_validator import ( NumericColumnsValidator, ) + v = NumericColumnsValidator(schema={"a": "INT"}) with patch.object(v, "_load_data", side_effect=RuntimeError("boom")): res = v.validate("x") @@ -259,6 +270,7 @@ def test_numeric_columns_load_bad_path_returns_none(): from tracebloc_ingestor.validators.numeric_columns_validator import ( NumericColumnsValidator, ) + assert NumericColumnsValidator()._load_data("/no/such.csv") is None @@ -266,11 +278,15 @@ def test_numeric_columns_load_bad_path_returns_none(): # time validators: outer except + _load_data None paths # =========================================================================== -@pytest.mark.parametrize("modname,clsname", [ - ("time_format_validator", "TimeFormatValidator"), - ("time_ordered_validator", "TimeOrderedValidator"), - ("time_before_today_validator", "TimeBeforeTodayValidator"), -]) + +@pytest.mark.parametrize( + "modname,clsname", + [ + ("time_format_validator", "TimeFormatValidator"), + ("time_ordered_validator", "TimeOrderedValidator"), + ("time_before_today_validator", "TimeBeforeTodayValidator"), + ], +) def test_time_validators_generic_exception(make_csv, modname, clsname): mod = importlib.import_module(f"tracebloc_ingestor.validators.{modname}") cls = getattr(mod, clsname) @@ -281,11 +297,14 @@ def test_time_validators_generic_exception(make_csv, modname, clsname): assert not res.is_valid -@pytest.mark.parametrize("modname,clsname", [ - ("time_format_validator", "TimeFormatValidator"), - ("time_ordered_validator", "TimeOrderedValidator"), - ("time_before_today_validator", "TimeBeforeTodayValidator"), -]) +@pytest.mark.parametrize( + "modname,clsname", + [ + ("time_format_validator", "TimeFormatValidator"), + ("time_ordered_validator", "TimeOrderedValidator"), + ("time_before_today_validator", "TimeBeforeTodayValidator"), + ], +) def test_time_validators_load_non_csv_none(modname, clsname): mod = importlib.import_module(f"tracebloc_ingestor.validators.{modname}") v = getattr(mod, clsname)() @@ -296,6 +315,7 @@ def test_time_before_today_empty_after_dropna(make_csv): from tracebloc_ingestor.validators.time_before_today_validator import ( TimeBeforeTodayValidator, ) + # all-invalid timestamps -> valid set empty -> skips future check, stays valid path = make_csv({"timestamp": ["not-a-date", "also-bad"]}) res = TimeBeforeTodayValidator().validate(str(path)) @@ -303,17 +323,26 @@ def test_time_before_today_empty_after_dropna(make_csv): def test_time_to_event_load_unsupported_type(): - from tracebloc_ingestor.validators.time_to_event_validator import TimeToEventValidator + from tracebloc_ingestor.validators.time_to_event_validator import ( + TimeToEventValidator, + ) + assert TimeToEventValidator()._load_data(12345, None) is None def test_time_to_event_load_non_csv(): - from tracebloc_ingestor.validators.time_to_event_validator import TimeToEventValidator + from tracebloc_ingestor.validators.time_to_event_validator import ( + TimeToEventValidator, + ) + assert TimeToEventValidator()._load_data("/x.parquet", None) is None def test_time_to_event_generic_exception(): - from tracebloc_ingestor.validators.time_to_event_validator import TimeToEventValidator + from tracebloc_ingestor.validators.time_to_event_validator import ( + TimeToEventValidator, + ) + v = TimeToEventValidator() with patch.object(v, "_load_data", side_effect=RuntimeError("boom")): res = v.validate(pd.DataFrame({"time": [1]})) @@ -324,8 +353,10 @@ def test_time_to_event_generic_exception(): # image_validator: PIL-not-available + small branches # =========================================================================== + def test_image_validator_pil_unavailable(monkeypatch, clean_env): from tracebloc_ingestor.validators import image_validator as iv + monkeypatch.setattr(iv, "PIL_AVAILABLE", False) clean_env.setenv("SRC_PATH", "/tmp") res = iv.ImageResolutionValidator().validate(None) @@ -335,6 +366,7 @@ def test_image_validator_pil_unavailable(monkeypatch, clean_env): def test_image_validator_list_with_non_image(tmp_path): from tracebloc_ingestor.validators.image_validator import ImageResolutionValidator + txt = tmp_path / "a.txt" txt.write_text("x") v = ImageResolutionValidator() @@ -344,6 +376,7 @@ def test_image_validator_list_with_non_image(tmp_path): def test_image_validator_list_with_non_path_item(): from tracebloc_ingestor.validators.image_validator import ImageResolutionValidator + v = ImageResolutionValidator() assert v._get_image_files([123], True, True) == [] @@ -356,7 +389,9 @@ def test_image_validator_list_with_non_path_item(): def _client(**ov): - d = dict(BACKEND_TOKEN="tok", CLIENT_USERNAME=None, CLIENT_PASSWORD=None, EDGE_ENV="prod") + d = dict( + BACKEND_TOKEN="tok", CLIENT_USERNAME=None, CLIENT_PASSWORD=None, EDGE_ENV="prod" + ) d.update(ov) return APIClient(Config(**d)) @@ -374,7 +409,9 @@ def test_logging_retry_increment(): def test_authenticate_error_with_response_text(): - cfg = Config(BACKEND_TOKEN=None, CLIENT_USERNAME="u", CLIENT_PASSWORD="p", EDGE_ENV="prod") + cfg = Config( + BACKEND_TOKEN=None, CLIENT_USERNAME="u", CLIENT_PASSWORD="p", EDGE_ENV="prod" + ) with patch("requests.Session.post", side_effect=_err_with_response()): with pytest.raises(ValueError, match="Authentication failed"): APIClient(cfg) @@ -401,16 +438,21 @@ def test_generate_edge_label_error_with_response_text(): def test_prepare_dataset_error_with_response_text(): client = _client() with patch.object(client.session, "get", side_effect=_err_with_response()): - assert client.prepare_dataset( - TaskCategory.IMAGE_CLASSIFICATION, "ing", "image", "train" - ) is False + assert ( + client.prepare_dataset( + TaskCategory.IMAGE_CLASSIFICATION, "ing", "image", "train" + ) + is False + ) def test_create_dataset_error_with_response_text(): client = _client() with patch.object(client.session, "post", side_effect=_err_with_response()): with pytest.raises(requests.exceptions.RequestException): - client.create_dataset(ingestor_id="ing", category=TaskCategory.IMAGE_CLASSIFICATION) + client.create_dataset( + ingestor_id="ing", category=TaskCategory.IMAGE_CLASSIFICATION + ) # =========================================================================== @@ -442,7 +484,9 @@ def _seed(src, sub, name): def test_image_transfer_copy_failure_raises(ft_dirs): src, _ = ft_dirs _seed(src, "images", "a.jpg") - with patch.object(file_transfer, "_copy_file_with_retry", side_effect=OSError("disk")): + with patch.object( + file_transfer, "_copy_file_with_retry", side_effect=OSError("disk") + ): with pytest.raises(ValueError): file_transfer.image_transfer({"filename": "a"}, {"extension": ".jpg"}) @@ -450,15 +494,21 @@ def test_image_transfer_copy_failure_raises(ft_dirs): def test_annotation_transfer_copy_failure_raises(ft_dirs): src, _ = ft_dirs s = _seed(src, "annotations", "a.xml") - with patch.object(file_transfer, "_copy_file_with_retry", side_effect=OSError("disk")): + with patch.object( + file_transfer, "_copy_file_with_retry", side_effect=OSError("disk") + ): with pytest.raises(ValueError): - file_transfer.annotation_transfer({"filename": "a"}, {}, ".xml", str(s), "a.xml") + file_transfer.annotation_transfer( + {"filename": "a"}, {}, ".xml", str(s), "a.xml" + ) def test_text_transfer_copy_failure_raises(ft_dirs): src, _ = ft_dirs _seed(src, "texts", "a.txt") - with patch.object(file_transfer, "_copy_file_with_retry", side_effect=OSError("disk")): + with patch.object( + file_transfer, "_copy_file_with_retry", side_effect=OSError("disk") + ): with pytest.raises(ValueError): file_transfer.text_transfer({"filename": "a"}, {"extension": ".txt"}) @@ -466,7 +516,9 @@ def test_text_transfer_copy_failure_raises(ft_dirs): def test_mask_transfer_copy_failure_raises(ft_dirs): src, _ = ft_dirs s = _seed(src, "masks", "m.png") - with patch.object(file_transfer, "_copy_file_with_retry", side_effect=OSError("disk")): + with patch.object( + file_transfer, "_copy_file_with_retry", side_effect=OSError("disk") + ): with pytest.raises(ValueError): file_transfer.mask_transfer({"filename": "x"}, str(s), ".png", "m") @@ -481,19 +533,29 @@ def test_image_transfer_missing_filename_returns_none(ft_dirs): def test_map_object_detection_missing_image_returns_none(ft_dirs): # no image seeded -> image-not-found branch in map_file_transfer - assert file_transfer.map_file_transfer( - TaskCategory.OBJECT_DETECTION, {"filename": "ghost"}, {"extension": ".jpg"} - ) is None + assert ( + file_transfer.map_file_transfer( + TaskCategory.OBJECT_DETECTION, {"filename": "ghost"}, {"extension": ".jpg"} + ) + is None + ) def test_map_semantic_segmentation_missing_image_returns_none(ft_dirs): - assert file_transfer.map_file_transfer( - TaskCategory.SEMANTIC_SEGMENTATION, - {"filename": "ghost", "mask_id": "m"}, {"extension": ".jpg"}, - ) is None + assert ( + file_transfer.map_file_transfer( + TaskCategory.SEMANTIC_SEGMENTATION, + {"filename": "ghost", "mask_id": "m"}, + {"extension": ".jpg"}, + ) + is None + ) def test_map_semantic_segmentation_missing_filename_returns_none(ft_dirs): - assert file_transfer.map_file_transfer( - TaskCategory.SEMANTIC_SEGMENTATION, {}, {"extension": ".jpg"} - ) is None + assert ( + file_transfer.map_file_transfer( + TaskCategory.SEMANTIC_SEGMENTATION, {}, {"extension": ".jpg"} + ) + is None + ) diff --git a/tests/test_file_transfer_transfers.py b/tests/test_file_transfer_transfers.py index e5e0ed58..41ddf5d8 100644 --- a/tests/test_file_transfer_transfers.py +++ b/tests/test_file_transfer_transfers.py @@ -205,19 +205,6 @@ def test_map_semantic_segmentation_missing_mask_id_returns_none(dirs): assert rec is None -def test_map_mlm_copies_tokenizer(dirs): - src, dest = dirs - _seed(src, "sequences", "seq.txt", b"tokens") - (src / "tokenizer.json").write_text("{}") - rec = file_transfer.map_file_transfer( - TaskCategory.MASKED_LANGUAGE_MODELING, - {"filename": "seq"}, - {"extension": ".txt"}, - ) - assert rec is not None - assert (dest / "tokenizer.json").exists() - - def test_map_token_classification(dirs): src, dest = dirs _seed(src, "texts", "doc.txt", b"John Smith") @@ -239,36 +226,3 @@ def test_map_keypoint_detection(dirs): def test_map_unknown_category_returns_none(dirs): assert file_transfer.map_file_transfer("weird", {"filename": "x"}, {}) is None - - -def test_map_text_classification_copies_optional_tokenizer(dirs): - src, dest = dirs - _seed(src, "texts", "doc.txt", b"hello world") - (src / "tokenizer.json").write_text("{}") - rec = file_transfer.map_file_transfer( - TaskCategory.TEXT_CLASSIFICATION, {"filename": "doc"}, {"extension": ".txt"} - ) - assert rec is not None - assert (dest / "tokenizer.json").exists() - - -def test_map_token_classification_copies_optional_tokenizer(dirs): - src, dest = dirs - _seed(src, "texts", "doc.txt", b"John Smith") - (src / "tokenizer.json").write_text("{}") - rec = file_transfer.map_file_transfer( - TaskCategory.TOKEN_CLASSIFICATION, {"filename": "doc"}, {"extension": ".txt"} - ) - assert rec is not None - assert (dest / "tokenizer.json").exists() - - -def test_map_token_classification_without_tokenizer_is_fine(dirs): - src, dest = dirs - _seed(src, "texts", "doc.txt", b"John Smith") # no tokenizer.json - rec = file_transfer.map_file_transfer( - TaskCategory.TOKEN_CLASSIFICATION, {"filename": "doc"}, {"extension": ".txt"} - ) - assert rec is not None - assert (dest / "doc.txt").exists() - assert not (dest / "tokenizer.json").exists() diff --git a/tests/test_ingestor_base.py b/tests/test_ingestor_base.py index 8f13eab7..3ee01d48 100644 --- a/tests/test_ingestor_base.py +++ b/tests/test_ingestor_base.py @@ -304,7 +304,9 @@ def test_process_record_does_not_carry_mask_id_when_not_in_schema(): ) assert rec is not None assert rec["filename"] == "image_001" - assert "mask_id" not in rec, f"mask_id (not in schema) must not be a column; got {rec}" + assert ( + "mask_id" not in rec + ), f"mask_id (not in schema) must not be a column; got {rec}" def test_process_record_stores_mask_id_when_declared_in_schema(): @@ -1002,88 +1004,3 @@ def test_check_src_path_required_for_token_classification(): from tracebloc_ingestor.ingestors.base import _FILE_BEARING_CATEGORIES assert TaskCategory.TOKEN_CLASSIFICATION in _FILE_BEARING_CATEGORIES - - -# --------------------------------------------------------------------------- -# #805 Task 2: tokenizer fingerprint registration on the global-metadata channel -# --------------------------------------------------------------------------- - -_FINGERPRINT = { - "vocab_size": 15, - "mask_token_id": 4, - "pad_token_id": 0, - "tokenizer_type": "WordLevel", -} - - -@pytest.mark.parametrize( - "category", - ["MASKED_LANGUAGE_MODELING", "TEXT_CLASSIFICATION", "TOKEN_CLASSIFICATION"], -) -def test_ingest_registers_tokenizer_fingerprint_for_nlp(category): - """For every NLP category, a shipped tokenizer's 4-integer fingerprint is - attached to file_options so it rides the existing global-metadata channel.""" - from tracebloc_ingestor.utils.constants import TaskCategory - - cat = getattr(TaskCategory, category) - records = [{"a": "1", "filename": "f1"}] - ing = make_ingestor(records=records, category=cat, label_column=None) - with patch.object(base_mod, "Session") as Sess, patch.object( - ing, "validate_data", return_value=True - ), patch.object( - base_mod, "map_file_transfer", side_effect=lambda c, r, o, cfg=None: r - ), patch.object( - base_mod, "get_shipped_tokenizer_metadata", return_value=_FINGERPRINT - ): - Sess.return_value.__enter__.return_value = MagicMock() - ing.ingest("src", batch_size=10) - args, _ = ing.api_client.send_global_meta_meta.call_args - assert args[2].get("tokenizer") == _FINGERPRINT - - -def test_ingest_warns_and_skips_tokenizer_when_absent_for_nlp(): - """A site that ships no tokenizer.json still registers cleanly — the - fingerprint is simply omitted (the epic's legacy/skipped path).""" - from tracebloc_ingestor.utils.constants import TaskCategory - - records = [{"a": "1", "filename": "f1"}] - ing = make_ingestor( - records=records, - category=TaskCategory.TEXT_CLASSIFICATION, - label_column="a", - ) - with patch.object(base_mod, "Session") as Sess, patch.object( - ing, "validate_data", return_value=True - ), patch.object( - base_mod, "map_file_transfer", side_effect=lambda c, r, o, cfg=None: r - ), patch.object( - base_mod, "get_shipped_tokenizer_metadata", return_value=None - ): - Sess.return_value.__enter__.return_value = MagicMock() - ing.ingest("src", batch_size=10) - args, _ = ing.api_client.send_global_meta_meta.call_args - assert "tokenizer" not in args[2] - # Registration still completes — absence is non-fatal. - ing.api_client.create_dataset.assert_called_once() - - -def test_ingest_does_not_register_tokenizer_for_non_nlp(): - """Non-NLP categories never touch the tokenizer path.""" - from tracebloc_ingestor.utils.constants import TaskCategory - - records = [{"a": "1", "filename": "f1"}] - ing = make_ingestor( - records=records, - category=TaskCategory.IMAGE_CLASSIFICATION, - label_column="a", - ) - with patch.object(base_mod, "Session") as Sess, patch.object( - ing, "validate_data", return_value=True - ), patch.object( - base_mod, "map_file_transfer", side_effect=lambda c, r, o, cfg=None: r - ), patch.object(base_mod, "get_shipped_tokenizer_metadata") as get_meta: - Sess.return_value.__enter__.return_value = MagicMock() - ing.ingest("src", batch_size=10) - get_meta.assert_not_called() - args, _ = ing.api_client.send_global_meta_meta.call_args - assert "tokenizer" not in args[2] diff --git a/tests/test_modality_registry.py b/tests/test_modality_registry.py index 8fc7bb04..4c59cf09 100644 --- a/tests/test_modality_registry.py +++ b/tests/test_modality_registry.py @@ -59,8 +59,8 @@ def test_derived_sets_match_spec_flags(): def test_nlp_categories_are_the_three_text_categories(): - """#805 Task 2: the tokenizer-fingerprint set is exactly the NLP text - categories (text/token classification + MLM) — never image/tabular.""" + """#805: the NLP set is exactly the text categories + (text/token classification + MLM) — never image/tabular.""" assert NLP_CATEGORIES == { TaskCategory.TEXT_CLASSIFICATION, TaskCategory.TOKEN_CLASSIFICATION, diff --git a/tests/test_tokenizer_fingerprint.py b/tests/test_tokenizer_fingerprint.py deleted file mode 100644 index 95f732c2..00000000 --- a/tests/test_tokenizer_fingerprint.py +++ /dev/null @@ -1,290 +0,0 @@ -"""Tests for the NLP tokenizer fingerprint extracted + registered at ingest. - -Covers issue #805 Task 2: the 4 structural integers (vocab_size / -mask_token_id / pad_token_id / tokenizer_type) extracted from a shipped -``tokenizer.json`` and shipped on the global-metadata channel. The FL -guardrail is that ONLY these integers leave the cluster — never vocabulary -content and never a hash. -""" - -from __future__ import annotations - -import json - -import pytest - -from tracebloc_ingestor import file_transfer -from tracebloc_ingestor.validators.tokenizer_validator import ( - _special_token_id, - extract_tokenizer_metadata, - load_tokenizer_metadata, -) - -# A BERT/WordPiece-style tokenizer.json with the full classification + -# MLM special-token set, special tokens declared in added_tokens. -_BERT_STYLE = { - "version": "1.0", - "model": { - "type": "WordPiece", - "vocab": { - "[PAD]": 0, - "[UNK]": 1, - "[CLS]": 2, - "[SEP]": 3, - "[MASK]": 4, - "hello": 5, - "world": 6, - }, - }, - "added_tokens": [ - {"id": 0, "content": "[PAD]", "special": True}, - {"id": 1, "content": "[UNK]", "special": True}, - {"id": 2, "content": "[CLS]", "special": True}, - {"id": 3, "content": "[SEP]", "special": True}, - {"id": 4, "content": "[MASK]", "special": True}, - ], -} - -# A classification tokenizer that has [PAD] but no [MASK] (no masking task). -_NO_MASK = { - "model": {"type": "WordPiece", "vocab": {"[PAD]": 0, "[UNK]": 1, "a": 2}}, - "added_tokens": [{"id": 0, "content": "[PAD]", "special": True}], -} - -# A Unigram tokenizer stores its vocab as a [token, score] list. -_UNIGRAM = { - "model": { - "type": "Unigram", - "vocab": [["[PAD]", 0.0], ["[MASK]", 0.0], ["x", -1.0], ["y", -2.0]], - }, - "added_tokens": [ - {"id": 0, "content": "[PAD]", "special": True}, - {"id": 1, "content": "[MASK]", "special": True}, - ], -} - - -# --------------------------------------------------------------------------- -# extract_tokenizer_metadata -# --------------------------------------------------------------------------- - - -def test_extract_full_fingerprint_bert_style(): - meta = extract_tokenizer_metadata(_BERT_STYLE) - assert meta == { - "vocab_size": 7, - "mask_token_id": 4, - "pad_token_id": 0, - "tokenizer_type": "WordPiece", - } - - -def test_extract_classification_without_mask_yields_none_mask_id(): - meta = extract_tokenizer_metadata(_NO_MASK) - assert meta["mask_token_id"] is None - assert meta["pad_token_id"] == 0 - assert meta["vocab_size"] == 3 - assert meta["tokenizer_type"] == "WordPiece" - - -def test_extract_unigram_list_vocab(): - meta = extract_tokenizer_metadata(_UNIGRAM) - assert meta["vocab_size"] == 4 - assert meta["mask_token_id"] == 1 - assert meta["pad_token_id"] == 0 - assert meta["tokenizer_type"] == "Unigram" - - -def test_fl_guardrail_only_four_scalar_keys(): - """No vocabulary content or hash may cross to the backend — only the 4 - scalar integers (one may be a type string / None).""" - meta = extract_tokenizer_metadata(_BERT_STYLE) - assert set(meta) == { - "vocab_size", - "mask_token_id", - "pad_token_id", - "tokenizer_type", - } - for value in meta.values(): - assert not isinstance(value, (dict, list)) - - -def test_extract_handles_empty_or_unknown_structure(): - meta = extract_tokenizer_metadata({}) - assert meta == { - "vocab_size": 0, - "mask_token_id": None, - "pad_token_id": None, - "tokenizer_type": None, - } - - -# --------------------------------------------------------------------------- -# Hardening — adversarial / malformed tokenizer.json (found by fuzzing). -# Every input must yield the 4 scalar keys and never raise (FL guardrail). -# --------------------------------------------------------------------------- - -_EMPTY = { - "vocab_size": 0, - "mask_token_id": None, - "pad_token_id": None, - "tokenizer_type": None, -} - - -@pytest.mark.parametrize("bad", [[], "x", 42, None, True, [1, 2, 3]]) -def test_extract_non_dict_top_level_is_safe(bad): - """A tokenizer.json that is valid JSON but not an object must not crash.""" - assert extract_tokenizer_metadata(bad) == _EMPTY - - -@pytest.mark.parametrize("model", [None, "wordpiece", 123, [], {"type": {"x": 1}}]) -def test_extract_malformed_model_is_safe(model): - """A null / non-dict ``model`` (or a non-string ``model.type``) must not - crash and must not leak a non-scalar ``tokenizer_type``.""" - meta = extract_tokenizer_metadata({"model": model}) - assert set(meta) == set(_EMPTY) - assert meta["tokenizer_type"] is None or isinstance(meta["tokenizer_type"], str) - - -def test_extract_non_int_added_token_id_is_dropped(): - """A non-int ``id`` (e.g. a nested object) must not become the token id — - the FL guardrail allows only scalar integers across the boundary.""" - data = { - "model": {"vocab": {"[PAD]": 0}}, - "added_tokens": [{"content": "[MASK]", "id": {"x": 1}}], - } - meta = extract_tokenizer_metadata(data) - assert meta["mask_token_id"] is None # the {"x": 1} id was rejected - assert meta["pad_token_id"] == 0 # vocab fallback still works - - -def test_extract_dict_model_type_coerced_to_none(): - meta = extract_tokenizer_metadata({"model": {"type": {"nested": 1}, "vocab": {}}}) - assert meta["tokenizer_type"] is None - - -def test_load_valid_json_non_object_returns_none(tmp_path): - """A tokenizer.json holding a JSON array/string/number is parseable but - not a tokenizer — load must return None, not crash.""" - p = tmp_path / "tokenizer.json" - p.write_text("[1, 2, 3]") - assert load_tokenizer_metadata(str(p)) is None - - -# --------------------------------------------------------------------------- -# _special_token_id -# --------------------------------------------------------------------------- - - -def test_special_token_id_prefers_added_tokens(): - data = { - "model": {"vocab": {"[PAD]": 99}}, - "added_tokens": [{"id": 7, "content": "[PAD]"}], - } - assert _special_token_id(data, "[PAD]") == 7 - - -def test_special_token_id_falls_back_to_vocab(): - data = {"model": {"vocab": {"[PAD]": 3}}, "added_tokens": []} - assert _special_token_id(data, "[PAD]") == 3 - - -def test_special_token_id_idless_added_token_falls_back_to_vocab(): - """A malformed added_tokens entry with no ``id`` must not shadow the - model.vocab mapping that does hold the id (bugbot).""" - data = { - "model": {"vocab": {"[PAD]": 5}}, - "added_tokens": [{"content": "[PAD]"}], # no "id" - } - assert _special_token_id(data, "[PAD]") == 5 - - -def test_special_token_id_absent_returns_none(): - assert _special_token_id(_NO_MASK, "[MASK]") is None - - -# --------------------------------------------------------------------------- -# load_tokenizer_metadata -# --------------------------------------------------------------------------- - - -def test_load_reads_file(tmp_path): - p = tmp_path / "tokenizer.json" - p.write_text(json.dumps(_BERT_STYLE)) - meta = load_tokenizer_metadata(str(p)) - assert meta["vocab_size"] == 7 - assert meta["mask_token_id"] == 4 - - -def test_load_missing_file_returns_none(tmp_path): - assert load_tokenizer_metadata(str(tmp_path / "nope.json")) is None - - -def test_load_malformed_json_returns_none(tmp_path): - p = tmp_path / "tokenizer.json" - p.write_text("{ not valid json ") - assert load_tokenizer_metadata(str(p)) is None - - -# --------------------------------------------------------------------------- -# file_transfer helpers (SRC_PATH / DEST_PATH backed) -# --------------------------------------------------------------------------- - - -@pytest.fixture -def dirs(tmp_path, monkeypatch): - """Point file_transfer's Config at tmp src + storage dirs (mirrors the - fixture in test_file_transfer_transfers.py).""" - src = tmp_path / "src" - storage = tmp_path / "storage" - src.mkdir() - storage.mkdir() - monkeypatch.setenv("SRC_PATH", str(src)) - monkeypatch.setenv("TABLE_NAME", "tbl") - monkeypatch.setattr(file_transfer.config, "STORAGE_PATH", str(storage)) - return src, storage / "tbl" - - -def test_get_shipped_tokenizer_metadata_reads_dest(dirs): - """Fingerprints the STAGED tokenizer (DEST) — the file the client uses.""" - _, dest = dirs - dest.mkdir(parents=True, exist_ok=True) - (dest / "tokenizer.json").write_text(json.dumps(_BERT_STYLE)) - meta = file_transfer.get_shipped_tokenizer_metadata() - assert meta["vocab_size"] == 7 - assert meta["pad_token_id"] == 0 - - -def test_get_shipped_tokenizer_metadata_none_when_absent(dirs): - assert file_transfer.get_shipped_tokenizer_metadata() is None - - -def test_get_shipped_fingerprints_dest_not_src(dirs): - """On a re-ingest the copy is skipped (DEST already exists), so the client - trains on DEST; the registered fingerprint must describe DEST, not a - changed SRC (bugbot).""" - src, dest = dirs - dest.mkdir(parents=True, exist_ok=True) - # SRC has a 3-token tokenizer; DEST (already staged) has the 7-token one. - (src / "tokenizer.json").write_text(json.dumps(_NO_MASK)) - (dest / "tokenizer.json").write_text(json.dumps(_BERT_STYLE)) - meta = file_transfer.get_shipped_tokenizer_metadata() - assert meta["vocab_size"] == 7 # DEST, not SRC's 3 - - -def test_copy_tokenizer_returns_fingerprint_on_copy(dirs): - src, dest = dirs - # In production text_transfer creates DEST_PATH before the tokenizer copy; - # mirror that here since we call the helper in isolation. - dest.mkdir(parents=True, exist_ok=True) - (src / "tokenizer.json").write_text(json.dumps(_BERT_STYLE)) - meta = file_transfer._copy_tokenizer_if_present() - assert (dest / "tokenizer.json").exists() - assert meta["vocab_size"] == 7 - # Already copied: a second call is a no-op and returns None. - assert file_transfer._copy_tokenizer_if_present() is None - - -def test_copy_tokenizer_none_when_absent(dirs): - assert file_transfer._copy_tokenizer_if_present() is None diff --git a/tests/test_tokenizer_validator.py b/tests/test_tokenizer_validator.py deleted file mode 100644 index cc2eeb58..00000000 --- a/tests/test_tokenizer_validator.py +++ /dev/null @@ -1,113 +0,0 @@ -"""Tests for TokenizerValidator — checks tokenizer.json at config.SRC_PATH.""" - -from __future__ import annotations - -import pytest - -from tracebloc_ingestor.validators.tokenizer_validator import TokenizerValidator - - -@pytest.fixture -def validator(): - return TokenizerValidator() - - -def test_passes_when_required_tokens_present(clean_env, validator, make_tokenizer): - src = make_tokenizer(vocab=["a", "b", "[MASK]", "[PAD]"]) - clean_env.setenv("SRC_PATH", str(src)) - result = validator.validate(None) - assert result.is_valid - assert result.metadata["vocab_size"] == 4 - - -def test_missing_file_fails(clean_env, validator, tmp_path): - clean_env.setenv("SRC_PATH", str(tmp_path)) - result = validator.validate(None) - assert not result.is_valid - assert "tokenizer.json not found" in result.errors[0] - - -def test_missing_required_token_fails(clean_env, validator, make_tokenizer): - src = make_tokenizer(vocab=["a", "b", "[PAD]"]) # no [MASK] - clean_env.setenv("SRC_PATH", str(src)) - result = validator.validate(None) - assert not result.is_valid - assert "[MASK]" in result.errors[0] - assert result.metadata["missing_tokens"] == ["[MASK]"] - - -def test_tokens_from_added_tokens_section(clean_env, validator, make_tokenizer): - # vocab lacks the special tokens; added_tokens supplies them. - src = make_tokenizer(vocab=["a", "b"], added=["[MASK]", "[PAD]"]) - clean_env.setenv("SRC_PATH", str(src)) - result = validator.validate(None) - assert result.is_valid - - -def test_unrecognized_structure_fails(clean_env, validator, make_tokenizer): - src = make_tokenizer(vocab=None) # empty model, no added_tokens - clean_env.setenv("SRC_PATH", str(src)) - result = validator.validate(None) - assert not result.is_valid - assert "Could not extract vocabulary" in result.errors[0] - - -def test_invalid_json_fails(clean_env, validator, make_tokenizer): - src = make_tokenizer(raw="{not valid json") - clean_env.setenv("SRC_PATH", str(src)) - result = validator.validate(None) - assert not result.is_valid - assert "not valid JSON" in result.errors[0] - - -def test_extract_vocab_unigram_list_form(): - data = {"model": {"vocab": [["[MASK]", -1.0], ["[PAD]", -2.0]]}} - vocab = TokenizerValidator._extract_vocab(data) - assert vocab == {"[MASK]", "[PAD]"} - - -def test_extract_vocab_added_tokens_string_form(): - data = {"added_tokens": ["[MASK]", "[PAD]"]} - vocab = TokenizerValidator._extract_vocab(data) - assert vocab == {"[MASK]", "[PAD]"} - - -def test_extract_vocab_returns_none_when_empty(): - assert TokenizerValidator._extract_vocab({}) is None - - -def test_custom_required_tokens(): - v = TokenizerValidator(required_tokens=("[CLS]",)) - assert v.required_tokens == {"[CLS]"} - - -# --------------------------------------------------------------------------- -# optional=True (text/token classification): missing tokenizer is a warning, -# not an error; a present tokenizer is still validated. -# --------------------------------------------------------------------------- - - -def test_optional_missing_file_warns_and_passes(clean_env, tmp_path): - clean_env.setenv("SRC_PATH", str(tmp_path)) # no tokenizer.json - v = TokenizerValidator(required_tokens=("[PAD]",), optional=True) - result = v.validate(None) - assert result.is_valid - assert result.warnings and "tokenizer.json" in result.warnings[0] - assert result.metadata.get("tokenizer_present") is False - - -def test_optional_present_without_pad_still_fails(clean_env, make_tokenizer): - src = make_tokenizer(vocab=("hello", "world")) # no [PAD] - clean_env.setenv("SRC_PATH", str(src)) - v = TokenizerValidator(required_tokens=("[PAD]",), optional=True) - result = v.validate(None) - assert not result.is_valid - assert "[PAD]" in result.errors[0] - - -def test_optional_present_with_pad_passes(clean_env, make_tokenizer): - src = make_tokenizer(vocab=("hello", "world", "[PAD]")) - clean_env.setenv("SRC_PATH", str(src)) - v = TokenizerValidator(required_tokens=("[PAD]",), optional=True) - result = v.validate(None) - assert result.is_valid diff --git a/tests/test_validators_mapping.py b/tests/test_validators_mapping.py index 100bd965..e4a31690 100644 --- a/tests/test_validators_mapping.py +++ b/tests/test_validators_mapping.py @@ -24,7 +24,6 @@ from tracebloc_ingestor.validators.keypoint_visibility_validator import ( KeypointVisibilityValidator, ) -from tracebloc_ingestor.validators.tokenizer_validator import TokenizerValidator from tracebloc_ingestor.validators.label_diversity_validator import ( LabelDiversityValidator, ) @@ -211,26 +210,10 @@ def test_time_to_event_without_schema(): assert DataValidator not in types -def test_masked_language_modeling_includes_tokenizer(): - v = map_validators(TaskCategory.MASKED_LANGUAGE_MODELING, {}) - assert TokenizerValidator in _types(v) - - def test_unknown_category_returns_empty(): assert map_validators("not_a_category", {}) == [] -def test_text_and_token_classification_include_optional_tokenizer_validator(): - from tracebloc_ingestor.validators.tokenizer_validator import TokenizerValidator - - for cat in (TaskCategory.TEXT_CLASSIFICATION, TaskCategory.TOKEN_CLASSIFICATION): - v = map_validators(cat, {}) - tok = [x for x in v if isinstance(x, TokenizerValidator)] - assert tok, f"{cat}: expected an (optional) TokenizerValidator" - assert tok[0].optional is True - assert tok[0].required_tokens == {"[PAD]"} - - # --- P4b: config injection seam ------------------------------------------- # map_validators is the single place the run's resolved Config is bound to # every validator it builds (BaseValidator.bind_config). These pin that the diff --git a/tracebloc_ingestor/file_transfer.py b/tracebloc_ingestor/file_transfer.py index 94c027c4..2889ab8f 100644 --- a/tracebloc_ingestor/file_transfer.py +++ b/tracebloc_ingestor/file_transfer.py @@ -30,7 +30,6 @@ FileExtension, TaskCategory, ) -from tracebloc_ingestor.validators.tokenizer_validator import load_tokenizer_metadata # Initialize config and configure logging config = Config() @@ -367,60 +366,6 @@ def mask_transfer( raise ValueError(f"{RED}Error processing mask file: {str(e)}{RESET}") -def _copy_tokenizer_if_present(cfg: Optional[Config] = None) -> Optional[Dict[str, Any]]: - """Copy a user-shipped ``tokenizer.json`` from SRC_PATH to DEST_PATH (once) - and return its structural fingerprint. - - Optional for NLP datasets: the training client looks for - ``DEST_PATH/tokenizer.json`` and uses it as a custom tokenizer; if it's - absent the client falls back to the HuggingFace tokenizer_id / default. - After the copy, the 4-integer fingerprint — ``vocab_size`` / - ``mask_token_id`` / ``pad_token_id`` / ``tokenizer_type`` — is extracted - and logged so it can be registered on the global-metadata channel for the - contributor-tokenizer cross-check at dataset linking (#805 Task 2). Only - those integers ever leave the cluster — never vocabulary content or a hash. - - Returns the fingerprint dict on the call that performs the copy, and - ``None`` thereafter (already copied this run) or when no tokenizer.json was - shipped. The authoritative fingerprint for registration is read once, - post-ingest, via :func:`get_shipped_tokenizer_metadata`. - """ - cfg = cfg or config - tokenizer_src = os.path.join(cfg.SRC_PATH, "tokenizer.json") - tokenizer_dest = os.path.join(cfg.DEST_PATH, "tokenizer.json") - if not os.path.isfile(tokenizer_src) or os.path.exists(tokenizer_dest): - return None - _copy_file_with_retry(tokenizer_src, tokenizer_dest) - metadata = load_tokenizer_metadata(tokenizer_dest) - logger.info( - f"{GREEN}Copied tokenizer.json to {cfg.DEST_PATH} " - f"(fingerprint: {metadata}){RESET}" - ) - return metadata - - -def get_shipped_tokenizer_metadata( - cfg: Optional[Config] = None, -) -> Optional[Dict[str, Any]]: - """Return the structural fingerprint of the staged ``tokenizer.json``. - - Reads ``DEST_PATH/tokenizer.json`` — the file the training client actually - loads — NOT ``SRC_PATH``. ``_copy_tokenizer_if_present`` copies the source - only once (it skips when a DEST copy already exists), so on a re-ingest - that ships a changed source the client keeps training on the existing DEST - file; fingerprinting DEST keeps the registered metadata matched to what is - trained on (else the contributor cross-check at linking would compare - against a tokenizer the site never used). Returns the 4 structural integers, - or ``None`` when no tokenizer was staged. Called once after ingestion to - register the fingerprint on the existing global-metadata channel (#805 - Task 2); only these integers cross the cluster boundary. ``cfg`` is the - run's resolved Config (threaded from the ingestor); ``None`` falls back to - the module-global ``config``. - """ - cfg = cfg or config - return load_tokenizer_metadata(os.path.join(cfg.DEST_PATH, "tokenizer.json")) - - # Per-row sidecar pointers that live on the RAW source record (not table # columns, so never on the cleaned DB-bound record). map_file_transfer lends # them to the transfer for the copy, then strips them (#212, P5). Currently diff --git a/tracebloc_ingestor/ingestors/base.py b/tracebloc_ingestor/ingestors/base.py index d9ef3e6f..25fb57bd 100644 --- a/tracebloc_ingestor/ingestors/base.py +++ b/tracebloc_ingestor/ingestors/base.py @@ -18,7 +18,7 @@ ) from ..utils import label_policy as label_policy_module from ..utils.validators_mapping import map_validators -from ..file_transfer import map_file_transfer, get_shipped_tokenizer_metadata +from ..file_transfer import map_file_transfer from ..reporting import ConsoleRenderer from . import preflight from .batch_writer import BatchWriter @@ -32,7 +32,6 @@ # (and their None/unknown -> False semantics are preserved). from ..modalities.registry import ( FILE_BEARING_CATEGORIES as _FILE_BEARING_CATEGORIES, - NLP_CATEGORIES as _NLP_CATEGORIES, TABULAR_FAMILY_CATEGORIES as _TABULAR_FAMILY_CATEGORIES, ) @@ -575,30 +574,6 @@ def _ingest_with_lock( "See the logged API error above." ) - # Register the contributor tokenizer fingerprint for NLP - # datasets (#805 Task 2). When a tokenizer.json was shipped - # (mandatory for MLM, optional for text/token classification), - # attach its 4 structural integers (vocab_size / mask_token_id / - # pad_token_id / tokenizer_type) to file_options so they ride - # the existing global-metadata channel below and the backend can - # cross-check a contributor tokenizer at dataset linking. Only - # these integers cross the cluster boundary — never vocabulary - # content or a hash (the FL guardrail). A site that ships none - # simply skips the cross-check (the epic's legacy/skipped path). - if self.category in _NLP_CATEGORIES: - tokenizer_meta = get_shipped_tokenizer_metadata( - self.database.config - ) - if tokenizer_meta: - self.file_options["tokenizer"] = tokenizer_meta - else: - logger.warning( - f"{YELLOW}No tokenizer.json shipped for NLP dataset " - f"'{self.table_name}'; no tokenizer fingerprint will " - f"be registered, so the contributor-tokenizer " - f"cross-check is skipped for this site.{RESET}" - ) - schema_dict = self.database.get_table_schema(self.table_name) if not self.api_client.send_global_meta_meta( self.table_name, schema_dict, self.file_options diff --git a/tracebloc_ingestor/modalities/registry.py b/tracebloc_ingestor/modalities/registry.py index 2dbb9045..e9e96fbe 100644 --- a/tracebloc_ingestor/modalities/registry.py +++ b/tracebloc_ingestor/modalities/registry.py @@ -160,6 +160,6 @@ def spec_for(category: str) -> ModalitySpec: SELF_SUPERVISED_CATEGORIES = frozenset( c for c, s in REGISTRY.items() if s.is_self_supervised ) -# NLP text categories that ship a tokenizer.json; their tokenizer fingerprint -# is registered on the global-metadata channel at ingest (#805 Task 2). +# NLP text categories (text / token classification, MLM). Gates NLP-specific +# ingest handling such as the data-derived text profile (#805). NLP_CATEGORIES = frozenset(c for c, s in REGISTRY.items() if s.is_nlp) diff --git a/tracebloc_ingestor/modalities/spec.py b/tracebloc_ingestor/modalities/spec.py index e683a04c..aa9a24de 100644 --- a/tracebloc_ingestor/modalities/spec.py +++ b/tracebloc_ingestor/modalities/spec.py @@ -61,12 +61,10 @@ class ModalitySpec: targets at train time (e.g. masked language modeling). The backend stores no edge-label metadata, so the edge-label call is skipped (#213). - is_nlp: an NLP text category (text / token classification, MLM) that - ships a ``tokenizer.json``. When one is present, its 4-integer - fingerprint (vocab_size / mask_token_id / pad_token_id / - tokenizer_type) is registered on the global-metadata channel at - ingest for the contributor-tokenizer cross-check at dataset linking - (#805 Task 2). Image / tabular / time-series are False. + is_nlp: an NLP text category (text / token classification, MLM). Gates + NLP-specific ingest handling — e.g. the data-derived text profile + shipped on the global-metadata channel for tokenizer-fit checks + (#805). Image / tabular / time-series are False. """ category: str diff --git a/tracebloc_ingestor/modalities/transfer.py b/tracebloc_ingestor/modalities/transfer.py index 2d708502..29367ba2 100644 --- a/tracebloc_ingestor/modalities/transfer.py +++ b/tracebloc_ingestor/modalities/transfer.py @@ -28,7 +28,6 @@ from ..config import Config from ..file_transfer import ( - _copy_tokenizer_if_present, _find_mask_src, _find_src, annotation_transfer, @@ -90,11 +89,7 @@ def object_detection( def text_classification( record: Dict[str, Any], options: Dict[str, Any], cfg: Optional[Config] = None ) -> Optional[Dict[str, Any]]: - result = text_transfer(record, options, cfg=cfg) - # Optional: ship a custom tokenizer.json so the client uses it instead of - # the HF default; absent is fine (handled by the optional validator). - _copy_tokenizer_if_present(cfg=cfg) - return result + return text_transfer(record, options, cfg=cfg) def token_classification( @@ -102,23 +97,13 @@ def token_classification( ) -> Optional[Dict[str, Any]]: # Same on-disk layout as text classification: one .txt per sample in the # ``texts`` subdir. BIO tags travel in the labels CSV, not on disk. - result = text_transfer(record, options, cfg=cfg) - # Optional custom tokenizer.json (same as text classification). - _copy_tokenizer_if_present(cfg=cfg) - return result + return text_transfer(record, options, cfg=cfg) def masked_language_modeling( record: Dict[str, Any], options: Dict[str, Any], cfg: Optional[Config] = None ) -> Optional[Dict[str, Any]]: - result = text_transfer(record, options, src_subdir="sequences", cfg=cfg) - # Copy the user's tokenizer.json so the MLM client uses it instead of - # falling back to bert-base-uncased (a vocab_size mismatch with the model's - # nn.Embedding would cause a CUDA device-side assert at training). For MLM - # the tokenizer is mandatory — its presence and [MASK]/[PAD] tokens are - # enforced by TokenizerValidator at validation. - _copy_tokenizer_if_present(cfg=cfg) - return result + return text_transfer(record, options, src_subdir="sequences", cfg=cfg) def semantic_segmentation( diff --git a/tracebloc_ingestor/modalities/validators.py b/tracebloc_ingestor/modalities/validators.py index 2039a263..7b2a08aa 100644 --- a/tracebloc_ingestor/modalities/validators.py +++ b/tracebloc_ingestor/modalities/validators.py @@ -28,7 +28,6 @@ from ..validators.time_format_validator import TimeFormatValidator from ..validators.time_ordered_validator import TimeOrderedValidator from ..validators.time_to_event_validator import TimeToEventValidator -from ..validators.tokenizer_validator import TokenizerValidator from ..validators.xml_validator import PascalVOCXMLValidator @@ -104,9 +103,6 @@ def text_classification(options: Dict[str, Any]) -> List[BaseValidator]: path="texts", ), ) - # Optional user-supplied tokenizer.json — warn (don't fail) if absent; - # if present, it must contain [PAD] (text classification pads batches). - validators.append(TokenizerValidator(required_tokens=("[PAD]",), optional=True)) # Add data validator if schema is provided if options.get("schema"): validators.append(DataValidator(schema=options["schema"])) @@ -135,9 +131,6 @@ def token_classification(options: Dict[str, Any]) -> List[BaseValidator]: label_column=options.get("label_column") or "label", ) ) - # Optional user-supplied tokenizer.json — warn (don't fail) if absent; - # if present, it must contain [PAD]. - validators.append(TokenizerValidator(required_tokens=("[PAD]",), optional=True)) # Add data validator if schema is provided if options.get("schema"): validators.append(DataValidator(schema=options["schema"])) @@ -244,8 +237,6 @@ def masked_language_modeling(options: Dict[str, Any]) -> List[BaseValidator]: path="sequences", ), ) - # Validate tokenizer.json has required special tokens ([MASK], [PAD]) - validators.append(TokenizerValidator()) # Add data validator if schema is provided if options.get("schema"): validators.append(DataValidator(schema=options["schema"])) diff --git a/tracebloc_ingestor/validators/__init__.py b/tracebloc_ingestor/validators/__init__.py index 7ee2eacf..0df71f6f 100644 --- a/tracebloc_ingestor/validators/__init__.py +++ b/tracebloc_ingestor/validators/__init__.py @@ -19,7 +19,6 @@ from .time_before_today_validator import TimeBeforeTodayValidator from .keypoint_annotation_validator import KeypointAnnotationValidator from .keypoint_visibility_validator import KeypointVisibilityValidator -from .tokenizer_validator import TokenizerValidator __all__ = [ "BaseValidator", @@ -36,5 +35,4 @@ "TimeBeforeTodayValidator", "KeypointAnnotationValidator", "KeypointVisibilityValidator", - "TokenizerValidator", ] diff --git a/tracebloc_ingestor/validators/tokenizer_validator.py b/tracebloc_ingestor/validators/tokenizer_validator.py deleted file mode 100644 index b7f23d6e..00000000 --- a/tracebloc_ingestor/validators/tokenizer_validator.py +++ /dev/null @@ -1,275 +0,0 @@ -"""Tokenizer Validator Module. - -Validates that a tokenizer.json file exists alongside the data and contains -the required special tokens ([MASK] and [PAD]) for masked language modeling. -Without these tokens the training client will fail with an embedding -out-of-bounds IndexError. -""" - -import json -import logging -import os -from pathlib import Path -from typing import Any, Dict, Optional - -from .base import BaseValidator, ValidationResult -from ..config import Config - -config = Config() -logger = logging.getLogger(__name__) -logger.setLevel(config.LOG_LEVEL) - - -def _special_token_id(tokenizer_data: dict, token: str) -> Optional[int]: - """Resolve a special token's id from a HuggingFace tokenizer JSON. - - Prefers the ``added_tokens`` list (where special tokens carry an explicit - id), then falls back to the ``model.vocab`` mapping. Returns ``None`` when - the token isn't present — e.g. classification tokenizers have no ``[MASK]``. - """ - added = tokenizer_data.get("added_tokens", []) - if isinstance(added, list): - for entry in added: - if isinstance(entry, dict) and entry.get("content") == token: - token_id = entry.get("id") - # Only trust an added_tokens entry that carries a real integer - # id (bool excluded — JSON true/false is not a token id). A - # missing or non-int id must not shadow the model.vocab mapping - # below, nor let a non-scalar value reach the backend. - if isinstance(token_id, int) and not isinstance(token_id, bool): - return token_id - model = tokenizer_data.get("model") - vocab = model.get("vocab") if isinstance(model, dict) else None - if isinstance(vocab, dict): - vid = vocab.get(token) - if isinstance(vid, int) and not isinstance(vid, bool): - return vid - return None - - -def extract_tokenizer_metadata(tokenizer_data: dict) -> Dict[str, Any]: - """Extract the 4 structural integers that fingerprint a tokenizer (#805). - - These — and only these — cross to the backend on the global-metadata - channel so a contributor tokenizer can be cross-checked at dataset - linking without ever shipping vocabulary content or a hash (the FL - guardrail): the vocabulary of a custom (e.g. clinical / knowledge-graph) - tokenizer is data-derived and must not be centrally fingerprinted. - - Returns a dict with: - - ``vocab_size``: number of distinct tokens (``model.vocab`` ∪ - ``added_tokens``) — the bound the model's embedding - table must cover; matches ``len(tokenizer)`` and the - SDK's upload-time vocab-fit check. - - ``mask_token_id``: id of ``[MASK]``, or ``None`` (classification has - no ``[MASK]``). - - ``pad_token_id``: id of ``[PAD]``, or ``None``. - - ``tokenizer_type``: ``model.type`` (``WordLevel`` / ``WordPiece`` / - ``BPE`` / ``Unigram``), or ``None``. - """ - # Totality: a tokenizer.json that is valid JSON but not an object (e.g. - # ``[]`` / ``"x"`` / ``null``) must not crash the post-ingest registration — - # treat it as empty. The TokenizerValidator gate already rejects such files - # upstream; this keeps the public helper safe for any caller. - if not isinstance(tokenizer_data, dict): - tokenizer_data = {} - vocab = TokenizerValidator._extract_vocab(tokenizer_data) or set() - model = tokenizer_data.get("model") - tok_type = model.get("type") if isinstance(model, dict) else None - # FL guardrail at the source: only scalar values ever leave the cluster. - # Coerce a malformed non-int id / non-string type to None so a hand-crafted - # tokenizer.json can't smuggle a nested object into the backend metadata. - return { - "vocab_size": len(vocab), - "mask_token_id": _special_token_id(tokenizer_data, "[MASK]"), - "pad_token_id": _special_token_id(tokenizer_data, "[PAD]"), - "tokenizer_type": tok_type if isinstance(tok_type, str) else None, - } - - -def load_tokenizer_metadata(tokenizer_path: str) -> Optional[Dict[str, Any]]: - """Read ``tokenizer_path`` and return its structural fingerprint, or ``None``. - - Returns ``None`` (with a warning) when the file is absent or unparseable so - callers on the post-ingest registration path never crash an already-committed - dataset over a tokenizer read — presence and validity are enforced upstream - by :class:`TokenizerValidator` at validation time. - """ - if not os.path.isfile(tokenizer_path): - return None - try: - with open(tokenizer_path, "r", encoding="utf-8") as f: - tokenizer_data = json.load(f) - except (OSError, ValueError) as e: - logger.warning(f"Could not read tokenizer.json at {tokenizer_path}: {e}") - return None - if not isinstance(tokenizer_data, dict): - logger.warning( - f"tokenizer.json at {tokenizer_path} is not a JSON object " - f"({type(tokenizer_data).__name__}); no fingerprint registered." - ) - return None - return extract_tokenizer_metadata(tokenizer_data) - - -class TokenizerValidator(BaseValidator): - """Validator for tokenizer.json special-token requirements. - - Ensures that a tokenizer.json file exists at the configured data path - and that its vocabulary includes all required special tokens. For MLM - the mandatory tokens are [MASK] (used to create training targets) and - [PAD] (used to pad variable-length sequences in a batch). - - Attributes: - required_tokens: Set of token strings that must appear in the vocab. - """ - - def __init__( - self, - required_tokens: tuple = ("[MASK]", "[PAD]"), - name: str = "Tokenizer Validator", - optional: bool = False, - ): - super().__init__(name) - self.required_tokens = set(required_tokens) - # When True, a missing tokenizer.json is a warning (not an error): - # the training client falls back to the HuggingFace tokenizer_id / - # default. Used by text/token classification, where the tokenizer is - # optional. MLM keeps optional=False — its vocab IS the prediction - # space, so a missing tokenizer must fail loud at ingest. - self.optional = optional - - def validate(self, data: Any, **kwargs) -> ValidationResult: - """Validate tokenizer.json at the configured source path. - - Args: - data: Unused (path is read from config.SRC_PATH). - **kwargs: Additional validation parameters. - - Returns: - ValidationResult with status and error details. - """ - try: - tokenizer_path = Path((self._config or config).SRC_PATH) / "tokenizer.json" - - if not tokenizer_path.exists(): - if self.optional: - warning = ( - f"No tokenizer.json found at {tokenizer_path}. This is " - "optional for this category — the training client will " - "use the HuggingFace tokenizer_id / model_id (default " - "bert-base-uncased). Ship a tokenizer.json here only if " - "you need a custom tokenizer." - ) - logger.warning(warning) - return self._create_result( - is_valid=True, - warnings=[warning], - metadata={ - "path_checked": str(tokenizer_path), - "tokenizer_present": False, - }, - ) - return self._create_result( - is_valid=False, - errors=[ - f"tokenizer.json not found at {tokenizer_path}. " - "MLM training requires a tokenizer.json file alongside " - "the sequence data." - ], - metadata={"path_checked": str(tokenizer_path)}, - ) - - with open(tokenizer_path, "r", encoding="utf-8") as f: - tokenizer_data = json.load(f) - - vocab = self._extract_vocab(tokenizer_data) - if vocab is None: - return self._create_result( - is_valid=False, - errors=[ - "Could not extract vocabulary from tokenizer.json. " - "Expected a 'model.vocab' mapping or an 'added_tokens' list." - ], - metadata={"path_checked": str(tokenizer_path)}, - ) - - missing = sorted(self.required_tokens - vocab) - if missing: - return self._create_result( - is_valid=False, - errors=[ - f"Tokenizer is missing required special tokens: " - f"{', '.join(missing)}. " - f"Without these tokens, training will fail with an " - f"embedding out-of-bounds error. " - f"Re-train or update the tokenizer to include them." - ], - metadata={ - "path_checked": str(tokenizer_path), - "missing_tokens": missing, - "required_tokens": sorted(self.required_tokens), - }, - ) - - return self._create_result( - is_valid=True, - metadata={ - "path_checked": str(tokenizer_path), - "required_tokens": sorted(self.required_tokens), - "vocab_size": len(vocab), - }, - ) - - except json.JSONDecodeError as e: - logger.error(f"Failed to parse tokenizer.json: {e}") - return self._create_result( - is_valid=False, - errors=[f"tokenizer.json is not valid JSON: {e}"], - ) - except Exception as e: - logger.error(f"Tokenizer validation error: {e}") - return self._create_result( - is_valid=False, - errors=[f"Tokenizer validation error: {str(e)}"], - ) - - @staticmethod - def _extract_vocab(tokenizer_data: dict): - """Extract the set of token strings from a HuggingFace tokenizer JSON. - - Checks both ``model.vocab`` (WordLevel / WordPiece / BPE) and - ``added_tokens`` (special tokens added after training). - - Returns: - Set of token strings, or None if the structure is unrecognised. - """ - if not isinstance(tokenizer_data, dict): - return None - - tokens = set() - - # model.vocab — the main vocabulary mapping - # WordLevel/WordPiece/BPE store vocab as {token: id} dict. - # Unigram stores vocab as [[token, score], ...] list. - model = tokenizer_data.get("model") - vocab = model.get("vocab") if isinstance(model, dict) else None - if isinstance(vocab, dict): - tokens.update(vocab.keys()) - elif isinstance(vocab, list): - for entry in vocab: - if isinstance(entry, (list, tuple)) and len(entry) >= 1: - tokens.add(str(entry[0])) - - # added_tokens — special tokens registered separately. Only count a - # string ``content`` — a malformed non-string would be unhashable - # (dict/list) or pollute the vocab count. - added_tokens = tokenizer_data.get("added_tokens", []) - if isinstance(added_tokens, list): - for entry in added_tokens: - if isinstance(entry, dict) and isinstance(entry.get("content"), str): - tokens.add(entry["content"]) - elif isinstance(entry, str): - tokens.add(entry) - - return tokens if tokens else None