diff --git a/testing/backend/unit/test_finding_intelligence_fingerprint.py b/testing/backend/unit/test_finding_intelligence_fingerprint.py new file mode 100644 index 000000000..f34b6911a --- /dev/null +++ b/testing/backend/unit/test_finding_intelligence_fingerprint.py @@ -0,0 +1,140 @@ +import sys + +sys.path.insert(0, ".") + +from backend.secuscan.finding_intelligence import ( + _fingerprint_score, + _finding_kind_for, + _typed_evidence, +) + + +class TestFingerprintScore: + def test_validated_yields_full_score(self): + finding = {"validated": True, "metadata": {}} + score, strength = _fingerprint_score(finding) + assert score == 1.0 + assert strength == "validated" + + def test_exact_match(self): + finding = {"metadata": {"match_strength": "exact"}} + score, strength = _fingerprint_score(finding) + assert score == 0.95 + assert strength == "exact" + + def test_strong_fuzzy(self): + finding = {"metadata": {"match_strength": "strong_fuzzy"}} + score, strength = _fingerprint_score(finding) + assert score == 0.8 + assert strength == "strong_fuzzy" + + def test_fuzzy(self): + finding = {"metadata": {"match_strength": "fuzzy"}} + score, strength = _fingerprint_score(finding) + assert score == 0.7 + assert strength == "fuzzy" + + def test_family(self): + finding = {"metadata": {"match_strength": "family"}} + score, strength = _fingerprint_score(finding) + assert score == 0.45 + assert strength == "family" + + def test_none_yields_default(self): + finding = {"metadata": {"match_strength": "none"}} + score, strength = _fingerprint_score(finding) + assert score == 0.25 + assert strength == "none" + + def test_missing_metadata_defaults(self): + finding = {} + score, strength = _fingerprint_score(finding) + assert score == 0.25 + assert strength == "none" + + def test_unknown_strength_defaults(self): + finding = {"metadata": {"match_strength": "unknown_strength"}} + score, strength = _fingerprint_score(finding) + assert score == 0.0 + assert strength == "unknown_strength" + + +class TestFindingKindFor: + def test_validated_high_severity_is_validated_issue(self): + finding = {"validated": True, "severity": "high", "category": "unknown"} + assert _finding_kind_for(finding) == "validated_issue" + + def test_observation_category_is_observation(self): + finding = {"category": "attack surface", "severity": "info"} + assert _finding_kind_for(finding) == "observation" + + def test_high_severity_is_suspected_issue(self): + finding = {"category": "unknown", "severity": "high"} + assert _finding_kind_for(finding) == "suspected_issue" + + def test_cve_is_suspected_issue(self): + finding = {"category": "info", "severity": "info", "cve": "CVE-2024-1"} + assert _finding_kind_for(finding) == "suspected_issue" + + def test_cpe_correlation_is_suspected_issue(self): + finding = { + "category": "info", + "severity": "info", + "validation_method": "cpe_cve_correlation", + } + assert _finding_kind_for(finding) == "suspected_issue" + + def test_low_severity_no_cve_is_observation(self): + finding = {"category": "unknown", "severity": "low"} + assert _finding_kind_for(finding) == "observation" + + def test_empty_category_defaults_to_observation(self): + finding = {"category": "", "severity": "info"} + assert _finding_kind_for(finding) == "observation" + + +class TestTypedEvidence: + def test_dict_item_returns_structured_evidence(self): + item = { + "type": "http_response", + "label": "HTTP Header", + "value": "X-Frame-Options: DENY", + "artifact_ref": "ref-1", + "source": "scanner", + "observed_at": "2024-01-01T00:00:00Z", + "confidence": 0.95, + } + result = _typed_evidence(item, source="base", observed_at="2024-01-01T00:00:00Z", confidence=0.8) + assert result["type"] == "http_response" + assert result["label"] == "HTTP Header" + assert result["value"] == "X-Frame-Options: DENY" + assert result["artifact_ref"] == "ref-1" + assert result["confidence"] == 0.95 + + def test_non_dict_item_returns_default_evidence(self): + result = _typed_evidence("some text", source="scanner", observed_at="t", confidence=0.75) + assert result["type"] == "evidence" + assert result["label"] == "Evidence" + assert result["value"] == "some text" + assert result["source"] == "scanner" + assert result["confidence"] == 0.75 + + def test_clamps_confidence_between_zero_and_one(self): + item = {"confidence": 1.5} + result = _typed_evidence(item, source="s", observed_at="t", confidence=0.5) + assert result["confidence"] == 1.0 + + def test_uses_base_confidence_when_item_has_none(self): + item = {} + result = _typed_evidence(item, source="s", observed_at="t", confidence=0.9) + assert result["confidence"] == 0.9 + + def test_observed_at_defaults_to_provided(self): + item = {} + result = _typed_evidence(item, source="s", observed_at="2024-06-01T00:00:00Z", confidence=0.5) + assert result["observed_at"] == "2024-06-01T00:00:00Z" + + def test_missing_label_uses_title_cased_type(self): + item = {"type": "http_header"} + result = _typed_evidence(item, source="s", observed_at="t", confidence=0.5) + assert result["label"] == "Http Header" diff --git a/testing/backend/unit/test_finding_intelligence_issue_signature.py b/testing/backend/unit/test_finding_intelligence_issue_signature.py new file mode 100644 index 000000000..49c5cd3ca --- /dev/null +++ b/testing/backend/unit/test_finding_intelligence_issue_signature.py @@ -0,0 +1,94 @@ +import sys + +sys.path.insert(0, ".") + +from backend.secuscan.finding_intelligence import _issue_signature + + +def test_cve_returns_cve_signature(): + finding = {"cve": "CVE-2024-1234"} + assert _issue_signature(finding) == "cve:cve-2024-1234" + + +def test_cve_normalizes_case(): + finding = {"cve": "CVE-2021-99999"} + result = _issue_signature(finding) + assert result.startswith("cve:") + assert "2021" in result + + +def test_no_cve_uses_metadata_template(): + finding = { + "category": "vulnerability", + "title": "SQL Injection", + "validation_method": "nuclei", + "metadata": {"template": "sqli-error"}, + } + result = _issue_signature(finding) + assert "vulnerability" in result + assert "sql-injection" in result + assert "nuclei" in result + assert "sqli-error" in result + + +def test_no_cve_uses_metadata_header(): + finding = { + "category": "web", + "title": "Open Redirect", + "metadata": {"header": "location"}, + } + result = _issue_signature(finding) + assert "open-redirect" in result + assert "location" in result + + +def test_no_cve_uses_metadata_service(): + finding = { + "category": "network", + "title": "SSL Issue", + "metadata": {"service": "https"}, + } + result = _issue_signature(finding) + assert "ssl-issue" in result + assert "https" in result + + +def test_no_metadata_falls_back_to_base_fields(): + finding = { + "category": "info", + "title": "Info Finding", + "validation_method": "scan", + } + result = _issue_signature(finding) + assert "info" in result + assert "info-finding" in result + assert "scan" in result + + +def test_empty_finding_returns_compact_pipes(): + finding = {} + result = _issue_signature(finding) + assert result == "||||" + + +def test_empty_metadata_defaults_to_empty(): + finding = { + "category": "x", + "title": "y", + "validation_method": "z", + "metadata": {}, + } + result = _issue_signature(finding) + assert "x" in result + assert "y" in result + assert "z" in result + + +def test_metadata_not_dict_ignored(): + finding = { + "category": "cat", + "title": "title", + "metadata": "not-a-dict", + } + result = _issue_signature(finding) + assert "cat" in result diff --git a/testing/backend/unit/test_plugin_health_dashboard.py b/testing/backend/unit/test_plugin_health_dashboard.py new file mode 100644 index 000000000..4cc6323d0 --- /dev/null +++ b/testing/backend/unit/test_plugin_health_dashboard.py @@ -0,0 +1,188 @@ +import os +import sys +import tempfile +from pathlib import Path + +sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "../../.."))) + +from scripts.plugin_health_dashboard import ( + safe_relative_path, + discover_plugins, + build_report, + format_markdown, +) + + +def test_safe_relative_path_normal_case(): + base = Path("/a/b") + path = Path("/a/b/c/d") + assert safe_relative_path(path, base) == "c/d" + + +def test_safe_relative_path_exact_match(): + base = Path("/a/b") + path = Path("/a/b") + assert safe_relative_path(path, base) == "." + + +def test_safe_relative_path_outside_returns_absolute(): + base = Path("/a/b") + path = Path("/a/x/y") + result = safe_relative_path(path, base) + assert result == str(path) + + +def test_safe_relative_path_deeply_nested(): + base = Path("/x") + path = Path("/x/y/z/w/v") + assert safe_relative_path(path, base) == "y/z/w/v" + + +def test_discover_plugins_returns_list(): + with tempfile.TemporaryDirectory() as tmpdir: + root = Path(tmpdir) + plugin_dir = root / "my_plugin" + plugin_dir.mkdir() + (plugin_dir / "parser.py").write_text("# parser") + (plugin_dir / "metadata.json").write_text( + '{"name": "My Plugin", "category": "recon"}' + ) + + result = discover_plugins(plugin_root=root) + assert isinstance(result, list) + assert len(result) == 1 + assert result[0]["name"] == "My Plugin" + assert result[0]["category"] == "recon" + assert result[0]["has_parser"] is True + + +def test_discover_plugins_detects_missing_parser(): + with tempfile.TemporaryDirectory() as tmpdir: + root = Path(tmpdir) + plugin_dir = root / "no_parser" + plugin_dir.mkdir() + (plugin_dir / "metadata.json").write_text('{"name": "No Parser"}') + + result = discover_plugins(plugin_root=root) + assert len(result) == 1 + assert result[0]["has_parser"] is False + + +def test_discover_plugins_handles_invalid_json(): + with tempfile.TemporaryDirectory() as tmpdir: + root = Path(tmpdir) + plugin_dir = root / "bad_json" + plugin_dir.mkdir() + (plugin_dir / "parser.py").write_text("# x") + (plugin_dir / "metadata.json").write_text("not json{{{") + + result = discover_plugins(plugin_root=root) + assert len(result) == 1 + assert result[0]["name"] == "bad_json" + + +def test_discover_plugins_sorts_by_name(): + with tempfile.TemporaryDirectory() as tmpdir: + root = Path(tmpdir) + z_dir = root / "zebra" + z_dir.mkdir() + (z_dir / "metadata.json").write_text('{"name": "Zebra"}') + a_dir = root / "alpha" + a_dir.mkdir() + (a_dir / "metadata.json").write_text('{"name": "Alpha"}') + + result = discover_plugins(plugin_root=root) + assert result[0]["name"] == "Alpha" + assert result[1]["name"] == "Zebra" + + +def test_discover_plugins_empty_directory(): + with tempfile.TemporaryDirectory() as tmpdir: + result = discover_plugins(plugin_root=Path(tmpdir)) + assert result == [] + + +def test_build_report_summary_counts(): + plugins = [ + {"name": "A", "category": "recon", "has_parser": True}, + {"name": "B", "category": "recon", "has_parser": False}, + {"name": "C", "category": "web", "has_parser": True}, + ] + report = build_report(plugins) + assert report["summary"]["total_plugins"] == 3 + assert report["summary"]["plugins_with_parser"] == 2 + assert report["summary"]["plugins_without_parser"] == 1 + assert report["categories"]["recon"] == 2 + assert report["categories"]["web"] == 1 + + +def test_build_report_empty(): + report = build_report([]) + assert report["summary"]["total_plugins"] == 0 + assert report["summary"]["plugins_with_parser"] == 0 + assert report["summary"]["plugins_without_parser"] == 0 + assert report["categories"] == {} + + +def test_format_markdown_contains_summary(): + report = { + "summary": { + "total_plugins": 3, + "plugins_with_parser": 2, + "plugins_without_parser": 1, + }, + "categories": {"recon": 2, "web": 1}, + "plugins": [ + { + "name": "Plugin A", + "category": "recon", + "has_parser": True, + "path": "recon/plugin_a", + } + ], + } + output = format_markdown(report) + assert "Total plugins: 3" in output + assert "Plugins with parser.py: 2" in output + assert "Plugins without parser.py: 1" in output + assert "recon: 2" in output + assert "web: 1" in output + assert "Plugin A" in output + assert "| Plugin | Category | Parser | Path |" in output + + +def test_format_markdown_categories_sorted(): + report = { + "summary": {"total_plugins": 2, "plugins_with_parser": 2, "plugins_without_parser": 0}, + "categories": {"z_cat": 1, "a_cat": 1}, + "plugins": [], + } + output = format_markdown(report) + assert "- a_cat: 1" in output + assert "- z_cat: 1" in output + assert output.index("- a_cat: 1") < output.index("- z_cat: 1") + + +def test_format_markdown_plugin_table_shows_parser_status(): + plugins = [ + { + "name": "With Parser", + "category": "web", + "has_parser": True, + "path": "web/with_parser", + }, + { + "name": "No Parser", + "category": "web", + "has_parser": False, + "path": "web/no_parser", + }, + ] + report = { + "summary": {"total_plugins": 2, "plugins_with_parser": 1, "plugins_without_parser": 1}, + "categories": {"web": 2}, + "plugins": plugins, + } + output = format_markdown(report) + assert "| With Parser | web | Yes | `web/with_parser` |" in output + assert "| No Parser | web | No | `web/no_parser` |" in output diff --git a/testing/backend/unit/test_risk_scoring_recency_detail.py b/testing/backend/unit/test_risk_scoring_recency_detail.py index 01670ae90..8e28e9ab5 100644 --- a/testing/backend/unit/test_risk_scoring_recency_detail.py +++ b/testing/backend/unit/test_risk_scoring_recency_detail.py @@ -1,70 +1,44 @@ -""" -Unit tests for backend/secuscan/risk_scoring.py's _recency_detail helper. +import sys +from datetime import datetime, timezone, timedelta -Tests the human-readable recency explanation generator. This is a pure -function, so these tests run with --noconftest (no FastAPI dependencies -needed) since risk_scoring.py only depends on stdlib. -""" +sys.path.insert(0, ".") -from datetime import datetime, timedelta, timezone +from backend.secuscan.risk_scoring import _recency_detail -import pytest -from backend.secuscan.risk_scoring import _recency_detail +def test_no_discovery_date_returns_default(): + result = _recency_detail(None, 5.0) + assert result == "No discovery date — assumed moderate recency" -class TestRecencyDetail: - def test_none_discovered_at_returns_moderate_recency_message(self): - """When discovered_at is None, a moderate-recency fallback message is returned.""" - result = _recency_detail(None, 5.0) - assert result == "No discovery date — assumed moderate recency" +def test_future_date_returns_future_message(): + future = datetime.now(timezone.utc) + timedelta(days=1) + result = _recency_detail(future, 10.0) + assert result == "Discovered in the future — treated as very recent" - def test_future_date_returns_very_recent_message(self): - """A discovered_at timestamp in the future is treated as very recent.""" - future = datetime.now(timezone.utc) + timedelta(days=3) - result = _recency_detail(future, 10.0) - assert result == "Discovered in the future — treated as very recent" - def test_today_returns_maximum_recency_message(self): - """A discovered_at timestamp from today (0 days ago) returns the max-score message.""" - now = datetime.now(timezone.utc) - result = _recency_detail(now, 10.0) - assert result == "Discovered today — maximum recency score" +def test_today_returns_today_message(): + today = datetime.now(timezone.utc) + result = _recency_detail(today, 10.0) + assert result == "Discovered today — maximum recency score" - def test_yesterday_returns_singular_day_message(self): - """Exactly 1 day ago uses singular 'day' wording with the recency score.""" - yesterday = datetime.now(timezone.utc) - timedelta(days=1) - result = _recency_detail(yesterday, 7.5) - assert result == "Discovered 1 day ago — recency score 7.5/10" - def test_multiple_days_ago_returns_plural_days_message(self): - """More than 1 day ago uses plural 'days' wording with the recency score.""" - ten_days_ago = datetime.now(timezone.utc) - timedelta(days=10) - result = _recency_detail(ten_days_ago, 5.0) - assert result == "Discovered 10 days ago — recency score 5.0/10" +def test_one_day_ago_returns_day_message(): + yesterday = datetime.now(timezone.utc) - timedelta(days=1) + result = _recency_detail(yesterday, 9.5) + assert "1 day ago" in result + assert "9.5/10" in result - def test_many_days_ago_returns_plural_days_message(self): - """A much older date (well past a year) still uses the plural days format.""" - long_ago = datetime.now(timezone.utc) - timedelta(days=400) - result = _recency_detail(long_ago, 1.0) - assert result == "Discovered 400 days ago — recency score 1.0/10" - def test_naive_datetime_is_treated_as_utc(self): - """A naive (non-tz-aware) datetime is assumed UTC and handled without error.""" - naive_yesterday = datetime.now(timezone.utc).replace(tzinfo=None) - timedelta(days=1) - result = _recency_detail(naive_yesterday, 7.5) - assert result == "Discovered 1 day ago — recency score 7.5/10" +def test_multiple_days_ago(): + five_days = datetime.now(timezone.utc) - timedelta(days=5) + result = _recency_detail(five_days, 8.0) + assert "5 days ago" in result + assert "8.0/10" in result - def test_non_utc_timezone_is_normalized_correctly(self): - """A tz-aware datetime in a non-UTC timezone is normalized before day-diffing.""" - ist = timezone(timedelta(hours=5, minutes=30)) - # 1 day ago in IST should still resolve to ~1 day ago once normalized to UTC. - one_day_ago_ist = datetime.now(ist) - timedelta(days=1) - result = _recency_detail(one_day_ago_ist, 7.5) - assert result == "Discovered 1 day ago — recency score 7.5/10" - def test_recency_score_is_formatted_to_one_decimal_place(self): - """The rv value passed in is always rendered with exactly one decimal place.""" - two_days_ago = datetime.now(timezone.utc) - timedelta(days=2) - result = _recency_detail(two_days_ago, 5) - assert result == "Discovered 2 days ago — recency score 5.0/10" \ No newline at end of file +def test_naive_datetime_gets_utc_tzinfo(): + naive = datetime.now() - timedelta(days=3) + result = _recency_detail(naive, 7.0) + assert "3 days ago" in result + assert "7.0/10" in result diff --git a/testing/backend/unit/test_validate_plugins_catalog.py b/testing/backend/unit/test_validate_plugins_catalog.py new file mode 100644 index 000000000..9da187ff4 --- /dev/null +++ b/testing/backend/unit/test_validate_plugins_catalog.py @@ -0,0 +1,295 @@ +import os +import sys +import tempfile +from pathlib import Path + +# Add root to sys.path +sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "../../.."))) + +from scripts.validate_plugins_catalog import ( + parse_plugins_md, + extract_counts_from_catalog, + extract_category_counts_from_catalog, + validate_catalog, +) + + +def test_parse_plugins_md_extracts_plugin_entries(): + catalog_content = """# Title + +## Plugin Index + +| Plugin | ID | Category | Safety | Primary Binary | Summary | +| --- | --- | --- | --- | --- | --- | +| Amass | `amass` | `recon` | `safe` | `amass` | Deep mapping. | +| Nuclei | `nuclei` | `vulnerability` | `intrusive` | `nuclei` | Fast scanner. | +""" + with tempfile.NamedTemporaryFile(mode="w", suffix=".md", delete=False) as f: + f.write(catalog_content) + path = Path(f.name) + + try: + result = parse_plugins_md(path) + assert "amass" in result + assert result["amass"]["name"] == "Amass" + assert result["amass"]["category"] == "recon" + assert result["amass"]["safety"] == "safe" + assert "nuclei" in result + assert result["nuclei"]["category"] == "vulnerability" + assert result["nuclei"]["safety"] == "intrusive" + finally: + path.unlink() + + +def test_parse_plugins_md_skips_header_row(): + catalog_content = """# Title + +| Plugin | ID | Category | Safety | Primary Binary | Summary | +| --- | --- | --- | --- | --- | --- | +| Plugin | ID | Category | Safety | Primary Binary | Summary | +| Real Plugin | `real` | `recon` | `safe` | `x` | desc. | +""" + with tempfile.NamedTemporaryFile(mode="w", suffix=".md", delete=False) as f: + f.write(catalog_content) + path = Path(f.name) + + try: + result = parse_plugins_md(path) + assert "real" in result + assert "ID" not in result + finally: + path.unlink() + + +def test_parse_plugins_md_empty_file(): + with tempfile.NamedTemporaryFile(mode="w", suffix=".md", delete=False) as f: + path = Path(f.name) + + try: + result = parse_plugins_md(path) + assert result == {} + finally: + path.unlink() + + +def test_extract_counts_from_catalog_parses_all_counts(): + catalog_content = """## At a Glance + +- Total plugins: 59 +- Safe plugins: 26 +- Intrusive plugins: 25 +- Exploit plugins: 8 +""" + with tempfile.NamedTemporaryFile(mode="w", suffix=".md", delete=False) as f: + f.write(catalog_content) + path = Path(f.name) + + try: + counts = extract_counts_from_catalog(path) + assert counts["total_plugins"] == 59 + assert counts["safe_plugins"] == 26 + assert counts["intrusive_plugins"] == 25 + assert counts["exploit_plugins"] == 8 + finally: + path.unlink() + + +def test_extract_counts_from_catalog_empty(): + with tempfile.NamedTemporaryFile(mode="w", suffix=".md", delete=False) as f: + path = Path(f.name) + + try: + counts = extract_counts_from_catalog(path) + assert counts == {} + finally: + path.unlink() + + +def test_extract_category_counts_from_catalog_parses_table(): + catalog_content = """## Category Summary + +| Category | Count | +| --- | ---: | +| `recon` | 17 | +| `vulnerability` | 12 | +| `web` | 5 | + +## Another Section +""" + with tempfile.NamedTemporaryFile(mode="w", suffix=".md", delete=False) as f: + f.write(catalog_content) + path = Path(f.name) + + try: + counts = extract_category_counts_from_catalog(path) + assert counts["recon"] == 17 + assert counts["vulnerability"] == 12 + assert counts["web"] == 5 + assert "Category" not in counts + finally: + path.unlink() + + +def test_extract_category_counts_from_catalog_stops_at_next_header(): + catalog_content = """## Category Summary + +| Category | Count | +| --- | ---: | +| `recon` | 17 | + +## Next Section + +| `web` | 5 | +""" + with tempfile.NamedTemporaryFile(mode="w", suffix=".md", delete=False) as f: + f.write(catalog_content) + path = Path(f.name) + + try: + counts = extract_category_counts_from_catalog(path) + assert counts["recon"] == 17 + assert "web" not in counts + finally: + path.unlink() + + +def test_validate_catalog_in_sync_returns_true(): + catalog_content = """# Title + +| Plugin | ID | Category | Safety | Primary Binary | Summary | +| --- | --- | --- | --- | --- | --- | +| Test Plugin | `test_plugin` | `recon` | `safe` | `x` | desc. | + +## At a Glance + +- Total plugins: 1 +- Safe plugins: 1 +- Intrusive plugins: 0 +- Exploit plugins: 0 + +## Category Summary + +| Category | Count | +| --- | ---: | +| `recon` | 1 | +""" + plugin_meta = '{"id": "test_plugin", "category": "recon", "safety": {"level": "safe"}}' + + with tempfile.TemporaryDirectory() as tmpdir: + catalog_path = Path(tmpdir) / "PLUGINS.md" + plugins_dir = Path(tmpdir) / "plugins" + plugins_dir.mkdir() + plugin_dir = plugins_dir / "test_plugin" + plugin_dir.mkdir() + (plugin_dir / "metadata.json").write_text(plugin_meta) + + catalog_path.write_text(catalog_content) + + valid, issues = validate_catalog(catalog_path, plugins_dir) + assert valid is True + assert issues == [] + + +def test_validate_catalog_missing_from_catalog_returns_false(): + catalog_content = """# Title + +| Plugin | ID | Category | Safety | Primary Binary | Summary | +| --- | --- | --- | --- | --- | --- | +| Listed Plugin | `listed` | `recon` | `safe` | `x` | desc. | + +## At a Glance + +- Total plugins: 1 +- Safe plugins: 1 +- Intrusive plugins: 0 +- Exploit plugins: 0 + +## Category Summary + +| Category | Count | +| --- | ---: | +| `recon` | 1 | +""" + plugin_meta = '{"id": "actual", "category": "recon", "safety": {"level": "safe"}}' + + with tempfile.TemporaryDirectory() as tmpdir: + catalog_path = Path(tmpdir) / "PLUGINS.md" + plugins_dir = Path(tmpdir) / "plugins" + plugins_dir.mkdir() + plugin_dir = plugins_dir / "actual" + plugin_dir.mkdir() + (plugin_dir / "metadata.json").write_text(plugin_meta) + + catalog_path.write_text(catalog_content) + + valid, issues = validate_catalog(catalog_path, plugins_dir) + assert valid is False + assert any("Missing from PLUGINS.md" in i for i in issues) + + +def test_validate_catalog_extra_in_catalog_returns_false(): + catalog_content = """# Title + +| Plugin | ID | Category | Safety | Primary Binary | Summary | +| --- | --- | --- | --- | --- | --- | +| Extra Plugin | `extra` | `recon` | `safe` | `x` | desc. | + +## At a Glance + +- Total plugins: 0 +- Safe plugins: 0 +- Intrusive plugins: 0 +- Exploit plugins: 0 + +## Category Summary + +| Category | Count | +| --- | ---: | +""" + with tempfile.TemporaryDirectory() as tmpdir: + catalog_path = Path(tmpdir) / "PLUGINS.md" + plugins_dir = Path(tmpdir) / "plugins" + plugins_dir.mkdir() + + catalog_path.write_text(catalog_content) + + valid, issues = validate_catalog(catalog_path, plugins_dir) + assert valid is False + assert any("In PLUGINS.md but not in plugins/" in i for i in issues) + + +def test_validate_catalog_count_mismatch_returns_false(): + catalog_content = """# Title + +| Plugin | ID | Category | Safety | Primary Binary | Summary | +| --- | --- | --- | --- | --- | --- | +| Test Plugin | `test` | `recon` | `safe` | `x` | desc. | + +## At a Glance + +- Total plugins: 99 +- Safe plugins: 99 +- Intrusive plugins: 0 +- Exploit plugins: 0 + +## Category Summary + +| Category | Count | +| --- | ---: | +| `recon` | 99 | +""" + plugin_meta = '{"id": "test", "category": "recon", "safety": {"level": "safe"}}' + + with tempfile.TemporaryDirectory() as tmpdir: + catalog_path = Path(tmpdir) / "PLUGINS.md" + plugins_dir = Path(tmpdir) / "plugins" + plugins_dir.mkdir() + plugin_dir = plugins_dir / "test" + plugin_dir.mkdir() + (plugin_dir / "metadata.json").write_text(plugin_meta) + + catalog_path.write_text(catalog_content) + + valid, issues = validate_catalog(catalog_path, plugins_dir) + assert valid is False + assert any("Total plugins mismatch" in i for i in issues)