feat(guardrails): semantic preservation constraint + holdout integrity (dedup, seeded splits)

evolution/core/config.py

@@ -30,12 +30,23 @@ class EvolutionConfig:
     max_tool_desc_size: int = 500  # chars
     max_param_desc_size: int = 200  # chars
     max_prompt_growth: float = 0.2  # 20% max growth over baseline
+    # Minimum content-term cosine similarity between baseline and evolved
+    # text (PLAN.md constraint 4, semantic preservation). Calibrated on
+    # real hermes-agent skills: same-skill rewrites score 0.84+, pairs of
+    # unrelated skills below 0.16. Set to 0 to disable.
+    min_semantic_similarity: float = 0.4
 
     # Eval dataset
     eval_dataset_size: int = 20  # Total examples to generate
     train_ratio: float = 0.5
     val_ratio: float = 0.25
     holdout_ratio: float = 0.25
+    # Holdout integrity: near-duplicate tasks are removed before the
+    # train/val/holdout split so memorized train examples cannot leak
+    # into the holdout set, and the split itself is seeded so a dataset
+    # always splits the same way.
+    dedup_jaccard: float = 0.9  # Token-set Jaccard above this = duplicate
+    dataset_split_seed: int = 13
 
     # Benchmark gating
     run_pytest: bool = True

evolution/core/constraints.py

@@ -4,13 +4,55 @@
 considered valid. Failed constraints = immediate rejection.
 """
 
+import math
+import re
 import subprocess
+from collections import Counter
 from pathlib import Path
 from dataclasses import dataclass
 from typing import Optional
 
 from evolution.core.config import EvolutionConfig
 
+# Common words that carry no topical signal. Deliberately small: the
+# 4+ char term filter already drops most function words, and an
+# aggressive stopword list would risk erasing domain vocabulary.
+_STOPWORDS = frozenset({
+    "about", "after", "also", "always", "been", "before", "being", "between",
+    "both", "cannot", "could", "does", "each", "either", "every", "from",
+    "have", "here", "instead", "into", "just", "like", "make", "more", "most",
+    "must", "never", "only", "other", "over", "should", "some", "such",
+    "than", "that", "them", "then", "there", "these", "they", "this",
+    "those", "under", "very", "want", "well", "were", "what", "when",
+    "where", "which", "while", "will", "with", "without", "would", "your",
+})
+
+
+def _content_terms(text: str) -> Counter:
+    """Frequency of topical terms: lowercase words of 4+ chars minus stopwords."""
+    words = re.findall(r"[a-z][a-z0-9_-]{3,}", text.lower())
+    return Counter(w for w in words if w not in _STOPWORDS)
+
+
+def semantic_similarity(baseline: str, evolved: str) -> float:
+    """Cosine similarity between the content-term frequencies of two texts.
+
+    A deterministic, dependency-free proxy for topical drift: a rewrite of
+    the same skill keeps most of its domain vocabulary (commands, API names,
+    concepts), while text that drifted to another purpose does not. Returns
+    a value in [0, 1].
+    """
+    a = _content_terms(baseline)
+    b = _content_terms(evolved)
+    if not a or not b:
+        return 1.0 if not a and not b else 0.0
+
+    shared = set(a) & set(b)
+    dot = sum(a[t] * b[t] for t in shared)
+    norm_a = math.sqrt(sum(v * v for v in a.values()))
+    norm_b = math.sqrt(sum(v * v for v in b.values()))
+    return dot / (norm_a * norm_b)
+
 
 @dataclass
 class ConstraintResult:
@@ -39,9 +81,10 @@ def validate_all(
         # 1. Size limits
         results.append(self._check_size(artifact_text, artifact_type))
 
-        # 2. Growth limit (if baseline provided)
+        # 2. Baseline-relative checks (growth + semantic preservation)
         if baseline_text:
             results.append(self._check_growth(artifact_text, baseline_text, artifact_type))
+            results.append(self._check_semantic_preservation(artifact_text, baseline_text))
 
         # 3. Non-empty
         results.append(self._check_non_empty(artifact_text))
@@ -133,6 +176,38 @@ def _check_growth(self, text: str, baseline: str, artifact_type: str) -> Constra
                 message=f"Growth exceeded: {growth:+.1%} (max {max_growth:+.1%})",
             )
 
+    def _check_semantic_preservation(self, text: str, baseline: str) -> ConstraintResult:
+        """Reject evolved text that drifted away from the baseline's purpose.
+
+        PLAN.md constraint 4: a skill for GitHub code review must still
+        perform code reviews, not drift into something else. Measured as
+        content-term cosine similarity against the baseline. Calibrated on
+        real hermes-agent skills: rewrites of the same skill score 0.84+,
+        while pairs of unrelated skills score below 0.16, so the default
+        threshold of 0.4 separates them with wide margin on both sides.
+        A threshold of 0 disables the check.
+        """
+        threshold = self.config.min_semantic_similarity
+        similarity = semantic_similarity(baseline, text)
+
+        if similarity >= threshold:
+            return ConstraintResult(
+                passed=True,
+                constraint_name="semantic_preservation",
+                message=f"Topic preserved: similarity {similarity:.2f} (min {threshold:.2f})",
+            )
+
+        # Show which topical vocabulary disappeared, for human review.
+        baseline_terms = _content_terms(baseline)
+        evolved_terms = _content_terms(text)
+        lost = [t for t, _ in baseline_terms.most_common(30) if t not in evolved_terms][:8]
+        return ConstraintResult(
+            passed=False,
+            constraint_name="semantic_preservation",
+            message=f"Topic drift: similarity {similarity:.2f} below minimum {threshold:.2f}",
+            details=f"Baseline terms missing from evolved text: {', '.join(lost)}" if lost else None,
+        )
+
     def _check_non_empty(self, text: str) -> ConstraintResult:
         if text.strip():
             return ConstraintResult(

evolution/core/dataset_builder.py

@@ -8,6 +8,7 @@
 
 import json
 import random
+import re
 from pathlib import Path
 from dataclasses import dataclass, field
 from typing import Optional
@@ -17,6 +18,68 @@
 from evolution.core.config import EvolutionConfig
 
 
+def _dedup_key(text: str) -> str:
+    """Normalize a task for duplicate detection: casefold, strip punctuation,
+    collapse whitespace."""
+    return " ".join(re.sub(r"[^a-z0-9\s]", " ", text.lower()).split())
+
+
+def _token_jaccard(a: str, b: str) -> float:
+    """Jaccard similarity between the token sets of two normalized tasks."""
+    tokens_a = set(_dedup_key(a).split())
+    tokens_b = set(_dedup_key(b).split())
+    if not tokens_a or not tokens_b:
+        return 1.0 if tokens_a == tokens_b else 0.0
+    return len(tokens_a & tokens_b) / len(tokens_a | tokens_b)
+
+
+def dedupe_examples(examples: list["EvalExample"], jaccard_threshold: float = 0.9) -> list["EvalExample"]:
+    """Drop duplicate and near-duplicate tasks, keeping the first occurrence.
+
+    LLM generation and mined session history both produce repeats. If two
+    copies of the same task land on opposite sides of the train/holdout
+    split, the holdout is contaminated and "improvement" can be pure
+    memorization. Exact duplicates are matched on the normalized task text;
+    near-duplicates on token-set Jaccard similarity.
+    """
+    kept: list["EvalExample"] = []
+    seen_keys: set[str] = set()
+    for example in examples:
+        key = _dedup_key(example.task_input)
+        if key in seen_keys:
+            continue
+        if any(_token_jaccard(example.task_input, k.task_input) >= jaccard_threshold for k in kept):
+            continue
+        seen_keys.add(key)
+        kept.append(example)
+    return kept
+
+
+def split_examples(
+    examples: list["EvalExample"],
+    train_ratio: float = 0.5,
+    val_ratio: float = 0.25,
+    seed: int = 13,
+) -> "EvalDataset":
+    """Deterministic shuffled train/val/holdout split.
+
+    Seeded so the same examples always split the same way: re-running a
+    build cannot silently move an example from holdout into train.
+    """
+    shuffled = list(examples)
+    random.Random(seed).shuffle(shuffled)
+    n = len(shuffled)
+    if n == 0:
+        return EvalDataset()
+    n_train = max(1, int(n * train_ratio))
+    n_val = max(1, int(n * val_ratio))
+    return EvalDataset(
+        train=shuffled[:n_train],
+        val=shuffled[n_train:n_train + n_val],
+        holdout=shuffled[n_train + n_val:],
+    )
+
+
 @dataclass
 class EvalExample:
     """A single evaluation example."""
@@ -156,16 +219,14 @@ def generate(
             if c.get("task_input") and c.get("expected_behavior")
         ]
 
-        # Shuffle and split
-        random.shuffle(examples)
-        n_total = len(examples)
-        n_train = max(1, int(n_total * self.config.train_ratio))
-        n_val = max(1, int(n_total * self.config.val_ratio))
-
-        return EvalDataset(
-            train=examples[:n_train],
-            val=examples[n_train:n_train + n_val],
-            holdout=examples[n_train + n_val:],
+        # Dedupe (LLMs generate near-identical cases) and split with a
+        # fixed seed so holdout membership is stable across runs.
+        examples = dedupe_examples(examples, self.config.dedup_jaccard)
+        return split_examples(
+            examples,
+            train_ratio=self.config.train_ratio,
+            val_ratio=self.config.val_ratio,
+            seed=self.config.dataset_split_seed,
         )
 
 
@@ -189,13 +250,4 @@ def load(path: Path) -> EvalDataset:
                 if line.strip():
                     examples.append(EvalExample.from_dict(json.loads(line)))
 
-        random.shuffle(examples)
-        n = len(examples)
-        n_train = max(1, int(n * 0.5))
-        n_val = max(1, int(n * 0.25))
-
-        return EvalDataset(
-            train=examples[:n_train],
-            val=examples[n_train:n_train + n_val],
-            holdout=examples[n_train + n_val:],
-        )
+        return split_examples(dedupe_examples(examples))

evolution/core/external_importers.py

@@ -33,7 +33,7 @@
 from rich.console import Console
 from rich.progress import Progress
 
-from evolution.core.dataset_builder import EvalExample, EvalDataset
+from evolution.core.dataset_builder import EvalExample, EvalDataset, dedupe_examples, split_examples
 
 console = Console()
 
@@ -668,17 +668,15 @@ def build_dataset_from_external(
             f"recommended for meaningful train/val/holdout split)[/yellow]"
         )
 
-    # Split into train/val/holdout (50/25/25)
-    random.shuffle(examples)
-    n = len(examples)
-    n_train = max(1, int(n * 0.5))
-    n_val = max(1, int(n * 0.25))
+    # Dedupe before splitting: session history repeats the same prompts,
+    # and a task present in both train and holdout contaminates the eval.
+    before = len(examples)
+    examples = dedupe_examples(examples)
+    if len(examples) < before:
+        console.print(f"  Removed {before - len(examples)} duplicate/near-duplicate tasks")
 
-    dataset = EvalDataset(
-        train=examples[:n_train],
-        val=examples[n_train:n_train + n_val],
-        holdout=examples[n_train + n_val:],
-    )
+    # Seeded split into train/val/holdout (50/25/25)
+    dataset = split_examples(examples)
 
     dataset.save(output_path)
     console.print(f"\n[bold]Saved to {output_path}/[/bold]")