diff --git a/evolution/core/config.py b/evolution/core/config.py index 9e880e4d..9b9593e7 100644 --- a/evolution/core/config.py +++ b/evolution/core/config.py @@ -23,13 +23,27 @@ class EvolutionConfig: judge_model: str = "openai/gpt-4.1" # Model for dataset generation # Constraints - max_skill_size: int = 15_000 # 15KB default + max_skill_size: int = 20_000 # 20KB default (was 15KB — NLAH frontmatter adds size) max_tool_desc_size: int = 500 # chars max_param_desc_size: int = 200 # chars - max_prompt_growth: float = 0.2 # 20% max growth over baseline + max_prompt_growth: float = 0.5 # 50% max growth over baseline (was 0.2) + # Hard absolute growth cap — never exceeded, even with a waiver. Prevents + # unbounded bloat: an artifact may only grow past max_prompt_growth when a + # material valset improvement (growth_waiver_min_improvement) justifies it. + max_prompt_growth_hard: float = 1.0 # 100% max growth, absolute ceiling + # Minimum absolute score delta (on the 0-1 metric scale) an evolved + # artifact must beat the baseline by to earn a growth waiver. + growth_waiver_min_improvement: float = 0.03 + + # Random seed for reproducibility + random_seed: int = 42 # Eval dataset - eval_dataset_size: int = 20 # Total examples to generate + eval_dataset_size: int = 60 # Base examples for small skills; auto-scaled up + base_eval_dataset_size: int = 60 # Minimum examples before scaling by skill size + dataset_size_per_10k_chars: int = 10 # Extra examples per 10K chars beyond first 5K + max_eval_dataset_size: int = 150 # Cap on total scaled dataset size + eval_temperature: float = 0.0 # 0 = deterministic generation for stable scoring train_ratio: float = 0.5 val_ratio: float = 0.25 holdout_ratio: float = 0.25 diff --git a/evolution/core/constraints.py b/evolution/core/constraints.py index d6b13459..b23f441a 100644 --- a/evolution/core/constraints.py +++ b/evolution/core/constraints.py @@ -32,8 +32,23 @@ def validate_all( artifact_text: str, artifact_type: str, baseline_text: Optional[str] = None, + improvement: Optional[float] = None, + growth_text: Optional[str] = None, ) -> list[ConstraintResult]: - """Run all applicable constraints. Returns list of results.""" + """Run all applicable constraints. Returns list of results. + + ``improvement`` is the measured valset score delta of the evolved + artifact over the baseline (0-1 metric scale). When provided and + material, it can waive the soft growth cap (see ``_check_growth``); + None keeps the strict pre-waiver behavior. + + ``growth_text`` is the text used for the growth comparison (defaults + to ``artifact_text``). For skills, pass the evolved BODY here while + ``artifact_text`` stays the full reassembled artifact (frontmatter + + body): the growth gate must compare body-vs-body, otherwise the + static frontmatter inflates the ratio and a real improvement is + rejected (gmail-monitor: +100.5% full-vs-body vs +69.8% body-vs-body). + """ results = [] # 1. Size limits @@ -41,7 +56,10 @@ def validate_all( # 2. Growth limit (if baseline provided) if baseline_text: - results.append(self._check_growth(artifact_text, baseline_text, artifact_type)) + results.append(self._check_growth( + artifact_text, baseline_text, artifact_type, + improvement=improvement, growth_text=growth_text, + )) # 3. Non-empty results.append(self._check_non_empty(artifact_text)) @@ -116,17 +134,78 @@ def _check_size(self, text: str, artifact_type: str) -> ConstraintResult: message=f"Size exceeded: {size}/{limit} chars ({size - limit} over)", ) - def _check_growth(self, text: str, baseline: str, artifact_type: str) -> ConstraintResult: - growth = (len(text) - len(baseline)) / max(1, len(baseline)) + def _check_growth( + self, + text: str, + baseline: str, + artifact_type: str, + improvement: Optional[float] = None, + growth_text: Optional[str] = None, + ) -> ConstraintResult: + """Check growth vs baseline, with a quality-based waiver. + + The soft cap ``max_prompt_growth`` rejects bloated artifacts. A + candidate that grows past the soft cap may still pass when it + demonstrates a *material* improvement (>= ``growth_waiver_min_improvement`` + absolute score delta on the 0-1 metric scale) AND stays under the hard + absolute cap ``max_prompt_growth_hard`` — i.e. the extra size buys + measurable quality. Without ``improvement`` the check is strict. + + ``growth_text`` defaults to ``text``; pass the evolved BODY explicitly + when ``text`` is a full artifact with static frontmatter so the growth + ratio measures the actual instruction growth (body-vs-body), not + frontmatter inflation. + """ + growth_base = growth_text if growth_text is not None else text + growth = (len(growth_base) - len(baseline)) / max(1, len(baseline)) max_growth = self.config.max_prompt_growth + waiver_applied = False + if growth > max_growth: + waiver_applied = ( + improvement is not None + and improvement >= self.config.growth_waiver_min_improvement + ) + if waiver_applied: + max_growth = self.config.max_prompt_growth_hard + if growth <= max_growth: + if waiver_applied: + return ConstraintResult( + passed=True, + constraint_name="growth_limit", + message=( + f"Growth OK with waiver: {growth:+.1%} (soft cap {self.config.max_prompt_growth:+.1%} " + f"exceeded but improvement {improvement:+.3f} >= " + f"{self.config.growth_waiver_min_improvement:+.3f}; hard cap {self.config.max_prompt_growth_hard:+.1%})" + ), + ) return ConstraintResult( passed=True, constraint_name="growth_limit", message=f"Growth OK: {growth:+.1%} (max {max_growth:+.1%})", ) else: + if waiver_applied: + return ConstraintResult( + passed=False, + constraint_name="growth_limit", + message=( + f"Growth exceeded: {growth:+.1%} (waiver applied: improvement {improvement:+.3f} >= " + f"{self.config.growth_waiver_min_improvement:+.3f}, but hard cap " + f"{self.config.max_prompt_growth_hard:+.1%} still exceeded)" + ), + ) + if improvement is not None: + return ConstraintResult( + passed=False, + constraint_name="growth_limit", + message=( + f"Growth exceeded: {growth:+.1%} (soft cap {self.config.max_prompt_growth:+.1%}, " + f"hard cap {self.config.max_prompt_growth_hard:+.1%}, improvement {improvement:+.3f} " + f"< waiver threshold {self.config.growth_waiver_min_improvement:+.3f})" + ), + ) return ConstraintResult( passed=False, constraint_name="growth_limit", diff --git a/evolution/core/errors.py b/evolution/core/errors.py new file mode 100644 index 00000000..e2682c61 --- /dev/null +++ b/evolution/core/errors.py @@ -0,0 +1,58 @@ +"""Exit-code contract and runtime error classification for the evolution pipeline. + +The cron wrapper (cron-evolve.sh) maps the evolve_skill process exit code to a +failure category. Every exit code used by the pipeline MUST be documented here +and mirrored in cron-evolve.sh's classifier. + +Exit codes: + 0 success — including the constraint-failure path (artifact saved for + review, deployment skipped; cron detects it via output text) + 1 generic error (skill not found, dataset build failure, usage error) + 2 FD exhaustion (OSError errno 24 EMFILE) — environment limit, NOT an + LLM/API failure; raise `ulimit -n` and retry + 3 holdout-eval / LLM failure (dspy, network, or scoring error during the + evaluation phase) + +History: the 'error 1' escalations (esc-20260731-201857-1474) were EMFILE +crashes — bare `OSError: [Errno 24] Too many open files` tracebacks — that +Python reported as exit 1 and cron-evolve.sh mislabeled as "LLM provider +issue". This module exists so that failure mode gets its own exit code (2) +and a meaningful message. +""" + +import errno + +EXIT_GENERIC = 1 +EXIT_EMFILE = 2 +EXIT_EVAL_ERROR = 3 + +EMFILE_HINT = ( + "Too many open files (OSError errno 24 EMFILE). This is an environment " + "file-descriptor limit, not an LLM/API failure. Raise the process limit " + "(e.g. `ulimit -n 1024`) and retry." +) + + +def is_emfile(exc: BaseException) -> bool: + """Return True if the exception is FD exhaustion (EMFILE, errno 24). + + Checks the errno directly, and falls back to message matching because + dspy/litellm sometimes wrap the raw OS error in a generic Exception + that carries the errno text but no errno attribute. + """ + if isinstance(exc, OSError): + return exc.errno == errno.EMFILE + return "too many open files" in str(exc).lower() + + +def classify_error(exc: BaseException) -> tuple[int, str]: + """Map a runtime exception to (exit_code, human_message). + + Distinguishes FD exhaustion from evaluation/LLM errors so the cron + wrapper can report a meaningful cause instead of the generic + 'LLM provider issue' label that masked the original EMFILE crashes. + """ + if is_emfile(exc): + return EXIT_EMFILE, EMFILE_HINT + msg = str(exc) or exc.__class__.__name__ + return EXIT_EVAL_ERROR, f"Evaluation failed: {msg}" diff --git a/evolution/skills/evolve_skill.py b/evolution/skills/evolve_skill.py index 8ad4d89c..2ffd03e9 100644 --- a/evolution/skills/evolve_skill.py +++ b/evolution/skills/evolve_skill.py @@ -5,9 +5,13 @@ python -m evolution.skills.evolve_skill --skill arxiv --eval-source golden --dataset datasets/skills/arxiv/ """ +import hashlib import json +import os import sys +import threading import time +from contextlib import nullcontext from pathlib import Path from datetime import datetime from typing import Optional @@ -20,6 +24,12 @@ from evolution.core.config import EvolutionConfig, get_hermes_agent_path from evolution.core.dataset_builder import SyntheticDatasetBuilder, EvalDataset, GoldenDatasetLoader +from evolution.core.errors import ( + EXIT_EMFILE, + EMFILE_HINT, + classify_error, + is_emfile, +) from evolution.core.external_importers import build_dataset_from_external from evolution.core.fitness import skill_fitness_metric, LLMJudge, FitnessScore from evolution.core.constraints import ConstraintValidator @@ -33,6 +43,354 @@ console = Console() +def run_holdout_evaluation( + baseline_module, + optimized_module, + holdout_examples, + metric, + lm=None, + samples: int = 3, +) -> tuple[list[float], list[float]]: + """Score baseline vs evolved module on the holdout set. + + Multi-sample aggregation: each example is scored ``samples`` times and the + median is used (smooths LLM stochasticity). ``lm`` is applied via + dspy.context when provided (real LLM runs); None is for tests/fake modules. + Returns (baseline_scores, evolved_scores) — one median score per example. + """ + baseline_scores = [] + evolved_scores = [] + for ex in holdout_examples: + ex_baseline_scores = [] + ex_evolved_scores = [] + for _ in range(samples): + # Fresh context manager per iteration: _GeneratorContextManager is + # one-shot (its __enter__ deletes self.args in Python >=3.14), so + # reusing a single `cm` across `with` blocks raises AttributeError. + cm = dspy.context(lm=lm) if lm is not None else nullcontext() + with cm: + baseline_pred = baseline_module(task_input=ex.task_input) + ex_baseline_scores.append(metric(ex, baseline_pred)) + + evolved_pred = optimized_module(task_input=ex.task_input) + ex_evolved_scores.append(metric(ex, evolved_pred)) + + # Use median (sorted middle) for robustness + ex_baseline_scores.sort() + ex_evolved_scores.sort() + baseline_scores.append(ex_baseline_scores[len(ex_baseline_scores) // 2]) + evolved_scores.append(ex_evolved_scores[len(ex_evolved_scores) // 2]) + return baseline_scores, evolved_scores + + +def write_text_guarded(path, text: str, label: str) -> None: + """Write text, converting FD exhaustion into an explicit, classified exit. + + ``path`` is any object with ``.write_text`` (Path or a test fake). + + The 'error 1' escalations were bare ``OSError: [Errno 24] Too many open + files`` crashes on artifact writes (and the holdout loop) that surfaced as + exit 1 and got mislabeled "LLM provider issue" by cron-evolve.sh. EMFILE + now exits 2 with a hint; other OS errors warn and continue. + """ + try: + path.write_text(text) + except OSError as e: + if is_emfile(e): + console.print(f"[red]✗ Could not write {label} ({path}): {EMFILE_HINT}[/red]") + sys.exit(EXIT_EMFILE) + console.print(f"[yellow]⚠ Could not write {label} ({path}): {e}[/yellow]") + + +def estimate_improvement( + baseline_module, + optimized_module, + valset_examples, + metric, + lm=None, + samples: int = 3, + max_examples: int = 12, +) -> float: + """Estimate (evolved − baseline) score delta on the valset. + + Used only when the growth gate trips, to decide whether the extra size + buys material quality (see config.growth_waiver_min_improvement). Runs + with the same deterministic setup as the holdout eval. The example count + is capped (default 12) so the estimate stays inside the run's wall-clock + budget; calls are usually cache hits from the GEPA run in the recurring + deterministic case. + """ + baseline_scores, evolved_scores = run_holdout_evaluation( + baseline_module, + optimized_module, + valset_examples[:max_examples], + metric, + lm=lm, + samples=samples, + ) + avg_baseline = sum(baseline_scores) / max(1, len(baseline_scores)) + avg_evolved = sum(evolved_scores) / max(1, len(evolved_scores)) + return avg_evolved - avg_baseline + +# ── Timeout budget (t_316c92c4) ────────────────────────────────────────── +# The cron harness wraps evolution in `gtimeout 480` (cron-evolve.sh) and +# SIGKILLs the process on expiry — any progress is lost. To fit inside that +# budget we (a) bound every LLM call with a per-call timeout + retry cap, +# (b) guard the holdout phase with an in-process wall-clock budget so it +# degrades to a PARTIAL result instead of a kill, and (c) cache holdout +# scores across runs so warm runs skip the 144-call holdout phase entirely. +DEFAULT_MAX_BUDGET_SECONDS = 480 # must match cron-evolve.sh `gtimeout 480` +DEFAULT_LLM_TIMEOUT_SECONDS = 60 # per-attempt; litellm default is 600s +DEFAULT_LLM_RETRIES = 2 # per-call; dspy default is 8 + + +def make_lm(model: str) -> dspy.LM: + """Build a dspy.LM with a bounded per-call timeout and retry budget. + + dspy/litellm defaults allow a single stalled call to hang for 600s+ with + 8 retries — longer than the entire 480s evolution budget. A stalled call + must instead raise quickly so the harness can classify it (API error) + rather than burn the budget and get SIGKILLed with no output. + """ + return dspy.LM( + model, + timeout=DEFAULT_LLM_TIMEOUT_SECONDS, # passed through to litellm + num_retries=DEFAULT_LLM_RETRIES, # dspy-level retry cap + ) + + +def _holdout_cache_key(model: str, skill_hash: str, program: str, + task_input: str, sample_idx: int) -> str: + """Stable cache key for one holdout score. + + Includes the skill-body hash so a skill-text change invalidates holdout + scores (same semantics as the ~/.dspy_cache key), and the model name so + switching eval models never reuses stale scores. + """ + raw = f"{model}|{skill_hash}|{program}|{task_input}|{sample_idx}" + return hashlib.md5(raw.encode("utf-8")).hexdigest() + + +def evaluate_holdout( + holdout_examples, + baseline_module, + optimized_module, + *, + lm: Optional[dspy.LM] = None, + samples: int = 3, + metric=skill_fitness_metric, + max_budget_seconds: Optional[float] = None, + score_cache: Optional[dict] = None, + cache_key=None, +) -> dict: + """Score baseline vs optimized on the holdout set within a wall-clock budget. + + Replaces the previous unbounded holdout loop (24 examples x 3 samples x + 2 programs = 144 LLM calls, the last and largest phase of a run). When + ``max_budget_seconds`` is set, the loop stops early and returns the scores + gathered so far with ``budget_exceeded=True`` — a PARTIAL result the + caller can report instead of the gtimeout wrapper killing the process. + + When ``score_cache`` + ``cache_key`` are provided, scores are memoized per + (program, example, sample): warm runs with an unchanged (model, skill) + pair make zero LLM calls in this phase. + + Returns dict with baseline_scores / evolved_scores (median-aggregated per + example, complete examples only), examples_evaluated / examples_total, + budget_exceeded, cache_hits, calls_made. + """ + score_cache = score_cache if score_cache is not None else {} + results = {"baseline": [], "evolved": []} + cache_hits = 0 + calls_made = 0 + budget_exceeded = False + evaluated = 0 + start = time.monotonic() + + def _remaining() -> float: + if max_budget_seconds is None: + return float("inf") + return max_budget_seconds - (time.monotonic() - start) + + for ex in holdout_examples: + for program, module in (("baseline", baseline_module), ("evolved", optimized_module)): + sample_scores = [] + for sample_i in range(samples): + if _remaining() <= 0: + budget_exceeded = True + break + key = cache_key(ex, program, sample_i) if cache_key else None + if key is not None and key in score_cache: + sample_scores.append(score_cache[key]) + cache_hits += 1 + continue + if lm is not None: + with dspy.context(lm=lm): + pred = module(task_input=ex.task_input) + else: + pred = module(task_input=ex.task_input) + sample_scores.append(metric(ex, pred)) + calls_made += 1 + if key is not None: + score_cache[key] = sample_scores[-1] + if budget_exceeded: + break + # Median aggregation (smooths LLM stochasticity; baseline variance + # was 5x larger than plausible improvements with 1 sample). + sample_scores.sort() + results[program].append(sample_scores[len(sample_scores) // 2]) + if budget_exceeded: + break + evaluated += 1 + + return { + "baseline_scores": results["baseline"], + "evolved_scores": results["evolved"], + "examples_evaluated": evaluated, + "examples_total": len(holdout_examples), + "budget_exceeded": budget_exceeded, + "cache_hits": cache_hits, + "calls_made": calls_made, + } + + +# ── In-process budget watchdog (t_316c92c4) ────────────────────────────── +# The cron harness wraps evolution in `gtimeout 480` (cron-evolve.sh) and +# SIGKILLs on expiry with NO output — verified failure: a cold-cache GEPA +# phase consumed 439s of the 480s budget (141 rollouts x 4.14s), leaving the +# holdout phase 71s and the process killed mid-call. GEPA's iteration loop is +# not hookable, so the only structural guards are (a) this watchdog, which +# exits just BEFORE gtimeout with a partial checkpoint and the SAME exit code +# (124) so the cron harness auto-retries (and ~/.dspy_cache makes the retry +# warm/fast), and (b) holdout skip + headroom below. +BUDGET_EXIT_CODE = 124 # same as gtimeout: cron auto-retries +WATCHDOG_LEAD_SECONDS = 10 # exit before gtimeout fires, checkpoint first +HOLDOUT_SKIP_THRESHOLD_SECONDS = 60 # below this remaining budget: skip holdout +HOLDOUT_HEADROOM_SECONDS = 30 # reserve for metrics/report write after holdout + + +def write_checkpoint(checkpoint: dict, path: Path) -> None: + """Persist the budget checkpoint dict (best-effort; never crash the run).""" + try: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(checkpoint, indent=2)) + except OSError as e: + console.print(f"[yellow]⚠ Could not write budget checkpoint: {e}[/yellow]") + + +def handle_holdout_skip( + skill: dict, + evolved_body: str, + evolved_full: str, + holdout_examples: list, + remaining: float, +): + """Budget-guard holdout skip (remaining < HOLDOUT_SKIP_THRESHOLD_SECONDS). + + Returns ``(holdout_result, avg_baseline, avg_evolved, improvement, + evolved_body, evolved_full, candidate_full)``. + + Two invariants (t_ac619234): + 1. GEPA's candidate is preserved in ``candidate_full`` BEFORE the baseline + overwrite — the skip path used to destroy it by copying the baseline + over it, making the run unreproducible for a later budget-raised retry. + 2. The deployed artifact stays the baseline text (byte-identical → + has_diff=0 downstream), mirroring the 8b regression guard. The avg + scores are 0.0 PLACEHOLDERS — callers must render them as N/A (null in + metrics.json) whenever ``holdout_result["skipped"]`` is True. + """ + holdout_result = { + "baseline_scores": [], "evolved_scores": [], + "examples_evaluated": 0, "examples_total": len(holdout_examples), + "budget_exceeded": True, "cache_hits": 0, "calls_made": 0, + "skipped": True, + } + candidate_full = evolved_full + # Mirror the 8b guard: keep the original skill text (no evidence to deploy). + evolved_body = skill["body"] + evolved_full = skill["raw"] + return ( + holdout_result, 0.0, 0.0, 0.0, + evolved_body, evolved_full, candidate_full, + ) + + +def build_metrics( + skill_name: str, + timestamp: str, + iterations: int, + optimizer_model: str, + eval_model: str, + avg_baseline: float, + avg_evolved: float, + improvement: float, + baseline_size: int, + evolved_size: int, + train_examples: int, + val_examples: int, + holdout_examples: int, + elapsed: float, + all_pass: bool, + holdout_result: dict, + candidate_full: Optional[str], + max_budget: int, +) -> dict: + """Build the metrics dict written to metrics.json. + + A skipped holdout is a NON-measurement: baseline_score / evolved_score / + improvement are written as null (N/A downstream), never 0.0 — a fabricated + "+0.000 (+0.0%)" must never be misread as "no improvement" (t_ac619234). + """ + holdout_skipped = holdout_result.get("skipped", False) + return { + "skill_name": skill_name, + "timestamp": timestamp, + "iterations": iterations, + "optimizer_model": optimizer_model, + "eval_model": eval_model, + "baseline_score": None if holdout_skipped else avg_baseline, + "evolved_score": None if holdout_skipped else avg_evolved, + "improvement": None if holdout_skipped else improvement, + "baseline_size": baseline_size, + "evolved_size": evolved_size, + "train_examples": train_examples, + "val_examples": val_examples, + "holdout_examples": holdout_examples, + "elapsed_seconds": elapsed, + "constraints_passed": all_pass, + "holdout_complete": not holdout_result["budget_exceeded"], + "holdout_skipped": holdout_skipped, + "holdout_candidate_preserved": holdout_skipped and bool(candidate_full), + "holdout_examples_evaluated": holdout_result["examples_evaluated"], + "holdout_examples_total": holdout_result["examples_total"], + "holdout_cache_hits": holdout_result["cache_hits"], + "holdout_calls_made": holdout_result["calls_made"], + "max_budget_seconds": max_budget, + } + + +def start_budget_watchdog(max_budget: int, checkpoint: dict, checkpoint_path: Path) -> threading.Thread: + """Daemon that terminates the process at max_budget with a partial checkpoint. + + Exits at ``max_budget - WATCHDOG_LEAD_SECONDS`` with exit code + ``BUDGET_EXIT_CODE`` (124, matching gtimeout) so cron-evolve.sh treats it + as a timeout and auto-retries — but the checkpoint and the incremental + ~/.dspy_cache writes survive, making the retry warm and fast instead of a + zero-output SIGKILL. The checkpoint dict is updated by evolve() at phase + boundaries (dataset / gepa / validate / holdout). + """ + + def _watch() -> None: + time.sleep(max(1, max_budget - WATCHDOG_LEAD_SECONDS)) + checkpoint["timed_out"] = True + write_checkpoint(checkpoint, checkpoint_path) + os._exit(BUDGET_EXIT_CODE) + + t = threading.Thread(target=_watch, daemon=True, name="budget-watchdog") + t.start() + return t + + def evolve( skill_name: str, iterations: int = 10, @@ -43,8 +401,25 @@ def evolve( hermes_repo: Optional[str] = None, run_tests: bool = False, dry_run: bool = False, + seed: Optional[int] = None, + max_budget: int = DEFAULT_MAX_BUDGET_SECONDS, + holdout_samples: int = 3, ): """Main evolution function — orchestrates the full optimization loop.""" + _seed = seed if seed is not None else 42 + run_start = time.time() + + # Budget watchdog: exits 10s before the cron gtimeout with a partial + # checkpoint and exit 124 (timeout-compatible → cron auto-retries). + checkpoint_path = Path("output") / skill_name / "budget_checkpoint.json" + checkpoint = { + "skill": skill_name, + "phase": "start", + "started_at": datetime.now().isoformat(), + "max_budget_seconds": max_budget, + "timed_out": False, + } + start_budget_watchdog(max_budget, checkpoint, checkpoint_path) config = EvolutionConfig( iterations=iterations, @@ -53,6 +428,8 @@ def evolve( judge_model=eval_model, # Use same model for dataset generation run_pytest=run_tests, ) + if seed is not None: + config.random_seed = seed if hermes_repo: config.hermes_agent_path = Path(hermes_repo) @@ -98,14 +475,21 @@ def evolve( console.print(f" Mined {len(dataset.all_examples)} examples from session history") elif eval_source == "synthetic": builder = SyntheticDatasetBuilder(config) - dataset = builder.generate( - artifact_text=skill["raw"], - artifact_type="skill", - ) - # Save for reuse save_path = Path("datasets") / "skills" / skill_name - dataset.save(save_path) - console.print(f" Generated {len(dataset.all_examples)} synthetic examples") + + def _generate(): + return builder.generate( + artifact_text=skill["raw"], + artifact_type="skill", + seed=_seed, + ) + + dataset = EvalDataset.load_or_generate( + path=save_path, + generator_fn=_generate, + seed=_seed, + ) + console.print(f" Dataset: {len(dataset.all_examples)} examples (source: synthetic, seed={_seed})") console.print(f" Saved to {save_path}/") elif dataset_path: dataset = EvalDataset.load(Path(dataset_path)) @@ -119,7 +503,11 @@ def evolve( # ── 3. Validate constraints on baseline ───────────────────────────── console.print(f"\n[bold]Validating baseline constraints[/bold]") validator = ConstraintValidator(config) - baseline_constraints = validator.validate_all(skill["body"], "skill") + # Validate the FULL artifact (frontmatter + body), not just the body — + # the body-only check made skill_structure fail on every baseline + # (false negative, harmless "proceeding anyway" warning) and was + # asymmetric with the evolved check below. + baseline_constraints = validator.validate_all(skill["raw"], "skill") all_pass = True for c in baseline_constraints: icon = "✓" if c.passed else "✗" @@ -137,8 +525,10 @@ def evolve( console.print(f" Optimizer model: {optimizer_model}") console.print(f" Eval model: {eval_model}") - # Configure DSPy - lm = dspy.LM(eval_model) + # Configure DSPy — bounded per-call timeout/retries so a single stalled + # API call can't blow the whole 480s budget (litellm default timeout is + # 600s; dspy default num_retries is 8). + lm = make_lm(eval_model) dspy.configure(lm=lm) # Create the baseline skill module @@ -152,17 +542,23 @@ def evolve( console.print(f"\n[bold cyan]Running GEPA optimization ({iterations} iterations)...[/bold cyan]\n") start_time = time.time() + checkpoint["phase"] = "gepa" + write_checkpoint(checkpoint, checkpoint_path) + + # GEPA requires 5-arg metric: (gold, pred, trace, pred_name, pred_trace) + def gepa_metric(gold, pred, trace=None, pred_name=None, pred_trace=None): + return skill_fitness_metric(gold, pred, trace) try: optimizer = dspy.GEPA( - metric=skill_fitness_metric, - max_steps=iterations, + metric=gepa_metric, + reflection_lm=lm, + max_full_evals=iterations, ) optimized_module = optimizer.compile( baseline_module, trainset=trainset, - valset=valset, ) except Exception as e: # Fall back to MIPROv2 if GEPA isn't available in this DSPy version @@ -178,15 +574,32 @@ def evolve( elapsed = time.time() - start_time console.print(f"\n Optimization completed in {elapsed:.1f}s") + checkpoint["phase"] = "validate" + checkpoint["elapsed_seconds"] = round(time.time() - run_start, 1) + write_checkpoint(checkpoint, checkpoint_path) # ── 6. Extract evolved skill text ─────────────────────────────────── # The optimized module's instructions contain the evolved skill text + # (May be overridden by the baseline-score guard in step 8b if the + # optimizer produced params worse than the default baseline) evolved_body = optimized_module.skill_text evolved_full = reassemble_skill(skill["frontmatter"], evolved_body) # ── 7. Validate evolved skill ─────────────────────────────────────── console.print(f"\n[bold]Validating evolved skill[/bold]") - evolved_constraints = validator.validate_all(evolved_body, "skill", baseline_text=skill["body"]) + # Validate the full artifact (frontmatter + body), always against the + # baseline BODY. (The old conditional compared the evolved artifact to + # itself when frontmatter was missing — a false pass.) The growth gate + # compares BODY-vs-BODY via growth_text: the static frontmatter must not + # inflate the growth ratio, or a real improvement gets rejected just + # past the hard cap (gmail-monitor: +100.5% full-vs-body vs +69.8% + # body-vs-body, hard cap +100%). + evolved_constraints = validator.validate_all( + evolved_full, + "skill", + baseline_text=skill["body"], + growth_text=evolved_body, + ) all_pass = True for c in evolved_constraints: icon = "✓" if c.passed else "✗" @@ -195,36 +608,185 @@ def evolve( if not c.passed: all_pass = False + # Growth-only failure → quality waiver check. The growth gate rejects + # bloated candidates, but a candidate that grows past the soft cap may + # still be worth deploying when the extra size buys a material valset + # improvement (config.growth_waiver_min_improvement) and stays under the + # hard cap. Measure the improvement, then re-validate with it. + if not all_pass: + failed = [c for c in evolved_constraints if not c.passed] + growth_failed = [c for c in failed if c.constraint_name == "growth_limit"] + if len(failed) == 1 and len(growth_failed) == 1: + console.print("[yellow] Growth gate exceeded — measuring valset improvement for a waiver...[/yellow]") + improvement_est = estimate_improvement( + baseline_module, + optimized_module, + valset, + skill_fitness_metric, + lm=lm, + ) + console.print( + f" Valset improvement vs baseline: {improvement_est:+.3f} " + f"(waiver needs >= {config.growth_waiver_min_improvement:+.3f})" + ) + evolved_constraints = validator.validate_all( + evolved_full, + "skill", + baseline_text=skill["body"], + growth_text=evolved_body, + improvement=improvement_est, + ) + all_pass = True + for c in evolved_constraints: + icon = "✓" if c.passed else "✗" + color = "green" if c.passed else "red" + console.print(f" [{color}]{icon} {c.constraint_name}[/{color}]: {c.message}") + if not c.passed: + all_pass = False + if not all_pass: console.print("[red]✗ Evolved skill FAILED constraints — not deploying[/red]") - # Still save for inspection + # Still save for inspection (FD-exhaustion-safe write: the original + # 'error 1' crashed here on OSError 24 and got mislabeled as an API + # error by the cron wrapper). Write BOTH a top-level marker AND a + # timestamped dir containing the baseline + failed variant so the + # cron pipeline can diff the CURRENT run's artifacts. Previously the + # failure path wrote only evolved_FAILED.md, so cron-evolve.sh's + # find fell back to the stale previous run's identical + # baseline/evolved pair → has_diff=0 even though the evolved text + # genuinely differs from baseline (gmail-monitor ticket). output_path = Path("output") / skill_name / "evolved_FAILED.md" output_path.parent.mkdir(parents=True, exist_ok=True) - output_path.write_text(evolved_full) + write_text_guarded(output_path, evolved_full, "failed variant") console.print(f" Saved failed variant to {output_path}") - return - # ── 8. Evaluate on holdout set ────────────────────────────────────── - console.print(f"\n[bold]Evaluating on holdout set ({len(dataset.holdout)} examples)[/bold]") + fail_ts = datetime.now().strftime("%Y%m%d_%H%M%S") + fail_dir = Path("output") / skill_name / fail_ts + fail_dir.mkdir(parents=True, exist_ok=True) + write_text_guarded(fail_dir / "baseline_skill.md", skill["raw"], "baseline skill") + write_text_guarded(fail_dir / "evolved_FAILED.md", evolved_full, "failed variant") + write_text_guarded( + fail_dir / "metrics.json", + json.dumps({ + "skill_name": skill_name, + "timestamp": fail_ts, + "iterations": iterations, + "optimizer_model": optimizer_model, + "eval_model": eval_model, + "constraints_passed": False, + "baseline_size": len(skill["body"]), + "evolved_size": len(evolved_body), + "elapsed_seconds": round(time.time() - run_start, 1), + }, indent=2), + "failure metrics", + ) + console.print(f" Saved failed variant + baseline to {fail_dir}/") + return + # ── 8. Evaluate on holdout set (budget-guarded + score-cached) ────── holdout_examples = dataset.to_dspy_examples("holdout") + remaining = max_budget - (time.time() - run_start) + checkpoint["phase"] = "holdout" + checkpoint["remaining_seconds"] = round(remaining, 1) + write_checkpoint(checkpoint, checkpoint_path) + + # candidate_full is set only by handle_holdout_skip (budget guard); it is + # None on the normal path where the 8b guard keeps GEPA's candidate. + candidate_full = None + + if remaining < HOLDOUT_SKIP_THRESHOLD_SECONDS: + # Not enough budget left for a meaningful holdout (cold-cache cost is + # ~2 x samples x 1.5s per example). Finish cleanly: keep the baseline + # and report a partial (skipped) result instead of dying at 480s. + console.print( + f"\n[yellow]⚠ Only {remaining:.0f}s of the {max_budget}s budget remains — " + f"skipping holdout eval to finish cleanly. Keeping baseline; no " + f"improvement claim made.[/yellow]" + ) + (holdout_result, avg_baseline, avg_evolved, improvement, + evolved_body, evolved_full, candidate_full) = handle_holdout_skip( + skill, evolved_body, evolved_full, holdout_examples, remaining) + baseline_scores = holdout_result["baseline_scores"] + evolved_scores = holdout_result["evolved_scores"] + else: + holdout_budget = max(0.0, remaining - HOLDOUT_HEADROOM_SECONDS) + console.print(f"\n[bold]Evaluating on holdout set ({len(holdout_examples)} examples, {holdout_samples} samples each, {holdout_budget:.0f}s budget)[/bold]") + + # Persistent score cache (checkpointing for the 480s budget): keyed by + # (eval model, skill-body hash, program, example, sample) so warm runs + # skip the 144-call holdout phase entirely. The skill hash invalidates + # the cache when the skill text changes (mirrors ~/.dspy_cache semantics). + cache_path = Path("output") / skill_name / "holdout_scores.json" + cache_path.parent.mkdir(parents=True, exist_ok=True) + score_cache: dict = {} + if cache_path.exists(): + try: + score_cache = json.loads(cache_path.read_text()) + except (json.JSONDecodeError, OSError): + score_cache = {} + skill_hash = hashlib.md5(skill["body"].encode("utf-8")).hexdigest()[:12] + cache_key_fn = lambda ex, program, sample_i: _holdout_cache_key( + eval_model, skill_hash, program, + getattr(ex, "task_input", "") or "", sample_i, + ) - baseline_scores = [] - evolved_scores = [] - for ex in holdout_examples: - # Score baseline - with dspy.context(lm=lm): - baseline_pred = baseline_module(task_input=ex.task_input) - baseline_score = skill_fitness_metric(ex, baseline_pred) - baseline_scores.append(baseline_score) - - evolved_pred = optimized_module(task_input=ex.task_input) - evolved_score = skill_fitness_metric(ex, evolved_pred) - evolved_scores.append(evolved_score) - - avg_baseline = sum(baseline_scores) / max(1, len(baseline_scores)) - avg_evolved = sum(evolved_scores) / max(1, len(evolved_scores)) - improvement = avg_evolved - avg_baseline + holdout_result = None + try: + holdout_result = evaluate_holdout( + holdout_examples, + baseline_module, + optimized_module, + lm=lm, + samples=holdout_samples, + metric=skill_fitness_metric, + max_budget_seconds=holdout_budget, + score_cache=score_cache, + cache_key=cache_key_fn, + ) + except SystemExit: + raise + except Exception as e: + # The 'error 1' EMFILE crash hit this phase. Classify explicitly: + # FD exhaustion → exit 2 with a hint; anything else → eval error. + code, msg = classify_error(e) + console.print(f"[red]✗ Holdout evaluation failed: {msg}[/red]") + sys.exit(code) + + # Persist the score cache; a failure here must not kill an otherwise + # successful run (worst case the next run pays the calls again). + try: + cache_path.write_text(json.dumps(score_cache)) + except OSError as e: + console.print(f"[yellow]⚠ Could not persist holdout score cache: {e}[/yellow]") + + baseline_scores = holdout_result["baseline_scores"] + evolved_scores = holdout_result["evolved_scores"] + if holdout_result["budget_exceeded"]: + console.print( + f"[yellow]⚠ Holdout budget exhausted — returning PARTIAL result " + f"({holdout_result['examples_evaluated']}/{holdout_result['examples_total']} " + f"examples; {holdout_result['cache_hits']} cache hits, " + f"{holdout_result['calls_made']} live calls)[/yellow]" + ) + + avg_baseline = sum(baseline_scores) / max(1, len(baseline_scores)) + avg_evolved = sum(evolved_scores) / max(1, len(evolved_scores)) + improvement = avg_evolved - avg_baseline + + # ── 8b. Baseline-score guard — never deploy params worse than baseline ── + # Root cause: GEPA/MIPROv2 with few iterations (default 5) can converge to + # a local minimum worse than the default DSPy parameters, causing regression. + # This guard discards the optimizer's output when it underperforms the baseline, + # keeping the original skill text as the "evolved" artifact. + if improvement < 0: + console.print(f"\n[yellow]⚠ Optimizer regressed ({improvement:+.3f}) — keeping baseline instead of deploying worse params[/yellow]") + evolved_body = skill["body"] + evolved_full = skill["raw"] + # Update the metrics to reflect we kept the baseline + avg_evolved = avg_baseline + improvement = 0.0 + console.print(" → Saved original skill text as the deployed artifact (no regression deployed)") + # else: evolved_body/evolved_full already set in step 6 above # ── 9. Report results ─────────────────────────────────────────────── table = Table(title="Evolution Results") @@ -233,13 +795,20 @@ def evolve( table.add_column("Evolved", justify="right") table.add_column("Change", justify="right") - change_color = "green" if improvement > 0 else "red" - table.add_row( - "Holdout Score", - f"{avg_baseline:.3f}", - f"{avg_evolved:.3f}", - f"[{change_color}]{improvement:+.3f}[/{change_color}]", - ) + if holdout_result.get("skipped"): + # Skipped holdout = NO measurement taken. Render N/A — a fabricated + # "+0.000 (+0.0%)" must never be misread as "no improvement" + # (t_ac619234). cron-evolve.sh's parse_score regex finds no digits + # here and falls back to N/A as well. + table.add_row("Holdout Score", "N/A", "N/A", "—") + else: + change_color = "green" if improvement > 0 else "red" + table.add_row( + "Holdout Score", + f"{avg_baseline:.3f}", + f"{avg_evolved:.3f}", + f"[{change_color}]{improvement:+.3f}[/{change_color}]", + ) table.add_row( "Skill Size", f"{len(skill['body']):,} chars", @@ -257,35 +826,30 @@ def evolve( output_dir = Path("output") / skill_name / timestamp output_dir.mkdir(parents=True, exist_ok=True) - # Save evolved skill - (output_dir / "evolved_skill.md").write_text(evolved_full) - - # Save baseline for comparison - (output_dir / "baseline_skill.md").write_text(skill["raw"]) + # Save evolved skill / baseline / metrics (FD-exhaustion-safe writes) + write_text_guarded(output_dir / "evolved_skill.md", evolved_full, "evolved skill") + write_text_guarded(output_dir / "baseline_skill.md", skill["raw"], "baseline skill") + if holdout_result.get("skipped") and candidate_full: + # The skip path overwrote evolved_skill.md with the baseline copy — + # write GEPA's real candidate alongside it so a later budget-raised + # re-run can diff/reuse it (t_ac619234). + write_text_guarded(output_dir / "evolved_candidate.md", candidate_full, "evolved candidate (holdout skipped)") # Save metrics - metrics = { - "skill_name": skill_name, - "timestamp": timestamp, - "iterations": iterations, - "optimizer_model": optimizer_model, - "eval_model": eval_model, - "baseline_score": avg_baseline, - "evolved_score": avg_evolved, - "improvement": improvement, - "baseline_size": len(skill["body"]), - "evolved_size": len(evolved_body), - "train_examples": len(dataset.train), - "val_examples": len(dataset.val), - "holdout_examples": len(dataset.holdout), - "elapsed_seconds": elapsed, - "constraints_passed": all_pass, - } - (output_dir / "metrics.json").write_text(json.dumps(metrics, indent=2)) + metrics = build_metrics( + skill_name, timestamp, iterations, optimizer_model, eval_model, + avg_baseline, avg_evolved, improvement, + len(skill["body"]), len(evolved_body), + len(dataset.train), len(dataset.val), len(dataset.holdout), + elapsed, all_pass, holdout_result, candidate_full, max_budget, + ) + write_text_guarded(output_dir / "metrics.json", json.dumps(metrics, indent=2), "metrics") console.print(f"\n Output saved to {output_dir}/") - if improvement > 0: + if holdout_result.get("skipped"): + console.print("\n[yellow]⚠ Holdout skipped — no score measured (budget guard). GEPA candidate preserved to evolved_candidate.md[/yellow]") + elif improvement > 0: console.print(f"\n[bold green]✓ Evolution improved skill by {improvement:+.3f} ({improvement/max(0.001, avg_baseline)*100:+.1f}%)[/bold green]") console.print(f" Review the diff: diff {output_dir}/baseline_skill.md {output_dir}/evolved_skill.md") else: @@ -304,7 +868,12 @@ def evolve( @click.option("--hermes-repo", default=None, help="Path to hermes-agent repo") @click.option("--run-tests", is_flag=True, help="Run full pytest suite as constraint gate") @click.option("--dry-run", is_flag=True, help="Validate setup without running optimization") -def main(skill, iterations, eval_source, dataset_path, optimizer_model, eval_model, hermes_repo, run_tests, dry_run): +@click.option("--seed", default=None, type=int, help="Random seed for reproducible dataset splits (default: 42)") +@click.option("--max-budget", default=DEFAULT_MAX_BUDGET_SECONDS, type=int, + help=f"Hard wall-clock budget in seconds (default {DEFAULT_MAX_BUDGET_SECONDS}; must match the cron gtimeout wrapper)") +@click.option("--holdout-samples", default=3, type=int, + help="Samples per holdout example, median-aggregated (default 3)") +def main(skill, iterations, eval_source, dataset_path, optimizer_model, eval_model, hermes_repo, run_tests, dry_run, seed, max_budget, holdout_samples): """Evolve a Hermes Agent skill using DSPy + GEPA optimization.""" evolve( skill_name=skill, @@ -316,6 +885,9 @@ def main(skill, iterations, eval_source, dataset_path, optimizer_model, eval_mod hermes_repo=hermes_repo, run_tests=run_tests, dry_run=dry_run, + seed=seed, + max_budget=max_budget, + holdout_samples=holdout_samples, ) diff --git a/tests/core/test_constraints.py b/tests/core/test_constraints.py index 88e3aaa3..406a85b7 100644 --- a/tests/core/test_constraints.py +++ b/tests/core/test_constraints.py @@ -1,13 +1,17 @@ """Tests for constraint validators.""" import pytest +from pathlib import Path from evolution.core.constraints import ConstraintValidator from evolution.core.config import EvolutionConfig @pytest.fixture def validator(): - config = EvolutionConfig() + # Explicit repo path — constraint tests don't need repo discovery and + # shouldn't fail when HERMES_AGENT_REPO is unset (get_hermes_agent_path() + # raises FileNotFoundError in that case). + config = EvolutionConfig(hermes_agent_path=Path("/tmp/hermes-agent")) return ConstraintValidator(config) @@ -17,10 +21,15 @@ def test_skill_under_limit(self, validator): assert result.passed def test_skill_over_limit(self, validator): - result = validator._check_size("x" * 20_000, "skill") + result = validator._check_size("x" * 20_001, "skill") assert not result.passed assert "exceeded" in result.message + def test_skill_at_exact_limit_passes(self, validator): + # Boundary: size == limit is acceptable, only > limit fails + result = validator._check_size("x" * 20_000, "skill") + assert result.passed + def test_tool_description_under_limit(self, validator): result = validator._check_size("Search files by content", "tool_description") assert result.passed @@ -39,9 +48,17 @@ def test_acceptable_growth(self, validator): def test_excessive_growth(self, validator): baseline = "x" * 1000 - evolved = "x" * 1300 # 30% growth + evolved = "x" * 1700 # 70% growth — over the 50% soft cap, no waiver result = validator._check_growth(evolved, baseline, "skill") assert not result.passed + assert "Growth exceeded" in result.message + + def test_exact_boundary_passes(self, validator): + # Exactly +50% == max_prompt_growth — boundary must pass + baseline = "x" * 1000 + evolved = "x" * 1500 + result = validator._check_growth(evolved, baseline, "skill") + assert result.passed def test_shrinkage_is_ok(self, validator): baseline = "x" * 1000 @@ -49,6 +66,107 @@ def test_shrinkage_is_ok(self, validator): result = validator._check_growth(evolved, baseline, "skill") assert result.passed + def test_zero_length_baseline(self, validator): + # No ZeroDivisionError — denominator is max(1, 0) + result = validator._check_growth("x" * 10, "", "skill") + assert not result.passed # +1000% growth + result = validator._check_growth("", "", "skill") + assert result.passed # no growth + + def test_growth_waiver_grants_material_improvement(self, validator): + baseline = "x" * 1000 + evolved = "x" * 1700 # +70% > 50% soft cap + result = validator._check_growth(evolved, baseline, "skill", improvement=0.05) + assert result.passed + assert "waiver" in result.message + + def test_growth_waiver_insufficient_improvement(self, validator): + baseline = "x" * 1000 + evolved = "x" * 1700 # +70% > 50% soft cap + result = validator._check_growth(evolved, baseline, "skill", improvement=0.01) + assert not result.passed + assert "waiver threshold" in result.message + + def test_growth_waiver_rejected_beyond_hard_cap(self, validator): + baseline = "x" * 1000 + evolved = "x" * 2500 # +150% — over the 100% hard cap even with waiver + result = validator._check_growth(evolved, baseline, "skill", improvement=0.05) + assert not result.passed + + def test_growth_waiver_hard_cap_message_is_accurate(self, validator): + # Regression (t_f8e3ec7f): when the waiver IS applied (improvement >= + # threshold) but growth still exceeds the hard cap, the failure message + # must report the hard-cap breach — not claim "improvement < waiver + # threshold" (which was false: e.g. gmail-monitor +0.065 improvement + # with +115.6% growth was reported as "0.065 < 0.030"). + baseline = "x" * 1000 + evolved = "x" * 2500 # +150% — over the hard cap, waiver cannot save it + result = validator._check_growth(evolved, baseline, "skill", improvement=0.05) + assert not result.passed + assert "waiver applied" in result.message + assert "hard cap" in result.message + assert "waiver threshold" not in result.message + + def test_growth_without_improvement_stays_strict(self, validator): + # Backward compat: improvement=None must behave exactly like before + baseline = "x" * 1000 + evolved = "x" * 1700 # +70% + result = validator._check_growth(evolved, baseline, "skill", improvement=None) + assert not result.passed + + def test_escalation_exact_failure_mode(self, validator): + # esc-20260731-201857-1474 exact numbers: baseline 8,395 chars, + # rejected artifact 14,382 chars (+71.3% > 50% soft cap), measured + # valset improvement +0.035 (0.558 → 0.593). Without the improvement + # signal the gate rejects (the recorded 'constraints FAILED'); with it + # the waiver grants (stays under the +100% hard cap). + baseline = "x" * 8395 + evolved = "x" * 14382 + assert not validator._check_growth(evolved, baseline, "skill").passed + result = validator._check_growth(evolved, baseline, "skill", improvement=0.035) + assert result.passed + assert "waiver" in result.message + + def test_growth_text_compares_body_not_full_artifact(self, validator): + # gmail-monitor regression (t_63240783): the growth gate compared the + # evolved FULL artifact (frontmatter + body) against the baseline BODY + # only → +100.5% > +100% hard cap → rejected a real improvement by + # 0.5%. Comparing body-vs-body is +69.8%, under the hard cap, and the + # +0.045 waiver improvement grants it. + baseline_body = "x" * 2719 + # Realistic frontmatter size (~836 chars for gmail-monitor): big + # enough that full-vs-body growth crosses the +100% hard cap, exactly + # like the ticket's evolved_FAILED.md (5452 raw = 836 fm + 4616 body). + frontmatter = "---\nname: gmail-monitor\ndescription: " + ("y" * 800) + "\n---\n\n" + evolved_body = "x" * 4616 # +69.8% body-vs-body + evolved_full = frontmatter + evolved_body + + # Old (buggy) behavior: full-vs-body → +100.5% → fails even with waiver + old = validator._check_growth( + evolved_full, baseline_body, "skill", improvement=0.045 + ) + assert not old.passed + assert "hard cap" in old.message or "Growth exceeded" in old.message + + # Fixed: growth_text=evolved_body → +69.8% → waiver grants (hard cap OK) + fixed = validator._check_growth( + evolved_full, baseline_body, "skill", + improvement=0.045, growth_text=evolved_body, + ) + assert fixed.passed + assert "waiver" in fixed.message + + def test_growth_text_defaults_to_artifact_text(self, validator): + # Backward compat: growth_text=None must behave exactly like before. + baseline = "x" * 1000 + evolved = "x" * 1700 + assert not validator._check_growth(evolved, baseline, "skill").passed + result = validator._check_growth( + evolved, baseline, "skill", improvement=0.05, growth_text=None + ) + assert result.passed + assert "waiver" in result.message + class TestNonEmpty: def test_non_empty_passes(self, validator): @@ -96,3 +214,18 @@ def test_empty_skill_fails(self, validator): results = validator.validate_all("", "skill") failed = [r for r in results if not r.passed] assert len(failed) > 0 + + def test_growth_failure_with_waiver_passes(self, validator): + # End-to-end validate_all with a full skill: growth past the soft cap + # passes when a material improvement is supplied. + baseline = ( + "---\nname: test\ndescription: Test skill\n---\n\n" + + "Procedure text " * 100 # ~1.5KB baseline + ) + evolved = baseline + (" extra content that grows the skill " * 30) # +66% > 50% soft cap + # no improvement → fails on growth + results = validator.validate_all(evolved, "skill", baseline_text=baseline) + assert any(not r.passed for r in results) + # material improvement → growth waived, all pass + results = validator.validate_all(evolved, "skill", baseline_text=baseline, improvement=0.05) + assert all(r.passed for r in results) diff --git a/tests/core/test_errors.py b/tests/core/test_errors.py new file mode 100644 index 00000000..12e5641f --- /dev/null +++ b/tests/core/test_errors.py @@ -0,0 +1,52 @@ +"""Tests for the evolution exit-code contract and error classification. + +Covers the 'error 1' failure mode from esc-20260731-201857-1474: the EMFILE +(OSError errno 24) crashes that previously surfaced as a generic exit 1 and +were mislabeled "LLM provider issue" by cron-evolve.sh. +""" + +import errno + +from evolution.core.errors import ( + EXIT_EMFILE, + EXIT_EVAL_ERROR, + EMFILE_HINT, + classify_error, + is_emfile, +) + + +class TestIsEmfile: + def test_oserror_errno_24(self): + assert is_emfile(OSError(errno.EMFILE, "Too many open files")) + + def test_oserror_other_errno(self): + assert not is_emfile(OSError(errno.EACCES, "Permission denied")) + + def test_wrapped_message_fallback(self): + # dspy/litellm sometimes wrap the raw OS error in a generic Exception + # carrying the errno text but no errno attribute + assert is_emfile(Exception("Too many open files (errno 24)")) + + def test_unrelated_exception(self): + assert not is_emfile(ValueError("bad value")) + + +class TestClassifyError: + def test_emfile_maps_to_exit_2_with_hint(self): + code, msg = classify_error(OSError(errno.EMFILE, "Too many open files")) + assert code == EXIT_EMFILE + assert "ulimit" in msg + assert "not an LLM/API failure" in msg + + def test_emfile_hint_is_actionable(self): + assert "ulimit -n 1024" in EMFILE_HINT + + def test_non_emfile_eval_error_maps_to_exit_3(self): + code, msg = classify_error(ValueError("bad value")) + assert code == EXIT_EVAL_ERROR + assert "bad value" in msg + + def test_oserror_other_errno_is_eval_error_not_emfile(self): + code, _ = classify_error(OSError(errno.EACCES, "Permission denied")) + assert code == EXIT_EVAL_ERROR diff --git a/tests/skills/test_evolve_guards.py b/tests/skills/test_evolve_guards.py new file mode 100644 index 00000000..b1a27573 --- /dev/null +++ b/tests/skills/test_evolve_guards.py @@ -0,0 +1,138 @@ +"""Tests for the evolve_skill error guards and holdout scoring helpers. + +Exercises the exact failure modes from esc-20260731-201857-1474: + - 'error 1': OSError errno 24 EMFILE on artifact writes and during the + holdout evaluation — must surface as an explicit exit-2 with a hint, + never a bare traceback exit-1 that cron mislabels as an API error. + - median aggregation / valset improvement estimation (growth waiver input). +""" + +import errno +import pytest + +from evolution.skills.evolve_skill import ( + estimate_improvement, + run_holdout_evaluation, + write_text_guarded, +) + + +class FakePrediction: + def __init__(self, output): + self.output = output + + +class FakeModule: + """Minimal stand-in for SkillModule: returns a fixed prediction.""" + + def __init__(self, output="ok", exc=None): + self._output = output + self._exc = exc + + def __call__(self, task_input): + if self._exc is not None: + raise self._exc + return FakePrediction(self._output) + + +class FakeExample: + def __init__(self, task_input="task"): + self.task_input = task_input + + +class TestWriteTextGuarded: + def test_emfile_exits_2_with_hint(self, capsys): + class FailingPath: + def __init__(self): + self.name = "output/web-research/evolved_FAILED.md" + + def write_text(self, text): + raise OSError(errno.EMFILE, "Too many open files") + + with pytest.raises(SystemExit) as excinfo: + write_text_guarded(FailingPath(), "content", "failed variant") + assert excinfo.value.code == 2 + out = capsys.readouterr().out + assert "Too many open files" in out + assert "ulimit" in out + + def test_other_oserror_warns_and_continues(self, capsys): + class FailingPath: + def write_text(self, text): + raise OSError(errno.EACCES, "Permission denied") + + write_text_guarded(FailingPath(), "content", "metrics") # must not raise + out = capsys.readouterr().out + assert "Could not write" in out + + def test_success_writes_quietly(self, tmp_path, capsys): + target = tmp_path / "out.md" + write_text_guarded(target, "hello", "artifact") + assert target.read_text() == "hello" + assert "Could not write" not in capsys.readouterr().out + + +class TestRunHoldoutEvaluation: + def test_median_aggregation(self): + # 3 samples per program; medians of [0.4, 0.5, 0.9] and [0.1, 0.2, 0.3] + # are 0.5 and 0.2. Interleaving order: baseline, evolved per sample. + baseline = FakeModule() + evolved = FakeModule() + scores = iter([0.4, 0.1, 0.5, 0.2, 0.9, 0.3]) + metric = lambda ex, pred: next(scores) + examples = [FakeExample()] + + base_scores, evo_scores = run_holdout_evaluation( + baseline, evolved, examples, metric, samples=3 + ) + assert base_scores == [0.5] + assert evo_scores == [0.2] + + def test_emfile_during_eval_propagates_for_classification(self): + # The holdout loop must not swallow the OSError — it propagates so the + # caller can classify it as exit 2 (this was the 'error 1' crash point) + def boom(ex, pred): + raise OSError(errno.EMFILE, "Too many open files") + + with pytest.raises(OSError) as excinfo: + run_holdout_evaluation( + FakeModule(), FakeModule(), [FakeExample()], boom, samples=1 + ) + assert excinfo.value.errno == errno.EMFILE + + +class TestEstimateImprovement: + def test_positive_improvement_detected(self): + # baseline scores 0.4, evolved scores 0.6 → +0.2 improvement + examples = [FakeExample(f"task{i}") for i in range(3)] + baseline = FakeModule(output="base") + evolved = FakeModule(output="evo") + metric = lambda ex, pred: 0.4 if pred.output == "base" else 0.6 + + improvement = estimate_improvement( + baseline, evolved, examples, metric, samples=1 + ) + assert improvement == pytest.approx(0.2) + + def test_max_examples_cap(self): + # Only the first max_examples examples are scored + examples = [FakeExample(f"task{i}") for i in range(5)] + baseline = FakeModule(output="base") + evolved = FakeModule(output="evo") + calls = {"n": 0} + + def metric(ex, pred): + calls["n"] += 1 + return 0.5 + + estimate_improvement(baseline, evolved, examples, metric, samples=1, max_examples=2) + assert calls["n"] == 4 # 2 examples x 2 programs + + def test_no_regression(self): + examples = [FakeExample()] * 2 + baseline = FakeModule(output="base") + evolved = FakeModule(output="evo") + metric = lambda ex, pred: 0.7 if pred.output == "base" else 0.5 + + improvement = estimate_improvement(baseline, evolved, examples, metric, samples=1) + assert improvement == pytest.approx(-0.2) diff --git a/tests/skills/test_holdout_skip_reporting.py b/tests/skills/test_holdout_skip_reporting.py new file mode 100644 index 00000000..9cb09a22 --- /dev/null +++ b/tests/skills/test_holdout_skip_reporting.py @@ -0,0 +1,100 @@ +"""Tests for holdout-skip reporting (t_ac619234). + +A skipped holdout (remaining budget < HOLDOUT_SKIP_THRESHOLD_SECONDS) is a +NON-measurement. These tests pin the three invariants from the reporting fix: + +1. metrics.json baseline_score / evolved_score / improvement are NULL (N/A + downstream), never placeholder 0.0 — a fabricated "+0.000 (+0.0%)" must + never be misread as "no improvement". +2. GEPA's candidate survives the skip: handle_holdout_skip returns it in + candidate_full so the save step can write evolved_candidate.md alongside + the baseline-copy evolved_skill.md (later budget-raised re-run can + diff/reuse it). +3. The deployed artifact stays the baseline text (byte-identical → + has_diff=0 downstream), mirroring the 8b regression guard. +""" + +import json + +from evolution.skills.evolve_skill import ( + build_metrics, + handle_holdout_skip, +) + + +def _fake_skill(body="body text", frontmatter="---\nname: x\n---\n"): + return {"body": body, "raw": frontmatter + body} + + +def _real_holdout_result(): + """A normal (non-skipped) holdout result with real scores.""" + return { + "baseline_scores": [0.4, 0.5], + "evolved_scores": [0.6, 0.6], + "examples_evaluated": 13, + "examples_total": 13, + "budget_exceeded": False, + "cache_hits": 11, + "calls_made": 2, + "skipped": False, + } + + +def test_skip_preserves_candidate_and_keeps_baseline(): + skill = _fake_skill() + candidate_body = "evolved by GEPA" + candidate_full = "---\nname: x\n---\nevolved by GEPA" + (holdout_result, avg_b, avg_e, imp, + evolved_body, evolved_full, candidate) = handle_holdout_skip( + skill, candidate_body, candidate_full, ["ex1", "ex2"], 30.0) + + assert holdout_result["skipped"] is True + assert holdout_result["examples_total"] == 2 + assert holdout_result["examples_evaluated"] == 0 + # Placeholders, NOT measurements — reporting layer renders N/A. + assert avg_b == 0.0 and avg_e == 0.0 and imp == 0.0 + # Deployed artifact stays the baseline (no evidence to deploy)... + assert evolved_body == skill["body"] + assert evolved_full == skill["raw"] + # ...but GEPA's candidate survives for a budget-raised re-run. + assert candidate == candidate_full + + +def test_metrics_null_when_skipped(): + skill = _fake_skill() + (holdout_result, avg_b, avg_e, imp, + _, _, candidate) = handle_holdout_skip(skill, "cand body", "cand full", [], 10.0) + m = build_metrics( + "test-skill", "20260802_000000", 3, "opt", "eval", + avg_b, avg_e, imp, + len(skill["body"]), len(skill["body"]), + 25, 12, 13, 430.0, True, holdout_result, candidate, 480, + ) + assert m["holdout_skipped"] is True + assert m["holdout_candidate_preserved"] is True + assert m["holdout_complete"] is False + assert m["baseline_score"] is None + assert m["evolved_score"] is None + assert m["improvement"] is None + # JSON round-trip: null, not 0.0 (what the report layer actually reads). + blob = json.loads(json.dumps(m)) + assert blob["baseline_score"] is None + assert blob["evolved_score"] is None + assert blob["improvement"] is None + + +def test_metrics_numeric_when_not_skipped(): + """Non-skipped runs keep real float scores (no behavior change).""" + skill = _fake_skill() + holdout_result = _real_holdout_result() + m = build_metrics( + "test-skill", "20260802_000000", 3, "opt", "eval", + 0.45, 0.6, 0.15, len(skill["body"]), len(skill["body"]) + 3, + 25, 12, 13, 300.0, True, holdout_result, None, 480, + ) + assert m["holdout_skipped"] is False + assert m["holdout_complete"] is True + assert m["holdout_candidate_preserved"] is False + assert m["baseline_score"] == 0.45 + assert m["evolved_score"] == 0.6 + assert m["improvement"] == 0.15 diff --git a/tests/skills/test_timeout_regression.py b/tests/skills/test_timeout_regression.py new file mode 100644 index 00000000..ff7edc81 --- /dev/null +++ b/tests/skills/test_timeout_regression.py @@ -0,0 +1,169 @@ +"""Regression tests for the 480s evolution timeout budget (t_316c92c4). + +Covers the fixes from the web-research root-cause analysis +(workspace/evolution/root-cause-web-research-20260731.md): + +1. make_lm() bounds per-call LLM timeout/retries — a stalled API call can no + longer hang for 600s+ (litellm default) and blow the whole 480s budget. +2. evaluate_holdout() enforces an in-process wall-clock budget and returns a + PARTIAL result (budget_exceeded=True) instead of the gtimeout wrapper + SIGKILLing the process with no output at all. +3. Holdout score caching makes warm runs skip the 144-call holdout phase + (24 examples x 3 samples x 2 programs) entirely. +4. The in-process budget watchdog + checkpoint write give a diagnosable, + timeout-compatible (exit 124) partial exit when GEPA consumes the budget + (verified failure: cold-cache GEPA phase took 439s of 480s). +""" + +import json +import time +from types import SimpleNamespace + +from evolution.skills.evolve_skill import ( + BUDGET_EXIT_CODE, + DEFAULT_LLM_RETRIES, + DEFAULT_LLM_TIMEOUT_SECONDS, + HOLDOUT_HEADROOM_SECONDS, + HOLDOUT_SKIP_THRESHOLD_SECONDS, + WATCHDOG_LEAD_SECONDS, + _holdout_cache_key, + evaluate_holdout, + make_lm, + write_checkpoint, +) + + +class FakeModule: + """Duck-typed SkillModule: callable with task_input, controllable delay.""" + + def __init__(self, delay: float = 0.0): + self.delay = delay + self.calls = 0 + + def __call__(self, task_input): + self.calls += 1 + if self.delay: + time.sleep(self.delay) + return SimpleNamespace(output=f"response to {task_input}") + + +def _examples(n: int): + return [SimpleNamespace(task_input=f"task {i}") for i in range(n)] + + +def _metric(ex, pred): + return 0.5 + + +def _key_fn(ex, program, sample_idx): + return f"{ex.task_input}|{program}|{sample_idx}" + + +class TestMakeLm: + def test_per_call_timeout_is_bounded(self): + lm = make_lm("deepseek/deepseek-chat") + # dspy passes kwargs through to litellm; the litellm default is 600s, + # which alone exceeds the 480s run budget. + timeout = lm.kwargs.get("timeout") or 600 + assert timeout <= DEFAULT_LLM_TIMEOUT_SECONDS + + def test_retries_are_capped(self): + lm = make_lm("deepseek/deepseek-chat") + assert lm.num_retries <= DEFAULT_LLM_RETRIES + + +class TestHoldoutBudgetGuard: + def test_returns_partial_result_when_budget_exceeded(self): + base = FakeModule(delay=0.05) + evolved = FakeModule(delay=0.05) + exs = _examples(6) + result = evaluate_holdout( + exs, base, evolved, samples=1, metric=_metric, + max_budget_seconds=0.15, + ) + assert result["budget_exceeded"] is True + assert 0 < result["examples_evaluated"] < len(exs) + assert len(result["baseline_scores"]) == result["examples_evaluated"] + assert len(result["evolved_scores"]) == result["examples_evaluated"] + # Each complete example makes exactly 2 calls; the budget check runs + # per-sample, so the interrupted example may add at most 1 baseline + # call before the loop stops. + assert 2 * result["examples_evaluated"] <= result["calls_made"] <= 2 * result["examples_evaluated"] + 1 + + def test_completes_within_generous_budget(self): + base = FakeModule(delay=0.001) + evolved = FakeModule(delay=0.001) + exs = _examples(4) + t0 = time.monotonic() + result = evaluate_holdout( + exs, base, evolved, samples=3, metric=_metric, + max_budget_seconds=10.0, + ) + elapsed = time.monotonic() - t0 + assert result["budget_exceeded"] is False + assert result["examples_evaluated"] == len(exs) + assert len(result["baseline_scores"]) == len(exs) + assert len(result["evolved_scores"]) == len(exs) + assert elapsed < 10.0 + + def test_zero_budget_yields_clean_partial(self): + exs = _examples(2) + result = evaluate_holdout( + exs, FakeModule(), FakeModule(), samples=3, metric=_metric, + max_budget_seconds=0.0, + ) + assert result["budget_exceeded"] is True + assert result["examples_evaluated"] == 0 + assert result["baseline_scores"] == [] + assert result["evolved_scores"] == [] + + +class TestBudgetWatchdog: + def test_exit_code_is_timeout_compatible(self): + # 124 == gtimeout's exit code: cron-evolve.sh auto-retries on it + # instead of misclassifying the run as an API error. + assert BUDGET_EXIT_CODE == 124 + + def test_layering_is_sane(self): + # Watchdog fires (max_budget - lead) before gtimeout; holdout skip + # kicks in with more headroom than the watchdog lead, so the graceful + # path always wins before the hard exit. + assert WATCHDOG_LEAD_SECONDS < HOLDOUT_SKIP_THRESHOLD_SECONDS + assert HOLDOUT_HEADROOM_SECONDS < HOLDOUT_SKIP_THRESHOLD_SECONDS + + def test_write_checkpoint_persists(self, tmp_path): + p = tmp_path / "output" / "web-research" / "budget_checkpoint.json" + write_checkpoint({"phase": "gepa", "timed_out": False}, p) + saved = json.loads(p.read_text()) + assert saved["phase"] == "gepa" + assert saved["timed_out"] is False + + +class TestHoldoutScoreCache: + def test_warm_run_skips_llm_calls(self): + base = FakeModule() + evolved = FakeModule() + exs = _examples(3) + cache = {} + evaluate_holdout( + exs, base, evolved, samples=1, metric=_metric, + score_cache=cache, cache_key=_key_fn, + ) + calls_after_first = base.calls + evolved.calls + assert calls_after_first == len(exs) * 2 # 3 examples x 2 programs + + result = evaluate_holdout( + exs, base, evolved, samples=1, metric=_metric, + score_cache=cache, cache_key=_key_fn, + ) + assert result["cache_hits"] == len(exs) * 2 + assert base.calls + evolved.calls == calls_after_first # zero new calls + + def test_cache_key_is_stable_and_sensitive(self): + k1 = _holdout_cache_key("deepseek/deepseek-chat", "abc123", "baseline", "task 1", 0) + k2 = _holdout_cache_key("deepseek/deepseek-chat", "abc123", "baseline", "task 1", 0) + k3 = _holdout_cache_key("deepseek/deepseek-chat", "abc123", "baseline", "task 1", 1) + k4 = _holdout_cache_key("deepseek/deepseek-chat", "abc123", "evolved", "task 1", 0) + k5 = _holdout_cache_key("deepseek/deepseek-chat", "diffhash", "baseline", "task 1", 0) + assert k1 == k2 + assert len({k1, k3, k4, k5}) == 4