fix(evolution): report skipped holdout as N/A + preserve GEPA candidate (t_ac619234)

evolution/core/config.py

@@ -23,13 +23,27 @@ class EvolutionConfig:
     judge_model: str = "openai/gpt-4.1"  # Model for dataset generation
 
     # Constraints
-    max_skill_size: int = 15_000  # 15KB default
+    max_skill_size: int = 20_000  # 20KB default (was 15KB — NLAH frontmatter adds size)
     max_tool_desc_size: int = 500  # chars
     max_param_desc_size: int = 200  # chars
-    max_prompt_growth: float = 0.2  # 20% max growth over baseline
+    max_prompt_growth: float = 0.5  # 50% max growth over baseline (was 0.2)
+    # Hard absolute growth cap — never exceeded, even with a waiver. Prevents
+    # unbounded bloat: an artifact may only grow past max_prompt_growth when a
+    # material valset improvement (growth_waiver_min_improvement) justifies it.
+    max_prompt_growth_hard: float = 1.0  # 100% max growth, absolute ceiling
+    # Minimum absolute score delta (on the 0-1 metric scale) an evolved
+    # artifact must beat the baseline by to earn a growth waiver.
+    growth_waiver_min_improvement: float = 0.03
+
+    # Random seed for reproducibility
+    random_seed: int = 42
 
     # Eval dataset
-    eval_dataset_size: int = 20  # Total examples to generate
+    eval_dataset_size: int = 60  # Base examples for small skills; auto-scaled up
+    base_eval_dataset_size: int = 60  # Minimum examples before scaling by skill size
+    dataset_size_per_10k_chars: int = 10  # Extra examples per 10K chars beyond first 5K
+    max_eval_dataset_size: int = 150  # Cap on total scaled dataset size
+    eval_temperature: float = 0.0  # 0 = deterministic generation for stable scoring
     train_ratio: float = 0.5
     val_ratio: float = 0.25
     holdout_ratio: float = 0.25

evolution/core/constraints.py

@@ -32,16 +32,34 @@ def validate_all(
         artifact_text: str,
         artifact_type: str,
         baseline_text: Optional[str] = None,
+        improvement: Optional[float] = None,
+        growth_text: Optional[str] = None,
     ) -> list[ConstraintResult]:
-        """Run all applicable constraints. Returns list of results."""
+        """Run all applicable constraints. Returns list of results.
+
+        ``improvement`` is the measured valset score delta of the evolved
+        artifact over the baseline (0-1 metric scale). When provided and
+        material, it can waive the soft growth cap (see ``_check_growth``);
+        None keeps the strict pre-waiver behavior.
+
+        ``growth_text`` is the text used for the growth comparison (defaults
+        to ``artifact_text``). For skills, pass the evolved BODY here while
+        ``artifact_text`` stays the full reassembled artifact (frontmatter +
+        body): the growth gate must compare body-vs-body, otherwise the
+        static frontmatter inflates the ratio and a real improvement is
+        rejected (gmail-monitor: +100.5% full-vs-body vs +69.8% body-vs-body).
+        """
         results = []
 
         # 1. Size limits
         results.append(self._check_size(artifact_text, artifact_type))
 
         # 2. Growth limit (if baseline provided)
         if baseline_text:
-            results.append(self._check_growth(artifact_text, baseline_text, artifact_type))
+            results.append(self._check_growth(
+                artifact_text, baseline_text, artifact_type,
+                improvement=improvement, growth_text=growth_text,
+            ))
 
         # 3. Non-empty
         results.append(self._check_non_empty(artifact_text))
@@ -116,17 +134,78 @@ def _check_size(self, text: str, artifact_type: str) -> ConstraintResult:
                 message=f"Size exceeded: {size}/{limit} chars ({size - limit} over)",
             )
 
-    def _check_growth(self, text: str, baseline: str, artifact_type: str) -> ConstraintResult:
-        growth = (len(text) - len(baseline)) / max(1, len(baseline))
+    def _check_growth(
+        self,
+        text: str,
+        baseline: str,
+        artifact_type: str,
+        improvement: Optional[float] = None,
+        growth_text: Optional[str] = None,
+    ) -> ConstraintResult:
+        """Check growth vs baseline, with a quality-based waiver.
+
+        The soft cap ``max_prompt_growth`` rejects bloated artifacts. A
+        candidate that grows past the soft cap may still pass when it
+        demonstrates a *material* improvement (>= ``growth_waiver_min_improvement``
+        absolute score delta on the 0-1 metric scale) AND stays under the hard
+        absolute cap ``max_prompt_growth_hard`` — i.e. the extra size buys
+        measurable quality. Without ``improvement`` the check is strict.
+
+        ``growth_text`` defaults to ``text``; pass the evolved BODY explicitly
+        when ``text`` is a full artifact with static frontmatter so the growth
+        ratio measures the actual instruction growth (body-vs-body), not
+        frontmatter inflation.
+        """
+        growth_base = growth_text if growth_text is not None else text
+        growth = (len(growth_base) - len(baseline)) / max(1, len(baseline))
         max_growth = self.config.max_prompt_growth
 
+        waiver_applied = False
+        if growth > max_growth:
+            waiver_applied = (
+                improvement is not None
+                and improvement >= self.config.growth_waiver_min_improvement
+            )
+            if waiver_applied:
+                max_growth = self.config.max_prompt_growth_hard
+
         if growth <= max_growth:
+            if waiver_applied:
+                return ConstraintResult(
+                    passed=True,
+                    constraint_name="growth_limit",
+                    message=(
+                        f"Growth OK with waiver: {growth:+.1%} (soft cap {self.config.max_prompt_growth:+.1%} "
+                        f"exceeded but improvement {improvement:+.3f} >= "
+                        f"{self.config.growth_waiver_min_improvement:+.3f}; hard cap {self.config.max_prompt_growth_hard:+.1%})"
+                    ),
+                )
             return ConstraintResult(
                 passed=True,
                 constraint_name="growth_limit",
                 message=f"Growth OK: {growth:+.1%} (max {max_growth:+.1%})",
             )
         else:
+            if waiver_applied:
+                return ConstraintResult(
+                    passed=False,
+                    constraint_name="growth_limit",
+                    message=(
+                        f"Growth exceeded: {growth:+.1%} (waiver applied: improvement {improvement:+.3f} >= "
+                        f"{self.config.growth_waiver_min_improvement:+.3f}, but hard cap "
+                        f"{self.config.max_prompt_growth_hard:+.1%} still exceeded)"
+                    ),
+                )
+            if improvement is not None:
+                return ConstraintResult(
+                    passed=False,
+                    constraint_name="growth_limit",
+                    message=(
+                        f"Growth exceeded: {growth:+.1%} (soft cap {self.config.max_prompt_growth:+.1%}, "
+                        f"hard cap {self.config.max_prompt_growth_hard:+.1%}, improvement {improvement:+.3f} "
+                        f"< waiver threshold {self.config.growth_waiver_min_improvement:+.3f})"
+                    ),
+                )
             return ConstraintResult(
                 passed=False,
                 constraint_name="growth_limit",

evolution/core/errors.py

@@ -0,0 +1,58 @@
+"""Exit-code contract and runtime error classification for the evolution pipeline.
+
+The cron wrapper (cron-evolve.sh) maps the evolve_skill process exit code to a
+failure category. Every exit code used by the pipeline MUST be documented here
+and mirrored in cron-evolve.sh's classifier.
+
+Exit codes:
+    0  success — including the constraint-failure path (artifact saved for
+       review, deployment skipped; cron detects it via output text)
+    1  generic error (skill not found, dataset build failure, usage error)
+    2  FD exhaustion (OSError errno 24 EMFILE) — environment limit, NOT an
+       LLM/API failure; raise `ulimit -n` and retry
+    3  holdout-eval / LLM failure (dspy, network, or scoring error during the
+       evaluation phase)
+
+History: the 'error 1' escalations (esc-20260731-201857-1474) were EMFILE
+crashes — bare `OSError: [Errno 24] Too many open files` tracebacks — that
+Python reported as exit 1 and cron-evolve.sh mislabeled as "LLM provider
+issue". This module exists so that failure mode gets its own exit code (2)
+and a meaningful message.
+"""
+
+import errno
+
+EXIT_GENERIC = 1
+EXIT_EMFILE = 2
+EXIT_EVAL_ERROR = 3
+
+EMFILE_HINT = (
+    "Too many open files (OSError errno 24 EMFILE). This is an environment "
+    "file-descriptor limit, not an LLM/API failure. Raise the process limit "
+    "(e.g. `ulimit -n 1024`) and retry."
+)
+
+
+def is_emfile(exc: BaseException) -> bool:
+    """Return True if the exception is FD exhaustion (EMFILE, errno 24).
+
+    Checks the errno directly, and falls back to message matching because
+    dspy/litellm sometimes wrap the raw OS error in a generic Exception
+    that carries the errno text but no errno attribute.
+    """
+    if isinstance(exc, OSError):
+        return exc.errno == errno.EMFILE
+    return "too many open files" in str(exc).lower()
+
+
+def classify_error(exc: BaseException) -> tuple[int, str]:
+    """Map a runtime exception to (exit_code, human_message).
+
+    Distinguishes FD exhaustion from evaluation/LLM errors so the cron
+    wrapper can report a meaningful cause instead of the generic
+    'LLM provider issue' label that masked the original EMFILE crashes.
+    """
+    if is_emfile(exc):
+        return EXIT_EMFILE, EMFILE_HINT
+    msg = str(exc) or exc.__class__.__name__
+    return EXIT_EVAL_ERROR, f"Evaluation failed: {msg}"