diff --git a/.github/workflows/python-test.yml b/.github/workflows/python-test.yml index 0e51c6de1..5b9dd70b6 100644 --- a/.github/workflows/python-test.yml +++ b/.github/workflows/python-test.yml @@ -7,9 +7,11 @@ on: jobs: build: - runs-on: ubuntu-latest + # ubuntu-latest (26.04) has no Python 3.9 build + runs-on: ubuntu-24.04 strategy: + fail-fast: false matrix: python-version: ['3.9', '3.10', '3.11', '3.12'] diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 6f5df3605..1e30d889d 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -13,7 +13,7 @@ on: jobs: release: - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 # ubuntu-latest (26.04) has no Python 3.9 build permissions: contents: write id-token: write diff --git a/adalflow/adalflow/optim/README.md b/adalflow/adalflow/optim/README.md index 9dc47a32b..7f3223079 100644 --- a/adalflow/adalflow/optim/README.md +++ b/adalflow/adalflow/optim/README.md @@ -38,8 +38,16 @@ and OpenEvolve; independent implementation. - `evolve` (`evolution/loop.py`): select parent -> propose -> evaluate -> archive, under a budget of evaluator calls; JSONL event log; stops on score, budget or stalls. - `Archive` (`evolution/archive.py`): per-instance Pareto parent selection (keeps - candidates that solve different cases), greedy `best`, or `resample` (always the - seed: a best-of-N baseline at the same budget). + candidates that solve different cases), greedy `best`, `map_elites` (one elite per + behavior cell from a user `descriptor`, parents uniform over cells), or `resample` + (always the seed: a best-of-N baseline at the same budget). +- `evolve(..., islands=k, migrate_every=m)`: sub-populations with periodic migration + of each island's best as the next island's inspiration. +- `evolve(..., confirm=n)` for noisy evaluators: a candidate that beats the best on + its first score is re-evaluated with the incumbent to `1 + n` samples each, and + only fully sampled candidates can be the best. In simulation this mainly makes + the reported best score honest; it improves the true quality found only when the + noise is large relative to the gains (`benchmarks/evolution/noise_sim.py`). - `evolve(..., holdout=...)` scores the seed and the final best on held-out instances after the search, so gains that do not generalize show up. - `LLMProposer` (`evolution/proposer.py`): `Generator`-based, fenced-block output, diff --git a/adalflow/adalflow/optim/evolution/__init__.py b/adalflow/adalflow/optim/evolution/__init__.py index 907d812cf..113ccc9f6 100644 --- a/adalflow/adalflow/optim/evolution/__init__.py +++ b/adalflow/adalflow/optim/evolution/__init__.py @@ -6,7 +6,13 @@ Independent implementation. """ -from adalflow.optim.evolution.archive import BEST, PARETO, RESAMPLE, Archive +from adalflow.optim.evolution.archive import ( + BEST, + MAP_ELITES, + PARETO, + RESAMPLE, + Archive, +) from adalflow.optim.evolution.loop import EvolutionResult, evolve from adalflow.optim.evolution.proposer import ( PROPOSE_TEMPLATE, @@ -18,13 +24,19 @@ render_feedback, ) from adalflow.optim.evolution.sandbox import PythonEvaluator, SandboxResult, run_python -from adalflow.optim.evolution.types import Candidate, Evaluation, to_evaluation +from adalflow.optim.evolution.types import ( + Candidate, + Evaluation, + combine, + to_evaluation, +) __all__ = [ "Archive", "BEST", "PARETO", "RESAMPLE", + "MAP_ELITES", "EvolutionResult", "evolve", "PROPOSE_TEMPLATE", @@ -39,5 +51,6 @@ "run_python", "Candidate", "Evaluation", + "combine", "to_evaluation", ] diff --git a/adalflow/adalflow/optim/evolution/archive.py b/adalflow/adalflow/optim/evolution/archive.py index a23a5e876..0991f5ea8 100644 --- a/adalflow/adalflow/optim/evolution/archive.py +++ b/adalflow/adalflow/optim/evolution/archive.py @@ -7,21 +7,29 @@ ``RESAMPLE`` always returns the seed, so every proposal is an independent sample from it: a best-of-N baseline at the same evaluation budget. + +``MAP_ELITES`` keeps the best candidate per behavior cell (``Candidate.cell``, set from +a user descriptor) and samples parents uniformly over cells (after Mouret & Clune, +2015), so a weak but different approach still gets improved. + +Every selection can be restricted to one island (``Candidate.island``); the seed +belongs to every island. """ from __future__ import annotations from random import Random -from typing import Dict, List, Optional +from typing import Dict, List, Optional, Tuple from adalflow.optim.evolution.types import Candidate -__all__ = ["Archive", "PARETO", "BEST", "RESAMPLE"] +__all__ = ["Archive", "PARETO", "BEST", "RESAMPLE", "MAP_ELITES"] PARETO = "pareto" BEST = "best" RESAMPLE = "resample" -STRATEGIES = (PARETO, BEST, RESAMPLE) +MAP_ELITES = "map_elites" +STRATEGIES = (PARETO, BEST, RESAMPLE, MAP_ELITES) _EPS = 1e-12 @@ -42,21 +50,41 @@ def add(self, candidate: Candidate) -> None: def has_text(self, text: str) -> bool: return text in self._texts - def evaluated(self) -> List[Candidate]: - return [c for c in self.candidates if c.ok] + @property + def seed(self) -> Candidate: + return self.candidates[0] + + def members(self, island: Optional[int] = None) -> List[Candidate]: + """Candidates of one island (the seed included), or all of them.""" + if island is None: + return list(self.candidates) + return [ + c for i, c in enumerate(self.candidates) if i == 0 or c.island == island + ] + + def evaluated(self, island: Optional[int] = None) -> List[Candidate]: + return [c for c in self.members(island) if c.ok] + + def best( + self, island: Optional[int] = None, min_samples: int = 1 + ) -> Optional[Candidate]: + """Highest mean score; the earliest candidate wins ties. - def best(self) -> Optional[Candidate]: - """Highest score; the earliest candidate wins ties.""" - ok = self.evaluated() - return max(ok, key=lambda c: c.score) if ok else None + With ``min_samples`` > 1, only candidates evaluated that many times compete + (falling back to all evaluated candidates when none has been). + """ + ok = self.evaluated(island) + confirmed = [c for c in ok if c.n_samples >= min_samples] + pool = confirmed or ok + return max(pool, key=lambda c: c.score) if pool else None - def _wins(self) -> Dict[str, int]: + @staticmethod + def _wins(pool: List[Candidate]) -> Dict[str, int]: """Candidate id -> number of instances where it has the (tied) best score.""" - ok = self.evaluated() - keys = sorted({k for c in ok for k in c.evaluation.instance_scores}) + keys = sorted({k for c in pool for k in c.evaluation.instance_scores}) wins: Dict[str, int] = {} for key in keys: - scored = [c for c in ok if key in c.evaluation.instance_scores] + scored = [c for c in pool if key in c.evaluation.instance_scores] top = max(c.evaluation.instance_scores[key] for c in scored) for c in scored: if c.evaluation.instance_scores[key] >= top - _EPS: @@ -72,16 +100,17 @@ def _dominates(a: Candidate, b: Candidate) -> bool: x > y + _EPS for x, y in zip(ra, rb) ) - def pareto_front(self) -> List[Candidate]: + def pareto_front(self, island: Optional[int] = None) -> List[Candidate]: """Non-dominated per-instance winners; ``[best]`` without instance scores. Among candidates with identical instance scores only the earliest is kept. """ - wins = self._wins() + pool = self.evaluated(island) + wins = self._wins(pool) if not wins: - best = self.best() + best = self.best(island) return [best] if best is not None else [] - winners = [c for c in self.evaluated() if c.id in wins] + winners = [c for c in pool if c.id in wins] front: List[Candidate] = [] for c in winners: if any(self._dominates(o, c) for o in winners if o is not c): @@ -94,27 +123,58 @@ def pareto_front(self) -> List[Candidate]: front.append(c) return front - def select_parent(self, rng: Random, strategy: str = PARETO) -> Candidate: + def elites(self, island: Optional[int] = None) -> Dict[Tuple, Candidate]: + """Behavior cell -> its best candidate (candidates without a cell skipped).""" + cells: Dict[Tuple, Candidate] = {} + for c in self.evaluated(island): + if c.cell is None: + continue + if c.cell not in cells or c.score > cells[c.cell].score: + cells[c.cell] = c + return cells + + def select_parent( + self, rng: Random, strategy: str = PARETO, island: Optional[int] = None + ) -> Candidate: """A parent to improve. With nothing evaluated yet, the latest candidate.""" if strategy not in STRATEGIES: raise ValueError(f"strategy must be one of {STRATEGIES}") if not self.candidates: raise ValueError("the archive is empty") if strategy == RESAMPLE: - return self.candidates[0] - if not self.evaluated(): - return self.candidates[-1] # e.g. a seed that crashed: fix it first + return self.seed + if not self.evaluated(island): + return self.members(island)[-1] # e.g. a seed that crashed: fix it first if strategy == BEST: - return self.best() - front = self.pareto_front() - wins = self._wins() + return self.best(island) + if strategy == MAP_ELITES: + elites = self.elites(island) + if elites: + return elites[rng.choice(sorted(elites, key=repr))] + return self.best(island) + front = self.pareto_front(island) + wins = self._wins(self.evaluated(island)) weights = [max(wins.get(c.id, 0), 1) for c in front] return rng.choices(front, weights=weights, k=1)[0] - def select_inspiration(self, rng: Random, parent: Candidate) -> Optional[Candidate]: - """Another strong candidate to show next to the parent, if there is one.""" - pool = [c for c in self.pareto_front() if c.id != parent.id] + def select_inspiration( + self, + rng: Random, + parent: Candidate, + island: Optional[int] = None, + strategy: str = PARETO, + ) -> Optional[Candidate]: + """Another strong candidate to show next to the parent, if there is one. + + Under MAP-Elites it is the elite of another cell, so the proposer sees a + different approach rather than a near copy of the parent. + """ + if strategy == MAP_ELITES: + pool = [c for c in self.elites(island).values() if c.id != parent.id] + if pool: + return rng.choice(sorted(pool, key=lambda c: c.id)) + pool = [c for c in self.pareto_front(island) if c.id != parent.id] if not pool: - ranked = sorted(self.evaluated(), key=lambda c: -c.score) + ranked = sorted(self.evaluated(island), key=lambda c: -c.score) pool = [c for c in ranked[:3] if c.id != parent.id] return rng.choice(pool) if pool else None diff --git a/adalflow/adalflow/optim/evolution/loop.py b/adalflow/adalflow/optim/evolution/loop.py index 66f6d6220..f8851f43c 100644 --- a/adalflow/adalflow/optim/evolution/loop.py +++ b/adalflow/adalflow/optim/evolution/loop.py @@ -7,9 +7,15 @@ from dataclasses import dataclass, field from pathlib import Path from random import Random -from typing import Any, Callable, Dict, List, Optional, Union +from typing import Any, Callable, Dict, List, Optional, Tuple, Union -from adalflow.optim.evolution.archive import PARETO, RESAMPLE, Archive +from adalflow.optim.evolution.archive import ( + MAP_ELITES, + PARETO, + RESAMPLE, + STRATEGIES, + Archive, +) from adalflow.optim.evolution.proposer import Proposer from adalflow.optim.evolution.types import Candidate, Evaluation, to_evaluation @@ -62,6 +68,10 @@ def evolve( max_stalls: int = 5, log_path: Union[str, Path, None] = None, holdout: Optional[Callable[[str], Any]] = None, + confirm: int = 0, + islands: int = 1, + migrate_every: int = 4, + descriptor: Optional[Callable[[Candidate], Tuple[Any, ...]]] = None, ) -> EvolutionResult: """Evolves ``seed`` with ``proposer`` under a budget of evaluator calls. @@ -70,20 +80,41 @@ def evolve( evaluate: ``text -> score | dict | Evaluation`` (e.g. ``PythonEvaluator``). Exceptions are recorded as failed evaluations. proposer: e.g. ``LLMProposer``. - max_evaluations: total evaluator calls, seed included. - selection: ``"pareto"`` (per-instance, GEPA-style), ``"best"`` (greedy) or + max_evaluations: total evaluator calls, seed and re-evaluations included. + selection: ``"pareto"`` (per-instance, GEPA-style), ``"best"`` (greedy), + ``"map_elites"`` (uniform over behavior cells; needs ``descriptor``) or ``"resample"`` (every proposal from the seed, no inspiration: the best-of-N baseline to compare a search strategy against). - stop_score: stop once a candidate reaches this score. + stop_score: stop once the best candidate reaches this (mean) score. max_stalls: stop after this many proposals in a row that failed or repeated an earlier candidate. - log_path: append one JSON line per event (candidate, failure, duplicate). + log_path: append one JSON line per event (candidate, sample, failure, + duplicate, migration, holdout). holdout: an evaluator on instances the search never sees. The seed and the final best are scored on it after the search (not counted in ``max_evaluations``), so overfitting to ``evaluate`` shows up as a gap. + confirm: for noisy evaluators. When a new candidate's first score beats the + best so far, both are re-evaluated until each has ``1 + confirm`` + samples, and the means decide. Only candidates with that many samples + can be the best, so one lucky score does not win the run. A proposal + starts only if the budget left can finish its comparison. + islands: number of sub-populations. Proposals rotate over islands, parents + and inspirations come from the current island, and every + ``migrate_every`` proposals the inspiration is the best of the next + island instead (0 never migrates). + descriptor: ``candidate -> tuple`` behavior cell, called after each + evaluation (it can read ``candidate.evaluation.metrics``). Required for + ``"map_elites"``. """ if max_evaluations < 1: raise ValueError("max_evaluations must be >= 1") + if selection not in STRATEGIES: + raise ValueError(f"selection must be one of {STRATEGIES}") + if selection == MAP_ELITES and descriptor is None: + raise ValueError("selection='map_elites' needs a descriptor") + if confirm < 0 or islands < 1 or migrate_every < 0: + raise ValueError("confirm and migrate_every must be >= 0, islands >= 1") + need = 1 + confirm rng = Random(random_seed) archive = Archive() events: List[Dict[str, Any]] = [] @@ -97,10 +128,20 @@ def record(event: Dict[str, Any]) -> None: with sink.open("a", encoding="utf-8") as f: f.write(json.dumps(event, default=str) + "\n") + def sample(candidate: Candidate) -> None: + candidate.add_sample(_evaluate(evaluate, candidate.text)) + result.evaluations += 1 + if descriptor is not None and candidate.ok: + try: + candidate.cell = tuple(descriptor(candidate)) + except Exception as exc: # a bad descriptor leaves the cell unset + log.warning("descriptor failed on %s: %s", candidate.id, exc) + candidate.cell = None + def add(candidate: Candidate) -> None: - candidate.evaluation = _evaluate(evaluate, candidate.text) + sample(candidate) archive.add(candidate) - record({"event": "candidate", **candidate.to_dict()}) + record({"event": "candidate", **_summary(candidate)}) ev = candidate.evaluation log.info( "evolve %s (parent %s): %s", @@ -109,17 +150,52 @@ def add(candidate: Candidate) -> None: f"score {ev.score:.4g}" if ev.ok else f"failed: {ev.error}", ) + def resample(candidate: Candidate) -> None: + sample(candidate) + record( + { + "event": "sample", + "id": candidate.id, + "n": candidate.n_samples, + "last": candidate.samples[-1].score, + "mean": candidate.score, + } + ) + + def race(challenger: Candidate) -> None: + """Re-evaluates a challenger and the incumbent until both have ``need``.""" + others = [c for c in archive.evaluated() if c is not challenger] + confirmed = [c for c in others if c.n_samples >= need] + pool = confirmed or others + incumbent = max(pool, key=lambda c: c.score) if pool else None + if not challenger.ok or incumbent is None: + return + if challenger.score <= incumbent.score: + return + while result.evaluations < max_evaluations and ( + challenger.n_samples < need or incumbent.n_samples < need + ): + target = ( + challenger if challenger.n_samples <= incumbent.n_samples else incumbent + ) + resample(target) + seed_candidate = Candidate(id="c0", text=seed, origin="seed") - add(seed_candidate) result = EvolutionResult( - best=None, seed=seed_candidate, archive=archive, evaluations=1, events=events + best=None, seed=seed_candidate, archive=archive, evaluations=0, events=events ) + add(seed_candidate) def reached() -> bool: - best = archive.best() - return stop_score is not None and best is not None and best.score >= stop_score + best = archive.best(min_samples=need) + return ( + stop_score is not None + and best is not None + and best.n_samples >= need + and best.score >= stop_score + ) - stalls = 0 + stalls = attempts = 0 while result.evaluations < max_evaluations: if reached(): result.stopped = "stop_score" @@ -127,10 +203,33 @@ def reached() -> bool: if stalls >= max_stalls: result.stopped = "stalled" break - parent = archive.select_parent(rng, selection) - inspiration = ( - None if selection == RESAMPLE else archive.select_inspiration(rng, parent) - ) + if confirm: + # Start a proposal only if its race can finish within the budget. + incumbent = archive.best(min_samples=need) + reserve = need + max(0, need - incumbent.n_samples) if incumbent else need + if max_evaluations - result.evaluations < reserve: + break + island = attempts % islands + attempts += 1 + scope = island if islands > 1 else None + parent = archive.select_parent(rng, selection, scope) + inspiration = None + if selection != RESAMPLE: + migrant = None + if islands > 1 and migrate_every and attempts % migrate_every == 0: + migrant = archive.best((island + 1) % islands, min_samples=need) + if migrant is not None and migrant.id != parent.id: + inspiration = migrant + record( + { + "event": "migration", + "id": migrant.id, + "from": (island + 1) % islands, + "to": island, + } + ) + else: + inspiration = archive.select_inspiration(rng, parent, scope, selection) try: proposal = proposer.propose(parent, inspiration) except Exception as exc: # model or parse failure: count it, try again @@ -150,23 +249,24 @@ def reached() -> bool: record({"event": "duplicate", "parent": parent.id}) continue stalls = 0 - add( - Candidate( - id=f"c{len(archive)}", - text=proposal.text, - generation=parent.generation + 1, - parent_id=parent.id, - inspiration_id=inspiration.id if inspiration is not None else None, - origin="llm", - rationale=proposal.rationale, - ) + child = Candidate( + id=f"c{len(archive)}", + text=proposal.text, + generation=parent.generation + 1, + parent_id=parent.id, + inspiration_id=inspiration.id if inspiration is not None else None, + origin="llm", + rationale=proposal.rationale, + island=island, ) - result.evaluations += 1 + add(child) + if confirm: + race(child) else: if reached(): result.stopped = "stop_score" - result.best = archive.best() + result.best = archive.best(min_samples=need) if holdout is not None: targets = {"seed": seed_candidate} if result.best is not None: @@ -184,3 +284,10 @@ def reached() -> bool: } ) return result + + +def _summary(candidate: Candidate) -> Dict[str, Any]: + """A candidate for the event log (its samples are logged as they arrive).""" + data = candidate.to_dict() + data.pop("samples", None) + return data diff --git a/adalflow/adalflow/optim/evolution/types.py b/adalflow/adalflow/optim/evolution/types.py index 1794118d1..9a696981b 100644 --- a/adalflow/adalflow/optim/evolution/types.py +++ b/adalflow/adalflow/optim/evolution/types.py @@ -3,9 +3,10 @@ from __future__ import annotations from dataclasses import asdict, dataclass, field -from typing import Any, Dict, Optional +import statistics +from typing import Any, Dict, List, Optional, Tuple -__all__ = ["Evaluation", "Candidate", "to_evaluation"] +__all__ = ["Evaluation", "Candidate", "combine", "to_evaluation"] @dataclass @@ -65,12 +66,47 @@ def to_evaluation(value: Any) -> Evaluation: ) +def combine(samples: List[Evaluation]) -> Evaluation: + """Averages repeated evaluations of one candidate (for noisy evaluators). + + Failed samples are left out of the means; the feedback and other metrics are the + latest sample's. ``metrics`` gets ``samples`` (scored samples) and, with two or + more, ``score_sd``. + """ + if not samples: + raise ValueError("no samples to combine") + if len(samples) == 1: + return samples[0] + ok = [s for s in samples if s.ok] + latest = samples[-1] + if not ok: + return latest + keys = sorted({k for s in ok for k in s.instance_scores}) + instances = {} + for key in keys: + values = [s.instance_scores[key] for s in ok if key in s.instance_scores] + instances[key] = statistics.mean(values) + scores = [s.score for s in ok] + metrics = dict(latest.metrics) + metrics["samples"] = len(ok) + if len(scores) > 1: + metrics["score_sd"] = statistics.stdev(scores) + return Evaluation( + score=statistics.mean(scores), + instance_scores=instances, + feedback=latest.feedback if latest.ok else ok[-1].feedback, + metrics=metrics, + ) + + @dataclass class Candidate: """One artifact in the archive. ``rationale`` is written by the proposer before the candidate is evaluated, so a - log of rationales never carries outcome annotations. + log of rationales never carries outcome annotations. ``evaluation`` is the mean + of ``samples`` when a candidate was evaluated more than once. ``island`` and + ``cell`` place it for island and MAP-Elites selection. """ id: str @@ -81,6 +117,19 @@ class Candidate: origin: str = "seed" rationale: str = "" evaluation: Optional[Evaluation] = None + samples: List[Evaluation] = field(default_factory=list) + island: int = 0 + cell: Optional[Tuple[Any, ...]] = None + + def add_sample(self, evaluation: Evaluation) -> None: + self.samples.append(evaluation) + self.evaluation = combine(self.samples) + + @property + def n_samples(self) -> int: + if self.samples: + return len(self.samples) + return 1 if self.evaluation is not None else 0 @property def ok(self) -> bool: diff --git a/adalflow/tests/test_evolution.py b/adalflow/tests/test_evolution.py index be9bdbdc9..3fd1f23bb 100644 --- a/adalflow/tests/test_evolution.py +++ b/adalflow/tests/test_evolution.py @@ -15,13 +15,16 @@ from adalflow.core.types import GeneratorOutput from adalflow.optim.evolution import ( BEST, + MAP_ELITES, Archive, Candidate, Evaluation, LLMProposer, + Proposal, ProposalError, RESAMPLE, PythonEvaluator, + combine, evolve, extract_candidate, run_python, @@ -322,3 +325,143 @@ def holdout(text): def test_sandbox_uses_the_current_interpreter(): run = run_python("import sys\nprint(sys.version_info[:2])") assert run.stdout.strip() == str(tuple(sys.version_info[:2])) + + +# ------------------------------------------------- noise and diversity +class ListProposer: + """Proposes the given texts in order and remembers what it was shown.""" + + def __init__(self, texts): + self.texts = list(texts) + self.calls = [] + + def propose(self, parent, inspiration=None): + self.calls.append((parent.id, inspiration.id if inspiration else None)) + return Proposal(text=self.texts.pop(0), rationale="scripted") + + +def noisy(table): + """An evaluator that returns each text's scores in turn (then repeats the last).""" + seen = {} + + def evaluate(text): + i = seen[text] = seen.get(text, -1) + 1 + scores = table[text] + return scores[min(i, len(scores) - 1)] + + return evaluate + + +def test_combine_averages_samples_and_skips_failures(): + a = Evaluation(score=0.2, instance_scores={"x": 0.0, "y": 0.4}, feedback="a") + b = Evaluation(score=0.6, instance_scores={"x": 1.0, "y": 0.2}, feedback="b") + out = combine([a, Evaluation(error="timeout"), b]) + assert out.score == pytest.approx(0.4) and out.feedback == "b" + assert out.instance_scores == pytest.approx({"x": 0.5, "y": 0.3}) + assert out.metrics["samples"] == 2 and out.metrics["score_sd"] > 0 + assert combine([a]) is a and not combine([Evaluation(error="x")]).ok + with pytest.raises(ValueError): + combine([]) + + +def test_confirm_rejects_a_lucky_first_score(): + table = {"seed": [0.5], "lucky": [0.9, 0.1, 0.1], "good": [0.7, 0.7, 0.7]} + plain = evolve("seed", noisy(table), ListProposer(["lucky"]), max_evaluations=2) + assert plain.best.text == "lucky" + + res = evolve( + "seed", + noisy(table), + ListProposer(["lucky", "good"]), + max_evaluations=20, + confirm=2, + max_stalls=1, + ) + by_text = {c.text: c for c in res.archive.candidates} + assert by_text["lucky"].n_samples == 3 and by_text["seed"].n_samples == 3 + assert by_text["lucky"].score == pytest.approx(1.1 / 3) + assert res.best.text == "good" and res.best.n_samples == 3 + # seed 3 + lucky 3 + good 3 samples; the run stops when proposals run out. + assert res.evaluations == 9 and res.stopped == "stalled" + assert sum(e["event"] == "sample" for e in res.events) == 6 + + +def test_confirm_reserves_budget_to_finish_each_comparison(): + table = {"seed": [0.5], "lucky": [0.9, 0.1]} + # After the seed, a proposal needs 3 samples of it and 2 more of the seed. + short = evolve( + "seed", noisy(table), ListProposer(["lucky"]), max_evaluations=5, confirm=2 + ) + assert short.evaluations == 1 and short.stopped == "budget" + res = evolve( + "seed", noisy(table), ListProposer(["lucky"]), max_evaluations=6, confirm=2 + ) + assert res.evaluations == 6 and res.best.text == "seed" + assert [c.n_samples for c in res.archive.candidates] == [3, 3] + + +def test_islands_rotate_and_migrate(): + texts = [f"t{i}" for i in range(6)] + table = {"seed": [0.1], **{t: [0.2 + 0.1 * i] for i, t in enumerate(texts)}} + proposer = ListProposer(texts) + res = evolve( + "seed", + noisy(table), + proposer, + max_evaluations=7, + selection=BEST, + islands=2, + migrate_every=4, + ) + assert [c.island for c in res.archive.candidates[1:]] == [0, 1, 0, 1, 0, 1] + parents = [p for p, _ in proposer.calls] + # Island 0 builds on c1, c3, ...; island 1 on c2, c4, ...; both start from c0. + assert parents == ["c0", "c0", "c1", "c2", "c3", "c4"] + migrations = [e for e in res.events if e["event"] == "migration"] + assert [(m["from"], m["to"], m["id"]) for m in migrations] == [(0, 1, "c3")] + assert proposer.calls[3] == ("c2", "c3") + assert res.best.text == "t5" + + +def test_map_elites_keeps_and_selects_every_cell(): + with pytest.raises(ValueError, match="descriptor"): + evolve("seed", noisy({"seed": [0.1]}), ListProposer([]), selection=MAP_ELITES) + + archive = Archive() + for cid, score, cell in [ + ("c0", 0.1, ("a",)), + ("c1", 0.9, ("a",)), + ("c2", 0.3, ("b",)), + ("c3", 0.2, ("b",)), + ("c4", 0.95, None), + ]: + c = Candidate(id=cid, text=cid, evaluation=Evaluation(score=score)) + c.cell = cell + archive.add(c) + elites = archive.elites() + assert {k: v.id for k, v in elites.items()} == {("a",): "c1", ("b",): "c2"} + rng = Random(0) + picks = {archive.select_parent(rng, MAP_ELITES).id for _ in range(30)} + assert picks == {"c1", "c2"} # the weaker cell's elite is still improved + assert ( + archive.select_inspiration(Random(0), elites[("a",)], strategy=MAP_ELITES).id + == "c2" + ) + + +def test_descriptor_sets_cells_and_tolerates_errors(): + def by_length(c): + if c.text == "boom": + raise RuntimeError("bad descriptor") + return (len(c.text) // 3,) + + table = {"seed": [0.1], "abcdef": [0.5], "boom": [0.2]} + res = evolve( + "seed", + noisy(table), + ListProposer(["abcdef", "boom"]), + max_evaluations=3, + selection=MAP_ELITES, + descriptor=by_length, + ) + assert [c.cell for c in res.archive.candidates] == [(1,), (2,), None] diff --git a/benchmarks/evolution/README.md b/benchmarks/evolution/README.md index 43f645653..50dcb0807 100644 --- a/benchmarks/evolution/README.md +++ b/benchmarks/evolution/README.md @@ -6,9 +6,12 @@ best on held-out instances the search never sees (`evolve(..., holdout=...)`). - `pareto` / `best`: build on earlier candidates (per-instance Pareto or greedy). - `resample`: every proposal is an independent sample from the seed (best-of-N). +- `map_elites`: one elite per behavior cell (tasks with a descriptor); `islands`: + `pareto` on 3 islands with migration. ```bash python benchmarks/evolution/run.py --check # seed scores, no model calls +python benchmarks/evolution/noise_sim.py # confirm= under simulated noise, no model python benchmarks/evolution/run.py --provider anthropic --model claude-haiku-5-5 \ --repeats 3 --budget 12 --out runs/evolution_bench ``` @@ -89,3 +92,45 @@ What they show: - Proposals are not very diverse: `resample` produced 100+ duplicate candidates per `binpack` run, nearly always best fit. Islands or quality-diversity archives (MAP-Elites) are the natural next step for open-ended tasks like this one. + +## Diversity: MAP-Elites and islands (2026-10-21, claude-haiku-5-5, 3 runs, budget 30) + +`map_elites` uses the `binpack` descriptor `binpack_cell`: how often a candidate's +decisions agree with best fit (5 bands) x code length (4 buckets). `islands` is +`pareto` on 3 islands, migrating every 4 proposals. + +| strategy | search best | held-out best | duplicates | cells reached | +|---|---|---|---|---| +| pareto | 0.954 [0.952, 0.959] | 0.948 [0.947, 0.949] | 0 | 4.3 | +| map_elites | 0.953 [0.951, 0.955] | 0.950 [0.947, 0.953] | 3.7 | 5.0 | +| islands | 0.952 [0.952, 0.952] | 0.950 [0.949, 0.951] | 1.7 | 4.3 | + +The differences are within run-to-run spread. The best single held-out result so +far came from `map_elites` (0.953), and `pareto` again overfit the most (its highest +search score, 0.9585, had its lowest held-out score). The 30 candidates of a run +fill only 4-6 of 20 cells, so the proposer's narrow output, not parent selection, +limits diversity here; more varied proposals (temperature, prompts that ask for a +different approach) are the next lever. + +## Noise: `confirm` in simulation + +`noise_sim.py` (no model) evolves a scalar "true quality" with mutations +N(-0.01, 0.03) and an evaluator that adds Gaussian noise; budget 60, greedy +selection, 200 runs per cell (± is the standard error). + +| noise sd | confirm | true quality of returned best | reported minus true | +|---|---|---|---| +| 0 | 0 | 0.953 ± 0.008 | +0.000 | +| 0 | 2 | 0.749 ± 0.004 | +0.000 | +| 0.02 | 0 | 0.742 ± 0.008 | +0.028 | +| 0.02 | 2 | 0.692 ± 0.005 | +0.009 | +| 0.05 | 0 | 0.557 ± 0.005 | +0.107 | +| 0.05 | 1 | 0.578 ± 0.005 | +0.066 | +| 0.1 | 0 | 0.511 ± 0.004 | +0.228 | +| 0.1 | 4 | 0.526 ± 0.003 | +0.078 | + +`confirm` always shrinks the winner's curse (the reported best overstates less), but +it spends budget on re-evaluation, so the true quality found goes down when noise is +small relative to the mutation steps and goes up only a little when noise is large. +Use it when an honest score for the returned candidate matters, not as a way to +search better. Full grid: `python benchmarks/evolution/noise_sim.py`. diff --git a/benchmarks/evolution/noise_sim.py b/benchmarks/evolution/noise_sim.py new file mode 100644 index 000000000..af93f7dbe --- /dev/null +++ b/benchmarks/evolution/noise_sim.py @@ -0,0 +1,90 @@ +"""Simulated noisy evolution: what does ``evolve(confirm=...)`` buy at a fixed budget? + +No model is called. A candidate is its true quality; a proposal perturbs the parent's +quality (most mutations are neutral or slightly harmful), and the evaluator returns +the true quality plus Gaussian noise. For each noise level and ``confirm`` setting the +script reports, over many runs, the true quality of the candidate ``evolve`` returns +and how far its reported score overstates it (the winner's curse). + + python benchmarks/evolution/noise_sim.py --runs 200 --budget 60 +""" + +from __future__ import annotations + +import argparse +import random +import statistics +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[2] / "adalflow")) + +from adalflow.optim.evolution import BEST, Proposal, evolve # noqa: E402 + + +def quality(text: str) -> float: + return float(text.split("=", 1)[1].split()[0]) + + +class DriftProposer: + """child quality = parent quality + N(mean_step, step_sd).""" + + def __init__(self, rng: random.Random, mean_step: float, step_sd: float) -> None: + self.rng, self.mean_step, self.step_sd = rng, mean_step, step_sd + self.count = 0 + + def propose(self, parent, inspiration=None) -> Proposal: + self.count += 1 + q = quality(parent.text) + self.rng.gauss(self.mean_step, self.step_sd) + return Proposal(text=f"q={q:.6f} #{self.count}") + + +def one(run: int, sigma: float, confirm: int, args) -> tuple: + noise = random.Random(10_000 + run) + + def evaluate(text: str) -> float: + return quality(text) + noise.gauss(0.0, sigma) + + res = evolve( + "q=0.5", + evaluate, + DriftProposer(random.Random(run), args.mean_step, args.step_sd), + max_evaluations=args.budget, + selection=BEST, + random_seed=run, + max_stalls=args.budget, + confirm=confirm, + ) + true_q = quality(res.best.text) + return true_q, res.best.score - true_q, len(res.archive) + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__.strip().splitlines()[0]) + parser.add_argument("--runs", type=int, default=200) + parser.add_argument("--budget", type=int, default=60) + parser.add_argument("--mean-step", type=float, default=-0.01) + parser.add_argument("--step-sd", type=float, default=0.03) + parser.add_argument("--sigmas", type=float, nargs="+", default=[0, 0.02, 0.05, 0.1]) + parser.add_argument("--confirms", type=int, nargs="+", default=[0, 1, 2, 4]) + args = parser.parse_args() + + print( + f"budget {args.budget}, {args.runs} runs per cell, mutation " + f"N({args.mean_step}, {args.step_sd}), seed quality 0.5\n" + ) + print("| noise sd | confirm | true quality of best | overstatement | candidates |") + print("|---|---|---|---|---|") + for sigma in args.sigmas: + for confirm in args.confirms: + rows = [one(r, sigma, confirm, args) for r in range(args.runs)] + q, gap, n = (list(x) for x in zip(*rows)) + se = statistics.stdev(q) / len(q) ** 0.5 + print( + f"| {sigma} | {confirm} | {statistics.mean(q):.4f} ± {se:.4f} " + f"| {statistics.mean(gap):+.4f} | {statistics.mean(n):.1f} |" + ) + + +if __name__ == "__main__": + main() diff --git a/benchmarks/evolution/run.py b/benchmarks/evolution/run.py index b34cf722e..2b84c5204 100644 --- a/benchmarks/evolution/run.py +++ b/benchmarks/evolution/run.py @@ -5,7 +5,9 @@ --budget 12 --out runs/evolution_bench ``resample`` is the baseline: every proposal is an independent sample from the seed -(best-of-N). ``pareto`` and ``best`` build on earlier candidates. Each run writes its +(best-of-N). ``pareto`` and ``best`` build on earlier candidates; ``map_elites`` keeps +one elite per behavior cell (tasks with a descriptor); ``islands`` is ``pareto`` on 3 +islands with migration every 4 proposals. Each run writes its event log to ``/--.jsonl``; ``summary.json`` and ``summary.md`` hold the per-run numbers and the aggregate table. Requires an API key for the chosen provider; not run in CI. ``--check`` scores each task's seed on both @@ -45,8 +47,21 @@ def make_client(provider: str): return OpenAIClient() +# strategy name -> evolve() keyword arguments +STRATEGIES: Dict[str, Dict[str, Any]] = { + "pareto": {"selection": "pareto"}, + "best": {"selection": "best"}, + "resample": {"selection": "resample"}, + "map_elites": {"selection": "map_elites"}, + "islands": {"selection": "pareto", "islands": 3, "migrate_every": 4}, +} + + def one_run(args, task_name: str, strategy: str, repeat: int) -> Dict[str, Any]: task = TASKS[task_name] + options = dict(STRATEGIES[strategy]) + if task.descriptor is not None: + options["descriptor"] = task.descriptor proposer = LLMProposer( task.objective, model_client=make_client(args.provider), @@ -59,12 +74,12 @@ def one_run(args, task_name: str, strategy: str, repeat: int) -> Dict[str, Any]: PythonEvaluator(task.search_evaluator, timeout=60), proposer, max_evaluations=args.budget, - selection=strategy, random_seed=repeat, stop_score=task.stop_score, max_stalls=args.budget, # keep sampling to the full budget log_path=log_path, holdout=PythonEvaluator(task.holdout_evaluator, timeout=60), + **options, ) curve, best = [], float("-inf") for c in res.archive.candidates: @@ -79,6 +94,7 @@ def one_run(args, task_name: str, strategy: str, repeat: int) -> Dict[str, Any]: "stopped": res.stopped, "proposal_failures": res.proposal_failures, "duplicates": res.duplicates, + "cells": len(res.archive.elites()) if task.descriptor else None, "search_seed": res.seed.score, "search_best": res.best.score if res.best else None, "holdout_seed": res.holdout["seed"].score, @@ -100,8 +116,8 @@ def _fmt(values: List[float]) -> str: def summarize(rows: List[Dict[str, Any]]) -> str: lines = [ "| task | strategy | runs | search best | held-out seed | held-out best " - "| best generation | evals used |", - "|---|---|---|---|---|---|---|---|", + "| best generation | evals used | duplicates | cells |", + "|---|---|---|---|---|---|---|---|---|---|", ] keys = sorted({(r["task"], r["strategy"]) for r in rows}) for task, strategy in keys: @@ -112,7 +128,9 @@ def summarize(rows: List[Dict[str, Any]]) -> str: f"| {_fmt([r['holdout_seed'] for r in group])} " f"| {_fmt([r['holdout_best'] for r in group])} " f"| {_fmt([r['best_generation'] for r in group])} " - f"| {_fmt([r['evaluations'] for r in group])} |" + f"| {_fmt([r['evaluations'] for r in group])} " + f"| {_fmt([r['duplicates'] for r in group])} " + f"| {_fmt([r['cells'] for r in group])} |" ) lines.append("") lines.append("Mean [min, max] over runs.") @@ -138,7 +156,10 @@ def main() -> None: parser.add_argument("--model") parser.add_argument("--tasks", nargs="+", default=sorted(TASKS), choices=TASKS) parser.add_argument( - "--strategies", nargs="+", default=["pareto", "best", "resample"] + "--strategies", + nargs="+", + default=["pareto", "best", "resample"], + choices=sorted(STRATEGIES), ) parser.add_argument("--repeats", type=int, default=3) parser.add_argument("--budget", type=int, default=12, help="evaluations per run") @@ -150,6 +171,9 @@ def main() -> None: return check(args.tasks) if not (args.provider and args.model): parser.error("--provider and --model are required (or pass --check)") + for task in args.tasks: + if "map_elites" in args.strategies and TASKS[task].descriptor is None: + parser.error(f"task {task!r} has no descriptor for map_elites") args.out.mkdir(parents=True, exist_ok=True) jobs = [ diff --git a/benchmarks/evolution/tasks.py b/benchmarks/evolution/tasks.py index 68553ebfe..64b72d29b 100644 --- a/benchmarks/evolution/tasks.py +++ b/benchmarks/evolution/tasks.py @@ -15,9 +15,9 @@ from __future__ import annotations from dataclasses import dataclass -from typing import Dict, List, Tuple +from typing import Any, Callable, Dict, List, Optional, Tuple -__all__ = ["Task", "TASKS"] +__all__ = ["Task", "TASKS", "binpack_cell"] @dataclass @@ -27,7 +27,9 @@ class Task: seed: str search_evaluator: str holdout_evaluator: str - stop_score: float = None # type: ignore[assignment] + stop_score: Optional[float] = None + # candidate -> behavior cell, for selection="map_elites" + descriptor: Optional[Callable[[Any], Tuple]] = None # ------------------------------------------------------------------ duration @@ -152,10 +154,16 @@ def _duration_evaluator(cases: List[DurationCase]) -> str: N_ITEMS, CAPACITY, LOW, HIGH = 300, 150, 20, 100 -def pack(choose_bin, items): +def best_fit(item, bins): + fits = [(r, i) for i, r in enumerate(bins) if r >= item] + return min(fits)[1] if fits else -1 + + +def pack(choose_bin, items, agree): bins = [] # remaining capacity of each open bin for item in items: i = choose_bin(item, list(bins), CAPACITY) + agree.append(i == best_fit(item, bins)) if i == -1: bins.append(CAPACITY - item) continue @@ -168,23 +176,37 @@ def pack(choose_bin, items): def evaluate(candidate): - scores, notes = {}, [] + scores, notes, agree = {}, [], [] for seed in SEEDS: rng = random.Random(seed) items = [rng.randint(LOW, HIGH) for _ in range(N_ITEMS)] bound = math.ceil(sum(items) / CAPACITY) name = f"instance-{seed}" try: - used = pack(candidate.choose_bin, items) + used = pack(candidate.choose_bin, items, agree) scores[name] = bound / used notes.append(f"{name}: {used} bins, lower bound {bound}") except Exception as exc: scores[name] = 0.0 notes.append(f"{name}: invalid packing: {type(exc).__name__}: {exc}") - return {"instance_scores": scores, "feedback": "\n".join(notes)} + return { + "instance_scores": scores, + "feedback": "\n".join(notes), + # Behavior, not quality: how often a decision matches best fit. + "bestfit_agreement": sum(agree) / len(agree) if agree else 0.0, + } """ +def binpack_cell(candidate) -> Tuple[int, int]: + """(agreement-with-best-fit bucket, code length bucket).""" + agreement = candidate.evaluation.metrics.get("bestfit_agreement", 0.0) + band = sum(agreement >= edge for edge in (0.6, 0.8, 0.9, 0.97)) + lines = len([ln for ln in candidate.text.splitlines() if ln.strip()]) + length = 0 if lines <= 8 else 1 if lines <= 20 else 2 if lines <= 40 else 3 + return (band, length) + + def _binpack_evaluator(seeds: List[int]) -> str: return _BINPACK_EVALUATOR.replace("__SEEDS__", repr(seeds)) @@ -205,6 +227,7 @@ def _binpack_evaluator(seeds: List[int]) -> str: """, search_evaluator=_binpack_evaluator([0, 1, 2, 3, 4]), holdout_evaluator=_binpack_evaluator(list(range(100, 110))), + descriptor=binpack_cell, ) TASKS: Dict[str, Task] = {t.name: t for t in (DURATION, BINPACK)} diff --git a/docs/source/new_tutorials/evolution.md b/docs/source/new_tutorials/evolution.md new file mode 100644 index 000000000..1f8eae4a9 --- /dev/null +++ b/docs/source/new_tutorials/evolution.md @@ -0,0 +1,236 @@ +
+ + GitHub + Open Source Code (Evolution examples) + +
+ +# Evolving Programs and Prompts + +`adalflow.optim.evolution` improves any text artifact (a Python function, a prompt, a +config) with an LLM in the loop. Each round picks a parent from an archive of +evaluated candidates, asks the model to rewrite it after reading how it was +evaluated (failing cases, tracebacks, error output), evaluates the rewrite, and adds +it to the archive. Parent selection follows GEPA's per-instance Pareto front +[[Agrawal et al., 2025]](https://arxiv.org/abs/2507.19457), and programs run in a +separate process in the spirit of OpenEvolve. + +You provide three things: + +| piece | what it is | +|---|---| +| seed | the starting text | +| evaluator | `text -> score`, or a dict with per-instance scores and feedback | +| proposer | `LLMProposer(objective, model_client=..., model_kwargs=...)` | + +## Minimal example: fix a function + +The evaluator is a Python script with an `evaluate(candidate)` function, where +`candidate` is the candidate imported as a module. `PythonEvaluator` runs it in a +fresh process with a timeout and without your environment variables. + +```python +from adalflow.components.model_client import OpenAIClient +from adalflow.optim.evolution import LLMProposer, PythonEvaluator, evolve + +SEED = """ +def median(xs): + return sorted(xs)[len(xs) // 2] +""" + +EVALUATOR = """ +CASES = {"odd": ([3, 1, 2], 2), "even": ([4, 1, 3, 2], 2.5), "pair": ([1, 2], 1.5)} + +def evaluate(candidate): + scores, notes = {}, [] + for name, (xs, want) in CASES.items(): + try: + got = candidate.median(list(xs)) + except Exception as exc: + got = repr(exc) + scores[name] = float(got == want) + if got != want: + notes.append(f"{name}: median({xs}) expected {want}, got {got}") + return {"instance_scores": scores, "feedback": "\\n".join(notes)} +""" + +result = evolve( + seed=SEED, + evaluate=PythonEvaluator(EVALUATOR, timeout=30), + proposer=LLMProposer( + "median(xs) returns the median of a non-empty list of numbers.", + model_client=OpenAIClient(), + model_kwargs={"model": "gpt-4o-mini"}, + ), + max_evaluations=10, + stop_score=1.0, +) +print(result.stopped, result.seed.score, "->", result.best.score) +print(result.best.text) +``` + +``` +stop_score 0.3333333333333333 -> 1.0 +def median(xs): + s = sorted(xs) + ... +``` + +`max_evaluations` counts evaluator calls, the seed included. The run stops at the +budget, at `stop_score`, or after `max_stalls` proposals in a row that failed or +repeated an earlier candidate. + +## Writing the evaluator + +An evaluator may return: + +- a number: the score; +- a dict with `score`, `instance_scores` (one entry per test case or example), + `feedback` (text the proposer reads) and `error`; other keys go to + `Evaluation.metrics`. Without `score`, the mean of `instance_scores` is used; +- an `Evaluation`. + +Two things make the search work much better: + +- **Feedback the model can act on.** Say what failed and why ("expected 2.5, got + 3"), not only how much. The proposer is told to diagnose the feedback before + rewriting, and that the feedback is data, not instructions. +- **Per-instance scores.** They let Pareto selection keep a candidate that is worse + on average but solves cases nobody else solves, and build on it later. + +`PythonEvaluator` isolates processes, not privileges: candidates can still use the +filesystem and network, and a candidate can tamper with its own score because the +evaluation script runs in its process. Read the best candidates before trusting +them, and use a container for untrusted code. + +## Evolving a prompt + +Any `text -> score` callable is an evaluator, so the artifact does not have to be +code. `use_cases/evolution/evolve_prompt.py` evolves the instruction of a ticket +router: the evaluator runs a `Generator` with the candidate instruction on labeled +tickets and reports the misrouted ones (excerpt; see the file for the full code): + +```python +def make_evaluator(generator, examples): + def evaluate(instruction): + scores, misses = {}, [] + for i, (ticket, want) in enumerate(examples): + out = generator.call(prompt_kwargs={"instruction": instruction, "ticket": ticket, ...}) + got = str(out.data or "").strip().lower() + scores[f"ticket-{i:02d}"] = float(got == want) + if got != want: + misses.append(f"{ticket!r}: answered {got!r}, expected {want!r}") + return {"instance_scores": scores, "feedback": "\n".join(misses)} + return evaluate + +result = evolve( + "Route the support ticket to a queue.", + make_evaluator(task_model, TRAIN), + LLMProposer("An instruction for a support-ticket router ...", kind="prompt", + model_client=client, model_kwargs={"model": model}), + max_evaluations=8, + stop_score=1.0, + holdout=make_evaluator(task_model, HOLDOUT), +) +``` + +```bash +python use_cases/evolution/evolve_prompt.py --provider openai --model gpt-4o-mini +``` + +One run with `claude-haiku-5-5` (2026-10-21): + +``` +stopped: stop_score after 4 evaluations + c0 (parent None): train accuracy 0.69 + c1 (parent c0): train accuracy 0.62 + c2 (parent c1): train accuracy 0.94 + c3 (parent c2): train accuracy 1.00 +held-out accuracy: seed 0.50 -> best 0.92 +``` + +c1 was worse on average but fixed some tickets the seed got wrong; Pareto selection +kept it as a parent, and its child was the first to reach 0.94. + +## Choosing a search strategy + +`selection=` and two keyword arguments control how parents are picked: + +| setting | what it does | use it for | +|---|---|---| +| `selection="pareto"` (default) | parents from the per-instance Pareto front | tasks with many test cases or examples | +| `selection="best"` | always the highest mean score | a quick greedy baseline | +| `selection="map_elites", descriptor=f` | one elite per behavior cell `f(candidate)`; parents uniform over cells | open-ended tasks where different approaches are worth keeping | +| `islands=3, migrate_every=4` | separate sub-populations; every 4 proposals the next island's best is shown as inspiration | long runs where one lineage would take over | +| `selection="resample"` | every proposal from the seed, no inspiration | the best-of-N baseline your strategy should beat | + +A descriptor maps a candidate to a small tuple of buckets. It is called after each +evaluation, so it can read evaluator metrics: + +```python +def cell(candidate): + agreement = candidate.evaluation.metrics["bestfit_agreement"] # from the evaluator + lines = len(candidate.text.splitlines()) + return (sum(agreement >= e for e in (0.6, 0.8, 0.9, 0.97)), min(lines // 10, 3)) + +result = evolve(seed, evaluator, proposer, max_evaluations=30, + selection="map_elites", descriptor=cell) +print(len(result.archive.elites()), "cells reached") +``` + +## Measuring honestly + +A score that went up on the cases the search could see is not yet an improvement. +Two built-in checks: + +- **`holdout=`**: an evaluator on cases the search never sees. After the run, the seed + and the final best are scored on it (`result.holdout["seed"]`, + `result.holdout["best"]`), outside the budget. A gap between search and held-out + scores means overfitting. +- **`selection="resample"`**: run it with the same budget. If independent samples + from the seed do as well, the search itself is not adding anything. + +`benchmarks/evolution/` runs these comparisons on two tasks. With `claude-haiku-5-5` +and 3 runs per strategy: on a duration-parsing task whose spec is revealed four +failures at a time, `pareto` reached 1.0 on held-out cases within 3-4 evaluations +in every run, while 12 independent samples from the seed reached 0.79-0.96. On online +bin packing, every strategy (including MAP-Elites and islands) plateaued near best +fit, within run-to-run spread. See the benchmark README for the full tables. + +## Noisy evaluators + +LLM judges and sampled benchmarks give a different score each time, and the +highest single score in an archive is usually a lucky one. `confirm=n` re-evaluates +a candidate that beats the best on its first score, together with the incumbent, +until both have `1 + n` samples, and compares the means; only fully sampled +candidates can be returned as the best. + +```python +result = evolve(seed, llm_judge, proposer, max_evaluations=40, confirm=2) +best = result.best +print(best.score, best.n_samples, best.evaluation.metrics.get("score_sd")) +``` + +Re-evaluations count toward `max_evaluations`. In simulation +(`benchmarks/evolution/noise_sim.py`), `confirm` consistently made the reported +best score closer to its true quality, but improved the true quality found only +when the noise was large compared with the gains between candidates; with a +deterministic evaluator it only spends budget. Use it when you need to trust the +returned score. + +## Results and logs + +`evolve` returns an `EvolutionResult`: + +- `best`, `seed`: candidates with `text`, `score`, `evaluation`, `generation`, + `parent_id`, `rationale` (written before evaluation); +- `archive`: every candidate, with `pareto_front()` and `elites()`; +- `evaluations`, `proposal_failures`, `duplicates`, `stopped`; +- `holdout`: held-out evaluations when `holdout=` was given. + +Pass `log_path="runs/x.jsonl"` to write one JSON line per event (`candidate`, +`sample`, `proposal_failure`, `duplicate`, `migration`, `holdout`). + +Each proposal is one `Generator` call (plus a retry when the reply has no fenced +block tagged `candidate` or repeats its parent). The proposer is built with +`use_cache=False`, so repeated proposals from one parent can differ. diff --git a/docs/source/new_tutorials/index.rst b/docs/source/new_tutorials/index.rst index a84d44088..80d867c3f 100644 --- a/docs/source/new_tutorials/index.rst +++ b/docs/source/new_tutorials/index.rst @@ -29,6 +29,7 @@ Tutorials :caption: Self-Improvement Optimizing Agent Harnesses + Evolving Programs and Prompts .. toctree:: :maxdepth: 1 diff --git a/use_cases/evolution/README.md b/use_cases/evolution/README.md index 3548219c2..a52cf100c 100644 --- a/use_cases/evolution/README.md +++ b/use_cases/evolution/README.md @@ -15,6 +15,18 @@ With the default `--provider scripted`, a stand-in model fixes exactly what the feedback reports, so the run shows the mechanics only: feedback reaching the proposer, lineage, the stop rule and the event log. The sandbox and the tests are real. +`evolve_prompt.py` evolves a prompt instead of a program: the instruction of a +support-ticket router, scored on labeled tickets with held-out tickets for the final +check. One `claude-haiku-5-5` run went from 0.69 to 1.0 on the training tickets in +three rewrites, and from 0.50 to 0.92 on the held-out ones. + +```bash +python use_cases/evolution/evolve_prompt.py --provider openai --model gpt-4o-mini +``` + +Tutorial: `docs/source/new_tutorials/evolution.md`. Strategy and noise benchmarks: +`benchmarks/evolution/`. + Your own task: ```python @@ -61,5 +73,6 @@ Notes: filesystem and network. Run LLM-written code in a container when that matters. - The evaluation script runs in the candidate's process, so a candidate can tamper with its own score. Read the best candidates before trusting their scores. -- Scores here are single measurements. With noisy evaluators (LLM judges, sampled - benchmarks), re-evaluate the best candidates before relying on the ranking. +- With noisy evaluators (LLM judges, sampled benchmarks), pass `confirm=2` (or more) + so the reported best is a mean over several evaluations rather than one lucky + score. It costs budget: with a deterministic evaluator leave it at 0. diff --git a/use_cases/evolution/evolve_prompt.py b/use_cases/evolution/evolve_prompt.py new file mode 100644 index 000000000..e8a6847be --- /dev/null +++ b/use_cases/evolution/evolve_prompt.py @@ -0,0 +1,155 @@ +""" +Evolve a prompt, not a program: a support-ticket routing instruction. + +The artifact is the instruction text given to a ``Generator``. The evaluator runs it +on labeled tickets and reports accuracy plus the misrouted tickets, which the +proposer reads to work out the routing policy (e.g. "delivered but not here" goes to +``fulfillment``, not ``logistics``). The final instruction is scored on held-out +tickets. + + python use_cases/evolution/evolve_prompt.py --provider openai --model gpt-4o-mini + python use_cases/evolution/evolve_prompt.py --provider anthropic --model claude-haiku-5-5 + +Requires an API key for the chosen provider. +""" + +from __future__ import annotations + +import argparse +import sys +from concurrent.futures import ThreadPoolExecutor +from pathlib import Path +from typing import Dict, List, Tuple + +PROJECT_ROOT = Path(__file__).resolve().parents[2] +if str(PROJECT_ROOT / "adalflow") not in sys.path: + sys.path.insert(0, str(PROJECT_ROOT / "adalflow")) + +from adalflow.core.generator import Generator # noqa: E402 +from adalflow.optim.evolution import LLMProposer, evolve # noqa: E402 + +LABELS = ["payments", "fulfillment", "logistics", "trust_safety", "privacy", "care"] + +# (ticket, queue). The routing policy, including its less obvious rules (missing +# deliveries go to fulfillment, email changes to privacy, promo codes and account +# merges to care), is only visible through these examples. +TRAIN: List[Tuple[str, str]] = [ + ("I was charged twice for order 4471.", "payments"), + ("The mug arrived cracked, I want my money back.", "fulfillment"), + ("Where is my package? Tracking hasn't moved in 5 days.", "logistics"), + ("I can't log in after resetting my password.", "trust_safety"), + ("Do you sell gift cards?", "care"), + ("Please cancel the subscription renewal that hit my card today.", "payments"), + ("Wrong size shoes were delivered, how do I send them back?", "fulfillment"), + ("Can I change the delivery address for an order that shipped?", "logistics"), + ("How do I update the email on my profile?", "privacy"), + ("Tracking says delivered but there is no parcel anywhere.", "fulfillment"), + ("My invoice shows a tax amount that looks wrong.", "payments"), + ("Someone logged into my account from another country.", "trust_safety"), + ("Delete my account and all my data.", "privacy"), + ("Why did my promo code not apply at checkout?", "care"), + ("I have two accounts, can you combine them?", "care"), + ("Is Saturday delivery available?", "logistics"), +] +HOLDOUT: List[Tuple[str, str]] = [ + ("You billed my card for an order I cancelled.", "payments"), + ("The jacket has a torn seam, I'd like a refund.", "fulfillment"), + ("My order shows delivered but nothing is at my door.", "fulfillment"), + ("Two-factor codes never reach my phone.", "trust_safety"), + ("What are your store opening hours?", "care"), + ("Please change the email address on my account.", "privacy"), + ("The discount code from your newsletter is rejected.", "care"), + ("Can you ship to a P.O. box?", "logistics"), + ("Please merge my old account into my new one.", "care"), + ("Send me a copy of all the personal data you hold on me.", "privacy"), + ("I was charged in euros instead of dollars.", "payments"), + ("I received someone else's order.", "fulfillment"), +] + +SEED = "Route the support ticket to a queue." + +TEMPLATE = r""" +{{ instruction }} +Reply with exactly one label from: {{ labels }}. + + +{{ ticket }} +""" + + +def make_client(provider: str): + if provider == "anthropic": + from adalflow.components.model_client import AnthropicAPIClient + + return AnthropicAPIClient() + from adalflow.components.model_client import OpenAIClient + + return OpenAIClient() + + +def make_evaluator(generator: Generator, examples: List[Tuple[str, str]]): + """instruction text -> {"instance_scores", "feedback"} on ``examples``.""" + + def classify(instruction: str, ticket: str) -> str: + out = generator.call( + prompt_kwargs={ + "instruction": instruction, + "labels": ", ".join(LABELS), + "ticket": ticket, + } + ) + return str(out.data or "").strip().strip(".").lower() + + def evaluate(instruction: str) -> Dict: + with ThreadPoolExecutor(max_workers=8) as pool: + answers = list(pool.map(lambda ex: classify(instruction, ex[0]), examples)) + scores, misses = {}, [] + for i, ((ticket, want), got) in enumerate(zip(examples, answers)): + scores[f"ticket-{i:02d}"] = float(got == want) + if got != want: + misses.append(f"{ticket!r}: answered {got!r}, expected {want!r}") + return {"instance_scores": scores, "feedback": "\n".join(misses)} + + return evaluate + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__.strip().splitlines()[0]) + parser.add_argument("--provider", choices=["openai", "anthropic"], default="openai") + parser.add_argument("--model", default="gpt-4o-mini") + parser.add_argument("--budget", type=int, default=8) + args = parser.parse_args() + + # The task model: classifies tickets with the candidate instruction. + task_model = Generator( + model_client=make_client(args.provider), + model_kwargs={"model": args.model}, + template=TEMPLATE, + use_cache=False, + ) + proposer = LLMProposer( + "An instruction for a support-ticket router. Make the routing policy " + "explicit so the router sends every ticket to the right queue.", + kind="prompt", + model_client=make_client(args.provider), + model_kwargs={"model": args.model}, + ) + result = evolve( + SEED, + make_evaluator(task_model, TRAIN), + proposer, + max_evaluations=args.budget, + stop_score=1.0, + holdout=make_evaluator(task_model, HOLDOUT), + ) + + print(f"stopped: {result.stopped} after {result.evaluations} evaluations") + for c in result.archive.candidates: + print(f" {c.id} (parent {c.parent_id}): train accuracy {c.score:.2f}") + seed, best = result.holdout["seed"], result.holdout.get("best") + print(f"held-out accuracy: seed {seed.score:.2f} -> best {best.score:.2f}") + print("\nbest instruction:\n" + result.best.text) + + +if __name__ == "__main__": + main()