From e718d1357adfec37110749ce4ef65279bb12d24a Mon Sep 17 00:00:00 2001 From: joshpineda17 Date: Mon, 24 Aug 2026 09:54:21 -0600 Subject: [PATCH 1/5] fix(skills): make the skill text an actually-optimizable parameter MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit SkillModule passed the skill body as a dspy InputField (`skill_instructions`) while DSPy optimizers only rewrite a predictor's *signature instructions* — they never touch input values. The skill text was therefore never mutated: GEPA proposed candidates every iteration, `optimized_module.skill_text` returned the untouched original, and the "evolved" skill was written to disk byte-identical to the baseline. Observed on a real run: GEPA logged 7 accepted proposals for `predictor.predict`, and the result was `improvement: 0.0` with `evolved_size == baseline_size`. Phase 1 could not produce an evolved skill at all. Seed the signature with `TaskWithSkill.with_instructions(skill_text)` so the skill body *is* the optimizable parameter, and expose `skill_text` as a property reading it back from the predictor. Callers are unchanged. Add TestSkillModuleIsOptimizable, which fails on the old wiring: it mutates the predictor's instructions the way an optimizer does and asserts `skill_text` follows. SkillModule previously had no tests. --- evolution/skills/skill_module.py | 42 ++++++++++++++++++++++-------- tests/skills/test_skill_module.py | 43 ++++++++++++++++++++++++++++++- 2 files changed, 73 insertions(+), 12 deletions(-) diff --git a/evolution/skills/skill_module.py b/evolution/skills/skill_module.py index 6d4d22ed..584ff4a7 100644 --- a/evolution/skills/skill_module.py +++ b/evolution/skills/skill_module.py @@ -84,9 +84,15 @@ def find_skill(skill_name: str, hermes_agent_path: Path) -> Optional[Path]: class SkillModule(dspy.Module): """A DSPy module that wraps a skill file for optimization. - The skill text (body) is the parameter that GEPA optimizes. + The skill text is the parameter that GEPA optimizes. Critically, it must + live in the predictor's **signature instructions**, not in an InputField: + DSPy optimizers mutate signature instructions and leave input *values* + alone. Wiring the skill as an InputField (the original design) meant GEPA + happily proposed improved variants that were then silently discarded, + producing an "evolved" skill byte-identical to the baseline. + On each forward pass, the module: - 1. Uses the skill text as instructions + 1. Uses the skill text as the predictor's instructions 2. Processes the task input 3. Returns the agent's response """ @@ -94,23 +100,37 @@ class SkillModule(dspy.Module): class TaskWithSkill(dspy.Signature): """Complete a task following the provided skill instructions. - You are an AI agent following specific skill instructions to complete a task. - Read the skill instructions carefully and follow the procedure described. + You are an AI agent following specific skill instructions to complete a + task. Read the skill instructions carefully and follow the procedure + described. """ - skill_instructions: str = dspy.InputField(desc="The skill instructions to follow") task_input: str = dspy.InputField(desc="The task to complete") output: str = dspy.OutputField(desc="Your response following the skill instructions") def __init__(self, skill_text: str): super().__init__() - self.skill_text = skill_text - self.predictor = dspy.ChainOfThought(self.TaskWithSkill) + # Seed the signature's instructions with the skill body so it becomes + # the optimizable parameter GEPA mutates. + self.predictor = dspy.ChainOfThought( + self.TaskWithSkill.with_instructions(skill_text) + ) + + @property + def skill_text(self) -> str: + """Current skill text, read back out of the signature instructions. + + After ``optimizer.compile(...)`` this returns the *evolved* text, which + is what callers persist to disk. + """ + # dspy.ChainOfThought wraps an inner Predict as `.predict`; GEPA names + # the component "predictor.predict". Fall back to the outer signature + # for other module shapes / DSPy versions. + inner = getattr(self.predictor, "predict", None) + sig = getattr(inner, "signature", None) or getattr(self.predictor, "signature", None) + return getattr(sig, "instructions", "") or "" def forward(self, task_input: str) -> dspy.Prediction: - result = self.predictor( - skill_instructions=self.skill_text, - task_input=task_input, - ) + result = self.predictor(task_input=task_input) return dspy.Prediction(output=result.output) diff --git a/tests/skills/test_skill_module.py b/tests/skills/test_skill_module.py index f4ad3c2c..bc1cdb4b 100644 --- a/tests/skills/test_skill_module.py +++ b/tests/skills/test_skill_module.py @@ -2,7 +2,7 @@ import pytest from pathlib import Path -from evolution.skills.skill_module import load_skill, reassemble_skill +from evolution.skills.skill_module import SkillModule, load_skill, reassemble_skill SAMPLE_SKILL = """--- @@ -90,3 +90,44 @@ def test_evolved_body_replaces_original(self): assert "EVOLVED" in result assert "New and improved" in result + + +class TestSkillModuleIsOptimizable: + """The skill text must be the parameter DSPy optimizers actually mutate. + + DSPy optimizers (GEPA, MIPROv2) rewrite a predictor's **signature + instructions**; they never touch InputField *values*. If the skill text is + wired as an InputField, optimization silently no-ops: GEPA proposes better + variants and the caller reads back the unchanged original. + """ + + BODY = "# Test Skill\n\n## Procedure\n1. Do the thing\n" + + def test_skill_text_seeds_predictor_instructions(self): + module = SkillModule(self.BODY) + + instructions = [ + predictor.signature.instructions + for _, predictor in module.named_predictors() + ] + assert instructions, "module exposes no optimizable predictors" + assert any(self.BODY.strip() == text.strip() for text in instructions), ( + "skill text is not in any predictor's instructions, so no DSPy " + "optimizer can mutate it" + ) + + def test_skill_text_reflects_optimizer_mutation(self): + module = SkillModule(self.BODY) + + # Simulate what GEPA/MIPROv2 do when they accept a candidate. + for _, predictor in module.named_predictors(): + predictor.signature = predictor.signature.with_instructions("# EVOLVED") + + assert module.skill_text.strip() == "# EVOLVED", ( + "mutating predictor instructions did not change skill_text — the " + "evolved variant would be discarded" + ) + + def test_skill_text_roundtrips_unmutated(self): + module = SkillModule(self.BODY) + assert module.skill_text.strip() == self.BODY.strip() From 012610f39d3631ef379f125c16ff290b887cbcbf Mon Sep 17 00:00:00 2001 From: joshpineda17 Date: Mon, 24 Aug 2026 09:54:46 -0600 Subject: [PATCH 2/5] fix(importers): read Hermes sessions from the SQLite state.db MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit HermesSessionImporter only understood `~/.hermes/sessions/*.json`, a layout modern Hermes no longer writes. Transcripts now live in a single SQLite database (`state.db`) with `sessions` and `messages` tables, so `--eval-source sessiondb` silently mined **0 messages** from Hermes on any current install — the one source that has richer signal than Claude Code (user-only) or Copilot. On the reporting machine: `~/.hermes/sessions/` did not exist, while `state.db` held 8,240 messages across 52 sessions. After this change the Hermes importer yields 424 user/assistant pairs where it previously yielded none. - Add `_candidate_db_paths()`: honors `HERMES_HOME`, then per-platform defaults (Windows `%LOCALAPPDATA%\hermes`, XDG, macOS Application Support, `~/.hermes`). - Add `_extract_from_sqlite()`: opens the DB read-only (`mode=ro`) so a live Hermes process is never disturbed, pairs each user turn with the next assistant turn *within the same session*, and skips rows flagged `compacted` so compression artifacts don't become training data. - Skip cron-injected preambles — agent scaffolding, not user tasks. - Keep the legacy JSON reader as a fallback so older installs still work. - Add `STATE_DB` for an explicit override, which also lets tests pin the database instead of picking up whatever install exists on the host. Tests: add TestHermesSessionImporterSQLite (pairing, session isolation, secret filtering, limit, cron skip, legacy fallback, unrelated schema), and isolate the existing legacy-JSON tests from the host's real state.db. --- evolution/core/external_importers.py | 150 +++++++++++++++++++++++-- tests/core/test_external_importers.py | 152 ++++++++++++++++++++++++++ 2 files changed, 295 insertions(+), 7 deletions(-) diff --git a/evolution/core/external_importers.py b/evolution/core/external_importers.py index 65fe0aaa..4c48b885 100644 --- a/evolution/core/external_importers.py +++ b/evolution/core/external_importers.py @@ -9,7 +9,8 @@ Supported sources: - Claude Code (~/.claude/history.jsonl) — user inputs only - GitHub Copilot (~/.copilot/session-state/*/events.jsonl) — full conversations - - Hermes Agent (~/.hermes/sessions/*.json) — user + assistant + tool context + - Hermes Agent (state.db, or legacy ~/.hermes/sessions/*.json) — user + + assistant + tool context Usage as standalone CLI: python -m evolution.core.external_importers \\ @@ -23,8 +24,10 @@ """ import json +import os import re import random +import sqlite3 from pathlib import Path from typing import Optional @@ -332,22 +335,150 @@ def _parse_copilot_events( class HermesSessionImporter: - """Import conversations from Hermes Agent session files. + """Import conversations from the Hermes Agent session store. - Hermes stores session transcripts as JSON files in ~/.hermes/sessions/. - Each file contains an OpenAI-format message list with user, assistant, - and tool messages — providing richer signal than Claude Code (user-only) - or Copilot (user+assistant without tool context). + Modern Hermes (>= v0.15) stores every transcript in a single SQLite + database (``state.db``) with ``sessions`` and ``messages`` tables — not as + one JSON file per session. Older builds used ``~/.hermes/sessions/*.json``. + This importer prefers the SQLite store and falls back to the legacy JSON + layout so both generations keep working. This mines user messages paired with the assistant's final response, giving the LLM judge both the task and how it was actually handled. """ + # Legacy layout (pre-SQLite Hermes builds). SESSION_DIR = Path.home() / ".hermes" / "sessions" + # Explicit SQLite store override. ``None`` means auto-discover; set it to a + # concrete path to pin the database (and to isolate tests from whatever + # Hermes install happens to exist on the machine running them). + STATE_DB: Optional[Path] = None + + # Cron-injected preamble — agent-generated scaffolding, not a user task. + _CRON_PREAMBLE = "[IMPORTANT: You are running as a scheduled cron job" + + @staticmethod + def _candidate_db_paths() -> list[Path]: + """Return plausible locations of the Hermes ``state.db``, best first. + + An explicit :attr:`STATE_DB` short-circuits discovery. Otherwise honors + ``HERMES_HOME``, then the per-platform defaults (Windows + ``%LOCALAPPDATA%\\hermes``, XDG data dir, macOS Application Support, + ``~/.hermes``). + """ + override = HermesSessionImporter.STATE_DB + if override is not None: + override = Path(override).expanduser() + return [override] if override.is_file() else [] + + candidates: list[Path] = [] + + hermes_home = os.environ.get("HERMES_HOME") + if hermes_home: + candidates.append(Path(hermes_home).expanduser()) + + home = Path.home() + local_appdata = os.environ.get("LOCALAPPDATA") + if local_appdata: + candidates.append(Path(local_appdata) / "hermes") + candidates.append(home / "AppData" / "Local" / "hermes") # Windows + candidates.append(home / ".local" / "share" / "hermes") # XDG + candidates.append(home / "Library" / "Application Support" / "hermes") # macOS + candidates.append(home / ".hermes") # legacy / custom + + seen: set[Path] = set() + paths: list[Path] = [] + for base in candidates: + db = base / "state.db" + if db in seen: + continue + seen.add(db) + if db.is_file(): + paths.append(db) + return paths + + @staticmethod + def _extract_from_sqlite(db_path: Path, limit: int = 0) -> list[dict]: + """Mine user/assistant pairs out of a Hermes ``state.db``.""" + messages: list[dict] = [] + + # Read-only URI so a live Hermes process is never disturbed. + try: + conn = sqlite3.connect(f"file:{db_path}?mode=ro", uri=True) + except sqlite3.Error: + return [] + + try: + conn.row_factory = sqlite3.Row + cols = {r[1] for r in conn.execute("PRAGMA table_info(messages)")} + if not {"session_id", "role", "content"} <= cols: + return [] + + order_col = "id" if "id" in cols else "rowid" + # Only real transcript turns; skip compacted/summary rows when the + # column exists so we don't train on compression artifacts. + where = "role IN ('user','assistant')" + if "compacted" in cols: + where += " AND COALESCE(compacted, 0) = 0" + + rows = conn.execute( + f"SELECT session_id, role, content FROM messages " + f"WHERE {where} ORDER BY session_id, {order_col}" + ).fetchall() + except sqlite3.Error: + return [] + finally: + conn.close() + + # Group by session, preserving order, then pair user -> next assistant. + by_session: dict[str, list[sqlite3.Row]] = {} + for row in rows: + by_session.setdefault(row["session_id"], []).append(row) + + for session_id, msg_list in by_session.items(): + for i, msg in enumerate(msg_list): + if msg["role"] != "user": + continue + user_text = msg["content"] or "" + if not isinstance(user_text, str) or len(user_text) < 10: + continue + if user_text.lstrip().startswith(HermesSessionImporter._CRON_PREAMBLE): + continue + if _contains_secret(user_text): + continue + + assistant_text = "" + for j in range(i + 1, len(msg_list)): + if msg_list[j]["role"] == "assistant": + content = msg_list[j]["content"] or "" + if content: + assistant_text = content + break + elif msg_list[j]["role"] == "user": + break + + if assistant_text and _contains_secret(assistant_text): + continue + + messages.append({ + "source": "hermes", + "task_input": user_text, + "assistant_response": assistant_text, + "session_id": session_id, + }) + + if limit and len(messages) >= limit: + return messages + + return messages + @staticmethod def extract_messages(limit: int = 0) -> list[dict]: - """Read user/assistant pairs from Hermes session files. + """Read user/assistant pairs from the Hermes session store. + + Tries the SQLite ``state.db`` first (modern Hermes), then the legacy + ``~/.hermes/sessions/*.json`` layout. Args: limit: Maximum messages to return (0 = no limit). @@ -356,6 +487,11 @@ def extract_messages(limit: int = 0) -> list[dict]: List of dicts with keys: source, task_input, assistant_response, session_id. """ + for db_path in HermesSessionImporter._candidate_db_paths(): + found = HermesSessionImporter._extract_from_sqlite(db_path, limit=limit) + if found: + return found + if not HermesSessionImporter.SESSION_DIR.exists(): return [] diff --git a/tests/core/test_external_importers.py b/tests/core/test_external_importers.py index adfacc07..55102e0a 100644 --- a/tests/core/test_external_importers.py +++ b/tests/core/test_external_importers.py @@ -15,6 +15,7 @@ """ import json +import sqlite3 from pathlib import Path from types import SimpleNamespace from unittest.mock import patch, MagicMock @@ -468,6 +469,14 @@ def test_parse_copilot_events_permission_error(self, tmp_path): class TestHermesSessionImporter: + @pytest.fixture(autouse=True) + def _isolate_state_db(self, tmp_path): + """Pin STATE_DB to a nonexistent file so these legacy-JSON tests never + pick up a real Hermes install's ``state.db`` on the machine running + them.""" + with patch.object(HermesSessionImporter, "STATE_DB", tmp_path / "absent-state.db"): + yield + def test_parses_session_json(self, tmp_path): session = { "session_id": "test-session", @@ -556,6 +565,149 @@ def test_respects_limit(self, tmp_path): assert len(msgs) == 3 +def _make_state_db(path, rows): + """Build a minimal Hermes-shaped state.db containing ``rows``. + + ``rows`` is a list of ``(session_id, role, content)`` tuples, inserted in + order so the ``id`` column reflects conversation order. + """ + conn = sqlite3.connect(path) + conn.execute( + "CREATE TABLE messages (" + " id INTEGER PRIMARY KEY AUTOINCREMENT," + " session_id TEXT," + " role TEXT," + " content TEXT," + " compacted INTEGER DEFAULT 0" + ")" + ) + conn.executemany( + "INSERT INTO messages (session_id, role, content) VALUES (?, ?, ?)", rows + ) + conn.commit() + conn.close() + return path + + +class TestHermesSessionImporterSQLite: + """Modern Hermes keeps transcripts in a single SQLite ``state.db``.""" + + def test_reads_user_assistant_pairs(self, tmp_path): + db = _make_state_db(tmp_path / "state.db", [ + ("sess-1", "user", "Fix the bug in auth.py"), + ("sess-1", "assistant", "I found the issue and fixed it."), + ("sess-1", "user", "Now run the tests please"), + ("sess-1", "assistant", "All 42 tests passed."), + ]) + + with patch.object(HermesSessionImporter, "STATE_DB", db): + msgs = HermesSessionImporter.extract_messages() + + assert len(msgs) == 2 + assert msgs[0]["source"] == "hermes" + assert msgs[0]["session_id"] == "sess-1" + assert msgs[0]["task_input"] == "Fix the bug in auth.py" + assert msgs[0]["assistant_response"] == "I found the issue and fixed it." + assert msgs[1]["assistant_response"] == "All 42 tests passed." + + def test_pairs_within_session_only(self, tmp_path): + """A user turn must not borrow the next session's assistant reply.""" + db = _make_state_db(tmp_path / "state.db", [ + ("sess-a", "user", "Question asked in session A"), + ("sess-b", "assistant", "Answer that belongs to session B"), + ]) + + with patch.object(HermesSessionImporter, "STATE_DB", db): + msgs = HermesSessionImporter.extract_messages() + + assert len(msgs) == 1 + assert msgs[0]["session_id"] == "sess-a" + assert msgs[0]["assistant_response"] == "" + + def test_skips_short_messages(self, tmp_path): + db = _make_state_db(tmp_path / "state.db", [ + ("s", "user", "hi"), + ("s", "assistant", "Hello!"), + ]) + + with patch.object(HermesSessionImporter, "STATE_DB", db): + msgs = HermesSessionImporter.extract_messages() + + assert msgs == [] + + def test_filters_secrets(self, tmp_path): + db = _make_state_db(tmp_path / "state.db", [ + ("s", "user", "Set ANTHROPIC_API_KEY=sk-ant-0123456789abcdef in the env"), + ("s", "assistant", "Done."), + ]) + + with patch.object(HermesSessionImporter, "STATE_DB", db): + msgs = HermesSessionImporter.extract_messages() + + assert msgs == [] + + def test_skips_cron_preamble(self, tmp_path): + """Cron-injected prompts are agent scaffolding, not user tasks.""" + db = _make_state_db(tmp_path / "state.db", [ + ("cron-1", "user", HermesSessionImporter._CRON_PREAMBLE + " ... do the thing]"), + ("cron-1", "assistant", "Report delivered."), + ("sess-1", "user", "A genuine question from the user"), + ("sess-1", "assistant", "A genuine answer."), + ]) + + with patch.object(HermesSessionImporter, "STATE_DB", db): + msgs = HermesSessionImporter.extract_messages() + + assert len(msgs) == 1 + assert msgs[0]["session_id"] == "sess-1" + + def test_respects_limit(self, tmp_path): + rows = [] + for i in range(10): + rows.append(("s", "user", f"Message number {i} with enough text")) + rows.append(("s", "assistant", f"Reply number {i}")) + db = _make_state_db(tmp_path / "state.db", rows) + + with patch.object(HermesSessionImporter, "STATE_DB", db): + msgs = HermesSessionImporter.extract_messages(limit=3) + + assert len(msgs) == 3 + + def test_missing_db_falls_back_to_legacy_json(self, tmp_path): + """No SQLite store → the legacy JSON layout still works.""" + session = { + "session_id": "legacy", + "messages": [ + {"role": "user", "content": "A legacy question from JSON"}, + {"role": "assistant", "content": "A legacy answer."}, + ], + } + legacy_dir = tmp_path / "sessions" + legacy_dir.mkdir() + (legacy_dir / "s.json").write_text(json.dumps(session)) + + with patch.object(HermesSessionImporter, "STATE_DB", tmp_path / "absent.db"), \ + patch.object(HermesSessionImporter, "SESSION_DIR", legacy_dir): + msgs = HermesSessionImporter.extract_messages() + + assert len(msgs) == 1 + assert msgs[0]["task_input"] == "A legacy question from JSON" + + def test_unrelated_schema_is_ignored(self, tmp_path): + """A state.db without the expected columns must not raise.""" + db = tmp_path / "state.db" + conn = sqlite3.connect(db) + conn.execute("CREATE TABLE messages (id INTEGER, unrelated TEXT)") + conn.commit() + conn.close() + + with patch.object(HermesSessionImporter, "STATE_DB", db), \ + patch.object(HermesSessionImporter, "SESSION_DIR", tmp_path / "nope"): + msgs = HermesSessionImporter.extract_messages() + + assert msgs == [] + + # ── Skill Name Matching ────────────────────────────────────────────────────── From 202937112bf19d46c98ab914111956ef9e156efb Mon Sep 17 00:00:00 2001 From: joshpineda17 Date: Mon, 24 Aug 2026 09:55:10 -0600 Subject: [PATCH 3/5] fix(skills): repair the GEPA call and the deploy-blocking constraint check MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three defects in the Phase 1 driver, each of which alone prevents a run from producing a deployable skill: 1. `dspy.GEPA(max_steps=...)` — no such argument in dspy 3.x. Every run raised TypeError and fell through to the MIPROv2 fallback, so GEPA never actually ran. Use `max_full_evals=iterations` (the closest analogue to "iterations") and pass the `reflection_lm` GEPA needs to propose mutations. 2. The MIPROv2 fallback then died on `ImportError: MIPROv2 requires optional dependency 'optuna'`, taking the whole run down. With GEPA fixed the fallback is rare, but `optuna` is now declared so the documented fallback path works instead of crashing. 3. `_check_skill_structure` asserts YAML frontmatter exists, but was handed `skill["body"]` — which `load_skill()` has already stripped of frontmatter. The check could therefore never pass. On the baseline it printed a spurious violation; on the evolved artifact it tripped "Evolved skill FAILED constraints — not deploying", discarding good variants. Validate the full artifact (`skill["raw"]` / `evolved_full`) so the structural check sees the frontmatter it is asserting on. Also add `_gepa_metric`, an arity-tolerant wrapper: GEPA invokes metrics as `(gold, pred, trace, pred_name, pred_trace)` while Evaluate uses `(gold, pred)`, and `skill_fitness_metric` accepts only three positional args. Normalizing at the call site keeps the metric's public signature intact. After these fixes a 6-iteration run completes end to end and produces a genuinely different evolved skill, correctly gated by the growth limit. --- evolution/skills/evolve_skill.py | 35 +++++++++++++++++++++++++++----- pyproject.toml | 5 +++++ 2 files changed, 35 insertions(+), 5 deletions(-) diff --git a/evolution/skills/evolve_skill.py b/evolution/skills/evolve_skill.py index 2a79a670..024e2a91 100644 --- a/evolution/skills/evolve_skill.py +++ b/evolution/skills/evolve_skill.py @@ -33,6 +33,21 @@ console = Console() +def _gepa_metric(*args, **kwargs) -> float: + """Arity-tolerant wrapper around :func:`skill_fitness_metric`. + + DSPy optimizers call metrics with varying signatures — GEPA passes + ``(gold, pred, trace, pred_name, pred_trace)`` while Evaluate passes + ``(gold, pred)``. ``skill_fitness_metric`` only accepts three positional + args, so calling it directly raises ``TypeError`` under GEPA. Normalize + here instead of changing the metric's public signature. + """ + example = args[0] if len(args) > 0 else kwargs.get("example") or kwargs.get("gold") + prediction = args[1] if len(args) > 1 else kwargs.get("prediction") or kwargs.get("pred") + trace = args[2] if len(args) > 2 else kwargs.get("trace") + return skill_fitness_metric(example, prediction, trace) + + def evolve( skill_name: str, iterations: int = 10, @@ -118,8 +133,13 @@ def evolve( # ── 3. Validate constraints on baseline ───────────────────────────── console.print(f"\n[bold]Validating baseline constraints[/bold]") validator = ConstraintValidator(config) - baseline_constraints = validator.validate_all(skill["body"], "skill") + # Validate the FULL skill file, not just the body: _check_skill_structure + # asserts YAML frontmatter exists, and load_skill() has already stripped it + # from skill["body"] — passing the body guarantees a false failure, which + # would later block a perfectly good evolved skill from deploying. + baseline_constraints = validator.validate_all(skill["raw"], "skill") all_pass = True + for c in baseline_constraints: icon = "✓" if c.passed else "✗" color = "green" if c.passed else "red" @@ -153,9 +173,14 @@ def evolve( start_time = time.time() try: + # dspy>=3.x GEPA has no `max_steps`; `max_full_evals` is the closest + # analogue to "iterations" (budgeted full passes over the valset), and + # it requires an explicit reflection LM to propose mutations. + reflection_lm = dspy.LM(optimizer_model, temperature=1.0, max_tokens=8000) optimizer = dspy.GEPA( - metric=skill_fitness_metric, - max_steps=iterations, + metric=_gepa_metric, + max_full_evals=iterations, + reflection_lm=reflection_lm, ) optimized_module = optimizer.compile( @@ -167,7 +192,7 @@ def evolve( # Fall back to MIPROv2 if GEPA isn't available in this DSPy version console.print(f"[yellow]GEPA not available ({e}), falling back to MIPROv2[/yellow]") optimizer = dspy.MIPROv2( - metric=skill_fitness_metric, + metric=_gepa_metric, auto="light", ) optimized_module = optimizer.compile( @@ -185,7 +210,7 @@ def evolve( # ── 7. Validate evolved skill ─────────────────────────────────────── console.print(f"\n[bold]Validating evolved skill[/bold]") - evolved_constraints = validator.validate_all(evolved_body, "skill", baseline_text=skill["body"]) + evolved_constraints = validator.validate_all(evolved_full, "skill", baseline_text=skill["raw"]) all_pass = True for c in evolved_constraints: icon = "✓" if c.passed else "✗" diff --git a/pyproject.toml b/pyproject.toml index dae7f461..8ba23ea0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -30,6 +30,11 @@ dev = [ darwinian = [ "darwinian-evolver", ] +# The MIPROv2 fallback in evolve_skill.py needs optuna; without it a GEPA +# failure turns into ImportError instead of a working fallback. +mipro = [ + "optuna>=3.0", +] [project.urls] Homepage = "https://github.com/NousResearch/hermes-agent-self-evolution" From 3247dad5df2440bd6684a5e64ef08b79822db139 Mon Sep 17 00:00:00 2001 From: joshpineda17 Date: Mon, 24 Aug 2026 09:55:51 -0600 Subject: [PATCH 4/5] test(config): fix tilde-expansion test on Windows `test_resolve_expands_user_home` monkeypatched only HOME, but `Path.expanduser()` reads USERPROFILE (and HOMEDRIVE/HOMEPATH) on Windows. The test asserted against `/home/example/...` while expansion resolved to the real user profile, so it failed on every Windows run independently of any of the fixes in this PR. Set USERPROFILE alongside HOME and clear HOMEDRIVE/HOMEPATH so the test exercises tilde expansion on both platforms. --- tests/core/test_config_repo_path.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/tests/core/test_config_repo_path.py b/tests/core/test_config_repo_path.py index 179064e1..599a1b32 100644 --- a/tests/core/test_config_repo_path.py +++ b/tests/core/test_config_repo_path.py @@ -46,8 +46,16 @@ def test_resolve_honors_explicit_path_without_default(tmp_path, monkeypatch): def test_resolve_expands_user_home(monkeypatch): from evolution.core.config import resolve_hermes_agent_path - monkeypatch.setenv("HOME", "/home/example") - assert resolve_hermes_agent_path("~/code/hermes-agent") == Path("/home/example/code/hermes-agent") + # Path.expanduser() reads HOME on POSIX but USERPROFILE on Windows, so set + # both — otherwise this test only exercises tilde expansion on POSIX and + # fails on Windows against the real user profile. + fake_home = Path("/home/example") + monkeypatch.setenv("HOME", str(fake_home)) + monkeypatch.setenv("USERPROFILE", str(fake_home)) + monkeypatch.delenv("HOMEDRIVE", raising=False) + monkeypatch.delenv("HOMEPATH", raising=False) + + assert resolve_hermes_agent_path("~/code/hermes-agent") == fake_home / "code" / "hermes-agent" def test_resolve_falls_back_to_env_var_when_no_override(tmp_path, monkeypatch): From 48fa6828840496582cc2ce61a723ecffa100e9c6 Mon Sep 17 00:00:00 2001 From: Raul Date: Tue, 25 Aug 2026 10:28:12 +0200 Subject: [PATCH 5/5] fix(importers): keep Hermes session importer tests isolated --- evolution/core/external_importers.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/evolution/core/external_importers.py b/evolution/core/external_importers.py index 4c48b885..79c0deed 100644 --- a/evolution/core/external_importers.py +++ b/evolution/core/external_importers.py @@ -492,6 +492,13 @@ def extract_messages(limit: int = 0) -> list[dict]: if found: return found + # An explicit existing STATE_DB pins the source (especially in tests). If + # that DB yields no usable messages, do not fall through into the user's + # real legacy session directory and contaminate the result. A missing + # override still allows legacy fallback for compatibility. + if HermesSessionImporter.STATE_DB is not None and Path(HermesSessionImporter.STATE_DB).expanduser().is_file(): + return [] + if not HermesSessionImporter.SESSION_DIR.exists(): return [] @@ -505,7 +512,7 @@ def extract_messages(limit: int = 0) -> list[dict]: for session_file in session_files: try: data = json.loads(session_file.read_text()) - except (json.JSONDecodeError, OSError): + except (UnicodeDecodeError, json.JSONDecodeError, OSError): continue msg_list = data.get("messages", [])