diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index afaac9b..61d369a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -65,4 +65,13 @@ jobs: - uses: astral-sh/setup-uv@v7 - run: uv sync --group docs - run: uv run python scripts/build_llms_full.py -o docs/llms-full.txt - - run: uv run zensical build + # The real guard for the API reference. zensical --strict does NOT cover + # griffe: verified against 0.0.51, the build prints every griffe warning + # and still reports "No issues found" and exits 0. Locally the warnings do + # not even print, because a warm ./.cache/ replays the cached + # api-reference render without re-collecting. Runs before the build so a + # docstring regression fails fast with a readable message. + - run: uv run python scripts/check_docstrings.py + # Kept for what it does catch -- zensical's own issues, e.g. broken links + # and missing anchors. Not a docstring guard; see the step above. + - run: uv run zensical build --strict diff --git a/scripts/check_docstrings.py b/scripts/check_docstrings.py new file mode 100644 index 0000000..a87f0ed --- /dev/null +++ b/scripts/check_docstrings.py @@ -0,0 +1,114 @@ +"""Fail if griffe emits any warning while parsing the package's docstrings. + +This is the regression guard for the API reference, and it exists because nothing +else actually guards it: + +* ``zensical build --strict`` does not. Verified against zensical 0.0.51: the flag + is forwarded to the Rust runtime, whose issue collector reports "No issues + found" and exits 0 even while griffe warnings are printed to the same output. + ``zensical serve --strict`` is blunter still -- it prints "Strict mode is + currently unsupported." +* A local build does not, because a warm ``./.cache/`` replays the cached + ``api-reference/*`` render, so the python handler never re-collects and griffe + never re-parses. The warnings simply stop appearing. + +What griffe warns about here is not cosmetic. A Google-style ``Args:`` section in +a *class* docstring is validated against ``Class.parameters``, which is literally +``all_members["__init__"].parameters``. Classes whose ``__init__`` is generated at +runtime -- pydantic models, ``NamedTuple``, a ``str`` subclass with only +``__new__`` -- have no static ``__init__``, so that list is empty: every documented +name warns, and the rendered table loses its type column and marks every field +``required``. Such classes must use ``Attributes:`` instead, which griffe resolves +from the real class-body annotation. ``@dataclass`` docstrings keep ``Args:``: +griffe's built-in dataclasses extension synthesises an ``__init__`` for them, so +they get correct types *and* a defaults column. + +Note the asymmetry this guard cannot cover: the ``Attributes:`` parser never +validates names against the class body, so a typo'd attribute name renders a +phantom row silently rather than warning. + +Usage:: + + python scripts/check_docstrings.py +""" + +from __future__ import annotations + +import logging +from pathlib import Path + +import griffe + +REPO_ROOT = Path(__file__).parent.parent +PACKAGE = "pytest_agent_eval" + + +class _WarningCollector(logging.Handler): + """Capture every warning-or-worse record griffe logs while parsing.""" + + def __init__(self) -> None: + super().__init__() + self.messages: list[str] = [] + + def emit(self, record: logging.LogRecord) -> None: + if record.levelno >= logging.WARNING: + self.messages.append(record.getMessage()) + + +def _parse_every_docstring(obj: object, seen: set[int]) -> None: + """Walk the tree, forcing each docstring to parse so warnings are emitted. + + ``Docstring.parsed`` is lazy, so merely loading the package warns about + nothing; the sections have to be materialised to exercise the parser. + """ + if id(obj) in seen: + return + seen.add(id(obj)) + docstring = getattr(obj, "docstring", None) + if docstring is not None: + _ = docstring.parsed + for member in getattr(obj, "members", {}).values(): + if not member.is_alias: + _parse_every_docstring(member, seen) + + +def collect_warnings() -> list[str]: + """Load the package with griffe and return every docstring warning raised.""" + collector = _WarningCollector() + root = logging.getLogger() + root.addHandler(collector) + previous_level = root.level + root.setLevel(logging.WARNING) + try: + package = griffe.load( + PACKAGE, + docstring_parser=griffe.Parser.google, + extensions=griffe.load_extensions(), + search_paths=[str(REPO_ROOT / "src")], + submodules=True, + ) + _parse_every_docstring(package, set()) + finally: + root.removeHandler(collector) + root.setLevel(previous_level) + return collector.messages + + +def main() -> int: + """Report any docstring warnings; return a non-zero exit code if there are any.""" + warnings = collect_warnings() + if not warnings: + print(f"griffe: no docstring warnings in {PACKAGE}") + return 0 + print(f"griffe: {len(warnings)} docstring warning(s) in {PACKAGE}:") + for message in warnings: + print(f" {message}") + print( + "\nA class whose __init__ is generated at runtime (pydantic, NamedTuple, " + "__new__-only) must document its fields under 'Attributes:', not 'Args:'." + ) + return 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/pytest_agent_eval/config.py b/src/pytest_agent_eval/config.py index 21d7312..612c7b4 100644 --- a/src/pytest_agent_eval/config.py +++ b/src/pytest_agent_eval/config.py @@ -25,7 +25,7 @@ class AgentEvalConfig(BaseModel): contrasts with ``[tool.agent_eval.groups]``: a typo there would silently disable a CI gate, whereas a typo here leaves a documented default in place. - Args: + Attributes: model: pydantic-ai model string used by JudgeEvaluator (e.g. "openai:gpt-4o"). judge_model: Dedicated judge model; takes priority over ``model``. threshold: Default fraction of runs that must pass (0.0-1.0). diff --git a/src/pytest_agent_eval/groups.py b/src/pytest_agent_eval/groups.py index ea72053..87c5e5b 100644 --- a/src/pytest_agent_eval/groups.py +++ b/src/pytest_agent_eval/groups.py @@ -19,7 +19,7 @@ class GroupConfig(BaseModel): Validated strictly, unlike the rest of [tool.agent_eval] where unknown keys are ignored: a typo'd key or threshold here would silently disable a CI gate. - Args: + Attributes: name: Group name (the table key, not a key inside the table). threshold: Fraction of matched, non-skipped tests that must pass (0.0-1.0). tags: Transcript tags selecting members (OR-combined with pytest_markers). diff --git a/src/pytest_agent_eval/models.py b/src/pytest_agent_eval/models.py index 616b508..e7edc3e 100644 --- a/src/pytest_agent_eval/models.py +++ b/src/pytest_agent_eval/models.py @@ -158,7 +158,7 @@ class AgentReply(NamedTuple): *contract* stays the wider plain tuple, so a hand-written ``async def agent(history) -> tuple[str, list[str]]`` remains valid. - Args: + Attributes: reply: The agent's text reply for this turn. tool_calls: Tools called during the turn. Plain strings are accepted; the runner normalises them to ``ToolCall`` with ``args=None``. @@ -184,11 +184,6 @@ class ToolCall(str): returning ``list[str]`` (the runner normalises those to ``ToolCall`` with ``args=None``). - Args: - name: The tool name. - args: The arguments the tool was called with, or None when the adapter - could not capture them. - Example: ```python call = ToolCall("book_slot", {"date": "tomorrow", "time": "10am"}) @@ -202,7 +197,13 @@ class ToolCall(str): args: ToolArgs | None def __new__(cls, name: str, args: ToolArgs | None = None) -> ToolCall: - """Create a ToolCall from a tool name and optional captured arguments.""" + """Create a ToolCall from a tool name and optional captured arguments. + + Args: + name: The tool name. + args: The arguments the tool was called with, or None when the adapter + could not capture them. + """ obj = super().__new__(cls, name) obj.args = args return obj @@ -353,7 +354,7 @@ def _valid_regex(pattern: str) -> str: class JudgeConfig(_StrictModel): """Judge configuration for a YAML transcript turn. - Args: + Attributes: rubric: The rubric string passed to the LLM judge. model: Optional pydantic-ai model ID override (e.g. "openai:gpt-4o"), or a pydantic-ai ``Model`` instance. Falls back to [tool.agent_eval] model if None. @@ -368,7 +369,7 @@ class JudgeConfig(_StrictModel): class ToolCallArgsConfig(_StrictModel): """One tool-argument assertion in a YAML transcript turn. - Args: + Attributes: tool: Name of the tool whose arguments to check. args: Expected arguments for the deterministic check, or None. mode: "subset" or "exact" (deterministic check only). @@ -397,9 +398,9 @@ def _needs_args_or_judge(self) -> ToolCallArgsConfig: class Expect(_StrictModel): """Expectations for a single transcript turn. - Args: - evaluators: Programmatic evaluators (Python API). Excluded from serialisation - and from the JSON schema, since they are Python objects. + Attributes: + evaluators (list[Evaluator]): Programmatic evaluators (Python API). Excluded from + serialisation and from the JSON schema, since they are Python objects. judge: YAML-defined judge config. tool_calls_include: Tool names that must appear in tool_calls. tool_calls_exclude: Tool names that must NOT appear in tool_calls. @@ -430,10 +431,10 @@ class Expect(_StrictModel): class Turn(_StrictModel): """A single turn in a transcript. - Args: + Attributes: user: The user message (also used as the transcript when ``audio`` is set). - audio: Optional path to a WAV file for voice adapters. Resolved relative to - the YAML file's directory when loaded from YAML. + audio (str | Path | None): Optional path to a WAV file for voice adapters. Resolved + relative to the YAML file's directory when loaded from YAML. expect: Expectations for the agent's reply. """ @@ -447,7 +448,7 @@ class Turn(_StrictModel): class Transcript(_StrictModel): """A multi-turn evaluation transcript. - Args: + Attributes: id: Unique identifier used as the pytest test name. turns: Ordered list of turns. threshold: Fraction of runs that must pass (0.0-1.0).