docs: fix the griffe docstring warnings and guard the API reference

.github/workflows/ci.yml

@@ -65,4 +65,13 @@ jobs:
       - uses: astral-sh/setup-uv@v7
       - run: uv sync --group docs
       - run: uv run python scripts/build_llms_full.py -o docs/llms-full.txt
-      - run: uv run zensical build
+      # The real guard for the API reference. zensical --strict does NOT cover
+      # griffe: verified against 0.0.51, the build prints every griffe warning
+      # and still reports "No issues found" and exits 0. Locally the warnings do
+      # not even print, because a warm ./.cache/ replays the cached
+      # api-reference render without re-collecting. Runs before the build so a
+      # docstring regression fails fast with a readable message.
+      - run: uv run python scripts/check_docstrings.py
+      # Kept for what it does catch -- zensical's own issues, e.g. broken links
+      # and missing anchors. Not a docstring guard; see the step above.
+      - run: uv run zensical build --strict

scripts/check_docstrings.py

@@ -0,0 +1,114 @@
+"""Fail if griffe emits any warning while parsing the package's docstrings.
+
+This is the regression guard for the API reference, and it exists because nothing
+else actually guards it:
+
+* ``zensical build --strict`` does not. Verified against zensical 0.0.51: the flag
+  is forwarded to the Rust runtime, whose issue collector reports "No issues
+  found" and exits 0 even while griffe warnings are printed to the same output.
+  ``zensical serve --strict`` is blunter still -- it prints "Strict mode is
+  currently unsupported."
+* A local build does not, because a warm ``./.cache/`` replays the cached
+  ``api-reference/*`` render, so the python handler never re-collects and griffe
+  never re-parses. The warnings simply stop appearing.
+
+What griffe warns about here is not cosmetic. A Google-style ``Args:`` section in
+a *class* docstring is validated against ``Class.parameters``, which is literally
+``all_members["__init__"].parameters``. Classes whose ``__init__`` is generated at
+runtime -- pydantic models, ``NamedTuple``, a ``str`` subclass with only
+``__new__`` -- have no static ``__init__``, so that list is empty: every documented
+name warns, and the rendered table loses its type column and marks every field
+``required``. Such classes must use ``Attributes:`` instead, which griffe resolves
+from the real class-body annotation. ``@dataclass`` docstrings keep ``Args:``:
+griffe's built-in dataclasses extension synthesises an ``__init__`` for them, so
+they get correct types *and* a defaults column.
+
+Note the asymmetry this guard cannot cover: the ``Attributes:`` parser never
+validates names against the class body, so a typo'd attribute name renders a
+phantom row silently rather than warning.
+
+Usage::
+
+    python scripts/check_docstrings.py
+"""
+
+from __future__ import annotations
+
+import logging
+from pathlib import Path
+
+import griffe
+
+REPO_ROOT = Path(__file__).parent.parent
+PACKAGE = "pytest_agent_eval"
+
+
+class _WarningCollector(logging.Handler):
+    """Capture every warning-or-worse record griffe logs while parsing."""
+
+    def __init__(self) -> None:
+        super().__init__()
+        self.messages: list[str] = []
+
+    def emit(self, record: logging.LogRecord) -> None:
+        if record.levelno >= logging.WARNING:
+            self.messages.append(record.getMessage())
+
+
+def _parse_every_docstring(obj: object, seen: set[int]) -> None:
+    """Walk the tree, forcing each docstring to parse so warnings are emitted.
+
+    ``Docstring.parsed`` is lazy, so merely loading the package warns about
+    nothing; the sections have to be materialised to exercise the parser.
+    """
+    if id(obj) in seen:
+        return
+    seen.add(id(obj))
+    docstring = getattr(obj, "docstring", None)
+    if docstring is not None:
+        _ = docstring.parsed
+    for member in getattr(obj, "members", {}).values():
+        if not member.is_alias:
+            _parse_every_docstring(member, seen)
+
+
+def collect_warnings() -> list[str]:
+    """Load the package with griffe and return every docstring warning raised."""
+    collector = _WarningCollector()
+    root = logging.getLogger()
+    root.addHandler(collector)
+    previous_level = root.level
+    root.setLevel(logging.WARNING)
+    try:
+        package = griffe.load(
+            PACKAGE,
+            docstring_parser=griffe.Parser.google,
+            extensions=griffe.load_extensions(),
+            search_paths=[str(REPO_ROOT / "src")],
+            submodules=True,
+        )
+        _parse_every_docstring(package, set())
+    finally:
+        root.removeHandler(collector)
+        root.setLevel(previous_level)
+    return collector.messages
+
+
+def main() -> int:
+    """Report any docstring warnings; return a non-zero exit code if there are any."""
+    warnings = collect_warnings()
+    if not warnings:
+        print(f"griffe: no docstring warnings in {PACKAGE}")
+        return 0
+    print(f"griffe: {len(warnings)} docstring warning(s) in {PACKAGE}:")
+    for message in warnings:
+        print(f"  {message}")
+    print(
+        "\nA class whose __init__ is generated at runtime (pydantic, NamedTuple, "
+        "__new__-only) must document its fields under 'Attributes:', not 'Args:'."
+    )
+    return 1
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

src/pytest_agent_eval/config.py

@@ -25,7 +25,7 @@ class AgentEvalConfig(BaseModel):
     contrasts with ``[tool.agent_eval.groups]``: a typo there would silently disable a
     CI gate, whereas a typo here leaves a documented default in place.
 
-    Args:
+    Attributes:
         model: pydantic-ai model string used by JudgeEvaluator (e.g. "openai:gpt-4o").
         judge_model: Dedicated judge model; takes priority over ``model``.
         threshold: Default fraction of runs that must pass (0.0-1.0).

src/pytest_agent_eval/groups.py

@@ -19,7 +19,7 @@ class GroupConfig(BaseModel):
     Validated strictly, unlike the rest of [tool.agent_eval] where unknown keys are
     ignored: a typo'd key or threshold here would silently disable a CI gate.
 
-    Args:
+    Attributes:
         name: Group name (the table key, not a key inside the table).
         threshold: Fraction of matched, non-skipped tests that must pass (0.0-1.0).
         tags: Transcript tags selecting members (OR-combined with pytest_markers).

src/pytest_agent_eval/models.py

@@ -158,7 +158,7 @@ class AgentReply(NamedTuple):
     *contract* stays the wider plain tuple, so a hand-written
     ``async def agent(history) -> tuple[str, list[str]]`` remains valid.
 
-    Args:
+    Attributes:
         reply: The agent's text reply for this turn.
         tool_calls: Tools called during the turn. Plain strings are accepted; the
             runner normalises them to ``ToolCall`` with ``args=None``.
@@ -184,11 +184,6 @@ class ToolCall(str):
     returning ``list[str]`` (the runner normalises those to ``ToolCall`` with
     ``args=None``).
 
-    Args:
-        name: The tool name.
-        args: The arguments the tool was called with, or None when the adapter
-            could not capture them.
-
     Example:
         ```python
         call = ToolCall("book_slot", {"date": "tomorrow", "time": "10am"})
@@ -202,7 +197,13 @@ class ToolCall(str):
     args: ToolArgs | None
 
     def __new__(cls, name: str, args: ToolArgs | None = None) -> ToolCall:
-        """Create a ToolCall from a tool name and optional captured arguments."""
+        """Create a ToolCall from a tool name and optional captured arguments.
+
+        Args:
+            name: The tool name.
+            args: The arguments the tool was called with, or None when the adapter
+                could not capture them.
+        """
         obj = super().__new__(cls, name)
         obj.args = args
         return obj
@@ -353,7 +354,7 @@ def _valid_regex(pattern: str) -> str:
 class JudgeConfig(_StrictModel):
     """Judge configuration for a YAML transcript turn.
 
-    Args:
+    Attributes:
         rubric: The rubric string passed to the LLM judge.
         model: Optional pydantic-ai model ID override (e.g. "openai:gpt-4o"), or a
             pydantic-ai ``Model`` instance. Falls back to [tool.agent_eval] model if None.
@@ -368,7 +369,7 @@ class JudgeConfig(_StrictModel):
 class ToolCallArgsConfig(_StrictModel):
     """One tool-argument assertion in a YAML transcript turn.
 
-    Args:
+    Attributes:
         tool: Name of the tool whose arguments to check.
         args: Expected arguments for the deterministic check, or None.
         mode: "subset" or "exact" (deterministic check only).
@@ -397,9 +398,9 @@ def _needs_args_or_judge(self) -> ToolCallArgsConfig:
 class Expect(_StrictModel):
     """Expectations for a single transcript turn.
 
-    Args:
-        evaluators: Programmatic evaluators (Python API). Excluded from serialisation
-            and from the JSON schema, since they are Python objects.
+    Attributes:
+        evaluators (list[Evaluator]): Programmatic evaluators (Python API). Excluded from
+            serialisation and from the JSON schema, since they are Python objects.
         judge: YAML-defined judge config.
         tool_calls_include: Tool names that must appear in tool_calls.
         tool_calls_exclude: Tool names that must NOT appear in tool_calls.
@@ -430,10 +431,10 @@ class Expect(_StrictModel):
 class Turn(_StrictModel):
     """A single turn in a transcript.
 
-    Args:
+    Attributes:
         user: The user message (also used as the transcript when ``audio`` is set).
-        audio: Optional path to a WAV file for voice adapters. Resolved relative to
-            the YAML file's directory when loaded from YAML.
+        audio (str | Path | None): Optional path to a WAV file for voice adapters. Resolved
+            relative to the YAML file's directory when loaded from YAML.
         expect: Expectations for the agent's reply.
     """
 
@@ -447,7 +448,7 @@ class Turn(_StrictModel):
 class Transcript(_StrictModel):
     """A multi-turn evaluation transcript.
 
-    Args:
+    Attributes:
         id: Unique identifier used as the pytest test name.
         turns: Ordered list of turns.
         threshold: Fraction of runs that must pass (0.0-1.0).