[chore]: improve public api checks and error reporting

AGENTS.md

@@ -100,6 +100,34 @@ def test_relu():
 These are the tests run by `nix run .#ci-test` (`pytest -m kernels_ci`), which
 is what kernels-community CI executes on a GPU runner.
 
+## Kernel versions
+
+Any change to a kernel's public API needs the `version` in the `[general]`
+section of its `build.toml` incremented. This covers *additions* — a new
+function, layer, or Torch operator — just as much as it covers removals and
+signature changes.
+
+Additions matter because only the last two Torch versions are rebuilt. Say
+`mykernel` version 1 exposes `a` and has builds for Torch 2.9 through 2.13.
+Adding `b` without a bump leaves version 1 advertising `b` while the 2.9-2.11
+variants, which are not rebuilt, still only carry `a`. Downstream code that
+starts calling `b` then fails on those Torch versions with a symbol that
+cannot be found. Bumping to version 2 instead means `kernels` simply reports
+that no build variant exists for the user's system.
+
+No bump is needed where every published variant gets refreshed anyway:
+
+- No-arch kernels (a `[torch-noarch]` build).
+- Torch stable-ABI kernels, as long as the CUDA versions built overlap with
+  the current build variants.
+- AoT-compiled kernels where a build replaces all variants.
+- Brand-new kernels, which only have builds for the latest two Torch versions.
+
+The `Check Public API` workflow enforces this via
+`scripts/check_public_api.py`, which recognises the no-arch, stable-ABI, and
+brand-new cases on its own. The AoT case is not visible from the repo, so bump
+the version there anyway - a bump is never wrong.
+
 # Kernel-specific instructions
 
 ## flash-attn3

scripts/check_public_api.py

@@ -18,9 +18,11 @@
 # ///
 import argparse
 import ast
+import enum
 import json
 import sys
 import tomllib
+from dataclasses import dataclass
 from pathlib import Path
 
 import pygit2
@@ -334,6 +336,41 @@ def extract_api(src: Source) -> dict:
     return api
 
 
+# Version and build-flavour metadata from a kernel's build.toml.
+
+
+def _build_toml(src: Source) -> dict:
+    if not src.is_file("build.toml"):
+        return {}
+    try:
+        return tomllib.loads(src.read("build.toml"))
+    except tomllib.TOMLDecodeError:
+        return {}
+
+
+def kernel_version(src: Source):
+    return _build_toml(src).get("general", {}).get("version")
+
+
+# Build flavours where adding a symbol cannot break already-published
+# variants, because every variant is refreshed by the next build: no-arch
+# kernels ship a single variant, and stable-ABI kernels keep one build working
+# across Torch versions. Removals and signature changes still need a bump.
+class Flavour(enum.StrEnum):
+    NO_ARCH = "no-arch"
+    STABLE_ABI = "stable-ABI"
+
+
+# The flavour that exempts this kernel from a bump on additions, if any.
+def additive_safe(src: Source) -> Flavour | None:
+    data = _build_toml(src)
+    if "torch-noarch" in data:
+        return Flavour.NO_ARCH
+    if "stable-abi" in data.get("torch", {}):
+        return Flavour.STABLE_ABI
+    return None
+
+
 # Resolve a branch/tag/SHA via libgit2.
 def resolve_ref(repo, ref: str):
     return repo.revparse_single(ref).peel(pygit2.Commit)
@@ -367,11 +404,36 @@ def _tree(items: list) -> None:
             print(f"     {'  ' if last else '| '} {d}")
 
 
+# __all__ is a single API entry per module, so exporting a new name shows up as a
+# changed entry rather than an added one. Detecting the superset case classifies
+# it as additive instead: it still needs a version bump, but it stays eligible for
+# the no-arch and stable-ABI exemptions, which removals and signature changes are
+# not.
+def _all_grew(base: str, head: str) -> bool:
+    try:
+        old = ast.literal_eval(base)
+        new = ast.literal_eval(head)
+    except (ValueError, SyntaxError):
+        return False
+    return isinstance(old, list) and isinstance(new, list) and set(old) < set(new)
+
+
+@dataclass(frozen=True)
+class ApiDiff:
+    breaking: bool
+    additive: bool
+
+
+# Breaking covers symbols that were removed or whose signature changed;
+# additive covers newly exposed ones. Both require a version bump.
 def report(
     kernel: str, base: dict, head: dict, limit: int = 20, preview: int = 6
-) -> bool:
+) -> ApiDiff:
+    # In the baseline, gone from the head.
     removed = sorted(k for k in base if k not in head)
+    # In both, but with a different signature.
     changed = sorted(k for k in base if k in head and base[k] != head[k])
+    # In the head only.
     added = sorted(k for k in head if k not in base)
 
     if not (removed or changed or added):
@@ -381,7 +443,7 @@ def report(
         if len(keys) > preview:
             items.append((f"... and {len(keys) - preview} more", []))
         _tree(items)
-        return False
+        return ApiDiff(breaking=False, additive=False)
 
     counts = ", ".join(
         f"{n} {label}"
@@ -401,7 +463,17 @@ def report(
     if len(items) > limit:
         items = items[:limit] + [(f"... and {len(items) - limit} more", [])]
     _tree(items)
-    return True
+
+    # Changed `__all__` entries that only gained names.
+    grown = [
+        k for k in changed if k.endswith(" __all__") and _all_grew(base[k], head[k])
+    ]
+    # Everything else that changed is a real signature change.
+    signature_changed = set(changed) - set(grown)
+    return ApiDiff(
+        breaking=bool(removed or signature_changed),
+        additive=bool(added or grown),
+    )
 
 
 def main() -> int:
@@ -456,22 +528,55 @@ def main() -> int:
         print("  (no kernel sources changed)")
         return 0
 
-    changed = False
+    unbumped = False
     for kernel in kernels:
         if not Path(kernel).is_dir():
             print(f"  [skip] {kernel}: directory not found")
             continue
-        head_api = extract_api(Source.from_disk(Path(kernel)))
-        if kernel not in base_tree:  # a kernel added by this change has no API to have changed
-            print(f"  [new] {kernel}: {len(head_api)} symbols")
+        base_src = Source.from_tree(repo, base_tree, kernel)
+        head_src = Source.from_disk(Path(kernel))
+        # A kernel added by this PR has no baseline API to diff against, and no
+        # published variants that could go stale.
+        if not base_src.is_file("build.toml"):
+            print(f"  [new] {kernel}: not in the base tree, nothing to compare")
             continue
-        base_api = extract_api(Source.from_tree(repo, base_tree, kernel))
-        if report(kernel, base_api, head_api):
-            changed = True
 
-    if changed:
+        diff = report(kernel, extract_api(base_src), extract_api(head_src))
+        if not (diff.breaking or diff.additive):
+            continue
+
+        old_version = kernel_version(base_src)
+        new_version = kernel_version(head_src)
+        if (
+            isinstance(old_version, int)
+            and isinstance(new_version, int)
+            and new_version > old_version
+        ):
+            print(f"     => version bumped {old_version} -> {new_version}")
+            continue
+
+        flavour = additive_safe(head_src)
+        if diff.additive and not diff.breaking and flavour is not None:
+            print(f"     => additions only on a {flavour} kernel, bump not required")
+            continue
+
+        print(f"     => version is still {old_version}, needs a bump")
+        unbumped = True
+
+    if unbumped:
         print(
-            "\nERROR: public API changed - bump the kernel version if intentional.",
+            "\nERROR: public API changed without a version bump in build.toml.\n"
+            "\n"
+            "Increment [general] version for every public API change, additions\n"
+            "included. Only the last two Torch versions get rebuilt, so a symbol\n"
+            "added under the current version is absent from the older variants\n"
+            "that still advertise it, and downstream callers hit an unresolved\n"
+            "symbol there. A bump instead makes kernels report that no build\n"
+            "variant matches the user's system.\n"
+            "\n"
+            "The cases where every published variant is refreshed anyway, and no\n"
+            "bump is needed - brand-new, no-arch, and Torch stable-ABI kernels -\n"
+            "are recognised here already.",
             file=sys.stderr,
         )
         return 1