fix: recover credentials a stacklet lost, without a reinstall

docs/stack-reference.md

@@ -106,6 +106,7 @@ requires    = ["core", "messages"]
 | `type` | string | `"docker"` | `"docker"` (default) or `"host"`. Host stacklets install native macOS software (brew, compiled binaries) alongside optional Docker containers. |
 | `requires` | list | `[]` | Stacklet IDs that must be enabled before this one. The runtime enforces ordering on `stack up` and prevents destroying dependencies. |
 | `build` | bool | false | If true, the stacklet has a local Dockerfile. `stack up` rebuilds the image on every run instead of pulling from a registry. Use for stacklets with custom code (bots, agents). |
+| `required_secrets` | list | `[]` | Secret names (unprefixed, as `ctx.secret()` reads them) the stacklet cannot work without. `stack doctor` reports any that are absent and points at `stack setup <id>`. Declare a secret here when it is minted by `on_install_success`, since that hook never runs again on an instance that is already installed. |
 
 ### Upstream
 

lib/stack/cli.py

@@ -838,13 +838,27 @@ def handle_doctor(stck, args):
     preferred = stck._cfg("core", "runtime", "orbstack")
     docker.init_runtime(preferred)
 
-    stacklets = sorted(s["id"] for s in stck.discover())
+    discovered = stck.discover()
+    stacklets = sorted(s["id"] for s in discovered)
+    manifests = {s["id"]: s["manifest"] for s in discovered}
+
+    def missing_secrets(stacklet_id):
+        """Declared credentials the secret store cannot produce.
+
+        `required_secrets` names them the way a stacklet's own hooks do,
+        without the namespace prefix, so the manifest reads the same as
+        the `ctx.secret("MEMORY_BOT_TOKEN")` call that consumes it.
+        """
+        required = manifests.get(stacklet_id, {}).get("required_secrets", [])
+        return [name for name in required if not stck.secret(stacklet_id, name)]
+
     findings = doctor.diagnose(
         stacklets,
         stck.env,
         docker.containers_for,
         docker.container_env,
         docker.image_env,
+        missing_secrets=missing_secrets,
     )
 
     print()

lib/stack/doctor.py

@@ -124,6 +124,40 @@ def check_exited(container: str, exit_code: int, since: str) -> Finding | None:
     )
 
 
+def check_missing_secrets(stacklet: str, missing: list[str]) -> Finding | None:
+    """A stacklet whose declared credentials were never provisioned.
+
+    Secrets are minted by `on_install_success`, which runs once, on the
+    first install. A stacklet that grows a new credential later leaves
+    every existing instance without it: the hook has already run and
+    will not run again, so the gap is permanent and silent. `memory`
+    did exactly that, and the symptom reached the operator as a vault
+    write failing with "Forgejo credentials missing" on a stack whose
+    containers were all green.
+
+    Generic on purpose, in the spirit of the rest of this module: the
+    stacklet says which keys it cannot work without (`required_secrets`
+    in its manifest) and the caller says which are absent. Nothing here
+    knows what a Forgejo token is, so the same rule covers whatever
+    credential the next stacklet adds.
+
+    Names only, never values - see `env_drift`.
+    """
+    if not missing:
+        return None
+    return Finding(
+        level=ERROR,
+        title=f"{stacklet} is missing credentials it needs",
+        detail=(
+            ", ".join(missing)
+            + " declared as required but absent from the secret store. "
+            "These are provisioned once, during install, so a stacklet "
+            "installed before it started needing one never gets it."
+        ),
+        fix=f"stack setup {stacklet}",
+    )
+
+
 def check_endpoint(name: str, url: str, reachable: bool) -> Finding | None:
     """A configured endpoint that does not answer.
 
@@ -142,12 +176,14 @@ def check_endpoint(name: str, url: str, reachable: bool) -> Finding | None:
 
 
 def diagnose(stacklets, rendered_env, containers_for, container_env,
-             image_env) -> list[Finding]:
+             image_env, *, missing_secrets=None) -> list[Finding]:
     """Run every check across the given stacklets.
 
-    The five collaborators are injected rather than imported so the whole
-    walk is testable with plain dicts - no Docker, no instance. Each is a
+    The collaborators are injected rather than imported so the whole walk
+    is testable with plain dicts - no Docker, no instance. Each is a
     callable taking a stacklet id (or container name) and returning facts.
+    `missing_secrets` is optional so a caller that has no secret store to
+    consult still gets the container checks.
 
     A stacklet whose env cannot be rendered is skipped rather than fatal:
     one misconfigured stacklet should not stop the others being diagnosed,
@@ -157,8 +193,15 @@ def diagnose(stacklets, rendered_env, containers_for, container_env,
     for stacklet in stacklets:
         containers = containers_for(stacklet)
         if not containers:
+            # Nothing running means the stacklet is not part of this
+            # instance, so its missing credentials are not yet a problem.
             continue
 
+        if missing_secrets:
+            found = check_missing_secrets(stacklet, missing_secrets(stacklet))
+            if found:
+                findings.append(found)
+
         try:
             rendered = rendered_env(stacklet)
         except Exception:

stacklets/docs/auth.py

@@ -0,0 +1,86 @@
+"""Getting, and keeping, the Paperless API token.
+
+Every write famstack makes to Paperless carries this token. The
+archivist files documents with it (`core` renders it into the bot
+runner as `PAPERLESS_TOKEN`), and the start hook seeds person tags and
+the category taxonomy with it. It is not a secret we invent: Paperless
+issues it against the admin's own credentials, which is what makes a
+lost one always recoverable and a stale one always detectable.
+
+That matters because the token used to be obtained during install and
+never again. Paperless binds a token to its database, so a
+`stack destroy docs` + `stack up docs` cycle invalidates it, and an
+instance installed before this stacklet stored one has none at all.
+Either way the install hook had already run for the last time: the
+archivist quietly stopped being able to file anything, `stack up docs`
+skipped its seeding without saying why, and the only cure was
+re-running setup by hand.
+
+Both hooks come through here now, so every `stack up docs` re-checks
+the token it is holding and asks for a new one when the answer is no.
+"""
+
+from __future__ import annotations
+
+DEFAULT_URL = "http://localhost:42020"
+
+
+def ensure_api_token(ctx) -> str:
+    """Return a token Paperless currently accepts, obtaining one if needed.
+
+    Returns "" when no token could be had, which happens two ways: there
+    are no admin credentials to authenticate as, or Paperless did not
+    answer. Callers treat that as "do nothing this run" rather than as a
+    failure, because the next start tries again and a stack coming up
+    with Paperless still migrating its database is ordinary.
+    """
+    url = ctx.env.get("PAPERLESS_URL", DEFAULT_URL)
+
+    stored = ctx.secret("API_TOKEN")
+    if stored and _token_accepted(ctx, url, stored):
+        return stored
+    if stored:
+        ctx.step("Stored API token is invalid — obtaining a new one")
+
+    username = ctx.env.get("ADMIN_USER", "")
+    password = ctx.secret("ADMIN_PASSWORD")
+    if not (username and password):
+        ctx.step("No admin credentials — skipping API token")
+        return ""
+
+    ctx.step("Obtaining API token...")
+    try:
+        data = ctx.http_post(
+            f"{url}/api/token/",
+            f"username={username}&password={password}",
+        )
+    except Exception as e:
+        ctx.step(f"Could not obtain API token: {e}")
+        return ""
+
+    fresh = data.get("token", "")
+    if not fresh:
+        ctx.step("Unexpected response from Paperless token endpoint")
+        return ""
+
+    ctx.secret("API_TOKEN", fresh)
+    ctx.step("API token saved")
+    return fresh
+
+
+def _token_accepted(ctx, url: str, token: str) -> bool:
+    """True when Paperless still answers to this token.
+
+    Deliberately cannot tell "rejected" from "unreachable", and does not
+    need to: the caller's response to both is to ask for a new token,
+    and that request fails too when Paperless is down. The run ends with
+    the stored token untouched either way.
+    """
+    try:
+        ctx.http_get(
+            f"{url}/api/documents/",
+            headers={"Authorization": f"Token {token}"},
+        )
+        return True
+    except Exception:
+        return False

stacklets/docs/hooks/on_install_success.py

@@ -7,67 +7,28 @@
 
 Also seeded on every `stack up docs` via on_start_ready.py
 so they stay in sync with users.toml and taxonomy.yaml changes.
+The token comes from `auth.ensure_api_token`, which both hooks share,
+so an instance that loses one does not have to wait for a reinstall.
 """
 
 import sys
 from pathlib import Path
 
-# seed.py lives one level up from hooks/
+# seed.py and auth.py live one level up from hooks/
 sys.path.insert(0, str(Path(__file__).parent.parent))
+from auth import ensure_api_token
 from seed import seed_person_tags, seed_taxonomy
 
 def run(ctx):
-    env = ctx.env
-    secret = ctx.secret
-    step = ctx.step
-    http_post = ctx.http_post
-    http_get = ctx.http_get
-
-    PAPERLESS_URL = env.get("PAPERLESS_URL", "http://localhost:42020")
-
-    # Verify existing token still works (a previous destroy + up cycle
-    # creates a fresh database, invalidating the old token in secrets.toml)
-    existing_token = secret("API_TOKEN")
-    token_valid = False
-    if existing_token:
-        try:
-            http_get(
-                f"{PAPERLESS_URL}/api/documents/",
-                headers={"Authorization": f"Token {existing_token}"},
-            )
-            token_valid = True
-        except Exception:
-            step("Stored API token is invalid — obtaining a new one")
-
-    if not token_valid:
-        username = env.get("ADMIN_USER", "")
-        password = secret("ADMIN_PASSWORD")
-        if not username or not password:
-            step("No admin credentials — skipping API token")
-            return
-
-        step("Obtaining API token...")
-        try:
-            data = http_post(
-                f"{PAPERLESS_URL}/api/token/",
-                f"username={username}&password={password}",
-            )
-            existing_token = data.get("token")
-            if existing_token:
-                secret("API_TOKEN", existing_token)
-                step("API token saved")
-            else:
-                step("Unexpected response from Paperless token endpoint")
-                return
-        except Exception as e:
-            step(f"Could not obtain API token: {e}")
-            return
+    token = ensure_api_token(ctx)
+    if not token:
+        return
 
     # ── Create admin-role users as superusers ────────────────────────
-    _create_admin_users(ctx, existing_token)
+    _create_admin_users(ctx, token)
 
     # ── Seed person tags + category taxonomy ───────────────────────────
-    _seed_taxonomy(ctx, existing_token)
+    _seed_taxonomy(ctx, token)
 
 
 def _create_admin_users(ctx, token):

stacklets/docs/hooks/on_start_ready.py

@@ -4,17 +4,26 @@
 person tags and category taxonomy stay in sync. Idempotent -- skips
 existing entries, creates new ones for users or categories added
 since last run.
+
+It also makes sure there is a working API token to seed with. This
+hook used to read one and give up silently when it found none, which
+is the state any instance predating the stored token was in: seeding
+skipped every start, and the archivist -- which gets the same token
+through rendered container env -- could not file a document. Paperless
+will issue a replacement whenever asked, so there is nothing to give
+up about. See auth.py.
 """
 
 import sys
 from pathlib import Path
 
 sys.path.insert(0, str(Path(__file__).parent.parent))
+from auth import ensure_api_token
 from seed import seed_person_tags, seed_taxonomy
 
 
 def run(ctx):
-    token = ctx.secret("API_TOKEN")
+    token = ensure_api_token(ctx)
     if not token:
         return
 

stacklets/memory/hooks/on_start_ready.py

@@ -8,9 +8,10 @@
 that URL rot on their own schedule, and they need different cures. The
 host part (a LAN IP baked in at clone time) is re-derived from the
 current config, because the answer is knowable. The embedded token is
-not: nothing anywhere holds a newer one, so a token Forgejo rejects is
-replaced with a freshly issued one rather than rewritten. Between them,
-a clone made months ago starts working again after a restart.
+not: nothing anywhere holds a newer one, so a token Forgejo rejects —
+or one that was never stored at all — is replaced with a freshly issued
+one rather than rewritten. Between them, a clone made months ago starts
+working again after a restart.
 """
 
 from __future__ import annotations
@@ -53,21 +54,39 @@ def remote_for(tok: str) -> str:
             vault_remote_url(code_url), BOT_USERNAME, tok,
         )
 
-    remote = remote_for(ctx.secret("MEMORY_BOT_TOKEN"))
+    token = ctx.secret("MEMORY_BOT_TOKEN")
+    remote = remote_for(token)
 
     # The token is minted once at install and read forever after, so a
     # token Forgejo has since rejected cannot be re-derived from
     # anything — only replaced. Until it is, every host-side write
     # (a todo tick, an ontology edit) fails 401 and a restart changes
     # nothing, because re-pointing the remote writes the dead token
     # back. Checked before the pull so the pull gets the good one.
-    if remote and remote_rejects_credentials(remote):
+    #
+    # A token that was never stored needs the same cure and used to get
+    # none: instances installed before this hook's sibling learned to
+    # persist one hold nothing, a missing token builds no remote, and
+    # the repair below only ran once there was a remote to test. Those
+    # instances answered "Forgejo credentials missing" to every vault
+    # write until someone re-ran setup by hand. Both causes reduce to
+    # "we hold no credential Forgejo accepts", so both mint one.
+    if not code_url:
+        reason = ""
+    elif not token:
+        reason = "Memory: no vault write token on file"
+    elif remote_rejects_credentials(remote):
+        reason = "Memory: Forgejo rejected the stored token"
+    else:
+        reason = ""
+
+    if reason:
         if fresh := reissue_write_token(code_url, admin_user, admin_password):
             ctx.secret("MEMORY_BOT_TOKEN", fresh)
             remote = remote_for(fresh)
-            ctx.step("Memory: Forgejo rejected the stored token; issued a new one")
+            ctx.step(f"{reason}; issued a new one")
         else:
-            ctx.step("Memory: Forgejo rejected the stored token and it could not be replaced")
+            ctx.step(f"{reason} and it could not be replaced")
 
     # If the vault never got cloned (install hook ran before code
     # stacklet was reachable, for example), try once more here. This

stacklets/memory/stacklet.toml

@@ -29,6 +29,12 @@ requires    = ["code"]
 # tweaks (`quartz/*.ts`) land without manual image management.
 build = true
 
+# The Forgejo write token every host-side vault write needs. It is minted
+# during install and stored as a secret; `stack doctor` reports it missing
+# so an instance that predates the token (or lost it with a rebuilt code
+# stacklet) says so plainly instead of failing one write at a time.
+required_secrets = ["MEMORY_BOT_TOKEN"]
+
 # LAN port for the wiki's Quartz preview server. Sits at the end of
 # the 42xxx range used by other stacklets so future infra ports stay
 # easy to scan.