From d509e8f07062f12434ed62f61bf64dc909156ce9 Mon Sep 17 00:00:00 2001 From: Timo Derstappen Date: Sun, 11 Oct 2026 12:19:33 +0200 Subject: [PATCH 1/3] fix(ateom): refuse a restore from the pre-per-volume durable layout Both the gVisor and the micro-VM ateom snapshot each durable-dir volume as its own durable-dir-.tar, and a restore skips a volume whose tar is missing, since it may have been added to the template after the snapshot. A snapshot written before that layout holds every volume in the single durable-dir.tar instead, which nothing reads any more: the restore reports success and the actor comes up with empty durable volumes, losing its data without an error. A workload that keeps its state there fails later with unrelated symptoms (a missing database table, a failed first turn). Refuse such a restore. A checkpoint no longer writes durable-dir.tar, so its presence in the restore directory marks the old layout: the restore now fails with FailedPrecondition, "snapshot written before the per-volume layout; re-snapshot from a running actor or reset the golden snapshot", before anything is extracted, and the actor crashes on the failed Resume instead of running with empty volumes. Signed-off-by: Timo Derstappen (cherry picked from commit 11a6bc4f, agent-substrate/substrate#2462; the micro-VM hunks are dropped: on this line the micro-VM ateom still writes the single durable-dir.tar, which its restore reads) --- cmd/ateom-gvisor/durable.go | 19 ++++++++++++- cmd/ateom-gvisor/durable_test.go | 49 ++++++++++++++++++++++++++++++++ 2 files changed, 67 insertions(+), 1 deletion(-) diff --git a/cmd/ateom-gvisor/durable.go b/cmd/ateom-gvisor/durable.go index f5fe1e0950..ea8e6af3ba 100644 --- a/cmd/ateom-gvisor/durable.go +++ b/cmd/ateom-gvisor/durable.go @@ -25,6 +25,7 @@ import ( "path/filepath" "strings" + "github.com/agent-substrate/substrate/internal/apierror" "github.com/agent-substrate/substrate/internal/proto/ateompb" "github.com/agent-substrate/substrate/internal/tarutil" ) @@ -42,6 +43,17 @@ func durableTarFile(volumeName string) (string, error) { return "durable-dir-" + volumeName + ".tar", nil } +// legacyDurableTarFile is the single tar of every durable-dir volume that +// snapshots held before the one-tar-per-volume layout. A checkpoint never +// writes it any more, so its presence marks a snapshot in that layout. +const legacyDurableTarFile = "durable-dir.tar" + +// errLegacySnapshotLayout refuses a restore from a snapshot in the layout +// before one tar per durable-dir volume: its data is in legacyDurableTarFile, +// which is never extracted, so restoring it would bring the actor up with +// empty volumes. +var errLegacySnapshotLayout = errors.New("snapshot written before the per-volume layout; re-snapshot from a running actor or reset the golden snapshot") + // hasDurableVolumes reports whether any container mounts a durable-dir volume. func hasDurableVolumes(containers []*ateompb.Container) bool { for _, c := range containers { @@ -79,8 +91,13 @@ func tarDurableVolumes(ctx context.Context, dir, checkpointDir string, volumes [ // untarDurableVolumes restores each durable-dir volume from the snapshot into // its directory under dir, which atelet has already created, empty. A volume -// with no tar in the snapshot (added to the template since) stays empty. +// with no tar in the snapshot (added to the template since) stays empty. A +// snapshot in the layout before one tar per volume is refused with +// errLegacySnapshotLayout. func untarDurableVolumes(dir, snapshotDir string, volumes []string) error { + if _, err := os.Stat(filepath.Join(snapshotDir, legacyDurableTarFile)); err == nil { + return apierror.FailedPrecondition("%w", errLegacySnapshotLayout) + } for _, vol := range volumes { name, err := durableTarFile(vol) if err != nil { diff --git a/cmd/ateom-gvisor/durable_test.go b/cmd/ateom-gvisor/durable_test.go index cf0483542c..142ebf663b 100644 --- a/cmd/ateom-gvisor/durable_test.go +++ b/cmd/ateom-gvisor/durable_test.go @@ -17,11 +17,16 @@ package main import ( + "archive/tar" + "errors" "os" "path/filepath" "slices" "testing" + "google.golang.org/grpc/codes" + + "github.com/agent-substrate/substrate/internal/apierror" "github.com/agent-substrate/substrate/internal/tarutil" ) @@ -120,6 +125,50 @@ func TestUntarDurableVolumesMissingTarIsEmpty(t *testing.T) { } } +// writeLegacyDurableTar writes the snapshot file of the layout before one tar +// per volume: a single durable-dir.tar whose entries are /... . +func writeLegacyDurableTar(t *testing.T, snapshotDir string, files map[string]string) { + t.Helper() + f, err := os.Create(filepath.Join(snapshotDir, legacyDurableTarFile)) + if err != nil { + t.Fatal(err) + } + defer f.Close() + tw := tar.NewWriter(f) + for name, body := range files { + if err := tw.WriteHeader(&tar.Header{Name: name, Mode: 0o600, Size: int64(len(body)), Typeflag: tar.TypeReg}); err != nil { + t.Fatal(err) + } + if _, err := tw.Write([]byte(body)); err != nil { + t.Fatal(err) + } + } + if err := tw.Close(); err != nil { + t.Fatal(err) + } +} + +// A snapshot in the layout before one tar per volume holds the actor's data in +// durable-dir.tar, which a restore never extracts: it is refused instead of +// restoring the volumes empty. +func TestUntarDurableVolumesRefusesLegacyLayout(t *testing.T) { + dst, snap := t.TempDir(), t.TempDir() + mkVolumeDirs(t, dst, "data") + writeLegacyDurableTar(t, snap, map[string]string{"data/app.db": "state"}) + + err := untarDurableVolumes(dst, snap, []string{"data"}) + if !errors.Is(err, errLegacySnapshotLayout) { + t.Fatalf("untarDurableVolumes = %v; want %v", err, errLegacySnapshotLayout) + } + if got := apierror.Code(err); got != codes.FailedPrecondition { + t.Errorf("code = %v; want %v", got, codes.FailedPrecondition) + } + entries, rerr := os.ReadDir(filepath.Join(dst, "data")) + if rerr != nil || len(entries) != 0 { + t.Errorf("data: ReadDir = %v, %v; want it untouched", entries, rerr) + } +} + func TestDurableTarFileRejectsBadNames(t *testing.T) { for _, v := range []string{"", ".", "..", "a/b", "/abs", "../x"} { if _, err := durableTarFile(v); err == nil { From e594f6c6c45faa0dce67593a7a1f12f6c7afd0d8 Mon Sep 17 00:00:00 2001 From: Timo Derstappen Date: Sun, 11 Oct 2026 12:22:40 +0200 Subject: [PATCH 2/3] docs(fork): carry the refusal of a pre-per-volume snapshot restore Signed-off-by: Timo Derstappen --- FORK.md | 1 + 1 file changed, 1 insertion(+) diff --git a/FORK.md b/FORK.md index add232661b..4b66965973 100644 --- a/FORK.md +++ b/FORK.md @@ -113,6 +113,7 @@ Everything on `giantswarm` that is not in the pin (`git log v0.4.0-alpha1..giant | `crashActor` terminates the sandbox on atelet, detaches the actor's volumes and only then releases the worker (a failed step leaves the actor `CRASHED` with its worker assignment, for `DeleteActor` or `RevertActor` to retry); `DeleteActor` refuses with `FailedPrecondition` while the sandbox cannot be stopped or a volume released, keeping the assignment for the retry; `CreateActorTemplate` refuses a DATA `on_pause` or `on_commit` scope unless a container mounts a durable-dir volume (`InvalidArgument` naming the field); the existing-volumes e2e suite covers the delete after a failed suspend | a failed checkpoint crashed the actor and released its worker with the sandbox still running on it, so `DeleteActor` found no worker to reach and removed the record: the sandbox, its existing-volume NFS mounts and the worker's capacity leaked until the worker pod restarted, and the PersistentVolume stayed `Released` on `directory not empty` ([giantswarm/substrate#237](https://github.com/giantswarm/substrate/issues/237); giantswarm/giantswarm#37742 row 162) | [#258](https://github.com/giantswarm/substrate/pull/258) (`fix(ateapi): tear a crashed actor's workload down before it is deleted`) | [agent-substrate/substrate#1953](https://github.com/agent-substrate/substrate/pull/1953) (open: the crash tears the workload down before `CRASHED` and keeps the assignment when it fails, the shape adapted here) and [agent-substrate/substrate#1972](https://github.com/agent-substrate/substrate/pull/1972) (open: refuses the DATA scope without a durable dir, for agent-substrate/substrate#1936); falls away at the re-pin onto the first release that carries both | | The snapshot takes one rootfs-upper tar per container and one tar per durable volume; the restore creates the layout before it extracts into it, so a snapshot no longer decides which host paths are mounted or deleted; `tarutil` writes and reads the tree through an `os.Root` (`fs.WalkDir` over `root.FS()`, `root.Lstat`, `root.Readlink`), component by component, never by the full host path (`internal/tarutil`, `cmd/ateom-gvisor/durable.go`, `cmd/ateom-microvm/rootfsupper.go`) | a workspace tree deeper than `PATH_MAX` crashed the actor at its checkpoint and lost the session on every harness: the tar walked and stat'ed each entry by its full path, which the kernel refuses with `ENAMETOOLONG`; the same tar named the host paths a restore mounted and deleted (upstream's agent-substrate/substrate#2083; giantswarm/giantswarm#38054; [#37742](https://github.com/giantswarm/giantswarm/issues/37742) row 165). **Snapshots written before it do not resume after it**, nor its own on a release before it: the roll needs the snapshot reset of `docs/snapshot-reset.md`, golden and session snapshots, in either direction | `3afd756f` ([#260](https://github.com/giantswarm/substrate/pull/260), rebase-merged 2026-10-10; first release 1.8.0-rc.6): `git cherry-pick -x` of upstream's squash commit `893b1edb`, clean | [agent-substrate/substrate#2110](https://github.com/agent-substrate/substrate/pull/2110) (merged 2026-10-06, after the pin). Falls away at the re-pin onto the first release that carries it | | A snapshot extract creates only `0:0` character devices (overlay whiteouts) and sets only `user.*` xattrs and the path-based overlay attributes (opaque, redirect, impure); `Create` skips other device nodes; on the micro-VM host the merged rootfs overlay and the binds into the virtio-fs share are `nosuid,nodev` (`setNosuidNodev`, recursive) | snapshot tars are extracted as root on the host, and the extract made any device node and set any `trusted.overlay.*` xattr the archive named; a stray device could make a snapshot unrestorable (upstream's agent-substrate/substrate#2090; giantswarm/giantswarm#38054; [#37742](https://github.com/giantswarm/giantswarm/issues/37742) row 166) | `eb04dfc1` ([#260](https://github.com/giantswarm/substrate/pull/260), rebase-merged 2026-10-10; first release 1.8.0-rc.6): `git cherry-pick -x` of upstream's squash commit `8da76493`. Adapted to the line: it still mounts through `mount(8)`, so `nosuid,nodev` joins the overlay's `-o` options and `BindIntoShare` calls `setNosuidNodev` after the rbind; the hunk's `RemountReadOnly` and `Unmount` tests cover helpers the line does not have and are left out | [agent-substrate/substrate#2111](https://github.com/agent-substrate/substrate/pull/2111) (merged 2026-10-06, after the pin). Falls away at the re-pin onto the first release that carries it | +| A gVisor restore refuses a snapshot in the layout before one tar per durable volume (`durable-dir.tar` in the restore directory) with `FailedPrecondition` "snapshot written before the per-volume layout; re-snapshot from a running actor or reset the golden snapshot", before it extracts anything (`cmd/ateom-gvisor/durable.go`) | the row above skips a missing `durable-dir-.tar` as a volume added since the snapshot, so a snapshot written before it restored with no error and an empty `/data`: the Go ADK failed with `no such table: app_states`, the claude Harness with `Harness runtime execution failed`, and a missed snapshot reset lost the sessions' data without a word ([#263](https://github.com/giantswarm/substrate/issues/263); [#37742](https://github.com/giantswarm/giantswarm/issues/37742) row 171). Now the Resume fails and the actor is CRASHED with that message | [#265](https://github.com/giantswarm/substrate/pull/265): `git cherry-pick -x` of `11a6bc4f`, adapted to the line: the micro-VM hunks are left out, its ateom still writes and reads the single `durable-dir.tar` (agent-substrate/substrate#2283 is after the pin) | [agent-substrate/substrate#2462](https://github.com/agent-substrate/substrate/pull/2462) (ours, open; both ateoms). Falls away at the re-pin onto the first release that carries it | Twenty-three patches change Substrate ahead of upstream — egress for an actor while it resumes, without which no skill-carrying agent of the platform boots, the atelet scheduling knobs, the keep policy on the CRD chart's From 833aff777f79005cad3c3ac665003143b959c2a1 Mon Sep 17 00:00:00 2001 From: Timo Derstappen Date: Sun, 11 Oct 2026 12:22:49 +0200 Subject: [PATCH 3/3] docs(fork): name the layout change the restore refusal row follows Signed-off-by: Timo Derstappen --- FORK.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/FORK.md b/FORK.md index 4b66965973..4d77583953 100644 --- a/FORK.md +++ b/FORK.md @@ -113,7 +113,7 @@ Everything on `giantswarm` that is not in the pin (`git log v0.4.0-alpha1..giant | `crashActor` terminates the sandbox on atelet, detaches the actor's volumes and only then releases the worker (a failed step leaves the actor `CRASHED` with its worker assignment, for `DeleteActor` or `RevertActor` to retry); `DeleteActor` refuses with `FailedPrecondition` while the sandbox cannot be stopped or a volume released, keeping the assignment for the retry; `CreateActorTemplate` refuses a DATA `on_pause` or `on_commit` scope unless a container mounts a durable-dir volume (`InvalidArgument` naming the field); the existing-volumes e2e suite covers the delete after a failed suspend | a failed checkpoint crashed the actor and released its worker with the sandbox still running on it, so `DeleteActor` found no worker to reach and removed the record: the sandbox, its existing-volume NFS mounts and the worker's capacity leaked until the worker pod restarted, and the PersistentVolume stayed `Released` on `directory not empty` ([giantswarm/substrate#237](https://github.com/giantswarm/substrate/issues/237); giantswarm/giantswarm#37742 row 162) | [#258](https://github.com/giantswarm/substrate/pull/258) (`fix(ateapi): tear a crashed actor's workload down before it is deleted`) | [agent-substrate/substrate#1953](https://github.com/agent-substrate/substrate/pull/1953) (open: the crash tears the workload down before `CRASHED` and keeps the assignment when it fails, the shape adapted here) and [agent-substrate/substrate#1972](https://github.com/agent-substrate/substrate/pull/1972) (open: refuses the DATA scope without a durable dir, for agent-substrate/substrate#1936); falls away at the re-pin onto the first release that carries both | | The snapshot takes one rootfs-upper tar per container and one tar per durable volume; the restore creates the layout before it extracts into it, so a snapshot no longer decides which host paths are mounted or deleted; `tarutil` writes and reads the tree through an `os.Root` (`fs.WalkDir` over `root.FS()`, `root.Lstat`, `root.Readlink`), component by component, never by the full host path (`internal/tarutil`, `cmd/ateom-gvisor/durable.go`, `cmd/ateom-microvm/rootfsupper.go`) | a workspace tree deeper than `PATH_MAX` crashed the actor at its checkpoint and lost the session on every harness: the tar walked and stat'ed each entry by its full path, which the kernel refuses with `ENAMETOOLONG`; the same tar named the host paths a restore mounted and deleted (upstream's agent-substrate/substrate#2083; giantswarm/giantswarm#38054; [#37742](https://github.com/giantswarm/giantswarm/issues/37742) row 165). **Snapshots written before it do not resume after it**, nor its own on a release before it: the roll needs the snapshot reset of `docs/snapshot-reset.md`, golden and session snapshots, in either direction | `3afd756f` ([#260](https://github.com/giantswarm/substrate/pull/260), rebase-merged 2026-10-10; first release 1.8.0-rc.6): `git cherry-pick -x` of upstream's squash commit `893b1edb`, clean | [agent-substrate/substrate#2110](https://github.com/agent-substrate/substrate/pull/2110) (merged 2026-10-06, after the pin). Falls away at the re-pin onto the first release that carries it | | A snapshot extract creates only `0:0` character devices (overlay whiteouts) and sets only `user.*` xattrs and the path-based overlay attributes (opaque, redirect, impure); `Create` skips other device nodes; on the micro-VM host the merged rootfs overlay and the binds into the virtio-fs share are `nosuid,nodev` (`setNosuidNodev`, recursive) | snapshot tars are extracted as root on the host, and the extract made any device node and set any `trusted.overlay.*` xattr the archive named; a stray device could make a snapshot unrestorable (upstream's agent-substrate/substrate#2090; giantswarm/giantswarm#38054; [#37742](https://github.com/giantswarm/giantswarm/issues/37742) row 166) | `eb04dfc1` ([#260](https://github.com/giantswarm/substrate/pull/260), rebase-merged 2026-10-10; first release 1.8.0-rc.6): `git cherry-pick -x` of upstream's squash commit `8da76493`. Adapted to the line: it still mounts through `mount(8)`, so `nosuid,nodev` joins the overlay's `-o` options and `BindIntoShare` calls `setNosuidNodev` after the rbind; the hunk's `RemountReadOnly` and `Unmount` tests cover helpers the line does not have and are left out | [agent-substrate/substrate#2111](https://github.com/agent-substrate/substrate/pull/2111) (merged 2026-10-06, after the pin). Falls away at the re-pin onto the first release that carries it | -| A gVisor restore refuses a snapshot in the layout before one tar per durable volume (`durable-dir.tar` in the restore directory) with `FailedPrecondition` "snapshot written before the per-volume layout; re-snapshot from a running actor or reset the golden snapshot", before it extracts anything (`cmd/ateom-gvisor/durable.go`) | the row above skips a missing `durable-dir-.tar` as a volume added since the snapshot, so a snapshot written before it restored with no error and an empty `/data`: the Go ADK failed with `no such table: app_states`, the claude Harness with `Harness runtime execution failed`, and a missed snapshot reset lost the sessions' data without a word ([#263](https://github.com/giantswarm/substrate/issues/263); [#37742](https://github.com/giantswarm/giantswarm/issues/37742) row 171). Now the Resume fails and the actor is CRASHED with that message | [#265](https://github.com/giantswarm/substrate/pull/265): `git cherry-pick -x` of `11a6bc4f`, adapted to the line: the micro-VM hunks are left out, its ateom still writes and reads the single `durable-dir.tar` (agent-substrate/substrate#2283 is after the pin) | [agent-substrate/substrate#2462](https://github.com/agent-substrate/substrate/pull/2462) (ours, open; both ateoms). Falls away at the re-pin onto the first release that carries it | +| A gVisor restore refuses a snapshot in the layout before one tar per durable volume (`durable-dir.tar` in the restore directory) with `FailedPrecondition` "snapshot written before the per-volume layout; re-snapshot from a running actor or reset the golden snapshot", before it extracts anything (`cmd/ateom-gvisor/durable.go`) | since the one-tar-per-volume layout (agent-substrate/substrate#2110, two rows up) a restore skips a missing `durable-dir-.tar` as a volume added since the snapshot, so a snapshot written before it restored with no error and an empty `/data`: the Go ADK failed with `no such table: app_states`, the claude Harness with `Harness runtime execution failed`, and a missed snapshot reset lost the sessions' data without a word ([#263](https://github.com/giantswarm/substrate/issues/263); [#37742](https://github.com/giantswarm/giantswarm/issues/37742) row 171). Now the Resume fails and the actor is CRASHED with that message | [#265](https://github.com/giantswarm/substrate/pull/265): `git cherry-pick -x` of `11a6bc4f`, adapted to the line: the micro-VM hunks are left out, its ateom still writes and reads the single `durable-dir.tar` (agent-substrate/substrate#2283 is after the pin) | [agent-substrate/substrate#2462](https://github.com/agent-substrate/substrate/pull/2462) (ours, open; both ateoms). Falls away at the re-pin onto the first release that carries it | Twenty-three patches change Substrate ahead of upstream — egress for an actor while it resumes, without which no skill-carrying agent of the platform boots, the atelet scheduling knobs, the keep policy on the CRD chart's