Skip to content

Commit 87e8441

Browse files
authored
fix(runsc): support filestore reflinks on older kernels (#11)
Signed-off-by: 月可 <hukeyang.hky@antgroup.com>
1 parent e6e6a6c commit 87e8441

5 files changed

Lines changed: 59 additions & 5 deletions

File tree

‎g3doc/user_guide/checkpoint_restore.md‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,7 @@
11
# Checkpoint/Restore
22

3+
For anonymous disk-backed filestores saved with `--skip-filestore-pages` and `--filestore-snapshot-dir`, older kernels may reject FICLONE between different mount-namespace copies of the same filesystem with `EXDEV`. Start the source with `--filestore-parent-mountns=true` to open its anonymous backing files in the runsc launcher's mount namespace, matching checkpoint destinations opened there. This option is disabled by default and does not change self-backed overlays. It retains a reference to the backing mount until the sandbox exits; drain all users before unmounting that filesystem. Restore with `--filestore-adopt-dir` and clone-on-adopt keeps artifacts reusable and already opens each private clone in the launcher namespace.
4+
35
[TOC]
46

57
gVisor has the ability to checkpoint a process, save its current state in a

‎runsc/config/config.go‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -120,6 +120,13 @@ type Config struct {
120120
// mutates the artifacts in place (e.g. fd-hold fallbacks on ext4).
121121
FilestoreCloneOnAdopt bool `flag:"filestore-clone-on-adopt"`
122122

123+
// FilestoreParentMountNS creates anonymous filestores in the runsc
124+
// launcher's mount namespace. Older kernels require source and destination
125+
// descriptors to reference the same mount for FICLONE, even when both mounts
126+
// expose the same filesystem. This permits checkpoints outside the gofer
127+
// namespace, but the backing mount remains pinned until the sandbox exits.
128+
FilestoreParentMountNS bool `flag:"filestore-parent-mountns"`
129+
123130
// FSGoferHostUDS is deprecated: use host-uds=all.
124131
FSGoferHostUDS bool `flag:"fsgofer-host-uds"`
125132

‎runsc/config/flags.go‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -146,6 +146,7 @@ func RegisterFlags(flagSet *flag.FlagSet) {
146146
flagSet.Var(hostUDSPtr(HostUDSNone), flagHostUDS, "controls permission to access host Unix-domain sockets. Values: none|open|create|all, default: none")
147147
flagSet.String(flagFilestoreAdoptDir, "", "directory containing host filestore files to adopt as the sandbox's writable-layer backing files, instead of creating new anonymous ones. Files are named 'filestore-<n>' in mount order (root first). Requires restoring a state checkpoint saved with --skip-filestore-pages (experimental)")
148148
flagSet.Bool(flagFilestoreCloneOnAdopt, true, "when adopting filestore artifacts (--filestore-adopt-dir), reflink-clone (FICLONE) a private copy of each artifact instead of consuming the artifact itself, keeping it a reusable read-only template. Requires a reflink-capable filesystem (e.g. XFS); false performs a consuming restore that mutates the artifacts in place (experimental)")
149+
flagSet.Bool("filestore-parent-mountns", false, "create anonymous filestores in the runsc launcher's mount namespace for same-mount FICLONE checkpoints on older kernels. The backing mount remains pinned until the sandbox exits; drain before unmounting it (experimental)")
149150
flagSet.Var(hostFifoPtr(HostFifoNone), "host-fifo", "controls permission to access host FIFOs (or named pipes). Values: none|open, default: none")
150151
flagSet.Bool("gvisor-marker-file", false, "enable the presence of the /proc/gvisor/kernel_is_gvisor file that can be used by applications to detect that gVisor is in use")
151152
flagSet.String("override-procs", "", "comma-separated list of proc files to override with stubs (e.g. kallsyms)")

‎runsc/container/container.go‎

Lines changed: 13 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1164,10 +1164,12 @@ func (c *Container) initGoferConfs(ovlConf config.Overlay2, mountHints *boot.Pod
11641164
// Precondition: gofer process must be running.
11651165
func (c *Container) createGoferFilestores(conf *config.Config, ovlConf config.Overlay2, mountHints *boot.PodMountHints) ([]*os.File, error) {
11661166
var goferFilestores []*os.File
1167-
// NOTE(gvisor.dev/issue/9834): Create the filestores in the gofer mount
1167+
// NOTE(gvisor.dev/issue/9834): By default, create filestores in the gofer mount
11681168
// namespace, so that they don't prevent the host mount points from being
11691169
// unmounted from the host's mount namespace. We will use /proc/pid/root
1170-
// to access gofer's mount namespace. See proc_pid_root(5).
1170+
// to access gofer's mount namespace. See proc_pid_root(5). The explicit
1171+
// FilestoreParentMountNS option instead pins the launcher mount for
1172+
// same-mount reflink snapshots on kernels that reject namespace copies.
11711173
goferRootfs := fmt.Sprintf("/proc/%d/root", c.GoferPid.Load())
11721174

11731175
// If set, adopt pre-existing host files (from a checkpoint artifact) as
@@ -1186,7 +1188,7 @@ func (c *Container) createGoferFilestores(conf *config.Config, ovlConf config.Ov
11861188

11871189
// Handle rootfs first.
11881190
rootfsConf := c.GoferMountConfs[0]
1189-
filestore, err := c.createGoferFilestore(goferRootfs, ovlConf, rootfsConf, c.Spec.Root.Path, "/", mountHints, adopter)
1191+
filestore, err := c.createGoferFilestore(goferRootfs, ovlConf, rootfsConf, c.Spec.Root.Path, "/", mountHints, adopter, conf.FilestoreParentMountNS)
11901192
if err != nil {
11911193
return nil, err
11921194
}
@@ -1202,7 +1204,7 @@ func (c *Container) createGoferFilestores(conf *config.Config, ovlConf config.Ov
12021204
}
12031205
mountConf := c.GoferMountConfs[mountIdx]
12041206
mountIdx++
1205-
filestore, err := c.createGoferFilestore(goferRootfs, ovlConf, mountConf, m.Source, m.Destination, mountHints, adopter)
1207+
filestore, err := c.createGoferFilestore(goferRootfs, ovlConf, mountConf, m.Source, m.Destination, mountHints, adopter, conf.FilestoreParentMountNS)
12061208
if err != nil {
12071209
return nil, err
12081210
}
@@ -1218,7 +1220,7 @@ func (c *Container) createGoferFilestores(conf *config.Config, ovlConf config.Ov
12181220
return goferFilestores, nil
12191221
}
12201222

1221-
func (c *Container) createGoferFilestore(goferRootfs string, ovlConf config.Overlay2, goferConf specutils.GoferMountConf, mountSrc string, mountDest string, mountHints *boot.PodMountHints, adopter *filestoreAdopter) (*os.File, error) {
1223+
func (c *Container) createGoferFilestore(goferRootfs string, ovlConf config.Overlay2, goferConf specutils.GoferMountConf, mountSrc string, mountDest string, mountHints *boot.PodMountHints, adopter *filestoreAdopter, parentMountNS bool) (*os.File, error) {
12221224
if !goferConf.IsFilestorePresent() {
12231225
return nil, nil
12241226
}
@@ -1236,6 +1238,12 @@ func (c *Container) createGoferFilestore(goferRootfs string, ovlConf config.Over
12361238
if adopter != nil {
12371239
return adopter.adopt(c, mountDest)
12381240
}
1241+
if parentMountNS {
1242+
// Match the mount used by the host-side checkpoint destinations.
1243+
// The usual gofer-root lookup retains a distinct mount object on
1244+
// older kernels and makes FICLONE fail with EXDEV on the same XFS.
1245+
goferRootfs = ""
1246+
}
12391247
return c.createGoferFilestoreInDir(goferRootfs, ovlConf.Medium().HostFileDir())
12401248
default:
12411249
return nil, fmt.Errorf("unexpected upper layer with filestore %s", goferConf)

‎runsc/container/filestore_snapshot_test.go‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,42 @@ import (
2727
"gvisor.dev/gvisor/runsc/specutils"
2828
)
2929

30+
func TestAnonymousFilestoreParentMountNamespace(t *testing.T) {
31+
dir := t.TempDir()
32+
goferRoot := t.TempDir()
33+
goferDir := filepath.Join(goferRoot, dir)
34+
if err := os.MkdirAll(goferDir, 0700); err != nil {
35+
t.Fatal(err)
36+
}
37+
var overlay config.Overlay2
38+
if err := overlay.Set("root:dir=" + dir); err != nil {
39+
t.Fatal(err)
40+
}
41+
c := &Container{}
42+
for _, parent := range []bool{false, true} {
43+
f, err := c.createGoferFilestore(goferRoot, overlay,
44+
specutils.GoferMountConf{Lower: specutils.NoneLower, Upper: specutils.AnonOverlay},
45+
"/", "/", nil, nil, parent)
46+
if err != nil {
47+
t.Fatal(err)
48+
}
49+
wantDir := goferDir
50+
if parent {
51+
wantDir = dir
52+
}
53+
if filepath.Dir(f.Name()) != wantDir {
54+
t.Errorf("parent=%v: backing file %q, want directory %q", parent, f.Name(), wantDir)
55+
}
56+
if _, err := os.Stat(f.Name()); !os.IsNotExist(err) {
57+
t.Errorf("filestore must remain anonymous: stat=%v", err)
58+
}
59+
if _, err := f.WriteString("private writable state"); err != nil {
60+
t.Error(err)
61+
}
62+
f.Close()
63+
}
64+
}
65+
3066
func TestFilestoreArtifactAdoption(t *testing.T) {
3167
for _, tc := range []struct {
3268
name string

0 commit comments

Comments
 (0)