feat(api,orch): add reboot escape hatch for sandbox resume

Adds an optional `reboot` flag on the resume endpoint that boots the
sandbox from its snapshot filesystem with fresh memory, discarding the
memory snapshot. Useful when memfile restore is slow or corrupted.

The flag flows via gRPC metadata to the orchestrator, which creates an
empty memfile and cold-boots the VM through systemd. Existing paths are
unchanged when the flag is not set.

Author: ValentaTomas

packages/api/internal/api/api.gen.go

@@ -257,6 +257,7 @@ func (s *SandboxService) ResumeSandbox(ctx context.Context, req *proxygrpc.Sandb
 		s.api.buildResumeSandboxData(sandboxID, nil),
 		&headers,
 		true,
+		false,
 		nil, // mcp
 	)
 	if apiErr != nil {

packages/api/internal/handlers/proxy_grpc.go

@@ -28,6 +28,7 @@ func (a *APIStore) startSandbox(
 	getSandboxData orchestrator.SandboxDataFetcher,
 	requestHeader *http.Header,
 	isResume bool,
+	rebootFromRootfs bool,
 	mcp api.Mcp,
 ) (*api.Sandbox, *api.APIError) {
 	sbx, apiErr := a.startSandboxInternal(
@@ -38,6 +39,7 @@ func (a *APIStore) startSandbox(
 		getSandboxData,
 		requestHeader,
 		isResume,
+		rebootFromRootfs,
 		mcp,
 	)
 	if apiErr != nil {
@@ -56,6 +58,7 @@ func (a *APIStore) startSandboxInternal(
 	getSandboxData orchestrator.SandboxDataFetcher,
 	requestHeader *http.Header,
 	isResume bool,
+	rebootFromRootfs bool,
 	mcp api.Mcp,
 ) (sandbox.Sandbox, *api.APIError) {
 	startTime := time.Now()
@@ -77,6 +80,7 @@ func (a *APIStore) startSandboxInternal(
 		timeout,
 		isResume,
 		creationMeta,
+		rebootFromRootfs,
 	)
 	if instanceErr != nil {
 		telemetry.ReportError(ctx, "error when creating instance", instanceErr.Err)

packages/api/internal/handlers/sandbox.go

@@ -152,6 +152,7 @@ func (a *APIStore) PostSandboxesSandboxIDConnect(c *gin.Context, sandboxID api.S
 		a.buildResumeSandboxData(sandboxID, nil),
 		&c.Request.Header,
 		true,
+		false,
 		nil, // mcp
 	)
 	if createErr != nil {

packages/api/internal/handlers/sandbox_connect.go

@@ -265,6 +265,7 @@ func (a *APIStore) PostSandboxes(c *gin.Context) {
 		getSandboxData,
 		&c.Request.Header,
 		false,
+		false,
 		mcp,
 	)
 	if createErr != nil {

packages/api/internal/handlers/sandbox_create.go

@@ -54,6 +54,7 @@ func (a *APIStore) PostSandboxesSandboxIDResume(c *gin.Context, sandboxID api.Sa
 	}
 
 	telemetry.ReportEvent(ctx, "Parsed body")
+	rebootFromRootfs := body.Reboot != nil && *body.Reboot
 
 	timeout := sandbox.SandboxTimeoutDefault
 	if body.Timeout != nil {
@@ -160,6 +161,7 @@ func (a *APIStore) PostSandboxesSandboxIDResume(c *gin.Context, sandboxID api.Sa
 		a.buildResumeSandboxData(sandboxID, body.AutoPause),
 		&c.Request.Header,
 		true,
+		rebootFromRootfs,
 		nil, // mcp
 	)
 	if createErr != nil {

packages/api/internal/handlers/sandbox_resume.go

@@ -10,6 +10,7 @@ import (
 	"go.opentelemetry.io/otel/attribute"
 	"go.opentelemetry.io/otel/metric"
 	"go.uber.org/zap"
+	"google.golang.org/grpc/metadata"
 	"google.golang.org/protobuf/types/known/timestamppb"
 
 	"github.com/e2b-dev/infra/packages/api/internal/api"
@@ -127,6 +128,7 @@ func (o *Orchestrator) CreateSandbox(
 	timeout time.Duration,
 	isResume bool,
 	creationMeta sandbox.CreationMetadata,
+	rebootFromRootfsOpt ...bool,
 ) (sbx sandbox.Sandbox, apiErr *api.APIError) {
 	ctx, childSpan := tracer.Start(ctx, "create-sandbox")
 	defer childSpan.End()
@@ -296,6 +298,10 @@ func (o *Orchestrator) CreateSandbox(
 		}
 	}
 
+	if len(rebootFromRootfsOpt) > 0 && rebootFromRootfsOpt[0] {
+		ctx = metadata.AppendToOutgoingContext(ctx, orchestrator.SandboxRebootFromRootfsGRPCMetadataKey, "true")
+	}
+
 	nodeClusterID := clusters.WithClusterFallback(team.ClusterID)
 	clusterNodes := o.GetClusterNodes(nodeClusterID)
 

packages/api/internal/orchestrator/create_instance.go

@@ -24,10 +24,16 @@ import (
 	"google.golang.org/protobuf/types/known/emptypb"
 	"google.golang.org/protobuf/types/known/timestamppb"
 
+	grpcmetadata "google.golang.org/grpc/metadata"
+
 	"github.com/e2b-dev/infra/packages/orchestrator/pkg/sandbox"
+	"github.com/e2b-dev/infra/packages/orchestrator/pkg/sandbox/block"
 	"github.com/e2b-dev/infra/packages/orchestrator/pkg/sandbox/fc"
 	sbxtemplate "github.com/e2b-dev/infra/packages/orchestrator/pkg/sandbox/template"
+	configpkg "github.com/e2b-dev/infra/packages/orchestrator/pkg/template/build/config"
+	"github.com/e2b-dev/infra/packages/orchestrator/pkg/template/constants"
 	"github.com/e2b-dev/infra/packages/orchestrator/pkg/template/metadata"
+	"github.com/e2b-dev/infra/packages/orchestrator/pkg/units"
 	"github.com/e2b-dev/infra/packages/shared/pkg/events"
 	"github.com/e2b-dev/infra/packages/shared/pkg/featureflags"
 	"github.com/e2b-dev/infra/packages/shared/pkg/grpc/orchestrator"
@@ -58,6 +64,66 @@ const (
 	killReasonUnknown = "unknown"
 )
 
+func rebootFromRootfsEnabled(ctx context.Context) bool {
+	values := grpcmetadata.ValueFromIncomingContext(ctx, orchestrator.SandboxRebootFromRootfsGRPCMetadataKey)
+	return len(values) > 0 && values[0] == "true"
+}
+
+func (s *Server) createSandboxFromRootfs(
+	ctx context.Context,
+	template sbxtemplate.Template,
+	config *sandbox.Config,
+	runtime sandbox.RuntimeMetadata,
+	req *orchestrator.SandboxCreateRequest,
+) (*sandbox.Sandbox, error) {
+	memfile, err := block.NewEmpty(
+		units.MBToBytes(config.RamMB),
+		configpkg.MemfilePageSize(config.HugePages),
+		uuid.MustParse(template.Files().BuildID),
+	)
+	if err != nil {
+		return nil, fmt.Errorf("create empty memfile: %w", err)
+	}
+	defer memfile.Close()
+
+	maskedTemplate := sbxtemplate.NewMaskTemplate(template, sbxtemplate.WithMemfile(memfile))
+
+	kvmClock, err := utils.IsGTEVersion(config.Envd.Version, "0.2.11")
+	if err != nil {
+		return nil, fmt.Errorf("compare envd version: %w", err)
+	}
+
+	timeout := req.GetEndTime().AsTime().Sub(req.GetStartTime().AsTime())
+	if timeout <= 0 {
+		timeout = s.config.EnvdTimeout
+	}
+
+	sbx, err := s.sandboxFactory.CreateSandbox(
+		ctx,
+		config,
+		runtime,
+		maskedTemplate,
+		timeout,
+		"",
+		fc.ProcessOptions{
+			InitScriptPath: constants.SystemdInitPath,
+			KvmClock:       kvmClock,
+		},
+		req.GetSandbox(),
+		nil,
+	)
+	if err != nil {
+		return nil, err
+	}
+
+	if err := sbx.WaitForEnvd(ctx, timeout); err != nil {
+		closeErr := sbx.Close(context.WithoutCancel(ctx))
+		return nil, errors.Join(fmt.Errorf("wait for envd after rootfs reboot: %w", err), closeErr)
+	}
+
+	return sbx, nil
+}
+
 func (s *Server) Create(ctx context.Context, req *orchestrator.SandboxCreateRequest) (_ *orchestrator.SandboxCreateResponse, createErr error) {
 	// set max request timeout for this request
 	ctx, cancel := context.WithTimeoutCause(ctx, requestTimeout, errors.New("request timed out"))
@@ -187,18 +253,22 @@ func (s *Server) Create(ctx context.Context, req *orchestrator.SandboxCreateRequ
 		SandboxType: sandbox.SandboxTypeSandbox,
 	}
 
-	sbx, err := s.sandboxFactory.ResumeSandbox(
-		ctx,
-		template,
-		config,
-		runtime,
-		req.GetStartTime().AsTime(),
-		req.GetEndTime().AsTime(),
-		req.GetSandbox(),
-	)
+	var sbx *sandbox.Sandbox
+	if rebootFromRootfsEnabled(ctx) {
+		sbx, err = s.createSandboxFromRootfs(ctx, template, config, runtime, req)
+	} else {
+		sbx, err = s.sandboxFactory.ResumeSandbox(
+			ctx,
+			template,
+			config,
+			runtime,
+			req.GetStartTime().AsTime(),
+			req.GetEndTime().AsTime(),
+			req.GetSandbox(),
+		)
+	}
 	if err != nil {
 		if errors.Is(err, storage.ErrObjectNotExist) {
-			// Snapshot data not found, let the API know the data aren't probably upload yet
 			telemetry.ReportError(ctx, "sandbox files not found", err, telemetry.WithSandboxID(req.GetSandbox().GetSandboxId()))
 
 			return nil, status.Errorf(codes.FailedPrecondition, "sandbox files for '%s' not found", req.GetSandbox().GetSandboxId())

packages/orchestrator/pkg/server/sandboxes.go

@@ -0,0 +1,3 @@
+package orchestrator
+
+const SandboxRebootFromRootfsGRPCMetadataKey = "x-e2b-reboot-from-rootfs"

packages/shared/pkg/grpc/orchestrator/internal_flags.go

@@ -724,6 +724,9 @@ components:
           type: boolean
           deprecated: true
           description: Automatically pauses the sandbox after the timeout
+        reboot:
+          type: boolean
+          description: Boot from the snapshot filesystem with fresh memory instead of restoring the memory snapshot.
 
     ConnectSandbox:
       type: object