feat(api,orch): add reboot escape hatch for sandbox resume

packages/api/internal/handlers/proxy_grpc.go

@@ -257,6 +257,7 @@ func (s *SandboxService) ResumeSandbox(ctx context.Context, req *proxygrpc.Sandb
 		s.api.buildResumeSandboxData(sandboxID, nil),
 		&headers,
 		true,
+		false,
 		nil, // mcp
 	)
 	if apiErr != nil {

packages/api/internal/handlers/sandbox.go

@@ -28,6 +28,7 @@ func (a *APIStore) startSandbox(
 	getSandboxData orchestrator.SandboxDataFetcher,
 	requestHeader *http.Header,
 	isResume bool,
+	reboot bool,
 	mcp api.Mcp,
 ) (*api.Sandbox, *api.APIError) {
 	sbx, apiErr := a.startSandboxInternal(
@@ -38,6 +39,7 @@ func (a *APIStore) startSandbox(
 		getSandboxData,
 		requestHeader,
 		isResume,
+		reboot,
 		mcp,
 	)
 	if apiErr != nil {
@@ -56,6 +58,7 @@ func (a *APIStore) startSandboxInternal(
 	getSandboxData orchestrator.SandboxDataFetcher,
 	requestHeader *http.Header,
 	isResume bool,
+	reboot bool,
 	mcp api.Mcp,
 ) (sandbox.Sandbox, *api.APIError) {
 	startTime := time.Now()
@@ -77,6 +80,7 @@ func (a *APIStore) startSandboxInternal(
 		timeout,
 		isResume,
 		creationMeta,
+		reboot,
 	)
 	if instanceErr != nil {
 		telemetry.ReportError(ctx, "error when creating instance", instanceErr.Err)

packages/api/internal/handlers/sandbox_connect.go

@@ -152,6 +152,7 @@ func (a *APIStore) PostSandboxesSandboxIDConnect(c *gin.Context, sandboxID api.S
 		a.buildResumeSandboxData(sandboxID, nil),
 		&c.Request.Header,
 		true,
+		false,
 		nil, // mcp
 	)
 	if createErr != nil {

packages/api/internal/handlers/sandbox_create.go

@@ -265,6 +265,7 @@ func (a *APIStore) PostSandboxes(c *gin.Context) {
 		getSandboxData,
 		&c.Request.Header,
 		false,
+		body.Reboot != nil && *body.Reboot,
 		mcp,
 	)
 	if createErr != nil {

packages/api/internal/handlers/sandbox_resume.go

@@ -54,6 +54,7 @@ func (a *APIStore) PostSandboxesSandboxIDResume(c *gin.Context, sandboxID api.Sa
 	}
 
 	telemetry.ReportEvent(ctx, "Parsed body")
+	reboot := body.Reboot != nil && *body.Reboot
 
 	timeout := sandbox.SandboxTimeoutDefault
 	if body.Timeout != nil {
@@ -160,6 +161,7 @@ func (a *APIStore) PostSandboxesSandboxIDResume(c *gin.Context, sandboxID api.Sa
 		a.buildResumeSandboxData(sandboxID, body.AutoPause),
 		&c.Request.Header,
 		true,
+		reboot,
 		nil, // mcp
 	)
 	if createErr != nil {

packages/api/internal/orchestrator/create_instance.go

@@ -127,6 +127,7 @@ func (o *Orchestrator) CreateSandbox(
 	timeout time.Duration,
 	isResume bool,
 	creationMeta sandbox.CreationMetadata,
+	reboot bool,
 ) (sbx sandbox.Sandbox, apiErr *api.APIError) {
 	ctx, childSpan := tracer.Start(ctx, "create-sandbox")
 	defer childSpan.End()
@@ -282,6 +283,7 @@ func (o *Orchestrator) CreateSandbox(
 		},
 		StartTime: timestamppb.New(startTime),
 		EndTime:   timestamppb.New(endTime),
+		Reboot:    reboot,
 	}
 
 	var node *nodemanager.Node

packages/api/internal/orchestrator/create_instance_events_test.go

@@ -141,6 +141,7 @@ func TestCreateSandbox_FreshCreate_FiresCallbackOnce(t *testing.T) {
 		now, now.Add(time.Hour), time.Hour,
 		false,
 		sandbox.CreationMetadata{IsResume: false, TeamName: "test-team"},
+		false,
 	)
 	require.Nil(t, apiErr)
 
@@ -169,6 +170,7 @@ func TestCreateSandbox_Resume_FiresCallbackOnceWithResumeFlag(t *testing.T) {
 		now, now.Add(time.Hour), time.Hour,
 		true,
 		sandbox.CreationMetadata{IsResume: true, TeamName: "test-team"},
+		false,
 	)
 	require.Nil(t, apiErr)
 
@@ -219,6 +221,7 @@ func TestCreateSandbox_ConcurrentRace_LoserPathSkipsCallback(t *testing.T) {
 			now, now.Add(time.Hour), time.Hour,
 			false,
 			sandbox.CreationMetadata{TeamName: "winner"},
+			false,
 		)
 	})
 
@@ -232,6 +235,7 @@ func TestCreateSandbox_ConcurrentRace_LoserPathSkipsCallback(t *testing.T) {
 			now, now.Add(time.Hour), time.Hour,
 			false,
 			sandbox.CreationMetadata{TeamName: "loser"},
+			false,
 		)
 	})
 
@@ -290,6 +294,7 @@ func TestCreateSandbox_ManyConcurrentRacers_OnlyOneCallback(t *testing.T) {
 			now, now.Add(time.Hour), time.Hour,
 			false,
 			sandbox.CreationMetadata{},
+			false,
 		)
 		if apiErr == nil {
 			successCount.Add(1)
@@ -307,6 +312,7 @@ func TestCreateSandbox_ManyConcurrentRacers_OnlyOneCallback(t *testing.T) {
 				now, now.Add(time.Hour), time.Hour,
 				false,
 				sandbox.CreationMetadata{},
+				false,
 			)
 			if apiErr == nil {
 				successCount.Add(1)
@@ -343,6 +349,7 @@ func TestCreateSandbox_SequentialDuplicate_SecondCallSkipsCallback(t *testing.T)
 		now, now.Add(time.Hour), time.Hour,
 		false,
 		sandbox.CreationMetadata{},
+		false,
 	)
 	require.Nil(t, apiErr)
 	require.Eventually(t, func() bool { return ec.get() == 1 }, time.Second, 10*time.Millisecond)
@@ -354,6 +361,7 @@ func TestCreateSandbox_SequentialDuplicate_SecondCallSkipsCallback(t *testing.T)
 		now, now.Add(time.Hour), time.Hour,
 		false,
 		sandbox.CreationMetadata{},
+		false,
 	)
 	require.Nil(t, apiErr)
 
@@ -376,6 +384,7 @@ func TestCreateSandbox_FetcherError_NoCallback(t *testing.T) {
 		now, now.Add(time.Hour), time.Hour,
 		false,
 		sandbox.CreationMetadata{},
+		false,
 	)
 	require.NotNil(t, apiErr)
 
@@ -399,6 +408,7 @@ func TestCreateSandbox_QuotaExceeded_NoCallback(t *testing.T) {
 		now, now.Add(time.Hour), time.Hour,
 		false,
 		sandbox.CreationMetadata{},
+		false,
 	)
 	require.Nil(t, apiErr)
 	require.Eventually(t, func() bool { return ec.get() == 1 }, time.Second, 10*time.Millisecond)
@@ -410,6 +420,7 @@ func TestCreateSandbox_QuotaExceeded_NoCallback(t *testing.T) {
 		now, now.Add(time.Hour), time.Hour,
 		false,
 		sandbox.CreationMetadata{},
+		false,
 	)
 	require.NotNil(t, apiErr)
 	assert.Equal(t, http.StatusTooManyRequests, apiErr.Code)

packages/api/internal/orchestrator/create_instance_test.go

@@ -152,6 +152,7 @@ func TestCreateSandbox_StaleDataAfterConcurrentPause(t *testing.T) {
 			time.Hour,
 			true,
 			sandbox.CreationMetadata{IsResume: true},
+			false,
 		)
 		require.Nil(t, apiErr)
 		assert.Equal(t, "tpl-v1", sbx1.TemplateID)
@@ -176,6 +177,7 @@ func TestCreateSandbox_StaleDataAfterConcurrentPause(t *testing.T) {
 			time.Hour,
 			true,
 			sandbox.CreationMetadata{IsResume: true},
+			false,
 		)
 		require.Nil(t, apiErr)
 

packages/orchestrator/orchestrator.proto

@@ -101,6 +101,8 @@ message SandboxCreateRequest {
 
   google.protobuf.Timestamp start_time = 2;
   google.protobuf.Timestamp end_time = 3;
+
+  bool reboot = 4;
 }
 
 message SandboxCreateResponse {

packages/orchestrator/pkg/sandbox/sandbox.go

@@ -320,6 +320,18 @@ func NewFactory(
 // on the host side.
 type PreBootFn func(ctx context.Context, rootfsPath string) error
 
+type createOptions struct {
+	deferMarkRunning bool
+}
+
+type CreateOption func(*createOptions)
+
+// WithDeferredMarkRunning skips marking the sandbox running inside CreateSandbox
+// so the caller can mark it only after envd is ready, matching ResumeSandbox.
+func WithDeferredMarkRunning() CreateOption {
+	return func(o *createOptions) { o.deferMarkRunning = true }
+}
+
 // CreateSandbox creates the sandbox.
 // IMPORTANT: You must Close() the sandbox after you are done with it.
 func (f *Factory) CreateSandbox(
@@ -332,11 +344,17 @@ func (f *Factory) CreateSandbox(
 	processOptions fc.ProcessOptions,
 	apiConfigToStore *orchestrator.SandboxConfig,
 	preBootFn PreBootFn,
+	opts ...CreateOption,
 ) (s *Sandbox, e error) {
 	ctx, span := tracer.Start(ctx, "create sandbox")
 	defer span.End()
 	defer handleSpanError(span, &e)
 
+	var createOpts createOptions
+	for _, opt := range opts {
+		opt(&createOpts)
+	}
+
 	execCtx, execSpan := startExecutionSpan(ctx)
 
 	exit := utils.NewErrorOnce()
@@ -559,7 +577,9 @@ func (f *Factory) CreateSandbox(
 		exit.SetError(errors.Join(err, fcErr))
 	}()
 
-	f.Sandboxes.MarkRunning(ctx, sbx)
+	if !createOpts.deferMarkRunning {
+		f.Sandboxes.MarkRunning(ctx, sbx)
+	}
 
 	return sbx, nil
 }

packages/orchestrator/pkg/server/sandboxes.go

@@ -25,9 +25,13 @@ import (
 	"google.golang.org/protobuf/types/known/timestamppb"
 
 	"github.com/e2b-dev/infra/packages/orchestrator/pkg/sandbox"
+	"github.com/e2b-dev/infra/packages/orchestrator/pkg/sandbox/block"
 	"github.com/e2b-dev/infra/packages/orchestrator/pkg/sandbox/fc"
 	sbxtemplate "github.com/e2b-dev/infra/packages/orchestrator/pkg/sandbox/template"
+	configpkg "github.com/e2b-dev/infra/packages/orchestrator/pkg/template/build/config"
+	"github.com/e2b-dev/infra/packages/orchestrator/pkg/template/constants"
 	"github.com/e2b-dev/infra/packages/orchestrator/pkg/template/metadata"
+	"github.com/e2b-dev/infra/packages/orchestrator/pkg/units"
 	"github.com/e2b-dev/infra/packages/shared/pkg/events"
 	"github.com/e2b-dev/infra/packages/shared/pkg/featureflags"
 	"github.com/e2b-dev/infra/packages/shared/pkg/grpc/orchestrator"
@@ -58,6 +62,83 @@ const (
 	killReasonUnknown = "unknown"
 )
 
+func (s *Server) createSandboxFromRootfs(
+	ctx context.Context,
+	template sbxtemplate.Template,
+	config *sandbox.Config,
+	runtime sandbox.RuntimeMetadata,
+	req *orchestrator.SandboxCreateRequest,
+) (*sandbox.Sandbox, error) {
+	buildID, err := uuid.Parse(template.Files().BuildID)
+	if err != nil {
+		return nil, fmt.Errorf("parse build ID: %w", err)
+	}
+
+	memfile, err := block.NewEmpty(
+		units.MBToBytes(config.RamMB),
+		configpkg.MemfilePageSize(config.HugePages),
+		buildID,
+	)
+	if err != nil {
+		return nil, fmt.Errorf("create empty memfile: %w", err)
+	}
+
+	maskedTemplate := sbxtemplate.NewMaskTemplate(template, sbxtemplate.WithMemfile(memfile))
+
+	kvmClock, err := utils.IsGTEVersion(config.Envd.Version, "0.2.11")
+	if err != nil {
+		return nil, fmt.Errorf("compare envd version: %w", err)
+	}
+
+	startTime := req.GetStartTime().AsTime()
+	endTime := req.GetEndTime().AsTime()
+	timeout := endTime.Sub(startTime)
+	honorRequestWindow := timeout > 0
+	if !honorRequestWindow {
+		timeout = s.config.EnvdTimeout
+	}
+
+	sbx, err := s.sandboxFactory.CreateSandbox(
+		ctx,
+		config,
+		runtime,
+		maskedTemplate,
+		timeout,
+		"",
+		fc.ProcessOptions{
+			InitScriptPath: constants.SystemdInitPath,
+			KvmClock:       kvmClock,
+		},
+		req.GetSandbox(),
+		nil,
+		// Defer marking running until envd is ready, matching ResumeSandbox so a
+		// rebooted sandbox isn't visible/routable before it can serve traffic.
+		sandbox.WithDeferredMarkRunning(),
+	)
+	if err != nil {
+		return nil, err
+	}
+
+	// CreateSandbox anchors the lifetime to now; honor the API's absolute
+	// window like the resume path so queue delay can't extend the end time.
+	if honorRequestWindow {
+		sbx.SetStartedAt(startTime)
+		sbx.SetEndAt(endTime)
+	}
+
+	if err := sbx.WaitForEnvd(ctx, s.config.EnvdTimeout); err != nil {
+		closeErr := sbx.Close(context.WithoutCancel(ctx))
+
+		return nil, errors.Join(fmt.Errorf("wait for envd after reboot: %w", err), closeErr)
+	}
+
+	s.sandboxFactory.Sandboxes.MarkRunning(ctx, sbx)
+
+	go sbx.Checks.Start(context.WithoutCancel(ctx))
+
+	return sbx, nil
+}
+
 func (s *Server) Create(ctx context.Context, req *orchestrator.SandboxCreateRequest) (_ *orchestrator.SandboxCreateResponse, createErr error) {
 	// set max request timeout for this request
 	ctx, cancel := context.WithTimeoutCause(ctx, requestTimeout, errors.New("request timed out"))
@@ -187,18 +268,22 @@ func (s *Server) Create(ctx context.Context, req *orchestrator.SandboxCreateRequ
 		SandboxType: sandbox.SandboxTypeSandbox,
 	}
 
-	sbx, err := s.sandboxFactory.ResumeSandbox(
-		ctx,
-		template,
-		config,
-		runtime,
-		req.GetStartTime().AsTime(),
-		req.GetEndTime().AsTime(),
-		req.GetSandbox(),
-	)
+	var sbx *sandbox.Sandbox
+	if req.GetReboot() {
+		sbx, err = s.createSandboxFromRootfs(ctx, template, config, runtime, req)
+	} else {
+		sbx, err = s.sandboxFactory.ResumeSandbox(
+			ctx,
+			template,
+			config,
+			runtime,
+			req.GetStartTime().AsTime(),
+			req.GetEndTime().AsTime(),
+			req.GetSandbox(),
+		)
+	}
 	if err != nil {
 		if errors.Is(err, storage.ErrObjectNotExist) {
-			// Snapshot data not found, let the API know the data aren't probably upload yet
 			telemetry.ReportError(ctx, "sandbox files not found", err, telemetry.WithSandboxID(req.GetSandbox().GetSandboxId()))
 
 			return nil, status.Errorf(codes.FailedPrecondition, "sandbox files for '%s' not found", req.GetSandbox().GetSandboxId())