diff --git a/.github/actions/python-distributions/action.yml b/.github/actions/python-distributions/action.yml index c2104458..fa3878cf 100644 --- a/.github/actions/python-distributions/action.yml +++ b/.github/actions/python-distributions/action.yml @@ -60,6 +60,12 @@ runs: import adx_environment import akernel_sdk + from akernel_sdk import ComposeProject, DockerRuntime, ServiceHandle, ServiceProcessHandle + from importlib.resources import files + + assert all(callable(value) for value in (ComposeProject, DockerRuntime, ServiceHandle, ServiceProcessHandle)) + for helper in ("_image_bridge_guest.py", "_docker_network_guest.py", "_service_process_guest.py"): + assert files("akernel_sdk").joinpath(helper).read_text() assert version("akernel-sdk") == os.environ["EXPECTED_VERSION"] assert any( diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1e1954d3..dbbe8ee3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -15,6 +15,28 @@ concurrency: cancel-in-progress: ${{ github.event_name == 'pull_request' }} jobs: + image-bridge: + name: Managed Docker image bridge + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - name: Check out repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - name: Set up Go + uses: actions/setup-go@924ae3a1cded613372ab5595356fb5720e22ba16 # v6 + with: + go-version: '1.25.5' + cache-dependency-path: builder/image-bridge/go.sum + - name: Test and build the host/guest helper + working-directory: builder/image-bridge + env: + CGO_ENABLED: '0' + run: | + test -z "$(gofmt -l .)" + go vet ./... + go test ./... + GOOS=linux GOARCH=amd64 go build -trimpath -o "$RUNNER_TEMP/akernel-image-bridge" . + sdk-unit-tests: name: Python SDK unit tests (${{ matrix.python-version }}) runs-on: ubuntu-latest diff --git a/AGENTS.md b/AGENTS.md index 21c97674..84236635 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -869,3 +869,16 @@ ADX profile, applying patches only to present roles. Resolve profile/role selectors with ADX's single-pass environment/default/escape rules while preserving source expressions for the authoritative loader. Severity queries consume Collector-normalized log metadata and plain bodies. + + +## Managed Docker runtime + +The source-preview SDK owns Compose image discovery/OCI metadata validation, ROFS mount plans and guest bridge/containerd/dockerd startup. Sources live in `builder/image-bridge`, `builder/dind`, and `sdk/python/akernel_sdk/_docker_runtime.py`. Callers supply original project files and keep service orchestration in their adapter. Snapshot restore must rebind mounts, runtime extra_config and local tunnels without rerunning startup. Reuse the existing sandboxd image provider; do not add a parallel layer cache or modify deployment/runtime pins for client refactoring. See the SDK and builder guides for managed-upper prerequisites and supported scope. + +Compose project lifecycle and Docker service/process operations are SDK extensions in `_compose.py` and `_service_process.py`; the dependency-free `_service_process_guest.py` runs in the selected service PID namespace. Keep project descriptors JSON-compatible and restore attachment free of fresh-boot operations. Preserve explicit build/network-policy/up boundaries, unknown-outcome identities, source fencing, and container/process start-time checks before signals. Callers retain task/harness semantics. Run the real-process helper unit tests with owned-process cleanup and the complete SDK quality gate when changing these contracts. This feature does not add Docker semantics to Agent DX or change runtime pins. + +Automatic Compose image preparation covers image-only services as well as Dockerfile bases. Verify pinned Docker references and committed ROFS snapshots instead of running native pulls, including during up. Keep digest-reference regression coverage and fail closed on missing registrations; never relax bridge extraction protection or populate the private upper with replacement lower layers. + +Managed discovery requires helper schema 2 and probes capabilities before sending project input. Confine client Dockerfile/build-context and `.env` reads to the frozen project; never inherit unspecified host environment variables. External include/extends files remain unsupported. Validate manifest/config/chain relationships and allow legitimate same-chain roots with different target digests. Keep the empty automatic-plan marker for scratch-only builds. Unknown command exits and failed cancellation retain their identities and cleanup errors without replay. + +Managed Compose defaults preserve native IPv4 bridge definitions. Keep forwarding/NAT and per-service tunnel exposure in the SDK/runtime, with daemon-only legacy iptables selection for Docker DNS. Restore attachment stays pure; explicit reconnect_network refreshes only owned chains for the new outer TAP before the harness resumes. Do not claim external TCP address migration. Keep the opt-in Compose network integration regression, including existing inner and published-port sockets, synchronized with these APIs. Docker 28 operator flags are prerequisites; client work must not change production or runtime bundle pins. diff --git a/builder/dind/Dockerfile b/builder/dind/Dockerfile new file mode 100644 index 00000000..b9496334 --- /dev/null +++ b/builder/dind/Dockerfile @@ -0,0 +1,12 @@ +# Build from the repository root: docker build -f builder/dind/Dockerfile . +ARG GO_BUILD_IMAGE=golang:1.25.5-bookworm +FROM ${GO_BUILD_IMAGE} AS bridge +WORKDIR /src +COPY builder/image-bridge/go.mod builder/image-bridge/go.sum ./ +RUN go mod download +COPY builder/image-bridge/ ./ +RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -trimpath -o /out/akernel-image-bridge . + +FROM docker:28.3.3-dind +RUN apk add --no-cache bash python3 coreutils util-linux shadow tar xz +COPY --from=bridge /out/akernel-image-bridge /opt/akernel-image-bridge diff --git a/builder/image-bridge/README.md b/builder/image-bridge/README.md new file mode 100644 index 00000000..4ff71479 --- /dev/null +++ b/builder/image-bridge/README.md @@ -0,0 +1,27 @@ +# Managed Docker image bridge + +This helper exposes sandboxd-provided immutable ROFS image mounts as committed containerd snapshots, with private upper/work directories owned by the sandbox. It does not introduce another registry layer cache. Metadata-only image registration and source Dockerfile/Compose discovery use containerd, compose-go and BuildKit parsers. + +The snapshotter originated in tianyuzhou95's agent-worker POC, commit `9b3434d36086d920b6bb7f3322da642381a254b8`. The managed-upper runtime contract was ported to sandboxd separately. Preserve this provenance when moving or publishing the helper. + +Build the host/guest helper on a Linux/amd64 client: + +```sh +cd builder/image-bridge +CGO_ENABLED=0 go test ./... +CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -trimpath -o /path/to/akernel-image-bridge . +``` + +The Python SDK's `prepare_compose_images()` invokes `--mode discover`, resolves tags and verifies OCI manifest/config bytes. Credentials stay on the client; the runtime image manager must independently be authorized to fetch the pinned images. Only metadata is uploaded. Layer bytes are mounted by the existing sandboxd OCI/ROFS provider, which owns node-wide caching and distribution. + +`DockerRuntime.sandbox_options()` supplies the ROFS mounts and `fileBackedTmpfsMounts` to fresh creation and snapshot restore. `DockerRuntime.start()` uploads the helper when configured, starts its private containerd and bridge, registers original image aliases, then starts dockerd. Call start only once on a fresh sandbox; never on a restored sandbox. Failed startup is cleaned up by the owner deleting the newly created sandbox, not by touching shared node state. + +Automatic Compose uses these mounted roots for image-only services as well as build bases. After registration, the SDK verifies aliases against their pinned target digests and checks the committed ROFS snapshots. It does not use native Compose pull for those inputs: Docker Compose can pull digest references even with `--policy missing`, which would incorrectly enter the ordinary layer extraction path. Automatic `up` also disables repeat pulls. An absent registration fails instead of bypassing the mount plan or silently copying image layers into private storage. Different image manifests/configurations can legitimately share one chain ID; the validated configuration and chain bind their ROFS roots before verification allows that sharing. + +The current SDK requires **discovery schema 2**. Check with `akernel-image-bridge --mode capabilities`; older helpers (including the earlier `72e706e2` attachment) must be replaced before using this SDK. The probe sends no project data. Discovery rejects Dockerfile/build-context paths or `.env` links escaping the frozen project and does not inherit unspecified client environment variables. External `include` / `extends.file`, remote contexts and platform emulation are outside the source-preview profile; pass explicit supported Compose files instead. The runtime bundle and Sandboxd contract do not change with this client-helper update. + +Use `DockerRuntime.default_image`, the public Docker Hub `docker.io/library/docker:28.3.3-dind` image. Fresh startup installs missing Alpine support packages, so no custom runtime image build or push is required. An explicitly supplied outer image must provide Docker 28.3.3, containerd/ctr, Python 3, findmnt/mount, Bash, iproute tools, iptables-legacy and iptables-legacy-restore. `builder/dind/Dockerfile` optionally packages these and the guest helper; use repository root as build context. The SDK can upload the matching helper from the client. No helper binary is downloaded automatically. + +The default Linux/amd64 runsc profile preserves default/custom IPv4 Compose bridge networks, DNS and TCP/UDP userland port mappings. The operator must enable `extra_args = ["--net-raw=true", "--allow-packet-socket-write=true"]` under `[plugin.runtime.runsc]` for Docker 28. The SDK owns in-sandbox legacy forwarding/SNAT and per-service tunnel entries; Docker owns bridges/IPAM/DNS. After restore call `ComposeProject.reconnect_network()` to rebind the new outer TAP without restarting Docker. Task Dockerfiles and Compose files remain unchanged. Host networking is an explicit compatibility option. Remote contexts, runtime-created unknown image references and platform emulation remain outside this profile. Ordinary native Docker builds still run inside the sandbox. This is not a complete shared Docker content store: exporting/saving/pushing an image may still fetch layers. + +The file-backed upper requires sandboxd managed tmpfs support and runsc checkpoint support for those mounts. It consumes the sandbox storage limit. Reflink must be configured by the deployment, on a compatible filesystem and same mount; the SDK does not change deployment durability policy or replace runtime bundles. Memory mode is an explicit alternative and counts file contents toward memory snapshots. diff --git a/builder/image-bridge/discover.go b/builder/image-bridge/discover.go new file mode 100644 index 00000000..bf569be4 --- /dev/null +++ b/builder/image-bridge/discover.go @@ -0,0 +1,272 @@ +package main + +// Discovery uses the Compose and Dockerfile implementations, not regular +// expressions over user files. The original project is still built by Docker. +import ( + "context" + "encoding/json" + "fmt" + "io" + "os" + "path/filepath" + "sort" + "strconv" + "strings" + + "github.com/compose-spec/compose-go/v2/cli" + "github.com/compose-spec/compose-go/v2/types" + "github.com/distribution/reference" + "github.com/moby/buildkit/frontend/dockerfile/instructions" + "github.com/moby/buildkit/frontend/dockerfile/parser" + "github.com/moby/buildkit/frontend/dockerfile/shell" + "gopkg.in/yaml.v3" +) + +func projectPath(root, path string) (string, error) { + if !filepath.IsAbs(path) { + path = filepath.Join(root, path) + } + resolved, err := filepath.EvalSymlinks(path) + if err != nil { + return "", fmt.Errorf("cannot resolve managed project input: %w", err) + } + relative, err := filepath.Rel(root, resolved) + if err != nil || relative == ".." || strings.HasPrefix(relative, ".."+string(filepath.Separator)) { + return "", fmt.Errorf("managed Compose inputs must stay within the project directory") + } + return resolved, nil +} + +func validateComposeSources(root string, files []string) error { + // Only explicitly supplied top-level files may live outside the frozen + // context (SDK overlays do). Never let user YAML load other client files. + for _, file := range files { + data, err := os.ReadFile(file) + if err != nil { + return err + } + var document map[string]any + if err = yaml.Unmarshal(data, &document); err != nil { + return err + } + if document["include"] != nil { + return fmt.Errorf("managed Compose discovery does not support include; supply explicit project files") + } + services, _ := document["services"].(map[string]any) + for _, value := range services { + service, _ := value.(map[string]any) + extends, _ := service["extends"].(map[string]any) + if extends["file"] != nil { + return fmt.Errorf("managed Compose discovery does not support extends.file; supply explicit project files") + } + } + } + if _, err := os.Lstat(filepath.Join(root, ".env")); err == nil { + _, err = projectPath(root, ".env") + return err + } else if !os.IsNotExist(err) { + return err + } + return nil +} + +func dockerfileImages(source string, build *types.BuildConfig) ([]string, error) { + parsed, err := parser.Parse(strings.NewReader(source)) + if err != nil { + return nil, err + } + stages, args, err := instructions.Parse(parsed.AST, nil) + if err != nil { + return nil, err + } + values := map[string]string{ + "BUILDPLATFORM": "linux/amd64", "BUILDOS": "linux", "BUILDARCH": "amd64", "BUILDVARIANT": "", + "TARGETPLATFORM": "linux/amd64", "TARGETOS": "linux", "TARGETARCH": "amd64", "TARGETVARIANT": "", + } + lex := shell.NewLex(parsed.EscapeToken) + expand := func(word string) (string, error) { + env := make([]string, 0, len(values)) + for k, v := range values { + env = append(env, k+"="+v) + } + value, _, err := lex.ProcessWord(word, shell.EnvsFromSlice(env)) + return value, err + } + for _, command := range args { + for _, arg := range command.Args { + if override, ok := build.Args[arg.Key]; ok && override != nil { + values[arg.Key] = *override + continue + } else if ok { + if value, present := os.LookupEnv(arg.Key); present { + values[arg.Key] = value + continue + } + } + if arg.Value != nil { + value, err := expand(*arg.Value) + if err != nil { + return nil, err + } + values[arg.Key] = value + } + } + } + var images []string + knownStages := map[string]bool{} + for _, stage := range stages { + knownStages[strings.ToLower(stage.Name)] = true + } + add := func(name string, allowStage bool) error { + if strings.EqualFold(name, "scratch") || (allowStage && knownStages[strings.ToLower(name)]) { + return nil + } + if allowStage { + if _, err := strconv.Atoi(name); err == nil { + return nil + } + } + if context, ok := build.AdditionalContexts[name]; ok { + if !strings.HasPrefix(context, "docker-image://") { + return fmt.Errorf("automatic image preparation needs a docker-image context for %s", name) + } + name = strings.TrimPrefix(context, "docker-image://") + } + ref, err := reference.ParseNormalizedNamed(name) + if err != nil { + return fmt.Errorf("invalid external image %q: %w", name, err) + } + images = append(images, reference.TagNameOnly(ref).String()) + return nil + } + previousStages := map[string]bool{} + for _, stage := range stages { + if stage.Platform != "" { + platform, err := expand(stage.Platform) + if err != nil || platform != "linux/amd64" { + return nil, fmt.Errorf("automatic image preparation supports linux/amd64, got %q", platform) + } + } + name, err := expand(stage.BaseName) + if err != nil { + return nil, err + } + if !previousStages[strings.ToLower(name)] { + if err = add(name, false); err != nil { + return nil, err + } + } + previousStages[strings.ToLower(stage.Name)] = true + for _, command := range stage.Commands { + if copy, ok := command.(*instructions.CopyCommand); ok && copy.From != "" { + if err = add(copy.From, true); err != nil { + return nil, err + } + } + if run, ok := command.(*instructions.RunCommand); ok { + for _, mount := range instructions.GetMounts(run) { + if mount.From != "" { + if err = add(mount.From, true); err != nil { + return nil, err + } + } + } + } + } + } + return images, nil +} + +func discover(input io.Reader, output io.Writer) error { + var request struct { + Directory string `json:"directory"` + Files []string `json:"files"` + } + if err := json.NewDecoder(input).Decode(&request); err != nil { + return err + } + root, err := filepath.Abs(request.Directory) + if err != nil { + return err + } + root, err = filepath.EvalSymlinks(root) + if err != nil { + return err + } + if err = validateComposeSources(root, request.Files); err != nil { + return err + } + options, err := cli.NewProjectOptions(request.Files, + cli.WithWorkingDirectory(request.Directory), cli.WithName("harbor"), + cli.WithOsEnv, cli.WithEnvFiles(), cli.WithDotEnv, + cli.WithoutEnvironmentResolution, + ) + if err != nil { + return err + } + project, err := options.LoadProject(context.Background()) + if err != nil { + return err + } + seen := map[string]bool{} + for _, service := range project.Services { + if service.Platform != "" && service.Platform != "linux/amd64" { + return fmt.Errorf("service %s requires unsupported platform", service.Name) + } + if service.Build == nil { + ref, err := reference.ParseNormalizedNamed(service.Image) + if err != nil { + return err + } + seen[reference.TagNameOnly(ref).String()] = true + continue + } + build := service.Build + contextPath, err := projectPath(root, build.Context) + if err != nil { + return fmt.Errorf("service %s build context: %w", service.Name, err) + } + build.Context = contextPath + build.Args = build.Args.Resolve(func(key string) (string, bool) { + value, ok := project.Environment[key] + return value, ok + }) + for _, platform := range build.Platforms { + if platform != "linux/amd64" { + return fmt.Errorf("service %s requires unsupported build platform", service.Name) + } + } + source := build.DockerfileInline + if source == "" { + name := build.Dockerfile + if name == "" { + name = "Dockerfile" + } + if !filepath.IsAbs(name) { + name = filepath.Join(build.Context, name) + } + name, err = projectPath(root, name) + if err != nil { + return fmt.Errorf("service %s Dockerfile: %w", service.Name, err) + } + data, err := os.ReadFile(name) + if err != nil { + return fmt.Errorf("read Dockerfile for service %s: %w", service.Name, err) + } + source = string(data) + } + images, err := dockerfileImages(source, build) + if err != nil { + return fmt.Errorf("discover bases for service %s: %w", service.Name, err) + } + for _, image := range images { + seen[image] = true + } + } + images := make([]string, 0, len(seen)) + for image := range seen { + images = append(images, image) + } + sort.Strings(images) + return json.NewEncoder(output).Encode(map[string]any{"references": images}) +} diff --git a/builder/image-bridge/discover_test.go b/builder/image-bridge/discover_test.go new file mode 100644 index 00000000..6dc9e223 --- /dev/null +++ b/builder/image-bridge/discover_test.go @@ -0,0 +1,172 @@ +package main + +import ( + "bytes" + "encoding/json" + "os" + "path/filepath" + "reflect" + "strings" + "testing" + + "github.com/compose-spec/compose-go/v2/types" +) + +func TestDockerfileImageDiscovery(t *testing.T) { + version := "bookworm" + build := &types.BuildConfig{Args: types.MappingWithEquals{"VERSION": &version}} + images, err := dockerfileImages(`ARG VERSION=alpine +ARG BASE=debian:${VERSION}-slim +FROM --platform=$TARGETPLATFORM ${BASE} AS build +RUN < 4 && fields[4] == seed.Target { + mounted = true + break + } + } + if !mounted { + if err = bindReadOnly(seed.Source, seed.Target); err != nil { + return err + } + } + } + } else if !os.IsNotExist(err) { + return err + } + if err = os.MkdirAll(filepath.Dir(socket), 0700); err != nil { + return err + } + listener, err := net.Listen("unix", socket) + if err != nil { + return err + } + server := grpc.NewServer() + snapshotsapi.RegisterSnapshotsServer(server, snapshotservice.FromSnapshotter(b)) + log.Printf("image bridge snapshotter ready: %s", socket) + return server.Serve(listener) +} + +func register(plan Plan, planDir, address string) error { + client, err := containerd.New(address) + if err != nil { + return err + } + defer client.Close() + ctx := namespaces.WithNamespace(context.Background(), "moby") + ctx, done, err := client.WithLease(ctx) + if err != nil { + return err + } + defer done(ctx) + cs := client.ContentStore() + for _, image := range plan.Images { + for _, blob := range image.Blobs { + if filepath.Base(blob.File) != blob.File { + return errors.New("invalid metadata file name") + } + data, err := os.ReadFile(filepath.Join(planDir, blob.File)) + if err != nil { + return err + } + labels := map[string]string{} + var object struct { + Config *ocispec.Descriptor `json:"config"` + Manifests []ocispec.Descriptor `json:"manifests"` + } + if err = json.Unmarshal(data, &object); err != nil { + return err + } + if object.Config != nil { + labels["containerd.io/gc.ref.content.config"] = object.Config.Digest.String() + } + for n, d := range object.Manifests { + labels[fmt.Sprintf("containerd.io/gc.ref.content.m.%d", n)] = d.Digest.String() + } + if blob.Descriptor.Digest == image.Config.Digest { + labels["containerd.io/gc.ref.snapshot.akernel"] = image.ChainID + } + if err = content.WriteBlob(ctx, cs, blob.Descriptor.Digest.String(), bytes.NewReader(data), blob.Descriptor, content.WithLabels(labels)); err != nil { + return err + } + } + sn := client.SnapshotService("akernel") + if _, err = sn.Stat(ctx, image.ChainID); errdefs.IsNotFound(err) { + key := "seed-" + strings.TrimPrefix(image.ChainID, "sha256:") + if _, err = sn.Prepare(ctx, key, "", snapshots.WithLabels(map[string]string{rootLabel: image.Root})); err != nil { + return err + } + if err = sn.Commit(ctx, image.ChainID, key, snapshots.WithLabels(map[string]string{rootLabel: image.Root})); err != nil { + return err + } + } else if err != nil { + return err + } + existing, err := client.ImageService().Get(ctx, image.Reference) + if errdefs.IsNotFound(err) { + _, err = client.ImageService().Create(ctx, images.Image{Name: image.Reference, Target: image.Target}) + } else if err == nil && existing.Target.Digest != image.Target.Digest { + return errors.New("image reference changed") + } + if err != nil { + return err + } + log.Printf("registered mounted image without layer transfer: %s chain=%s", image.Reference, image.ChainID) + } + return nil +} + +func main() { + mode := flag.String("mode", "serve", "serve, register or discover") + planPath := flag.String("plan", "", "mounted image plan JSON") + root := flag.String("root", "/var/lib/docker/akernel-snapshotter", "snapshot metadata and writable layers") + socket := flag.String("socket", "/run/akernel-image-bridge/snapshotter.sock", "snapshotter socket") + address := flag.String("containerd", "/run/akernel-image-bridge/containerd.sock", "containerd socket") + flag.Parse() + if *mode == "capabilities" { + returnErr := json.NewEncoder(os.Stdout).Encode(map[string]any{"discover_schema": 2}) + if returnErr != nil { + log.Fatal(returnErr) + } + return + } + if *mode == "discover" { + if err := discover(os.Stdin, os.Stdout); err != nil { + log.Fatal(err) + } + return + } + plan, err := readPlan(*planPath) + if err != nil { + log.Fatal(err) + } + if *mode == "serve" { + err = serve(plan, *root, *socket) + } else if *mode == "register" { + err = register(plan, filepath.Dir(*planPath), *address) + } else { + err = errors.New("unknown mode") + } + if err != nil { + log.Fatal(err) + } +} diff --git a/builder/image-bridge/main_test.go b/builder/image-bridge/main_test.go new file mode 100644 index 00000000..a81f4fbf --- /dev/null +++ b/builder/image-bridge/main_test.go @@ -0,0 +1,26 @@ +package main + +import ( + "context" + "testing" + + "github.com/containerd/containerd/errdefs" + "github.com/containerd/containerd/snapshots" +) + +func TestUnregisteredLayersCannotFallBackToDownload(t *testing.T) { + // A nil underlying snapshotter proves rejection precedes ordinary Prepare. + b := &bridge{allowed: map[string]bool{}} + _, err := b.Prepare(context.Background(), "layer", "", snapshots.WithLabels(map[string]string{"containerd.io/snapshot.ref": "sha256:abc"})) + if !errdefs.IsFailedPrecondition(err) { + t.Fatalf("Prepare()=%v, want failed precondition", err) + } +} + +func TestUnregisteredRootCannotBeMounted(t *testing.T) { + b := &bridge{allowed: map[string]bool{}} + _, err := b.Prepare(context.Background(), "layer", "", snapshots.WithLabels(map[string]string{rootLabel: "/etc"})) + if !errdefs.IsInvalidArgument(err) { + t.Fatalf("Prepare()=%v, want invalid argument", err) + } +} diff --git a/sdk/python/README.md b/sdk/python/README.md index 31c8e2a6..3ab7fe04 100644 --- a/sdk/python/README.md +++ b/sdk/python/README.md @@ -1387,3 +1387,184 @@ results retain `outcome="unknown"` and `retry="same_operation"`; preserve the original identity when querying or retrying. Do not infer execution or cleanup from a timeout message. `status_code` describes the service failure (including an SSE final event's error code); it need not equal the stream's HTTP status. + + +### Managed Docker/Compose runtime (source preview) + +This API requires the SDK source revision containing managed Docker support; published `0.2.0rc3` does not provide it. It builds on the reusable snapshot API already merged into `develop`. Prepare a compatible runsc deployment with sandboxd managed file-backed tmpfs support. Select reflink checkpoint mode and durable sync in deployment configuration; the SDK never modifies cluster settings. See [runtime prerequisites](../../builder/image-bridge/README.md). + +```python +from pathlib import Path +from akernel_sdk import DockerRuntime, Sandbox, prepare_compose_images + +images = prepare_compose_images( + "./project", ["./project/docker-compose.yaml"], + output_dir="./diagnostics/images", + bridge_binary="/opt/tools/akernel-image-bridge", +) +runtime = DockerRuntime( + images=images, bridge_binary=Path("/opt/tools/akernel-image-bridge"), + writable_mb=2048, writable_backing="file", +) +sandbox = Sandbox( + image=DockerRuntime.default_image, cpu=4000, memory=8192, + storage_mb=30720, cwd="/", failover=False, + **runtime.sandbox_options(), +) +try: + runtime.start(sandbox, network="bridge") + # Upload the original project; run ordinary docker compose build/up here. + # Quiesce tools and synchronize any client-side harness state before saving. + snapshot = sandbox.create_snapshot(timeout=300) + restored = Sandbox.from_snapshot( + snapshot, cwd="/", failover=False, **runtime.sandbox_options(), + # Explicitly rebind any reverse_tunnel and current network_policy here. + ) + try: + runtime.reconnect_network(restored) # new outer TAP; no daemon restart + print(restored.commands.run("docker ps", check=True).stdout) + # Continue using restored, which wraps the captured processes. Do not + # call runtime.start(), rerun builds, or restart the agent command. + finally: + restored.kill() + Sandbox.delete_snapshot(snapshot.snapshot_id) +finally: + sandbox.kill() +``` + +The SDK prepares manifest/config metadata only. Sandboxd's existing OCI/ROFS image manager supplies immutable shared lower roots; containerd uses a private managed upper. Credentials for metadata discovery and node layer fetching are separate. No task image build/push or user image plan is required. Original definitions are read without rewriting them. Managed bridge networking preserves original default/custom IPv4 bridge networks, service DNS and TCP/UDP userland port mappings; Harbor continues to own its normal service, bind, resource and verifier overlays. + +`from_snapshot(..., extra_config=...)` and `create(snapshot_id=..., extra_config=...)` forward validated, defensively copied runtime configuration. Supply the same managed-upper geometry and image mounts on restore. Native ADX DTOs, client connection/TLS handling and reconstructed commands/files/tunnel handles stay inside the SDK. Snapshot restore does not capture local Python harness state or external services; callers own alignment, fencing the previous execution and deleting snapshots. + +`DockerRuntime.default_image` is the public Docker Hub image +`docker.io/library/docker:28.3.3-dind`. Fresh runtime startup installs missing +Alpine support packages (Python 3, Bash, util-linux and related tools). This +requires access to Docker Hub and Alpine package repositories; no custom runtime +image build or registry push is needed. A compatible prebuilt image can still be +selected explicitly to avoid installing packages during startup. Restore never +repeats runtime installation or starts another Docker daemon. + +### Compose projects and service processes (source preview) + +`ComposeProject`, `ServiceHandle` and `ServiceProcessHandle` are provided by this source revision. They use the same managed Docker runtime and existing ROFS provider described above. Their backend-neutral operations use the existing Agent DX API. The managed file-backed upper profile additionally requires compatible Sandboxd support; installing this Python SDK alone does not enable it. The framework/runtime boundary and integration order are described in [managed Compose integration](docs/managed-compose.md). A complete client-side example is in [compose_checkpoint.py](examples/compose_checkpoint.py). + +`ComposeProject.prepare()` freezes a private local copy of the selected context and Compose bytes, discovers image inputs from that copy and produces sandbox mount/config options. Put `output_dir` outside the input context. Keep build contexts inside the selected directory. The default `network="bridge"` preserves original Compose network definitions, Docker IPAM, DNS and published TCP/UDP ports. The explicit `network="host"` compatibility profile still adds host/loopback aliases and rejects explicit Compose networks. Scaled service process handles are rejected. Task files are not rewritten. Generic SDK defaults contain no Harbor service or artifact names. + +In automatic image mode, both Dockerfile bases and image-only services use pinned ROFS mounts. `pull()` verifies their Docker references and committed snapshots; `up()` verifies them again and prevents native repeat pulls. This includes `image: name:tag@sha256:...`, for which native Compose's missing policy can still request layer extraction. Tags are resolved once during preparation; missing or changed registrations fail without fallback. Without automatic images, native Compose pull behavior is retained. + +This revision requires an image bridge reporting `discover_schema: 2` via +`--mode capabilities`; replace older helpers before upgrading the SDK. Discovery +confines build/Dockerfile and `.env` reads to the frozen context and receives +only explicitly supplied project variables. `include` / `extends.file` and +remote contexts are not supported in this source preview. Valid shared chains +across different manifests/configs are supported after OCI configuration checks. + +```python +from pathlib import Path +from akernel_sdk import ComposeProject, DockerRuntime, Sandbox, ServiceProcessHandle + +project = ComposeProject.prepare( + "./project", ["./project/docker-compose.yaml"], + runtime=DockerRuntime(bridge_binary=Path("/opt/tools/akernel-image-bridge"), + writable_mb=2048, writable_backing="file"), + auto_images=True, output_dir="./artifacts/compose", +) +sandbox = Sandbox(image=DockerRuntime.default_image, cpu=4000, + memory=8192, storage_mb=30720, cwd="/", failover=False, + **project.sandbox_options()) +try: + project.start_runtime(sandbox) # fresh creation only + project.build(timeout=600) + project.pull(timeout=600) + # Apply the application's running network policy here if required. + project.up(wait_timeout=600) + worker = project.service("worker") + print(worker.exec("uname -s").stdout) # POSIX sh by default + process = worker.start_process("exec python3 worker.py", cwd="/app") + process.signal("stop") + snapshot = sandbox.create_snapshot(timeout=300) + project_state, process_state = project.descriptor, process.descriptor + # Persist these JSON descriptors with application/harness state as needed. + sandbox.kill() # fence source before continuing clone + restored = Sandbox.from_snapshot(snapshot, cwd="/", failover=False, + **project.sandbox_options()) + try: + attached = ComposeProject.attach(restored, project_state) + attached.reconnect_network() # rebind owned NAT/forwarding rules only + original = ServiceProcessHandle.attach(attached.service("worker"), process_state) + original.signal("continue") # original PID/process, no build/up/start + finally: + restored.kill() + Sandbox.delete_snapshot(snapshot.snapshot_id) +finally: + sandbox.kill() +``` + +`ServiceHandle` provides exec, upload/download (including directories), stop and process start. Select shell, login behavior, cwd, user and environment explicitly; the SDK does not assign a special main service. `ServiceProcessHandle` uses a stable process token, Docker container ID/start time and Linux process start time. Its in-service monitor and identity record are captured with the sandbox. It supports inspect, tree stop/continue/terminate/kill and exit-code wait. Python 3 is required only for managed service processes, not ordinary service exec. The process is detached from the short-lived docker-exec submission; outer command PIDs are not used to signal the inner process. + +Unknown start failures retain `error.process_descriptor`; attach using that token to observe the original execution instead of submitting it again. A restarted/replaced container or changed process identity is rejected. A partial runtime startup is fresh-only and cannot be replayed on the same project object; delete the owned sandbox after failure. Compose failures retain phase, exit code, stdout/stderr and diagnostic logs. The SDK never silently retries a side-effecting Compose operation with an unknown outcome. + +`ComposeProject.attach()` only reconstructs client bindings. It does not upload files, start daemons, build images or recreate services. Snapshot restore must retain the same mount/upper options, rebind tunnels and the current network policy, and fence the previous execution before resuming it. Application conversation state and checkpoint timing remain with the application. Descriptors may contain explicit project environment values; store them with the same protection as application configuration. + + +### Native Compose networking and model tunnels (source preview) + +The validated Linux/amd64 runsc profile requires Docker 28's operator-owned +`--net-raw=true` and `--allow-packet-socket-write=true` flags. The outer runtime +image supplies `ip`, `iptables-legacy` and `iptables-legacy-restore`. No task +Dockerfile or Compose edit is needed. These are node/runtime prerequisites; +the SDK never changes a deployment. Existing shared ROFS lower and file-backed +upper mounts are independent of the network choice. + +The SDK enables IPv4 forwarding/SNAT inside the sandbox and keeps independent +bridges from routing directly to each other. Docker owns bridge creation, IPAM, +embedded DNS and TCP/UDP userland port proxies. Global dockerd iptables handling +is disabled because this runsc lacks Docker's `addrtype` matcher. A daemon-only +PATH selects legacy tools for Docker's per-container DNS DNAT; system symlinks +and original user files are preserved. IPv6-only egress, non-bridge drivers and +all Docker firewall/driver options have not been qualified by this regression. + +After restoring to a new outer TAP/address, call `project.reconnect_network()` +(or `DockerRuntime.reconnect_network(restored)` for the low-level API) before +resuming work. It replaces only SDK-owned forwarding/NAT chains, retaining +conntrack and the captured bridges, containers, DNS/proxies and inner sockets. +`attach()` itself remains free of remote operations. Established connections +inside the sandbox, including through its published-port proxy, are tested. +Connections to external peers do not acquire address-migration guarantees. + +An `HttpReverseTunnel` listens on the **outer** loopback. For a service on its +own bridge, use the service-specific controlled entry point: + +```python +# Pass the same HttpReverseTunnel on fresh creation and Sandbox.from_snapshot. +url = project.service("worker").expose_tunnel(tunnel) +process = project.service("worker").start_process( + "exec python3 worker.py", envs={"MODEL_BASE_URL": url}, +) +``` + +`expose_tunnel()` binds an ephemeral port only on an attached IPv4 gateway and +accepts only that service's IP. It relays TCP bytes without changing URLs, +authentication headers, HTTP or SSE. An internal bridge can use its local +gateway without gaining general egress. Host-network services retain the +original loopback URL. The relay and JSON descriptor enter the sandbox snapshot; +restored projects reuse the entry point without starting another listener. +Container replacement, a missing relay descriptor on restore, or ambiguous +launch outcomes fail closed. Network-none and IPv6-only services cannot use this +IPv4 entry point. + +Run the opt-in real network regression only against an owned test instance: + +```sh +AKERNEL_RUN_COMPOSE_TESTS=1 \ +AKERNEL_TEST_DIND_IMAGE='' \ +AKERNEL_TEST_IMAGE_BRIDGE_BINARY='/absolute/path/akernel-image-bridge' \ +python -m unittest tests.integration.test_compose_network -v +``` + +It checks default/custom DNS, aliases, disjoint-network isolation, TCP/UDP port +mappings, peer-restricted model HTTP/auth/SSE, new egress, and original service +and published-port TCP sockets across two snapshot/delete/restore cycles. All +owned environments and snapshots are deleted; no deployment is changed. + +Service process trees freeze waiters before descendants and thaw descendants before waiters. This avoids Bash treating a still-stopped child as an exited tool (status 147); the regression uses a real job-control Bash waiter with a forced scheduler switch. Numeric PID order is not used as ancestry. diff --git a/sdk/python/akernel_sdk/__init__.py b/sdk/python/akernel_sdk/__init__.py index df3ba4cc..856271e9 100644 --- a/sdk/python/akernel_sdk/__init__.py +++ b/sdk/python/akernel_sdk/__init__.py @@ -15,6 +15,14 @@ """Public API for the AKernel Python SDK.""" import importlib +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from ._compose import ComposeOperationError, ComposeProject, ServiceHandle + from ._docker_runtime import DockerRuntime + from ._image_plan import ImageMountPlan + from ._image_resolver import prepare_compose_images + from ._service_process import ServiceProcessHandle from ._backends.errors import ( AuthenticationException, @@ -106,9 +114,23 @@ "DockerfileBuildError", "DockerfileParseError", "BuildInstruction", + "ComposeProject", + "ComposeOperationError", + "ServiceHandle", + "ServiceProcessHandle", + "DockerRuntime", + "ImageMountPlan", + "prepare_compose_images", ] _LAZY_IMPORTS = { + "ComposeProject": ("._compose", "ComposeProject"), + "ComposeOperationError": ("._compose", "ComposeOperationError"), + "ServiceHandle": ("._compose", "ServiceHandle"), + "ServiceProcessHandle": ("._service_process", "ServiceProcessHandle"), + "DockerRuntime": ("._docker_runtime", "DockerRuntime"), + "ImageMountPlan": ("._image_plan", "ImageMountPlan"), + "prepare_compose_images": ("._image_resolver", "prepare_compose_images"), "Client": ("._client", "Client"), "Sandbox": (".sandbox", "Sandbox"), "CommandHandle": (".commands", "CommandHandle"), diff --git a/sdk/python/akernel_sdk/_backends/adx.py b/sdk/python/akernel_sdk/_backends/adx.py index e1e62cb6..f90a5860 100644 --- a/sdk/python/akernel_sdk/_backends/adx.py +++ b/sdk/python/akernel_sdk/_backends/adx.py @@ -756,6 +756,7 @@ def fork( command_cwd=self._spec.command_cwd, port_forwardings=self._spec.port_forwardings, mounts=self._spec.mounts, + extra_config=self._spec.extra_config, reverse_tunnel=self._spec.reverse_tunnel, detached=False, failover=self._spec.failover, @@ -1433,6 +1434,7 @@ def _restore_snapshot(spec: SnapshotRestoreSpec, connection: Any) -> BackendSess cwd=spec.command_cwd, port_forwardings=list(spec.port_forwardings), mounts=[_native_mount(mount) for mount in spec.mounts], + extra_config=dict(spec.extra_config), upstream=tunnel.target if tunnel else None, proxy_port=tunnel.listen_port if tunnel else _DEFAULT_LISTEN_PORT, tunnel_connect_timeout=tunnel.connect_timeout if tunnel else None, diff --git a/sdk/python/akernel_sdk/_backends/base.py b/sdk/python/akernel_sdk/_backends/base.py index 4a426231..bdfda33c 100644 --- a/sdk/python/akernel_sdk/_backends/base.py +++ b/sdk/python/akernel_sdk/_backends/base.py @@ -103,6 +103,7 @@ class SnapshotRestoreSpec: failover: bool node_id: str | None network_policy: NetworkPolicy | None + extra_config: Mapping[str, object] = field(default_factory=dict) xpu: str | None = None storage_mb: int | None = None inherit_entrypoint: bool = False @@ -122,6 +123,7 @@ class SandboxConnectSpec: schedule_timeout: int = 30 network_policy: NetworkPolicy | None = None mounts: tuple[Mount, ...] = () + extra_config: Mapping[str, object] = field(default_factory=dict) xpu: str | None = None storage_mb: int | None = None inherit_entrypoint: bool = False diff --git a/sdk/python/akernel_sdk/_compose.py b/sdk/python/akernel_sdk/_compose.py new file mode 100644 index 00000000..0c5e8ad5 --- /dev/null +++ b/sdk/python/akernel_sdk/_compose.py @@ -0,0 +1,736 @@ +"""Compose workloads inside a Sandbox; independent of any evaluation harness.""" + +from __future__ import annotations + +import ipaddress +import json +import re +import shlex +import shutil +import tempfile +import time +from dataclasses import replace +from pathlib import Path +from threading import Event +from typing import TYPE_CHECKING, Any +from urllib.parse import urlsplit +from uuid import uuid4 + +import yaml + +from ._backends.errors import ( + BackendOperationError, + TimeoutException, + retain_failure_context, +) +from ._docker_runtime import DockerRuntime +from ._image_plan import ImageMountPlan +from ._image_resolver import prepare_compose_images +from .types import CommandResult, HttpReverseTunnel, Mount + +if TYPE_CHECKING: + from collections.abc import Mapping, Sequence + + from ._service_process import ServiceProcessHandle + from .sandbox import Sandbox + + +class ComposeOperationError(BackendOperationError): + """Compose returned a failure, retaining phase and command output.""" + + def __init__(self, phase: str, result: CommandResult) -> None: + super().__init__( + f"Compose {phase} failed (exit_code={result.exit_code})", + code="COMPOSE_FAILED", + outcome="unknown" if result.exit_code is None else "terminal", + ) + self.phase = phase + self.result = result + self.stdout, self.stderr = result.stdout, result.stderr + self.exit_code = result.exit_code + + +def _services(contents: Sequence[str]) -> dict[str, Any]: + services: dict[str, Any] = {} + for content in contents: + document = yaml.safe_load(content) or {} + for name, config in document.get("services", {}).items(): + services.setdefault(name, {}).update(config or {}) + if not services: + raise ValueError("Compose project requires at least one service") + return services + + +def host_network_overlay( + services: Mapping[str, Any], *, prebuilt: Sequence[str] = () +) -> dict[str, Any]: + """Explicit runsc host-network compatibility profile, preserving task files.""" + if any(config.get("networks") for config in services.values()): + raise ValueError("Host-network Compose does not support explicit networks") + hostnames = { + name: config.get("hostname") or name for name, config in services.items() + } + aliases = list(dict.fromkeys([*services, *hostnames.values()])) + return { + "services": { + name: { + "network_mode": "host", + "hostname": hostnames[name], + "extra_hosts": [f"{alias}:127.0.0.1" for alias in aliases], + **( + {"build": {"network": "host"}} + if config.get("build") is not None and name not in prebuilt + else {} + ), + } + for name, config in services.items() + } + } + + +class ComposeProject: + """Prepared client descriptor and remote Compose operations. + + prepare() freezes the project context and Compose bytes before resolving + image inputs. No server is changed until start_runtime(). Restore with + attach(): it performs no upload, build, up, or daemon startup. The caller + owns the outer Sandbox and its deletion after a partial startup failure. + """ + + _descriptor: dict[str, Any] + _runtime: DockerRuntime | None + _context: Path | None + _output: Path | None + _contents: list[str] + _sandbox: Sandbox | None + _startup_attempted: bool + + @classmethod + def prepare( + cls, + directory: str | Path, + files: Sequence[str | Path] = (), + *, + project_name: str = "akernel", + base: Mapping[str, Any] | None = None, + overrides: Sequence[Mapping[str, Any]] = (), + runtime: DockerRuntime | None = None, + auto_images: bool = False, + output_dir: str | Path | None = None, + environment: Mapping[str, str] | None = None, + network: str = "bridge", + buildkit: bool = True, + prebuilt_images: Mapping[str, str] | None = None, + remote_directory: str = "/opt/akernel-compose/project", + directories: Mapping[str, int] | None = None, + ) -> ComposeProject: + if re.fullmatch(r"[a-z0-9][a-z0-9_-]*", project_name) is None: + raise ValueError("Invalid Compose project_name") + if auto_images and not buildkit: + raise ValueError("Automatic images require BuildKit") + if network not in {"host", "bridge"}: + raise ValueError("Compose network must be host or bridge") + if not remote_directory.startswith("/") or ".." in Path(remote_directory).parts: + raise ValueError("remote_directory must be an absolute normalized path") + for path, mode in (directories or {}).items(): + if ( + not path.startswith("/") + or ".." in Path(path).parts + or isinstance(mode, bool) + or not isinstance(mode, int) + or not 0 <= mode <= 0o777 + ): + raise ValueError("Invalid bind directory initialization") + source = Path(directory).resolve(strict=True) + output = ( + Path(output_dir).resolve() + if output_dir + else Path(tempfile.mkdtemp(prefix="akernel-compose-")) + ) + if output == source or source in output.parents: + raise ValueError("Compose output_dir must be outside the source context") + output.mkdir(parents=True, exist_ok=True, mode=0o700) + context = output / "context" + if context.exists(): + raise ValueError("Compose output_dir already contains a prepared context") + contents: list[str] = [] + names: list[str] = [] + if base is not None: + names.append("base.json") + contents.append(json.dumps(base)) + for index, file in enumerate(files): + names.append(f"source-{index}.yaml") + contents.append(Path(file).read_text()) + for index, override in enumerate(overrides): + names.append(f"override-{index}.json") + contents.append(json.dumps(override)) + services = _services(contents) + # Preserve original definitions; compatibility policy is an extra file. + selected = set(prebuilt_images or {}) & services.keys() + if network == "host": + names.append("host.json") + contents.append( + json.dumps(host_network_overlay(services, prebuilt=sorted(selected))) + ) + # A private local copy makes discovery and upload see the same context. + shutil.copytree(source, context, symlinks=True) + local_files = [] + for name, content in zip(names, contents, strict=True): + local_path = output / name + local_path.write_text(content) + local_files.append(local_path) + runtime = runtime or DockerRuntime() + if auto_images: + if runtime.bridge_binary is None: + raise ValueError("Automatic images require a matching bridge binary") + images = prepare_compose_images( + context, + local_files, + output_dir=output / "images", + bridge_binary=runtime.bridge_binary, + env=environment, + ) + runtime = replace(runtime, images=images) + if selected: + lines = ["services:"] + for name in sorted(selected): + lines += [ + f" {json.dumps(name)}:", + " build: !reset null", + f" image: {json.dumps((prebuilt_images or {})[name])}", + ] + names.append("prebuilt.yaml") + contents.append("\n".join(lines) + "\n") + result = cls.__new__(cls) + result._runtime = runtime + result._context = context + result._contents = contents + result._output = output + result._sandbox = None + result._startup_attempted = False + result._descriptor = { + "schema_version": 1, + "project_name": project_name, + "directory": remote_directory, + "files": [remote_directory + ".config/" + n for n in names], + "services": list(services), + "network": network, + "buildkit": buildkit, + "prebuilt_only": runtime.images is not None and not auto_images, + "automatic_images": auto_images, + "environment": dict(environment or {}), + "directories": dict(directories or {}), + } + options = runtime.sandbox_options() + result._descriptor["sandbox_options"] = { + **options, + "mounts": [ + {"target": m.target, "image_url": m.image_url} + for m in options.get("mounts", []) + ], + } + return result + + @classmethod + def attach(cls, sandbox: Sandbox, descriptor: Mapping[str, Any]) -> ComposeProject: + """Rebind captured remote state only, with no fresh-boot operations.""" + data = json.loads(json.dumps(descriptor, allow_nan=False)) + if data.get("schema_version") != 1 or not isinstance( + data.get("services"), list + ): + raise ValueError("Invalid Compose project descriptor") + if re.fullmatch(r"[a-z0-9][a-z0-9_-]*", data.get("project_name", "")) is None: + raise ValueError("Invalid Compose project name") + paths = [data.get("directory"), *data.get("files", [])] + if not all( + isinstance(p, str) and p.startswith("/") and ".." not in Path(p).parts + for p in paths + ): + raise ValueError("Invalid Compose project paths") + result = cls.__new__(cls) + result._descriptor = data + result._sandbox = sandbox + result._startup_attempted = True + result._output = None + result._runtime = None + result._context = None + result._contents = [] + return result + + @property + def descriptor(self) -> dict[str, Any]: + return json.loads(json.dumps(self._descriptor)) + + @property + def images(self) -> ImageMountPlan | None: + return self._runtime.images if self._runtime else None + + def sandbox_options(self) -> dict[str, Any]: + options = json.loads(json.dumps(self._descriptor["sandbox_options"])) + options["mounts"] = [Mount(**value) for value in options.get("mounts", [])] + return options + + def _require_sandbox(self) -> Sandbox: + if self._sandbox is None: + raise RuntimeError("Compose project is not bound to a Sandbox") + return self._sandbox + + def start_runtime(self, sandbox: Sandbox) -> None: + if ( + self._sandbox is not None + or self._runtime is None + or self._startup_attempted + ): + raise RuntimeError("Compose runtime startup is fresh-only") + self._startup_attempted = True + self._runtime.start(sandbox, network=self._descriptor["network"]) + self._sandbox = sandbox + directory = self._descriptor["directory"] + sandbox.files.copy_from_local(str(self._context), directory) + sandbox.files.make_dir(directory + ".config") + for remote, content in zip( + self._descriptor["files"], self._contents, strict=True + ): + sandbox.files.write(remote, content) + for path, mode in self._descriptor["directories"].items(): + sandbox.commands.run( + shlex.join(["install", "-d", "-m", oct(mode)[2:], path]), + timeout=20, + check=True, + ) + + def reconnect_network(self) -> None: + """Rebind managed egress rules after restoring to a new outer TAP. + + Call after attach and before resuming work. Does not start a daemon, + build/up a project, replace containers or recreate tunnel listeners. + Only SDK-owned IPv4 forwarding/NAT chains can change. Inner network + identities and existing connections inside the sandbox remain intact; + connections to external peers do not gain IP-migration guarantees. + """ + DockerRuntime.reconnect_network( + self._require_sandbox(), network=self._descriptor["network"] + ) + + def command(self, args: Sequence[str]) -> str: + if not args: + raise ValueError("Compose subcommand is required") + parts = [ + "docker", + "compose", + "-p", + self._descriptor["project_name"], + "--project-directory", + self._descriptor["directory"], + ] + for file in self._descriptor["files"]: + parts += ["-f", file] + prefix = ( + f"DOCKER_BUILDKIT={int(self._descriptor['buildkit'])} " + if args[0] == "build" + else "" + ) + # Interpolation gets only explicit project variables, never host secrets. + if self._descriptor["environment"]: + prefix += ( + "env " + + shlex.join( + [f"{k}={v}" for k, v in self._descriptor["environment"].items()] + ) + + " " + ) + return prefix + shlex.join([*parts, *args]) + + def run( + self, + args: Sequence[str], + *, + timeout: int = 60, + check: bool = False, + cancellation: Event | None = None, + ) -> CommandResult: + result = self._execute( + self.command(args), timeout=timeout, cancellation=cancellation + ) + if check and result.exit_code != 0: + raise ComposeOperationError(args[0], result) + return result + + def _execute( + self, command: str, *, timeout: int, cancellation: Event | None = None + ) -> CommandResult: + sandbox = self._require_sandbox() + if cancellation is None and timeout != 0: + result = sandbox.commands.run(command, cwd="/", timeout=timeout) + else: + # A zero execution budget means no deadline. Waits remain bounded + # so an async caller can cancel without abandoning remote work. + handle = sandbox.commands.run(command, cwd="/", background=True) + deadline = time.monotonic() + timeout if timeout > 0 else None + while True: + cancelled = cancellation is not None and cancellation.is_set() + expired = deadline is not None and time.monotonic() >= deadline + if cancelled or expired: + failure = TimeoutException( + "Compose command cancelled" + if cancelled + else "Compose command timed out", + command_id=handle.command_id, + outcome="unknown", + ) + try: + handle.kill() + except BaseException as cleanup_error: + retain_failure_context(failure, sandbox.id, cleanup_error) + raise failure + result = handle.wait(timeout=1) + if result.exit_code is not None: + break + return result + + def _operation(self, args: Sequence[str], timeout: int) -> CommandResult: + try: + result = self.run(args, timeout=timeout) + except BackendOperationError: + # Unknown outcomes are never retried by the Compose layer. + raise + if self._output is not None: + (self._output / (args[0] + ".log")).write_text( + result.stdout + result.stderr + ) + if result.exit_code != 0: + if args[0] == "up": + try: + logs = self.run(["logs", "--no-color", "--tail", "200"], timeout=30) + if self._output is not None: + (self._output / "services.log").write_text( + logs.stdout + logs.stderr + ) + except Exception: + pass + raise ComposeOperationError(args[0], result) + return result + + def build(self, *, no_cache: bool = False, timeout: int = 600) -> None: + if not self._descriptor["prebuilt_only"]: + self._operation(["build", *(["--no-cache"] if no_cache else [])], timeout) + + def pull(self, *, timeout: int = 600) -> None: + """Prepare service images, verifying existing ROFS inputs in auto mode. + + Automatic discovery already resolved and mounted all external inputs. + Native Compose pull can still unpack digest references despite its + missing policy, so use the checked mounted images rather than pull again. + """ + if self._descriptor["automatic_images"]: + self._verify_images("pull", timeout) + elif not self._descriptor["prebuilt_only"]: + self._operation(["pull", "--ignore-buildable"], timeout) + + def _verify_images(self, phase: str, timeout: int) -> None: + result = self._execute( + "python3 /opt/akernel-docker-runtime/start.py --verify " + "--plan /opt/akernel-docker-runtime/metadata/plan.json", + timeout=timeout, + ) + if self._output is not None: + (self._output / (phase + "-images.log")).write_text( + result.stdout + result.stderr + ) + if phase == "pull": + (self._output / "pull.log").write_text(result.stdout + result.stderr) + if result.exit_code != 0: + raise ComposeOperationError(phase, result) + + def up(self, *, wait_timeout: int = 600, timeout: int = 600) -> None: + if self._descriptor["automatic_images"]: + self._verify_images("up", timeout) + self._operation( + [ + "up", + "-d", + "--wait", + "--wait-timeout", + str(wait_timeout), + *( + ["--no-build", "--pull", "never"] + if self._descriptor["prebuilt_only"] + else ["--pull", "never"] + if self._descriptor["automatic_images"] + else [] + ), + ], + timeout, + ) + + def ps(self) -> list[dict[str, Any]]: + output = self.run(["ps", "-a", "--format", "json"], check=True).stdout + rows = [] + for line in output.splitlines(): + parsed = json.loads(line) + rows.extend(parsed if isinstance(parsed, list) else [parsed]) + return rows + + def logs(self, *, tail: int = 200) -> str: + return self.run(["logs", "--no-color", "--tail", str(tail)], check=True).stdout + + def down(self, *, timeout: int = 60) -> None: + self.run(["down"], timeout=timeout, check=True) + + def service(self, name: str) -> ServiceHandle: + if name not in self._descriptor["services"]: + raise ValueError(f"Unknown Compose service: {name}") + return ServiceHandle(self, name) + + +class ServiceHandle: + """Operations in one Compose service; no hardcoded main or log paths.""" + + def __init__(self, project: ComposeProject, name: str) -> None: + self.project, self.name = project, name + + def identity(self) -> dict[str, str]: + ids = self.project.run(["ps", "-a", "-q", self.name], check=True).stdout.split() + if len(ids) != 1: + raise BackendOperationError( + "Service must have exactly one container", code="SERVICE_UNAVAILABLE" + ) + result = self.project._require_sandbox().commands.run( + shlex.join(["docker", "inspect", ids[0]]), timeout=20, check=True + ) + data = json.loads(result.stdout)[0] + return {"container_id": data["Id"], "started_at": data["State"]["StartedAt"]} + + def expose_tunnel(self, tunnel: HttpReverseTunnel) -> str: + """Return this service's reachable URL for an outer reverse tunnel. + + Host-network services use the existing loopback listener. For bridge + networks, a private TCP relay binds only to an attached gateway and + accepts only this service's IP. HTTP, credentials and SSE pass through + unchanged. The daemon and descriptor survive whole-sandbox snapshots; + attach never restarts a relay. Requires IPv4 on one attached network. + An ambiguous launch is never retried automatically. + """ + if not isinstance(tunnel, HttpReverseTunnel): + raise TypeError("tunnel must be an HttpReverseTunnel") + identity = self.identity() + key = f"{self.name}:{tunnel.listen_port}" + endpoints = self.project._descriptor.setdefault("tunnel_endpoints", {}) + if key in endpoints: + record = endpoints[key] + if any(record[k] != identity[k] for k in identity): + raise BackendOperationError( + "Tunnel service was replaced or restarted", + code="SERVICE_TUNNEL_STALE", + outcome="not_started", + ) + if "url" not in record: + raise BackendOperationError( + "Tunnel launch outcome unknown; do not resubmit", + code="SERVICE_TUNNEL_UNKNOWN", + outcome="unknown", + ) + return str(record["url"]) + if self.project._runtime is None: + raise RuntimeError("Restored projects can only attach existing tunnels") + sandbox = self.project._require_sandbox() + result = sandbox.commands.run( + shlex.join(["docker", "inspect", identity["container_id"]]), + timeout=20, + check=True, + ) + data = json.loads(result.stdout)[0] + if data["HostConfig"]["NetworkMode"] == "host": + endpoints[key] = {**identity, "url": tunnel.url} + return tunnel.url + networks = data["NetworkSettings"].get("Networks") or {} + connected = [] + for name, value in sorted(networks.items()): + if not value.get("IPAddress"): + continue + gateway = value.get("Gateway") + if not gateway: + # Internal Docker networks intentionally omit a default route; + # their local bridge gateway can still host a controlled tunnel. + inspect = sandbox.commands.run( + shlex.join(["docker", "network", "inspect", name]), + timeout=20, + check=True, + ) + config = json.loads(inspect.stdout)[0]["IPAM"]["Config"] + gateway = next( + ( + v["Gateway"] + for v in config + if v.get("Gateway") and ":" not in v["Gateway"] + ), + None, + ) + if gateway: + connected.append({"Gateway": gateway, "IPAddress": value["IPAddress"]}) + if not connected: + raise ValueError("Service tunnel requires an attached IPv4 bridge network") + network = connected[0] + address = str(ipaddress.IPv4Address(network["Gateway"])) + peer = str(ipaddress.IPv4Address(network["IPAddress"])) + if ( + ipaddress.IPv4Address(address).is_unspecified + or ipaddress.IPv4Address(address).is_loopback + ): + raise ValueError("Service tunnel requires a non-loopback bridge gateway") + operation_id = uuid4().hex + # Reserve before dispatch so a lost reply cannot create a second relay. + record = {**identity, "operation_id": operation_id} + endpoints[key] = record + script = Path(__file__).with_name("_docker_network_guest.py").read_text() + result = sandbox.commands.run( + shlex.join( + [ + "python3", + "-c", + script, + "expose", + "/opt/akernel-docker-runtime/tunnels/" + operation_id, + json.dumps( + { + "address": address, + "allowed": [peer], + "target_port": tunnel.listen_port, + } + ), + ] + ), + timeout=20, + check=True, + ) + value = json.loads(result.stdout) + url = urlsplit(value["url"]) + if ( + url.scheme != "http" + or url.hostname != address + or not url.port + or url.username + or url.path + or url.query + or url.fragment + ): + raise ValueError("Runtime returned an invalid service tunnel endpoint") + record["url"] = value["url"] + return str(record["url"]) + + def _args( + self, + command: str, + *, + cwd: str | None, + envs: Mapping[str, str] | None, + user: str | int | None, + shell: str, + login: bool = False, + ) -> list[str]: + args = ["exec", "-T"] + if cwd: + args += ["-w", cwd] + if user is not None: + args += ["-u", str(user)] + for key, value in (envs or {}).items(): + args += ["-e", f"{key}={value}"] + return [*args, self.name, shell, "-lc" if login else "-c", command] + + def exec( + self, + command: str, + *, + cwd: str | None = None, + envs: Mapping[str, str] | None = None, + user: str | int | None = None, + shell: str = "/bin/sh", + login: bool = False, + timeout: int = 60, + cancellation: Event | None = None, + ) -> CommandResult: + return self.project.run( + self._args( + command, cwd=cwd, envs=envs, user=user, shell=shell, login=login + ), + timeout=timeout, + cancellation=cancellation, + ) + + def _copy(self, source: str, target: str, timeout: int) -> None: + self.project.run(["cp", source, target], timeout=timeout, check=True) + + def upload( + self, + source: str | Path, + target: str, + *, + directory: bool = False, + timeout: int = 120, + ) -> None: + sandbox = self.project._require_sandbox() + temp = "/tmp/akernel-compose-" + uuid4().hex + failure: BaseException | None = None + try: + sandbox.files.copy_from_local(str(source), temp) + self._copy( + temp + ("/." if directory else ""), f"{self.name}:{target}", timeout + ) + except BaseException as error: + failure = error + raise + finally: + try: + sandbox.files.remove(temp) + except Exception as cleanup_error: + if failure is None: + raise + retain_failure_context(failure, sandbox.id, cleanup_error) + + def download( + self, + source: str, + target: str | Path, + *, + directory: bool = False, + timeout: int = 120, + ) -> None: + sandbox = self.project._require_sandbox() + temp = "/tmp/akernel-compose-" + uuid4().hex + failure: BaseException | None = None + try: + if directory: + sandbox.files.make_dir(temp) + self._copy( + f"{self.name}:{source}" + ("/." if directory else ""), temp, timeout + ) + Path(target).parent.mkdir(parents=True, exist_ok=True) + sandbox.files.copy_to_local(temp, str(target)) + except BaseException as error: + failure = error + raise + finally: + try: + sandbox.files.remove(temp) + except Exception as cleanup_error: + if failure is None: + raise + retain_failure_context(failure, sandbox.id, cleanup_error) + + def stop(self, *, timeout: int = 60) -> None: + self.project.run(["stop", self.name], timeout=timeout, check=True) + + def start_process( + self, + command: str, + *, + cwd: str | None = None, + envs: Mapping[str, str] | None = None, + user: str | int | None = None, + shell: str = "/bin/bash", + ) -> ServiceProcessHandle: + from ._service_process import ServiceProcessHandle + + return ServiceProcessHandle.start( + self, command, cwd=cwd, envs=envs, user=user, shell=shell + ) diff --git a/sdk/python/akernel_sdk/_docker_network_guest.py b/sdk/python/akernel_sdk/_docker_network_guest.py new file mode 100644 index 00000000..8120892f --- /dev/null +++ b/sdk/python/akernel_sdk/_docker_network_guest.py @@ -0,0 +1,161 @@ +"""Dependency-free networking helper, executed in the outer sandbox only. + +The runtime owns forwarding/NAT; Docker owns bridges, IPAM, DNS and its +userland port proxies. No rules or listeners are installed on the real host. +""" + +import ipaddress +import json +import os +import shutil +import socket +import subprocess +import sys +import threading +from pathlib import Path + +STATE = Path("/opt/akernel-docker-runtime/network.json") + + +def configure(*, reconnect=False): + route = subprocess.check_output( + ["ip", "-4", "route", "get", "1.1.1.1"], text=True + ).split() + device = route[route.index("dev") + 1] + source = str(ipaddress.IPv4Address(route[route.index("src") + 1])) + executable = shutil.which("iptables-legacy") + restore = shutil.which("iptables-legacy-restore") + if executable is None or restore is None: + raise RuntimeError( + "Managed bridge networking requires iptables-legacy " + "and iptables-legacy-restore" + ) + record = {"uplink": device, "source": source, "iptables": executable} + if reconnect: + previous = json.loads(STATE.read_text()) + if previous == record: + return record + else: + # Docker's embedded DNS still programs per-container DNAT rules with + # --iptables=false. Put legacy tools in only the daemon's PATH. + directory = STATE.parent / "network-bin" + directory.mkdir(parents=True, exist_ok=True) + for name in ("iptables", "ip6tables"): + legacy = shutil.which(name + "-legacy") + if legacy is not None: + (directory / name).symlink_to(legacy) + Path("/proc/sys/net/ipv4/ip_forward").write_text("1\n") + subprocess.run([executable, "-w", "-P", "FORWARD", "DROP"], check=True) + for table, hook, chain in ( + ("filter", "FORWARD", "AKERNEL_DIND_FWD"), + ("nat", "POSTROUTING", "AKERNEL_DIND_NAT"), + ): + prefix = [executable, "-w", "-t", table] + subprocess.run([*prefix, "-N", chain], check=True) + subprocess.run([*prefix, "-A", hook, "-j", chain], check=True) + # A restored sandbox has a new outer TAP name/address. Replace only owned + # chains, atomically per table, keeping Docker's bridges, DNS, proxies and + # existing inner connections alive. Never restart the daemon or Compose. + rules = ( + "*filter\n:AKERNEL_DIND_FWD - [0:0]\n" + f"-A AKERNEL_DIND_FWD -i {device} -j ACCEPT\n" + f"-A AKERNEL_DIND_FWD -o {device} -j ACCEPT\nCOMMIT\n" + "*nat\n:AKERNEL_DIND_NAT - [0:0]\n" + f"-A AKERNEL_DIND_NAT -o {device} -p tcp -j SNAT --to-source {source}\n" + f"-A AKERNEL_DIND_NAT -o {device} -p udp -j SNAT --to-source {source}\n" + "COMMIT\n" + ) + subprocess.run([restore, "--wait", "--noflush"], input=rules, text=True, check=True) + temporary = STATE.with_suffix(".tmp") + temporary.write_text(json.dumps(record)) + temporary.replace(STATE) + return record + + +def forward(peer, target): + """Relay bytes and half-closes without interpreting HTTP, auth or SSE.""" + try: + upstream = socket.create_connection(target, timeout=10) + except OSError: + peer.close() + return + + def pump(source, destination): + try: + while chunk := source.recv(65536): + destination.sendall(chunk) + except OSError: + pass + finally: + try: + destination.shutdown(socket.SHUT_WR) + except OSError: + pass + + with peer, upstream: + peer.settimeout(None) + upstream.settimeout(None) + other = threading.Thread(target=pump, args=(upstream, peer), daemon=True) + other.start() + pump(peer, upstream) + other.join() + + +def serve(listener, target, allowed): + while True: + try: + peer, address = listener.accept() + except OSError: + return + if address[0] not in allowed: + peer.close() + continue + threading.Thread(target=forward, args=(peer, target), daemon=True).start() + + +def expose(root, config): + # The caller reserves this identity before dispatch. Never replay an + # ambiguous start, or expose a wildcard/unrestricted outer listener. + root.mkdir(mode=0o700, parents=True, exist_ok=False) + address = str(ipaddress.IPv4Address(config["address"])) + allowed = {str(ipaddress.IPv4Address(v)) for v in config["allowed"]} + if not allowed or address in {"0.0.0.0", "127.0.0.1"}: + raise ValueError("A service gateway and explicit peer addresses are required") + port = config["target_port"] + if isinstance(port, bool) or not isinstance(port, int) or not 1 <= port <= 65535: + raise ValueError("Invalid loopback tunnel port") + listener = socket.socket() + listener.bind((address, 0)) + listener.listen(64) + record = {"url": f"http://{address}:{listener.getsockname()[1]}"} + pid = os.fork() + if pid == 0: + os.setsid() + # The listener must outlive the SDK command's stdin/stdout transport. + if os.fork() != 0: + os._exit(0) + descriptor = os.open("/dev/null", os.O_RDWR) + for number in (0, 1, 2): + os.dup2(descriptor, number) + if descriptor > 2: + os.close(descriptor) + serve(listener, ("127.0.0.1", port), allowed) + os._exit(0) + listener.close() + _, status = os.waitpid(pid, 0) + if status != 0: + raise RuntimeError("Tunnel relay could not detach; launch outcome unknown") + (root / "endpoint.json").write_text(json.dumps(record)) + return record + + +if __name__ == "__main__": + if sys.argv[1] == "bootstrap": + result = configure() + elif sys.argv[1] == "reconnect": + result = configure(reconnect=True) + elif sys.argv[1] == "expose": + result = expose(Path(sys.argv[2]), json.loads(sys.argv[3])) + else: + raise ValueError("Unknown Docker networking operation") + print(json.dumps(result)) diff --git a/sdk/python/akernel_sdk/_docker_runtime.py b/sdk/python/akernel_sdk/_docker_runtime.py new file mode 100644 index 00000000..53775dfe --- /dev/null +++ b/sdk/python/akernel_sdk/_docker_runtime.py @@ -0,0 +1,229 @@ +"""Managed Docker runtime using sandboxd's shared ROFS and private upper mounts.""" + +from __future__ import annotations + +import json +import shlex +import time +from dataclasses import dataclass +from pathlib import Path +from typing import TYPE_CHECKING, Any, ClassVar + +from ._image_plan import ImageMountPlan +from .types import Mount + +if TYPE_CHECKING: + from .sandbox import Sandbox + + +_PREPARE_RUNTIME = r"""set -eu +for tool in docker dockerd containerd ctr; do + command -v "$tool" >/dev/null 2>&1 || { + echo "Docker runtime image is missing $tool" >&2 + exit 127 + } +done +ready=true +for tool in bash python3 findmnt mount tar xz realpath usermod; do + command -v "$tool" >/dev/null 2>&1 || ready=false +done +if [ "$ready" = false ]; then + if ! command -v apk >/dev/null 2>&1; then + printf '%s\n' 'Missing Docker support tools; install bash, python3,' \ + 'util-linux, coreutils, shadow, tar and xz in the runtime image.' >&2 + exit 127 + fi + apk add --no-cache bash python3 coreutils util-linux shadow tar xz +fi +docker compose version >/dev/null +""" + + +@dataclass(frozen=True) +class DockerRuntime: + """Prepare a private Docker daemon inside a fresh sandbox. + + Pass sandbox_options() to both fresh creation and from_snapshot(). Call + start() only on fresh creation: snapshot restore retains the actual daemon, + containers, sockets and bridge processes. The public default_image provides + Docker, containerd and Compose; fresh startup installs missing Alpine support + packages. Custom images must provide these tools. The matching helper is + source-built from builder/image-bridge, or preinstalled by the operator at + /opt/akernel-image-bridge. No code is downloaded at runtime. + """ + + default_image: ClassVar[str] = "docker.io/library/docker:28.3.3-dind" + + images: ImageMountPlan | None = None + bridge_binary: Path | None = None + writable_mb: int = 1024 + writable_backing: str = "file" + log_dir: Path | None = None + + def __post_init__(self) -> None: + if ( + isinstance(self.writable_mb, bool) + or not isinstance(self.writable_mb, int) + or self.writable_mb < 1 + ): + raise ValueError("writable_mb must be a positive integer") + if self.writable_backing not in {"file", "memory"}: + raise ValueError("writable_backing must be file or memory") + if self.bridge_binary is not None and not Path(self.bridge_binary).is_file(): + raise ValueError("bridge_binary must be an existing matching executable") + + def sandbox_options(self) -> dict[str, Any]: + """Return public SDK mount/config options; also required during restore.""" + if self.images is None: + return {} + options: dict[str, Any] = {"mounts": self.images.mounts(Mount)} + if self.writable_backing == "file": + options["extra_config"] = { + "fileBackedTmpfsMounts": [ + { + "target": "/var/lib/akernel-image-snapshots", + "sizeMB": self.writable_mb, + } + ] + } + return options + + def _log(self, name: str, content: str) -> None: + if self.log_dir is not None: + self.log_dir.mkdir(parents=True, exist_ok=True) + (self.log_dir / name).write_text(content) + + def _bridge(self, sandbox: Sandbox) -> list[str]: + plan = self.images + if plan is None: + return ["--storage-driver=vfs"] + remote = "/opt/akernel-docker-runtime" + sandbox.files.make_dir(remote + "/metadata") + sandbox.files.write( + remote + "/metadata/plan.json", + json.dumps( + { + "images": plan.images, + "services": plan.services, + "aliases": plan.aliases, + "automatic": plan.automatic, + } + ), + ) + for name in sorted( + {b["file"] for image in plan.images for b in image["blobs"]} + ): + sandbox.files.copy_from_local( + str(plan.directory / name), remote + "/metadata/" + name + ) + if self.bridge_binary is not None: + sandbox.files.copy_from_local( + str(self.bridge_binary), "/opt/akernel-image-bridge" + ) + sandbox.commands.run( + "chmod 755 /opt/akernel-image-bridge", timeout=20, check=True + ) + # A packaged module is a wheel/sdist resource, not a Harbor source path. + sandbox.files.copy_from_local( + str(Path(__file__).with_name("_image_bridge_guest.py")), + remote + "/start.py", + ) + command = shlex.join( + [ + "python3", + remote + "/start.py", + "--root", + remote, + "--plan", + remote + "/metadata/plan.json", + "--writable-tmpfs-mb", + str(self.writable_mb), + "--writable-backing", + self.writable_backing, + ] + ) + try: + result = sandbox.commands.run(command, timeout=150, check=True) + self._log("startup.log", result.stdout + result.stderr) + except Exception: + try: + logs = sandbox.commands.run( + f"tail -100 {remote}/snapshotter.log {remote}/containerd.log " + f"{remote}/image-registration.log", + timeout=20, + check=True, + ) + self._log("services.log", logs.stdout + logs.stderr) + except Exception: + pass + raise + flags = json.loads(result.stdout) + if not isinstance(flags, list) or not all(isinstance(x, str) for x in flags): + raise ValueError("Image bridge returned invalid Docker arguments") + return flags + + def start(self, sandbox: Sandbox, *, network: str = "bridge") -> None: + """Start bridge, containerd and dockerd once; caller owns sandbox cleanup. + + Never call on a connected/restored sandbox. Fail closed on an existing + daemon so a retry cannot replace its mounted image/upper state. + """ + if network not in {"host", "bridge"}: + raise ValueError("Docker network must be host or bridge") + sandbox.commands.run("test ! -S /var/run/docker.sock", timeout=20, check=True) + sandbox.commands.run( + shlex.join(["sh", "-c", _PREPARE_RUNTIME]), timeout=300, check=True + ) + flags = self._bridge(sandbox) + if network == "host": + flags += ["--iptables=false", "--ip6tables=false", "--bridge=none"] + else: + # gVisor supports legacy SNAT, but not all of Docker's netfilter + # matchers. Keep Docker's bridges/IPAM/DNS and userland port proxies; + # configure forwarding inside this sandbox rather than rewriting + # service networks or installing rules on the node. + script = Path(__file__).with_name("_docker_network_guest.py").read_text() + sandbox.commands.run( + shlex.join(["python3", "-c", script, "bootstrap"]), + timeout=20, + check=True, + ) + flags += ["--iptables=false", "--ip6tables=false", "--userland-proxy=true"] + sandbox.commands.run( + "PATH=/opt/akernel-docker-runtime/network-bin:" + "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin " + + shlex.join(["dockerd", *flags]) + + " > /var/log/akernel-dockerd.log 2>&1", + background=True, + ) + for _ in range(30): + try: + sandbox.commands.run("docker info", timeout=10, check=True) + return + except Exception: + time.sleep(2) + logs = sandbox.commands.run( + "cat /var/log/akernel-dockerd.log", timeout=10, check=True + ) + self._log("dockerd.log", logs.stdout + logs.stderr) + raise RuntimeError( + "Docker daemon readiness failed; inspect /var/log/akernel-dockerd.log" + ) + + @staticmethod + def reconnect_network(sandbox: Sandbox, *, network: str = "bridge") -> None: + """Refresh owned egress rules after a new outer TAP is assigned. + + Call after snapshot restore, before resuming work. This never starts + dockerd, containers, image helpers or listeners. It preserves inner + connections; external connections do not gain IP-migration guarantees. + """ + if network not in {"host", "bridge"}: + raise ValueError("Docker network must be host or bridge") + if network == "bridge": + script = Path(__file__).with_name("_docker_network_guest.py").read_text() + sandbox.commands.run( + shlex.join(["python3", "-c", script, "reconnect"]), + timeout=20, + check=True, + ) diff --git a/sdk/python/akernel_sdk/_image_bridge_guest.py b/sdk/python/akernel_sdk/_image_bridge_guest.py new file mode 100644 index 00000000..1a5ee4e3 --- /dev/null +++ b/sdk/python/akernel_sdk/_image_bridge_guest.py @@ -0,0 +1,240 @@ +"""Start a private containerd and snapshotter for sandboxd-mounted images.""" + +import json +import subprocess +import time +from pathlib import Path + + +def verify_image_bridge(plan): + """Check pinned image records and mounted roots without pulling or unpacking. + + Docker's digest pull path does not treat metadata-only registration as a + complete local content store. The committed snapshot is our filesystem + source; missing aliases or snapshots must fail instead of falling back. + """ + data = json.loads(plan.read_text()) + references = {} + roots_by_chain: dict[str, set[str]] = {} + for image in data["images"]: + references[image["reference"]] = image["target"]["digest"] + roots_by_chain.setdefault(image["chain_id"], set()).add(image["root"]) + for chain, roots in roots_by_chain.items(): + result = subprocess.run( + [ + "ctr", + "--address", + "/run/akernel-image-bridge/containerd.sock", + "--namespace", + "moby", + "snapshots", + "--snapshotter", + "akernel", + "info", + chain, + ], + check=True, + capture_output=True, + text=True, + timeout=30, + ) + info = json.loads(result.stdout) + if ( + info.get("Kind") != "Committed" + or info.get("Labels", {}).get( + "containerd.io/snapshot/agent-service.image-root" + ) + not in roots + ): + raise RuntimeError("Prepared image snapshot does not match its ROFS mount") + for alias, reference in data.get("aliases", {}).items(): + references[alias] = references[reference] + for reference, expected in references.items(): + result = subprocess.run( + [ + "docker", + "image", + "inspect", + "--format", + "{{.Descriptor.Digest}}", + reference, + ], + check=True, + capture_output=True, + text=True, + timeout=30, + ) + if result.stdout.strip() != expected: + raise RuntimeError("Prepared image reference changed: " + reference) + print("Verified pinned ROFS images; no layer pull or extraction required") + + +def start_image_bridge( + root, plan, children, writable_tmpfs_mb=0, writable_backing="memory" +): + if writable_tmpfs_mb < 0: + raise ValueError("writable tmpfs size must be nonnegative") + if writable_backing not in {"memory", "file"}: + raise ValueError("writable backing must be memory or file") + snapshots = Path("/var/lib/akernel-image-snapshots") + use_snapshots = writable_tmpfs_mb > 0 or writable_backing == "file" + if writable_backing == "file": + result = subprocess.run( + [ + "findmnt", + "--noheadings", + "--output", + "FSTYPE", + "--mountpoint", + str(snapshots), + ], + check=True, + capture_output=True, + text=True, + ) + if result.stdout.strip() != "tmpfs": + raise RuntimeError("sandboxd must provide the file-backed tmpfs mount") + if writable_tmpfs_mb and writable_backing == "memory": + # This POC uses a separate tmpfs to avoid using the root overlay as upper. + # Writable host binds also work on the tested runsc, but need a separate + # storage snapshot/rollback integration before replacing this path. + # This is within the Sandbox memory budget and survives whole-Sandbox C/R. + # Docker bind-mounts its data root at startup; keep the tmpfs outside it. + snapshots = Path("/var/lib/akernel-image-snapshots") + snapshots.mkdir(parents=True, exist_ok=True) + if any(snapshots.iterdir()): + raise RuntimeError( + "refusing to hide existing snapshotter state under tmpfs" + ) + subprocess.run( + [ + "mount", + "-t", + "tmpfs", + "-o", + f"size={writable_tmpfs_mb}m", + "worker-image-writes", + str(snapshots), + ], + check=True, + ) + directory = Path("/run/akernel-image-bridge") + directory.mkdir(mode=0o700, exist_ok=True) + config = root / "containerd-image-bridge.toml" + config.write_text("""version = 2 +root = "/var/lib/docker/containerd-bridge" +state = "/run/akernel-image-bridge/state" +disabled_plugins = ["io.containerd.grpc.v1.cri"] +[grpc] + address = "/run/akernel-image-bridge/containerd.sock" +[proxy_plugins.akernel] + type = "snapshot" + address = "/run/akernel-image-bridge/snapshotter.sock" +""") + for name, command in [ + ( + "snapshotter", + [ + "/opt/akernel-image-bridge", + "--plan", + str(plan), + *(["--root", str(snapshots)] if use_snapshots else []), + ], + ), + ("containerd", ["containerd", "--config", str(config)]), + ]: + with (root / (name + ".log")).open("ab") as log: + process = subprocess.Popen( + command, stdout=log, stderr=subprocess.STDOUT, start_new_session=True + ) + children.append(process) + deadline = time.monotonic() + 60 + while True: + if any(p.poll() is not None for p in children): + raise RuntimeError( + "image bridge exited; inspect snapshotter/containerd logs" + ) + try: + result = subprocess.run( + ["ctr", "--address", str(directory / "containerd.sock"), "version"], + check=False, + capture_output=True, + text=True, + timeout=3, + ) + if result.returncode == 0 and (directory / "snapshotter.sock").exists(): + break + except subprocess.TimeoutExpired: + pass + if time.monotonic() > deadline: + raise TimeoutError("image bridge readiness timeout") + time.sleep(0.2) + with (root / "image-registration.log").open("ab") as log: + subprocess.run( + ["/opt/akernel-image-bridge", "--mode", "register", "--plan", str(plan)], + stdout=log, + stderr=subprocess.STDOUT, + check=True, + timeout=120, + ) + for alias, reference in json.loads(plan.read_text()).get("aliases", {}).items(): + subprocess.run( + [ + "ctr", + "--address", + str(directory / "containerd.sock"), + "--namespace", + "moby", + "images", + "tag", + "--force", + reference, + alias, + ], + stdout=log, + stderr=subprocess.STDOUT, + check=True, + timeout=30, + ) + config = root / "docker-image-bridge.json" + config.write_text( + json.dumps( + { + "features": {"containerd-snapshotter": True}, + "containerd": str(directory / "containerd.sock"), + "containerd-namespace": "moby", + "storage-driver": "akernel", + } + ) + ) + return ["--config-file", str(config)] + + +if __name__ == "__main__": + import argparse + + parser = argparse.ArgumentParser() + parser.add_argument( + "--root", type=Path, default=Path("/opt/akernel-docker-runtime") + ) + parser.add_argument("--plan", type=Path, required=True) + parser.add_argument("--writable-tmpfs-mb", type=int, default=0) + parser.add_argument("--verify", action="store_true") + parser.add_argument( + "--writable-backing", choices=["memory", "file"], default="memory" + ) + args = parser.parse_args() + if args.verify: + verify_image_bridge(args.plan) + else: + print( + json.dumps( + start_image_bridge( + args.root, + args.plan, + [], + args.writable_tmpfs_mb, + args.writable_backing, + ) + ) + ) diff --git a/sdk/python/akernel_sdk/_image_plan.py b/sdk/python/akernel_sdk/_image_plan.py new file mode 100644 index 00000000..9e646639 --- /dev/null +++ b/sdk/python/akernel_sdk/_image_plan.py @@ -0,0 +1,175 @@ +"""Startup-declared, immutable Compose images mounted by AKernel. + +The snapshotter and guest bootstrap originate from tianyuzhou95's image-bridge +POC, agent-worker commit 9b3434d36086d920b6bb7f3322da642381a254b8. +Docker keeps ownership of container lifecycle; this module stages the mount plan. +""" + +from __future__ import annotations + +import hashlib +import json +import re +from dataclasses import dataclass, field +from pathlib import Path +from typing import TYPE_CHECKING, Any + +if TYPE_CHECKING: + from collections.abc import Callable + + +def _digest(value: Any) -> str: + if ( + not isinstance(value, str) + or re.fullmatch(r"sha256:[0-9a-f]{64}", value) is None + ): + raise ValueError("Image bridge requires sha256 image and metadata digests") + return value + + +def _chain_id(diffs: list[str]) -> str: + if not diffs: + raise ValueError("Mounted external images require a nonempty rootfs") + chain = _digest(diffs[0]) + for diff in diffs[1:]: + chain = ( + "sha256:" + hashlib.sha256(f"{chain} {_digest(diff)}".encode()).hexdigest() + ) + return chain + + +def _validate_image_config(image: dict[str, Any], payloads: dict[str, Any]) -> None: + """Bind the mounted chain to the selected immutable OCI configuration.""" + try: + manifest = payloads[image["target"]["digest"]] + for _ in range(4): + if "manifests" not in manifest: + break + matches = [ + d + for d in manifest["manifests"] + if d.get("platform", {}).get("os") == "linux" + and d.get("platform", {}).get("architecture") == "amd64" + ] + if len(matches) != 1: + raise ValueError("Image plan requires one linux/amd64 manifest") + manifest = payloads[_digest(matches[0]["digest"])] + expected = manifest["config"] + if any( + expected[k] != image["config"][k] for k in ("digest", "size", "mediaType") + ): + raise ValueError("Image config is not selected by its target manifest") + config = payloads[_digest(expected["digest"])] + if config.get("os") != "linux" or config.get("architecture") != "amd64": + raise ValueError("Image config must match linux/amd64") + rootfs = config["rootfs"] + diffs = rootfs["diff_ids"] + if ( + rootfs.get("type") != "layers" + or not isinstance(diffs, list) + or len(diffs) != len(manifest["layers"]) + ): + raise ValueError("Image rootfs and manifest layer list disagree") + if _chain_id(diffs) != image["chain_id"]: + raise ValueError("Image chain ID does not match its rootfs configuration") + except (KeyError, TypeError) as error: + raise ValueError( + "Image plan has incomplete OCI manifest/config metadata" + ) from error + + +@dataclass(frozen=True) +class ImageMountPlan: + directory: Path + images: list[dict[str, Any]] + services: dict[str, str] + aliases: dict[str, str] = field(default_factory=dict) + automatic: bool = False + + @classmethod + def load(cls, path: str | Path, *, bases_only: bool = False) -> ImageMountPlan: + path = Path(path).resolve() + data = json.loads(path.read_text()) + images, services = data.get("images"), data.get("services") + automatic = data.get("automatic", False) + if not isinstance(automatic, bool): + raise ValueError("Image plan automatic marker must be a boolean") + if not isinstance(images, list) or ( + not images and not (bases_only and automatic) + ): + raise ValueError("Image plan requires a nonempty images list") + if not isinstance(services, dict) or ( + "main" not in services and not bases_only + ): + raise ValueError("Image plan requires a services map including main") + by_reference = {} + for image in images: + reference = image["reference"] + digest = _digest(image["target"]["digest"]) + if ( + not reference.endswith("@" + digest) + or image["root"] != "/opt/akernel-images/" + digest[7:] + or reference in by_reference + ): + raise ValueError("Image reference, digest and immutable mount disagree") + _digest(image["chain_id"]) + by_reference[reference] = image + descriptors = set() + payloads = {} + for blob in image["blobs"]: + name = blob["file"] + if not isinstance(name, str) or Path(name).name != name: + raise ValueError("Image metadata must be a sibling filename") + metadata = path.parent / name + if metadata.is_symlink(): + raise ValueError("Image metadata must not be a symlink") + payload = metadata.read_bytes() + descriptor = blob["descriptor"] + actual = "sha256:" + hashlib.sha256(payload).hexdigest() + if ( + actual != _digest(descriptor["digest"]) + or len(payload) != descriptor["size"] + ): + raise ValueError("Image metadata digest or size mismatch") + descriptors.add(actual) + payloads[actual] = json.loads(payload) + if not {digest, _digest(image["config"]["digest"])} <= descriptors: + raise ValueError("Image plan is missing its manifest or config") + _validate_image_config(image, payloads) + for service, reference in services.items(): + if ( + not isinstance(service, str) + or not service + or reference not in by_reference + ): + raise ValueError( + "Every planned service must reference a declared image" + ) + aliases = data.get("aliases", {}) + if not isinstance(aliases, dict) or any( + not isinstance(alias, str) or not alias or reference not in by_reference + for alias, reference in aliases.items() + ): + raise ValueError("Image aliases must reference declared immutable images") + selected = ( + list(by_reference) if bases_only else list(dict.fromkeys(services.values())) + ) + return cls( + path.parent, + [by_reference[r] for r in selected], + dict(services), + { + alias: reference + for alias, reference in aliases.items() + if reference in selected + }, + automatic, + ) + + def mounts(self, factory: Callable[..., Any]) -> list[Any]: + # Different registry names can refer to the same immutable image root. + by_root = {i["root"]: i for i in self.images} + return [ + factory(target=i["root"], image_url=i["reference"]) + for i in by_root.values() + ] diff --git a/sdk/python/akernel_sdk/_image_resolver.py b/sdk/python/akernel_sdk/_image_resolver.py new file mode 100644 index 00000000..8d152fc3 --- /dev/null +++ b/sdk/python/akernel_sdk/_image_resolver.py @@ -0,0 +1,246 @@ +"""Automatically prepare immutable image roots; Docker still performs the build. + +The bridge's host-side discovery command uses compose-go and BuildKit's actual +Dockerfile parser. Only OCI manifests/configs pass through the SDK client. +Layer data is supplied by AKernel's image manager through read-only mounts. +""" + +from __future__ import annotations + +import base64 +import hashlib +import json +import os +import re +import subprocess +from collections.abc import Mapping, Sequence +from pathlib import Path +from typing import Any +from urllib.parse import urlsplit + +import httpx + +from ._image_plan import ImageMountPlan, _chain_id, _digest + +ACCEPT = ( + "application/vnd.oci.image.index.v1+json, " + "application/vnd.oci.image.manifest.v1+json, " + "application/vnd.docker.distribution.manifest.list.v2+json, " + "application/vnd.docker.distribution.manifest.v2+json" +) + + +def _docker_credentials(host: str) -> tuple[str, str] | None: + config = Path(os.environ.get("DOCKER_CONFIG", str(Path.home() / ".docker"))) + path = config / "config.json" + data = json.loads(path.read_text()) if path.exists() else {} + server = "https://index.docker.io/v1/" if host == "docker.io" else host + helper = data.get("credHelpers", {}).get(host) or data.get("credsStore") + if helper: + result = subprocess.run( + ["docker-credential-" + helper, "get"], + input=server, + text=True, + capture_output=True, + timeout=20, + check=False, + ) + if result.returncode: + raise RuntimeError(f"Docker credential helper failed for {host}") + credential = json.loads(result.stdout) + username, secret = credential["Username"], credential["Secret"] + if not isinstance(username, str) or not isinstance(secret, str): + raise ValueError("Invalid Docker credential helper response") + return username, secret + for name in (host, server, "https://" + host): + auth = data.get("auths", {}).get(name, {}).get("auth") + if auth: + username, password = ( + base64.b64decode(auth, validate=True).decode().split(":", 1) + ) + return username, password + return None + + +class RegistryMetadata: + def __init__(self, client: httpx.Client, directory: Path): + self.client = client + self.directory = directory + self.tokens: dict[tuple[str, str], str] = {} + + def fetch(self, host: str, repository: str, kind: str, selector: str) -> bytes: + endpoint = "registry-1.docker.io" if host == "docker.io" else host + url = f"https://{endpoint}/v2/{repository}/{kind}/{selector}" + key = (host, repository) + headers = {"Accept": ACCEPT} + if key in self.tokens: + headers["Authorization"] = "Bearer " + self.tokens[key] + response = self.client.get(url, headers=headers) + if response.status_code == 401: + challenge = response.headers.get("www-authenticate", "") + credentials = _docker_credentials(host) + if challenge.lower().startswith("bearer "): + fields = dict(re.findall(r'(\w+)="([^"]*)"', challenge)) + realm = fields.get("realm", "") + if urlsplit(realm).scheme != "https": + raise ValueError("Registry token service must use HTTPS") + token = self.client.get( + realm, + params={k: fields[k] for k in ("service", "scope") if k in fields}, + auth=credentials, + ) + token.raise_for_status() + value = token.json().get("token") or token.json().get("access_token") + if not isinstance(value, str) or not value: + raise ValueError("Registry returned no bearer token") + self.tokens[key] = value + headers["Authorization"] = "Bearer " + value + response = self.client.get(url, headers=headers) + elif challenge.lower().startswith("basic ") and credentials: + response = self.client.get(url, headers=headers, auth=credentials) + response.raise_for_status() + payload = response.content + actual = "sha256:" + hashlib.sha256(payload).hexdigest() + expected = ( + selector + if selector.startswith("sha256:") + else response.headers.get("docker-content-digest") + ) + if expected and actual != _digest(expected): + raise ValueError("Registry metadata digest mismatch") + return payload + + def prepare(self, reference: str) -> dict[str, Any]: + repository, separator, selector = reference.partition("@") + if not separator: + repository, selector = reference.rsplit(":", 1) + elif ":" in repository.rsplit("/", 1)[-1]: + repository = repository.rsplit(":", 1)[0] + host, name = repository.split("/", 1) + blobs = [] + + def fetch(kind: str, selected: str, descriptor: dict[str, Any] | None = None): + payload = self.fetch(host, name, kind, selected) + obj = json.loads(payload) + digest = "sha256:" + hashlib.sha256(payload).hexdigest() + if descriptor is not None and len(payload) != descriptor["size"]: + raise ValueError("Registry metadata size mismatch") + media_type = (descriptor or {}).get("mediaType") or obj["mediaType"] + result = {"mediaType": media_type, "digest": digest, "size": len(payload)} + filename = digest.replace(":", "-") + ".json" + (self.directory / filename).write_bytes(payload) + blobs.append({"descriptor": result, "file": filename}) + return obj, result + + manifest, target = fetch("manifests", selector) + for _ in range(4): + if "manifests" not in manifest: + break + matches = [ + d + for d in manifest["manifests"] + if d.get("platform", {}).get("os") == "linux" + and d.get("platform", {}).get("architecture") == "amd64" + ] + if len(matches) != 1: + raise ValueError("Image must contain exactly one linux/amd64 manifest") + manifest, _ = fetch("manifests", _digest(matches[0]["digest"]), matches[0]) + if "manifests" in manifest: + raise ValueError("Image index nesting is too deep") + descriptor = manifest["config"] + config, config_descriptor = fetch( + "blobs", _digest(descriptor["digest"]), descriptor + ) + if config.get("os") != "linux" or config.get("architecture") != "amd64": + raise ValueError("Image config must match linux/amd64") + diffs = [_digest(d) for d in config["rootfs"]["diff_ids"]] + if not diffs: + raise ValueError("An external image with an empty rootfs cannot be mounted") + chain = _chain_id(diffs) + return { + "reference": repository + "@" + target["digest"], + "root": "/opt/akernel-images/" + target["digest"][7:], + "chain_id": chain, + "target": target, + "config": config_descriptor, + "blobs": blobs, + } + + +def prepare_compose_images( + directory: str | Path, + files: Sequence[str | Path], + *, + output_dir: str | Path, + bridge_binary: str | Path, + env: Mapping[str, str] | None = None, +) -> ImageMountPlan: + """Discover Compose/Dockerfile image inputs without building or pulling layers. + + The operator supplies a source-built, version-matched image-bridge helper. + Credentials stay on the client. Sandboxd's existing image manager owns layer + distribution, ROFS mounts and shared caching; this stores metadata only. + Original project files are read without modification. + """ + output = Path(output_dir).resolve() + output.mkdir(parents=True, exist_ok=True) + request = { + "directory": str(Path(directory).resolve()), + "files": [str(Path(p).resolve()) for p in files], + } + binary = str(Path(bridge_binary).resolve()) + # Probe without project data before invoking a possibly older helper. The + # schema version requires client-file containment before reading Dockerfiles. + capability = subprocess.run( + [binary, "--mode", "capabilities"], + env=dict(env or {}), + text=True, + capture_output=True, + timeout=10, + check=False, + ) + try: + supported = ( + capability.returncode == 0 + and json.loads(capability.stdout).get("discover_schema") == 2 + ) + except (ValueError, AttributeError): + supported = False + if not supported: + raise ValueError( + "Image bridge helper must support discovery schema 2; " + "install the helper matching this SDK" + ) + result = subprocess.run( + [binary, "--mode", "discover"], + input=json.dumps(request), + env=dict(env or {}), + text=True, + capture_output=True, + timeout=60, + check=False, + ) + if result.returncode: + raise ValueError("Automatic image discovery failed: " + result.stderr[-1500:]) + references = json.loads(result.stdout)["references"] + aliases: dict[str, str] = {} + images: dict[str, dict[str, Any]] = {} + with httpx.Client(follow_redirects=True, timeout=60) as client: + registry = RegistryMetadata(client, output) + for reference in references: + image = registry.prepare(reference) + images.setdefault(image["reference"], image) + aliases[reference] = image["reference"] + path = output / "plan.json" + path.write_text( + json.dumps( + { + "images": list(images.values()), + "services": {}, + "aliases": aliases, + "automatic": True, + }, + indent=2, + ) + ) + return ImageMountPlan.load(path, bases_only=True) diff --git a/sdk/python/akernel_sdk/_service_process.py b/sdk/python/akernel_sdk/_service_process.py new file mode 100644 index 00000000..0337818f --- /dev/null +++ b/sdk/python/akernel_sdk/_service_process.py @@ -0,0 +1,190 @@ +"""Recoverable identity of an actual process inside a Compose service.""" + +from __future__ import annotations + +import json +import re +import shlex +import time +from pathlib import Path +from typing import TYPE_CHECKING, Any +from uuid import uuid4 + +from ._backends.errors import BackendOperationError, TimeoutException + +if TYPE_CHECKING: + from collections.abc import Mapping + + from ._compose import ServiceHandle + + +class ServiceProcessHandle: + """A service process, independent of its short-lived docker-exec submission. + + The in-service monitor and identity record are included in whole-sandbox + snapshots. attach() never relaunches a command. Container identity/start time + and Linux process start time are checked before signalling. Python 3 is + required inside this service; ordinary ServiceHandle.exec needs no Python. + """ + + def __init__(self, service: ServiceHandle, descriptor: Mapping[str, Any]) -> None: + self.service = service + self._descriptor = json.loads(json.dumps(descriptor, allow_nan=False)) + if ( + self._descriptor.get("schema_version") != 1 + or re.fullmatch(r"[0-9a-f]{32}", self._descriptor.get("process_id", "")) + is None + or self._descriptor.get("service") != service.name + or self._descriptor.get("project_name") + != service.project.descriptor["project_name"] + ): + raise ValueError("Invalid service process descriptor") + + @property + def descriptor(self) -> dict[str, Any]: + return json.loads(json.dumps(self._descriptor)) + + @property + def process_id(self) -> str: + return str(self._descriptor["process_id"]) + + @property + def pid(self) -> int: + return int(self._descriptor["pid"]) + + def _verify_container(self) -> None: + identity = self.service.identity() + if any( + identity[k] != self._descriptor[k] for k in ("container_id", "started_at") + ): + raise BackendOperationError( + "Service container was replaced or restarted", + code="SERVICE_PROCESS_STALE", + outcome="not_started", + ) + + def _invoke( + self, + operation: str, + argument: str, + *, + user: str | int | None = "root", + cwd: str | None = None, + envs: Mapping[str, str] | None = None, + ) -> Any: + script = Path(__file__).with_name("_service_process_guest.py").read_text() + command = ( + "for executable in python3 python /usr/local/bin/python3 " + "/usr/bin/python3; do " + 'process_python=$(command -v "$executable") || continue; ' + '"$process_python" -c "import sys;sys.exit(sys.version_info[0]!=3)" ' + "> /dev/null 2>&1 || continue; " + 'exec "$process_python" -c ' + + shlex.quote(script) + + " " + + shlex.join([operation, self.process_id, argument]) + + '; done; echo "Service process handles need Python 3" >&2; exit 127' + ) + result = self.service.exec(command, cwd=cwd, envs=envs, user=user, timeout=30) + if result.exit_code != 0: + failure = BackendOperationError( + "Service process operation failed: " + result.stderr[-500:], + code="SERVICE_PROCESS_CONFIGURATION" + if result.exit_code == 127 + else "SERVICE_PROCESS_OBSERVATION", + outcome="unknown" if operation == "start" else None, + ) + failure.process_descriptor = self.descriptor # type: ignore[attr-defined] + raise failure + return json.loads(result.stdout) + + @classmethod + def start( + cls, + service: ServiceHandle, + command: str, + *, + cwd: str | None = None, + envs: Mapping[str, str] | None = None, + user: str | int | None = None, + shell: str = "/bin/bash", + ) -> ServiceProcessHandle: + if ( + not isinstance(command, str) + or not command.strip() + or not shell.startswith("/") + ): + raise ValueError("Process command and absolute shell are required") + identity = service.identity() + descriptor = { + "schema_version": 1, + "process_id": uuid4().hex, + "project_name": service.project.descriptor["project_name"], + "service": service.name, + **identity, + } + result = cls(service, descriptor) + try: + row = result._invoke( + "start", + json.dumps({"command": command, "shell": shell}), + cwd=cwd, + envs=envs, + user=user, + ) + except BaseException as failure: + # The remote process may already exist. Retain its recovery token; + # never repeat a side-effecting start after an unknown response. + failure.process_descriptor = result.descriptor # type: ignore[attr-defined] + raise + result._descriptor.update(pid=row["pid"], start_time=row["start_time"]) + return result + + @classmethod + def attach( + cls, service: ServiceHandle, descriptor: Mapping[str, Any] + ) -> ServiceProcessHandle: + result = cls(service, descriptor) + result._verify_container() + status = result._invoke("status", "") + root = status["identity"] + if "pid" in result._descriptor and ( + root["pid"] != result.pid + or root["start_time"] != result._descriptor["start_time"] + ): + raise BackendOperationError( + "Service process identity changed", code="SERVICE_PROCESS_STALE" + ) + result._descriptor.update(pid=root["pid"], start_time=root["start_time"]) + return result + + def inspect(self) -> list[dict[str, Any]]: + return self.signal("inspect") + + def signal(self, action: str) -> list[dict[str, Any]]: + if action not in {"inspect", "stop", "continue", "terminate", "kill"}: + raise ValueError("Unsupported service process action") + self._verify_container() + return self._invoke( + "signal", + json.dumps( + { + "action": action, + "pid": self.pid, + "start_time": self._descriptor["start_time"], + } + ), + ) + + def wait(self, *, timeout: float = 60) -> int: + deadline = time.monotonic() + timeout + while True: + self._verify_container() + status = self._invoke("status", "") + if status["exit_code"] is not None: + return int(status["exit_code"]) + if time.monotonic() >= deadline: + raise TimeoutException( + "Service process wait timed out", outcome="running" + ) + time.sleep(0.1) diff --git a/sdk/python/akernel_sdk/_service_process_guest.py b/sdk/python/akernel_sdk/_service_process_guest.py new file mode 100644 index 00000000..0c9c4f4b --- /dev/null +++ b/sdk/python/akernel_sdk/_service_process_guest.py @@ -0,0 +1,170 @@ +"""Dependency-free helper executed inside the selected service's PID namespace.""" + +import json +import os +import signal +import sys +import time +from pathlib import Path + + +def stat(pid): + try: + value = Path("/proc/" + str(pid) + "/stat").read_text() + fields = value[value.rindex(")") + 2 :].split() + return { + "pid": pid, + "ppid": int(fields[1]), + "state": fields[0], + "comm": value[value.index("(") + 1 : value.rindex(")")], + "start_time": fields[19], + } + except (OSError, ValueError): + return None + + +def write(path, data): + temporary = path.with_suffix(".tmp") + temporary.write_text(json.dumps(data)) + temporary.replace(path) + + +def start(root, config): + # Exclusive reservation prevents replay of an ambiguous launch. + root.mkdir(mode=0o700, exist_ok=False) + monitor = os.fork() + if monitor == 0: + os.setsid() + null = os.open("/dev/null", os.O_RDWR) + for number in (0, 1, 2): + os.dup2(null, number) + os.close(null) + pid = os.fork() + if pid == 0: + try: + for number, name in ((1, "stdout"), (2, "stderr")): + descriptor = os.open( + str(root / name), os.O_WRONLY | os.O_CREAT | os.O_APPEND, 0o600 + ) + os.dup2(descriptor, number) + os.close(descriptor) + os.execvpe( + config["shell"], + [config["shell"], "-c", config["command"]], + os.environ, + ) + except BaseException: + os._exit(127) + record = stat(pid) + if record is not None: + record["monitor_pid"] = os.getpid() + write(root / "identity.json", record) + _, status = os.waitpid(pid, 0) + code = ( + os.WEXITSTATUS(status) + if os.WIFEXITED(status) + else 128 + os.WTERMSIG(status) + ) + write(root / "exit.json", {"exit_code": code}) + os._exit(0) + deadline = time.monotonic() + 10 + while not (root / "identity.json").exists(): + if time.monotonic() >= deadline: + raise RuntimeError("process launch outcome unknown; do not resubmit") + time.sleep(0.02) + return json.loads((root / "identity.json").read_text()) + + +def inspect(root): + record = json.loads((root / "identity.json").read_text()) + current = stat(record["pid"]) + if current is None or current["state"] == "Z": + return [] + if current["start_time"] != record["start_time"]: + raise RuntimeError("process identity mismatch; refusing reused PID") + records = {} + for name in os.listdir("/proc"): + if name.isdigit(): + row = stat(int(name)) + if row is not None: + records[row["pid"]] = row + selected = {record["pid"]} + while True: + children = { + pid for pid, row in records.items() if row["ppid"] in selected + } - selected + if not children: + break + selected.update(children) + return [records[pid] for pid in sorted(selected) if pid in records] + + +def operate(root, action): + rows = inspect(root) + if action == "inspect": + return rows + signals = { + "stop": signal.SIGSTOP, + "continue": signal.SIGCONT, + "terminate": signal.SIGTERM, + "kill": signal.SIGKILL, + } + if action not in signals: + raise ValueError("unsupported process action") + for _ in range(3 if action == "stop" else 1): + parents = {row["pid"]: row["ppid"] for row in rows} + + def depth(row, parents=parents): + pid, visited = row["pid"], set() + while pid in parents and pid not in visited: + visited.add(pid) + pid = parents[pid] + return len(visited) + + # Freeze waiters before children. Thaw children before waiters so Bash + # cannot consume a pending child-stop notification as exit status 147. + # PID numeric ordering does not encode ancestry after PID reuse/wrap. + for row in sorted(rows, key=depth, reverse=action == "continue"): + current = stat(row["pid"]) + if current is not None and current["start_time"] == row["start_time"]: + try: + os.kill(row["pid"], signals[action]) + except ProcessLookupError: + pass + if action == "stop": + time.sleep(0.03) + rows = inspect(root) + return rows + + +def main(): + operation, identity, argument = sys.argv[1:4] + if len(identity) != 32 or any(c not in "0123456789abcdef" for c in identity): + raise ValueError("invalid process identity") + root = Path("/tmp/akernel-service-process-" + identity) + if operation == "start": + result = start(root, json.loads(argument)) + elif operation == "signal": + request = json.loads(argument) + expected = json.loads((root / "identity.json").read_text()) + if ( + request.get("pid") != expected["pid"] + or request.get("start_time") != expected["start_time"] + ): + raise RuntimeError("process identity record changed") + result = operate(root, request["action"]) + elif operation == "status": + result = { + "processes": inspect(root), + "identity": json.loads((root / "identity.json").read_text()), + "exit_code": None, + } + if (root / "exit.json").exists(): + result.update(json.loads((root / "exit.json").read_text())) + else: + raise ValueError("unsupported process operation") + print(json.dumps(result), flush=True) + + +if __name__ == "__main__": + main() diff --git a/sdk/python/akernel_sdk/sandbox.py b/sdk/python/akernel_sdk/sandbox.py index 01b651fc..cd214e69 100644 --- a/sdk/python/akernel_sdk/sandbox.py +++ b/sdk/python/akernel_sdk/sandbox.py @@ -757,6 +757,7 @@ def create( cwd: str | None = None, port_forwardings: Sequence[int] | None = None, mounts: Sequence[Mount] | None = None, + extra_config: Mapping[str, object] | None = None, reverse_tunnel: HttpReverseTunnel | None = None, detached: bool = False, failover: bool = False, @@ -797,6 +798,7 @@ def create( cwd=cwd, port_forwardings=port_forwardings, mounts=mounts, + extra_config=extra_config, reverse_tunnel=reverse_tunnel, detached=detached, failover=failover, @@ -835,6 +837,7 @@ def create( cwd=cwd, port_forwardings=port_forwardings, mounts=mounts, + extra_config=extra_config, reverse_tunnel=reverse_tunnel, detached=detached, failover=failover, @@ -854,6 +857,7 @@ def from_snapshot( cwd: str | None = None, port_forwardings: Sequence[int] | None = None, mounts: Sequence[Mount] | None = None, + extra_config: Mapping[str, object] | None = None, reverse_tunnel: HttpReverseTunnel | None = None, detached: bool = False, failover: bool = False, @@ -868,6 +872,9 @@ def from_snapshot( Inherits captured runtime, resource geometry and process state; does not rerun Dockerfile setup or the image entrypoint. The source is unaffected. Rebind client-side tunnels, ports and mounts explicitly as needed. + ``extra_config`` is defensively copied JSON runtime configuration. + Restore the same managed-mount geometry used by the captured runtime; + the backend owns compatibility checks and does not rerun fresh setup. ``cwd`` sets a command default without creating directories. Supply ``network_policy`` explicitly to restore restrictions: the pinned ADX backend uses its default if omitted or empty. ``timeout`` is the logical @@ -915,6 +922,7 @@ def from_snapshot( command_cwd=cwd, port_forwardings=tuple(ports), mounts=tuple(mount_list), + extra_config=_normalize_extra_config(extra_config), reverse_tunnel=reverse_tunnel, detached=detached, failover=failover, diff --git a/sdk/python/docs/managed-compose.md b/sdk/python/docs/managed-compose.md new file mode 100644 index 00000000..021ff9d0 --- /dev/null +++ b/sdk/python/docs/managed-compose.md @@ -0,0 +1,61 @@ +# Managed Compose integration + +The SDK manages Docker and Compose inside one outer AKernel sandbox. Callers can keep original Dockerfiles and Compose definitions. It is an opt-in Linux/amd64 runsc profile, separate from the lightweight Dockerfile direct-launch parser. + +## Component boundary + +| Layer | Responsibility | +| --- | --- | +| Evaluation framework such as Harbor | Translate task resources, bind paths and verifier isolation; run the model proxy; choose checkpoint boundaries; persist aligned conversation state and fence the previous execution before recovery. | +| AKernel SDK | Discover startup-declared images, validate immutable OCI metadata, prepare mounts, bootstrap Docker/containerd/bridge once, build/up projects, expose service I/O and process identities, reconnect client tunnels and owned network rules. | +| Agent DX | Existing environment creation, exec/files/tunnel transport and reusable snapshot control. No Compose-aware server API is introduced. | +| Sandboxd | Existing shared ROFS image provider plus private, quota-accounted file-backed tmpfs mounts. Validate and clone managed filestores during checkpoint/restore. | +| runsc | Capture the entire outer sandbox, including dockerd, services, process memory, filesystem metadata and sockets. Reflink shares file-backed data; it does not turn RAM capture into a dirty-page incremental snapshot. | + +The image bridge downloads only OCI manifests/configs on the client. Sandboxd's existing OCI/ROFS provider supplies shared lower roots and owns layer distribution/cache. Each sandbox has a separate writable upper. Client metadata credentials and node image-pull credentials are separate. There is no second shared layer cache in the SDK. + +## Integration order and prerequisites + +This implementation is based on the reusable snapshot/lifecycle SDK already merged into `develop`. It retains that implementation's creation ownership checks, atomic command identity, cancellation and rollback semantics. It does not include the older preview's metadata/lifetime/callback API extensions. + +1. Review and merge Sandboxd's `fileBackedTmpfsMounts` support ([Sandboxd #83](https://github.com/inclusionAI/sandboxd/pull/83)). The AKernel consumer must then pin the merged Sandboxd revision and qualify a normal build before offering the file-backed profile. This SDK patch does not replace the existing gitlink or runtime bundle. +2. Configure runsc reflink mode, durable output sync, sufficient writable quota and a compatible backing/checkpoint filesystem. On Linux 5.10 these paths must use the same mount object. The separate [AKernel #126](https://github.com/inclusionAI/AKernel/pull/126) adds an explicit Helm profile without changing defaults. Docker 28 also requires operator-enabled `--net-raw=true` and `--allow-packet-socket-write=true`. +3. Build the matching [image bridge](../../../builder/image-bridge/README.md). Use `DockerRuntime.default_image` (public `docker.io/library/docker:28.3.3-dind`); fresh startup installs missing Alpine support packages. An explicitly supplied compatible outer image may instead preinstall Docker 28.3.3, containerd/ctr, Python 3, Bash, mount/findmnt, iproute and legacy iptables tools. No user task Dockerfile changes or task image publication are needed. +4. Install the SDK revision containing `ComposeProject`, then use [the maintained example](../examples/compose_checkpoint.py). An independent verifier prepares its own project and outer sandbox; it does not attach to the agent sandbox. + +The SDK never updates node configuration or deployments. The snapshot/restore API has no client switch that can enable reflink on an unconfigured backend. + +Use a helper advertising `{"discover_schema":2}` through `--mode capabilities`. +The SDK checks this before dispatching project data; the earlier `72e706e2` +binary is insufficient for this revision. Discovery keeps Dockerfile/build +context and project `.env` reads inside the frozen project and receives only +explicit project variables. `include` and `extends.file` are not supported by +this preview. Pass explicit supported Compose overlays; do not assume complete +Compose syntax compatibility. Original kv-live-surgery definitions need no edit. + +## Fresh creation and recovery + +`ComposeProject.prepare()` creates a frozen local context and resolves image inputs before sandbox creation. Pass `project.sandbox_options()` into creation, then call `start_runtime()`, `build()`, `pull()` and `up()` at explicit boundaries. A partial startup is not replayable on that project object; the owner must clean up its newly created sandbox. + +With `auto_images=True`, image-only services (including `image: name:tag@sha256:...`) use the same prepared ROFS roots as Dockerfile bases. `pull()` verifies each pinned Docker reference and committed bridge snapshot instead of issuing a second native pull. `up()` repeats that verification and disables native pulls; builds remain enabled. Tags are resolved once during preparation. Missing snapshots or changed references fail visibly without downloading/extracting a replacement into the private upper. Ordinary Compose without automatic images retains native pull behavior. Original project files and image declarations are unchanged. + +Plans validate the selected linux/amd64 manifest/config and derive its chain ID +from the rootfs diff IDs. Different config or index/manifest digests may share +the same chain. Scratch-only automatic builds carry an explicit empty-plan +marker; unused aliases from a manually selected service plan are not registered. +An absent command exit status remains an unknown outcome. A timeout/cancel +retains command identity and secondary kill errors, without replaying execution. + +For checkpointing, quiesce tools/model requests, stop the real process tree using `ServiceProcessHandle`, and call `sandbox.create_snapshot()`. Persist the snapshot ID together with the project descriptor, process descriptor and application state. Resume the source tree if the source continues. + +To recover, use `Sandbox.from_snapshot(..., **project.sandbox_options())` with the original upper geometry, mounts, current network policy and reverse tunnel. Fence the old execution before continuing the restored process. `ComposeProject.attach()` reconstructs local bindings only; `reconnect_network()` updates SDK-owned forwarding/NAT for the new outer TAP. `ServiceProcessHandle.attach()` checks Docker container ID/start time and Linux PID/start time. Continue the captured process without rerunning `build`, `up` or its launch command. Unknown process-launch outcomes retain a token for observation and must not be blindly resubmitted. + +Harbor's Python memory and local model proxy are outside the remote snapshot. Their aligned state remains the framework's responsibility. A service-specific controlled TCP entry makes the existing outer reverse tunnel reachable from a bridge container without changing HTTP, auth headers or SSE. + +## Supported scope and verification + +The default profile retains native default/custom IPv4 bridges, service DNS/aliases and TCP/UDP userland port mappings. The opt-in `tests/integration/test_compose_network.py` checks existing inner and published-port TCP sockets across two delete/restore cycles, as well as peer-restricted model HTTP/auth/SSE and new egress. It requires a dedicated compatible test endpoint and is not run against a shared production deployment by CI. External TCP address migration, IPv6-only egress, non-bridge drivers, remote build contexts, arbitrary runtime-created images and complete Docker content-store/export compatibility are not claimed. Managed process handles require Python 3 in that service; ordinary service exec does not. + +The earlier Harbor prototype passed original-process Claude C/R and an independent verifier on unchanged kv-live-surgery inputs. Those measurements predate this migration onto the current `develop` SDK and do not qualify the new revision or its newly pinned upstream bundle. One earlier prototype checkpoint returned no result before its 300-second deadline despite completed serialization; the later successful cycles did not establish a root cause. Keep fresh isolated end-to-end qualification as a merge/release requirement. + +Local review checks are `make sdk-check` and `CGO_ENABLED=0 go test ./...` under `builder/image-bridge`; CI also vets and builds the bridge and validates clean wheel/sdist installs. Existing upstream ownership/fork/rollback tests stay in place. diff --git a/sdk/python/examples/compose_checkpoint.py b/sdk/python/examples/compose_checkpoint.py new file mode 100644 index 00000000..7312ab48 --- /dev/null +++ b/sdk/python/examples/compose_checkpoint.py @@ -0,0 +1,71 @@ +"""Run a Compose service process, snapshot it, and attach the original process. + +Requires an owned endpoint with the managed Compose runtime prerequisites. +Example: python sdk/python/examples/compose_checkpoint.py ./project \ + --file ./project/compose.yaml \ + --bridge /absolute/path/akernel-image-bridge --service worker \ + --command 'exec python3 worker.py' +""" + +import argparse +import tempfile +from pathlib import Path + +from akernel_sdk import ComposeProject, DockerRuntime, Sandbox, ServiceProcessHandle + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("directory", type=Path) + parser.add_argument("--file", action="append", required=True, type=Path) + parser.add_argument("--image", default=DockerRuntime.default_image) + parser.add_argument("--bridge", required=True, type=Path) + parser.add_argument("--service", required=True) + parser.add_argument("--command", required=True) + args = parser.parse_args() + snapshot = None + with tempfile.TemporaryDirectory(prefix="akernel-compose-example-") as output: + project = ComposeProject.prepare( + args.directory, + args.file, + runtime=DockerRuntime(bridge_binary=args.bridge, writable_mb=2048), + auto_images=True, + output_dir=Path(output), + ) + try: + with Sandbox( + image=args.image, + cpu=4000, + memory=8192, + storage_mb=30720, + failover=False, + **project.sandbox_options(), + ) as source: + project.start_runtime(source) + project.build() + project.pull() + project.up() + process = project.service(args.service).start_process(args.command) + process.signal("stop") + # An application also saves its aligned client-side state here. + snapshot = source.create_snapshot(timeout=300) + project_state, process_state = project.descriptor, process.descriptor + # Context exit acknowledges deletion before the clone continues. + with Sandbox.from_snapshot( + snapshot, failover=False, **project.sandbox_options() + ) as restored: + attached = ComposeProject.attach(restored, project_state) + attached.reconnect_network() + original = ServiceProcessHandle.attach( + attached.service(args.service), process_state + ) + original.signal("continue") + print("Restored original service PID", original.pid) + print("Exit code:", original.wait(timeout=300)) + finally: + if snapshot is not None: + Sandbox.delete_snapshot(snapshot) + + +if __name__ == "__main__": + main() diff --git a/sdk/python/pyproject.toml b/sdk/python/pyproject.toml index 24174966..a2ad27d4 100644 --- a/sdk/python/pyproject.toml +++ b/sdk/python/pyproject.toml @@ -30,6 +30,8 @@ classifiers = [ dependencies = [ "adx-environment==0.1.0rc3", "idna>=3,<4", + "httpx>=0.27,<1", + "PyYAML>=6,<7", "websockets>=10.0", "dockerfile-parse>=2.0.1", ] @@ -42,6 +44,7 @@ dev = [ "build>=1.2,<2", "mypy>=1.10,<2", "ruff>=0.11,<1", + "types-PyYAML>=6,<7", ] [project.scripts] diff --git a/sdk/python/tests/integration/test_compose_network.py b/sdk/python/tests/integration/test_compose_network.py new file mode 100644 index 00000000..fb38bbd2 --- /dev/null +++ b/sdk/python/tests/integration/test_compose_network.py @@ -0,0 +1,335 @@ +"""Opt-in managed DinD IPv4 network and original-process C/R regression. + +Requires an isolated compatible runtime (Docker 28 runsc flags, managed upper, +reflink and durable sync). Never updates a deployment. Set +AKERNEL_RUN_COMPOSE_TESTS=1, AKERNEL_TEST_DIND_IMAGE and +AKERNEL_TEST_IMAGE_BRIDGE_BINARY. The caller supplies ordinary SDK connection +variables. AKERNEL_TEST_EGRESS_HOST defaults to 1.1.1.1:80 and must be reachable. +""" + +import http.server +import json +import os +import shlex +import tempfile +import threading +import time +import unittest +from pathlib import Path + +import yaml + +from akernel_sdk import ( + ComposeProject, + DockerRuntime, + HttpReverseTunnel, + Sandbox, + ServiceProcessHandle, +) + +_ENABLED = os.environ.get("AKERNEL_RUN_COMPOSE_TESTS") == "1" +_IMAGE = os.environ.get("AKERNEL_TEST_DIND_IMAGE", "") +_BRIDGE = os.environ.get("AKERNEL_TEST_IMAGE_BRIDGE_BINARY", "") +_SSE = b'event: message_stop\ndata: {"type":"message_stop"}\n\n' +_SERVER = """ +import socket,threading +listener=socket.socket();listener.bind(('0.0.0.0',8000));listener.listen(64) +def echo(peer): + with peer: + while chunk:=peer.recv(65536):peer.sendall(chunk) +def udp(): + peer=socket.socket(socket.AF_INET,socket.SOCK_DGRAM);peer.bind(('0.0.0.0',8001)) + while True: + chunk,address=peer.recvfrom(65536);peer.sendto(chunk,address) +threading.Thread(target=udp,daemon=True).start() +while True: + peer,_=listener.accept();threading.Thread(target=echo,args=(peer,),daemon=True).start() +""" +_PROBE = """ +import socket,pathlib,time,os,json +peer=socket.create_connection(('api',8000),5);peer.settimeout(None);cookie=41 +while True: + if pathlib.Path('/tmp/mutate').exists():cookie=99 + peer.sendall(b'original');assert peer.recv(8)==b'original' + value={'pid':os.getpid(),'cookie':cookie,'local':peer.getsockname(), + 'remote':peer.getpeername(),'time':time.time()} + path=pathlib.Path('/tmp/witness.tmp');path.write_text(json.dumps(value)) + path.replace('/tmp/witness.json');time.sleep(.1) +""" + + +@unittest.skipUnless(_ENABLED and _IMAGE and _BRIDGE, "enable managed Compose tests") +class ComposeNetworkIntegrationTest(unittest.TestCase): + def setUp(self): + self.environments = [] + self.snapshots = [] + self.addCleanup(self.cleanup_remote) + temporary = tempfile.TemporaryDirectory() + self.addCleanup(temporary.cleanup) + self.root = Path(temporary.name) + self.requests = [] + requests = self.requests + + class Handler(http.server.BaseHTTPRequestHandler): + def do_POST(self): + requests.append( + (self.path, {"authorization": self.headers.get("Authorization")}) + ) + self.rfile.read(int(self.headers.get("content-length", "0"))) + if self.headers.get("x-api-key") != "trial-fixture": + self.send_error(401) + return + self.send_response(200) + self.send_header("Content-Type", "text/event-stream") + self.send_header("Transfer-Encoding", "chunked") + self.end_headers() + for chunk in (_SSE[:13], _SSE[13:]): + self.wfile.write(f"{len(chunk):x}\r\n".encode() + chunk + b"\r\n") + self.wfile.flush() + self.wfile.write(b"0\r\n\r\n") + + def log_message(self, *_args): + pass + + server = http.server.ThreadingHTTPServer(("127.0.0.1", 0), Handler) + server.daemon_threads = True + threading.Thread(target=server.serve_forever, daemon=True).start() + self.addCleanup(server.server_close) + self.addCleanup(server.shutdown) + self.tunnel = HttpReverseTunnel(f"http://127.0.0.1:{server.server_port}") + context = self.root / "context" + context.mkdir() + services = { + name: {"image": "python:3.12-slim", "command": ["sleep", "infinity"]} + for name in ("main", "loadgen", "standard", "private") + } + services["main"].update( + networks={"default": {}, "custom": {"aliases": ["api"]}}, + ports=["127.0.0.1:18090:8000/tcp", "127.0.0.1:18091:8001/udp"], + ) + services["loadgen"]["networks"] = ["custom"] + services["private"]["networks"] = ["private"] + self.compose = context / "compose.yaml" + self.original = yaml.safe_dump( + { + "services": services, + "networks": { + "custom": {"ipam": {"config": [{"subnet": "172.28.41.0/24"}]}}, + "private": {"internal": True}, + }, + } + ) + self.compose.write_text(self.original) + self.project = ComposeProject.prepare( + context, + [self.compose], + output_dir=self.root / "prepared", + network="bridge", + auto_images=True, + runtime=DockerRuntime(bridge_binary=Path(_BRIDGE), writable_mb=2048), + ) + self.options = self.project.sandbox_options() + self.options["reverse_tunnel"] = self.tunnel + self.sandbox = Sandbox( + image=_IMAGE, + cpu=2000, + memory=4096, + idle_timeout=1200, + schedule_timeout=180, + storage_mb=8192, + **self.options, + ) + self.environments.append(self.sandbox.id) + self.project.start_runtime(self.sandbox) + self.project.up(wait_timeout=60) + for name in ("main", "private"): + self.project.service(name).start_process( + shlex.join(["python3", "-c", _SERVER]), shell="/bin/sh" + ) + # Confirm the negative isolation test has a real listening target. + self.assertEqual(self.python("private", self.echo("127.0.0.1")), "ok") + self.process = self.project.service("loadgen").start_process( + shlex.join(["python3", "-c", _PROBE]), shell="/bin/sh" + ) + self.endpoint = self.project.service("main").expose_tunnel(self.tunnel) + self.sandbox.commands.run( + shlex.join( + ["python3", "-c", _PROBE.replace("('api',8000)", "('127.0.0.1',18090)")] + ), + background=True, + ) + + def cleanup_remote(self): + for identity in self.environments: + Sandbox.delete(sandbox_id=identity) + for identity in self.snapshots: + Sandbox.delete_snapshot(identity) + + def python(self, service, script): + result = self.project.service(service).exec( + shlex.join(["python3", "-c", script]), timeout=15 + ) + self.assertEqual(result.exit_code, 0, result.stderr) + return json.loads(result.stdout) + + @staticmethod + def echo(host): + return ( + "import socket,json; " + f"s=socket.create_connection(({host!r},8000),3); " + "s.sendall(b'ok');print(json.dumps(s.recv(2).decode()))" + ) + + def witness(self): + started = time.time() + for _ in range(60): + value = self.python( + "loadgen", + "import pathlib; " + "p=pathlib.Path('/tmp/witness.json'); " + "print(p.read_text() if p.exists() else 'null')", + ) + if value and value.pop("time") > started: + return value + time.sleep(0.1) + self.fail("Existing inner TCP connection did not resume") + + def outer_witness(self): + started = time.time() + for _ in range(60): + result = self.sandbox.commands.run( + shlex.join( + [ + "python3", + "-c", + "import pathlib; p=pathlib.Path('/tmp/witness.json'); " + "print(p.read_text() if p.exists() else 'null')", + ] + ), + timeout=15, + check=True, + ) + value = json.loads(result.stdout) + if value and value.pop("time") > started: + return value + time.sleep(0.1) + self.fail("Existing published-port TCP connection did not resume") + + def check_network(self): + self.assertEqual(self.python("loadgen", self.echo("api")), "ok") + self.assertEqual(self.python("standard", self.echo("main")), "ok") + result = self.sandbox.commands.run( + shlex.join( + [ + "python3", + "-c", + """ +import socket,json +s=socket.create_connection(('127.0.0.1',18090),3);s.sendall(b'tcp');a=s.recv(3) +u=socket.socket(socket.AF_INET,socket.SOCK_DGRAM);u.settimeout(3) +u.sendto(b'udp',('127.0.0.1',18091)) +print(json.dumps([a.decode(),u.recv(3).decode()])) +""", + ] + ), + timeout=15, + check=True, + ) + self.assertEqual(json.loads(result.stdout), ["tcp", "udp"]) + for name in ("main", "private"): + endpoint = self.project.service(name).expose_tunnel(self.tunnel) + self.assertEqual( + self.python( + name, + "import urllib.request,json; " + f"r=urllib.request.Request({endpoint!r}+'/v1/messages?trace=1'," + "data=b'{}',headers={'x-api-key':'trial-fixture'," + "'Authorization':'Bearer opaque'}); " + "print(json.dumps(urllib.request.urlopen(r,timeout=5).read().decode()))", + ), + _SSE.decode(), + ) + host = os.environ.get("AKERNEL_TEST_EGRESS_HOST", "1.1.1.1") + self.assertTrue( + self.python( + "loadgen", + "import socket,json; " + f"s=socket.create_connection(({host!r},80),3);" + "print(json.dumps(bool(s.getsockname())));s.close()", + ) + ) + inspect = self.project.service("private").identity() + result = self.sandbox.commands.run( + shlex.join(["docker", "inspect", inspect["container_id"]]), check=True + ) + networks = json.loads(result.stdout)[0]["NetworkSettings"]["Networks"] + address = next(iter(networks.values()))["IPAddress"] + self.assertTrue( + self.python( + "main", + "import socket,json\ntry:\n " + f"s=socket.create_connection(({address!r},8000),1); print('false')" + "\nexcept (OSError,TimeoutError): print('true')", + ) + ) + # A different service must be rejected at TCP, even with valid model auth. + self.assertTrue( + self.python( + "loadgen", + "import socket,json,urllib.parse; " + f"u=urllib.parse.urlsplit({self.endpoint!r}); " + "s=socket.create_connection((u.hostname,u.port),3);s.settimeout(3)\n" + "try:\n s.sendall(b'GET / HTTP/1.0\\r\\n\\r\\n'); " + "print(json.dumps(s.recv(100)==b''))\n" + "except ConnectionResetError: print('true')", + ) + ) + + def test_native_network_and_two_restores(self): + self.project.service("private").expose_tunnel(self.tunnel) + identities = { + name: self.project.service(name).identity() + for name in ("main", "loadgen", "standard", "private") + } + before = self.witness() + published_before = self.outer_witness() + self.assertEqual(before["cookie"], 41) + self.check_network() + for _ in range(2): + snapshot = self.sandbox.create_snapshot() + self.snapshots.append(snapshot.snapshot_id) + self.project.service("loadgen").exec("touch /tmp/mutate") + self.assertEqual(self.witness()["cookie"], 99) + self.sandbox.kill() + self.sandbox = Sandbox.from_snapshot( + snapshot.snapshot_id, + idle_timeout=1200, + schedule_timeout=180, + **self.options, + ) + self.environments.append(self.sandbox.id) + self.project = ComposeProject.attach(self.sandbox, self.project.descriptor) + self.project.reconnect_network() + self.process = ServiceProcessHandle.attach( + self.project.service("loadgen"), self.process.descriptor + ) + self.assertEqual(self.witness(), before) + self.assertEqual(self.outer_witness(), published_before) + self.assertEqual( + {name: self.project.service(name).identity() for name in identities}, + identities, + ) + self.check_network() + self.assertEqual(self.compose.read_text(), self.original) + self.assertTrue( + all( + path == "/v1/messages?trace=1" + and headers.get("authorization") == "Bearer opaque" + for path, headers in self.requests + ), + self.requests, + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/sdk/python/tests/unit/test_compose.py b/sdk/python/tests/unit/test_compose.py new file mode 100644 index 00000000..340786be --- /dev/null +++ b/sdk/python/tests/unit/test_compose.py @@ -0,0 +1,306 @@ +"""Compose phases, immutable inputs, failure diagnostics and restore attachment.""" + +import json +import tempfile +import unittest +from pathlib import Path +from unittest.mock import MagicMock, patch + +from akernel_sdk import ComposeOperationError, ComposeProject, DockerRuntime +from akernel_sdk._backends.errors import BackendOperationError +from akernel_sdk.types import CommandResult + + +class ComposeTest(unittest.TestCase): + def setUp(self): + temp = tempfile.TemporaryDirectory() + self.addCleanup(temp.cleanup) + self.root = Path(temp.name) + self.context = self.root / "source" + self.context.mkdir() + (self.context / "Dockerfile").write_text("FROM scratch\n") + self.file = self.context / "docker-compose.yaml" + self.file.write_text( + "services:\n api:\n build: .\n load:\n image: alpine:3.20\n" + ) + self.sandbox = MagicMock(id="source") + self.sandbox.commands.run.return_value = CommandResult( + stdout="", stderr="", exit_code=0 + ) + + def prepare(self, **kwargs): + kwargs.setdefault("network", "host") + return ComposeProject.prepare( + self.context, [self.file], output_dir=self.root / "prepared", **kwargs + ) + + def test_frozen_inputs_drive_both_discovery_and_remote_upload(self): + binary = self.root / "bridge" + binary.write_bytes(b"fixture") + images = MagicMock() + images.mounts.return_value = [] + with patch( + "akernel_sdk._compose.prepare_compose_images", return_value=images + ) as discover: + project = self.prepare( + auto_images=True, runtime=DockerRuntime(bridge_binary=binary) + ) + frozen = discover.call_args.args[0] + self.assertEqual((frozen / "Dockerfile").read_text(), "FROM scratch\n") + (self.context / "Dockerfile").write_text("FROM changed\n") + self.file.write_text("services: {different: {image: wrong}}") + with patch.object(DockerRuntime, "start"): + project.start_runtime(self.sandbox) + uploaded = Path(self.sandbox.files.copy_from_local.call_args.args[0]) + self.assertEqual((uploaded / "Dockerfile").read_text(), "FROM scratch\n") + staged = "".join(c.args[1] for c in self.sandbox.files.write.call_args_list) + self.assertIn("load:", staged) + self.assertNotIn("different:", staged) + self.assertNotIn("harbor", project.command(["ps"])) + + def test_unknown_build_outcome_is_not_replayed(self): + project = self.prepare() + with patch.object(DockerRuntime, "start"): + project.start_runtime(self.sandbox) + self.sandbox.commands.run.reset_mock() + failure = BackendOperationError( + "lost reply", outcome="unknown", request_id="same-request" + ) + self.sandbox.commands.run.side_effect = failure + with self.assertRaises(BackendOperationError) as raised: + project.build() + self.assertIs(raised.exception, failure) + self.assertEqual(self.sandbox.commands.run.call_count, 1) + + def test_build_and_up_are_separate_policy_boundaries(self): + project = self.prepare(buildkit=False) + with patch.object(DockerRuntime, "start"): + project.start_runtime(self.sandbox) + self.sandbox.commands.run.reset_mock() + project.build(no_cache=True) + commands = [c.args[0] for c in self.sandbox.commands.run.call_args_list] + self.assertEqual(len(commands), 1) + self.assertTrue(commands[0].startswith("DOCKER_BUILDKIT=0 ")) + self.assertIn("build --no-cache", commands[0]) + self.assertNotIn(" up ", commands[0]) + project.up() + self.assertIn("up -d --wait", self.sandbox.commands.run.call_args.args[0]) + + def test_automatic_images_verify_pinned_inputs_and_never_pull_them_again(self): + binary = self.root / "bridge" + binary.write_bytes(b"fixture") + images = MagicMock() + images.mounts.return_value = [] + original = ( + "services:\n api:\n build: .\n load:\n image: " + "registry.example/load:release@sha256:" + "a" * 64 + "\n" + ) + self.file.write_text(original) + with patch("akernel_sdk._compose.prepare_compose_images", return_value=images): + project = self.prepare( + auto_images=True, runtime=DockerRuntime(bridge_binary=binary) + ) + attached = ComposeProject.attach(self.sandbox, project.descriptor) + attached.pull() + self.assertIn("start.py --verify", self.sandbox.commands.run.call_args.args[0]) + self.sandbox.commands.run.reset_mock() + attached.build(no_cache=True) + attached.up() + commands = [c.args[0] for c in self.sandbox.commands.run.call_args_list] + self.assertIn("build --no-cache", commands[0]) + self.assertIn("start.py --verify", commands[1]) + self.assertIn("--pull never", commands[2]) + self.assertNotIn("--no-build", commands[2]) + self.assertFalse(any(" pull " in c for c in commands)) + self.assertEqual(self.file.read_text(), original) + self.assertEqual((self.root / "prepared/source-0.yaml").read_text(), original) + + def test_automatic_image_verification_failure_is_not_hidden_or_retried(self): + project = self.prepare() + descriptor = project.descriptor + descriptor["automatic_images"] = True + attached = ComposeProject.attach(self.sandbox, descriptor) + self.sandbox.commands.run.return_value = CommandResult( + stdout="", stderr="missing ROFS snapshot", exit_code=1 + ) + for phase in ("pull", "up"): + with self.subTest(phase=phase): + self.sandbox.commands.run.reset_mock() + with self.assertRaises(ComposeOperationError) as raised: + getattr(attached, phase)() + self.assertEqual(raised.exception.phase, phase) + self.assertEqual(raised.exception.stderr, "missing ROFS snapshot") + self.assertEqual(self.sandbox.commands.run.call_count, 1) + + def test_unmanaged_pull_keeps_native_compose_behavior(self): + project = ComposeProject.attach(self.sandbox, self.prepare().descriptor) + project.pull() + self.assertTrue( + self.sandbox.commands.run.call_args.args[0].endswith( + "pull --ignore-buildable" + ) + ) + + def test_missing_exit_status_is_an_unknown_operation_outcome(self): + failure = ComposeOperationError( + "build", CommandResult(stdout="partial", stderr="", exit_code=None) + ) + self.assertEqual(failure.outcome, "unknown") + + def test_zero_timeout_image_verification_waits_without_foreground_timeout(self): + project = self.prepare() + descriptor = project.descriptor + descriptor["automatic_images"] = True + project = ComposeProject.attach(self.sandbox, descriptor) + handle = self.sandbox.commands.run.return_value = MagicMock( + command_id="verify-id" + ) + handle.wait.side_effect = [ + CommandResult(stdout="", stderr="", exit_code=None), + CommandResult(stdout="verified", stderr="", exit_code=0), + ] + project.pull(timeout=0) + self.assertIs(self.sandbox.commands.run.call_args.kwargs["background"], True) + handle.kill.assert_not_called() + + def test_failed_cancel_keeps_original_timeout_and_command_identity(self): + from threading import Event + + from akernel_sdk import TimeoutException + + project = ComposeProject.attach(self.sandbox, self.prepare().descriptor) + handle = self.sandbox.commands.run.return_value = MagicMock( + command_id="owned-command" + ) + cleanup = RuntimeError("kill response lost") + handle.kill.side_effect = cleanup + cancellation = Event() + cancellation.set() + with self.assertRaises(TimeoutException) as raised: + project.run(["ps"], timeout=0, cancellation=cancellation) + self.assertEqual(raised.exception.command_id, "owned-command") + self.assertEqual(raised.exception.cleanup_errors, (cleanup,)) + + def test_failed_up_retains_output_and_collects_logs_without_restarting(self): + project = self.prepare() + with patch.object(DockerRuntime, "start"): + project.start_runtime(self.sandbox) + self.sandbox.commands.run.reset_mock() + self.sandbox.commands.run.side_effect = [ + CommandResult( + stdout="service partial", stderr="health failed", exit_code=7 + ), + CommandResult(stdout="diagnostic", stderr="", exit_code=0), + ] + with self.assertRaises(ComposeOperationError) as raised: + project.up() + self.assertEqual(raised.exception.phase, "up") + self.assertEqual(raised.exception.exit_code, 7) + self.assertEqual(raised.exception.stderr, "health failed") + self.assertEqual(self.sandbox.commands.run.call_count, 2) + self.assertEqual( + (self.root / "prepared/services.log").read_text(), "diagnostic" + ) + + def test_attach_rebinds_only_and_retains_managed_upper_geometry(self): + project = self.prepare(runtime=DockerRuntime(writable_mb=2048)) + with patch.object(DockerRuntime, "start"): + project.start_runtime(self.sandbox) + other = MagicMock(id="restored") + attached = ComposeProject.attach(other, project.descriptor) + other.commands.run.assert_not_called() + other.files.copy_from_local.assert_not_called() + self.assertEqual(attached.sandbox_options(), project.sandbox_options()) + with self.assertRaises(RuntimeError): + attached.start_runtime(other) + attached.service("load").exec("echo restored", user="user", envs={"A": "x y"}) + self.assertIn("exec -T -u user -e", other.commands.run.call_args.args[0]) + self.assertIn("load /bin/sh -c", other.commands.run.call_args.args[0]) + + def test_explicit_network_fails_before_remote_mutations(self): + self.file.write_text("services: {api: {networks: [private]}}") + with self.assertRaisesRegex(ValueError, "explicit networks"): + self.prepare() + self.sandbox.commands.run.assert_not_called() + + def test_service_identity_rejects_scaled_service(self): + project = self.prepare() + attached = ComposeProject.attach(self.sandbox, project.descriptor) + self.sandbox.commands.run.return_value = CommandResult( + stdout="one\ntwo\n", stderr="", exit_code=0 + ) + with self.assertRaisesRegex(BackendOperationError, "exactly one"): + attached.service("api").identity() + + def test_prebuilt_images_do_not_add_undeclared_services_or_builds(self): + images = MagicMock() + images.mounts.return_value = [] + project = self.prepare( + runtime=DockerRuntime(images=images), + prebuilt_images={"api": "registry/api:seed", "absent": "unused"}, + ) + other = ComposeProject.attach(self.sandbox, project.descriptor) + other.build() + self.sandbox.commands.run.assert_not_called() + other.up() + self.assertIn( + "--no-build --pull never", self.sandbox.commands.run.call_args.args[0] + ) + self.assertNotIn("absent", project.descriptor["services"]) + text = (self.root / "prepared/host.json").read_text() + self.assertNotIn('"build"', text) + + def test_custom_hostnames_are_preserved_in_compatibility_overlay(self): + self.file.write_text( + "services: {worker: {image: alpine, hostname: custom.example}}" + ) + self.prepare() + hosts = json.loads((self.root / "prepared/host.json").read_text())["services"] + self.assertEqual(hosts["worker"]["hostname"], "custom.example") + self.assertIn("custom.example:127.0.0.1", hosts["worker"]["extra_hosts"]) + + def test_partial_runtime_start_is_not_replayed(self): + project = self.prepare() + with patch.object( + DockerRuntime, "start", side_effect=BackendOperationError("partial") + ) as start: + with self.assertRaises(BackendOperationError): + project.start_runtime(self.sandbox) + with self.assertRaises(RuntimeError): + project.start_runtime(self.sandbox) + start.assert_called_once() + + def test_copy_cleanup_failure_does_not_replace_transfer_failure(self): + project = self.prepare() + attached = ComposeProject.attach(self.sandbox, project.descriptor) + failure = BackendOperationError("copy unavailable", request_id="transfer") + cleanup = BackendOperationError("cleanup unavailable", request_id="cleanup") + self.sandbox.files.copy_from_local.side_effect = failure + self.sandbox.files.remove.side_effect = cleanup + with self.assertRaises(BackendOperationError) as raised: + attached.service("api").upload("local", "/target") + self.assertIs(raised.exception, failure) + self.assertEqual(failure.cleanup_errors, (cleanup,)) + + def test_zero_timeout_waits_without_deadline_and_cancellation_keeps_identity(self): + from threading import Event + + from akernel_sdk import TimeoutException + + project = self.prepare() + attached = ComposeProject.attach(self.sandbox, project.descriptor) + handle = self.sandbox.commands.run.return_value = MagicMock( + command_id="stable-command" + ) + handle.wait.side_effect = [ + CommandResult(stdout="", stderr="", exit_code=None), + CommandResult(stdout="finished", stderr="", exit_code=0), + ] + self.assertEqual(attached.run(["ps"], timeout=0).stdout, "finished") + self.assertTrue(self.sandbox.commands.run.call_args.kwargs["background"]) + cancellation = Event() + cancellation.set() + with self.assertRaises(TimeoutException) as raised: + attached.run(["ps"], timeout=0, cancellation=cancellation) + self.assertEqual(raised.exception.command_id, "stable-command") + handle.kill.assert_called_once() diff --git a/sdk/python/tests/unit/test_compose_network.py b/sdk/python/tests/unit/test_compose_network.py new file mode 100644 index 00000000..0875a752 --- /dev/null +++ b/sdk/python/tests/unit/test_compose_network.py @@ -0,0 +1,246 @@ +"""Container-accessible tunnel ownership, restore rebinding and byte transport.""" + +import json +import socket +import tempfile +import threading +import unittest +from pathlib import Path +from unittest.mock import MagicMock, patch + +from akernel_sdk import ComposeProject, HttpReverseTunnel +from akernel_sdk import _docker_network_guest as guest +from akernel_sdk._backends.errors import BackendOperationError +from akernel_sdk._compose import ServiceHandle +from akernel_sdk.types import CommandResult + + +class ComposeNetworkTest(unittest.TestCase): + def setUp(self): + temporary = tempfile.TemporaryDirectory() + self.addCleanup(temporary.cleanup) + root = Path(temporary.name) + source = root / "source" + source.mkdir() + self.original = ( + "services: {worker: {image: test, networks: [private]}}\n" + "networks: {private: {internal: true}}\n" + ) + compose = source / "compose.yaml" + compose.write_text(self.original) + self.project = ComposeProject.prepare( + source, [compose], network="bridge", output_dir=root / "out" + ) + self.sandbox = MagicMock() + self.project._sandbox = self.sandbox + self.service = self.project.service("worker") + self.identity = {"container_id": "full-id", "started_at": "original-start"} + identity = patch.object( + ServiceHandle, "identity", return_value=self.identity.copy() + ) + self.mock_identity = identity.start() + self.addCleanup(identity.stop) + self.tunnel = HttpReverseTunnel("http://127.0.0.1:12345") + self.inspect = { + "HostConfig": {"NetworkMode": "project_private"}, + "NetworkSettings": { + "Networks": { + "project_private": { + "Gateway": "172.28.41.1", + "IPAddress": "172.28.41.2", + } + } + }, + } + self.sandbox.commands.run.side_effect = [ + CommandResult(stdout=json.dumps([self.inspect]), stderr="", exit_code=0), + CommandResult( + stdout='{"url":"http://172.28.41.1:19000"}', stderr="", exit_code=0 + ), + ] + + def test_native_prepare_has_no_network_overlay(self): + self.assertEqual(self.project._contents, [self.original]) + self.assertEqual(self.project.descriptor["network"], "bridge") + self.assertNotIn("127.0.0.1", "".join(self.project._contents)) + + def test_gateway_endpoint_is_peer_restricted_and_restore_never_relaunches(self): + url = self.service.expose_tunnel(self.tunnel) + self.assertEqual(url, "http://172.28.41.1:19000") + command = self.sandbox.commands.run.call_args.args[0] + self.assertIn("172.28.41.2", command) + descriptor = self.project.descriptor + self.assertNotIn(self.tunnel.target, json.dumps(descriptor)) + restored = MagicMock() + attached = ComposeProject.attach(restored, descriptor) + self.assertEqual(attached.service("worker").expose_tunnel(self.tunnel), url) + restored.commands.run.assert_not_called() + attached.reconnect_network() + self.assertIn("reconnect", restored.commands.run.call_args.args[0]) + self.assertNotIn("dockerd", restored.commands.run.call_args.args[0]) + self.assertEqual(restored.commands.run.call_count, 1) + + def test_missing_default_gateway_uses_internal_bridge_ipam(self): + self.inspect["NetworkSettings"]["Networks"]["project_private"]["Gateway"] = "" + self.sandbox.commands.run.side_effect = [ + CommandResult(stdout=json.dumps([self.inspect]), stderr="", exit_code=0), + CommandResult( + stdout=json.dumps( + [ + { + "IPAM": { + "Config": [ + { + "Subnet": "172.28.41.0/24", + "Gateway": "172.28.41.1", + } + ] + } + } + ] + ), + stderr="", + exit_code=0, + ), + CommandResult( + stdout='{"url":"http://172.28.41.1:19000"}', stderr="", exit_code=0 + ), + ] + self.assertEqual( + self.service.expose_tunnel(self.tunnel), "http://172.28.41.1:19000" + ) + + def test_host_mode_retains_loopback_without_starting_a_proxy(self): + self.inspect["HostConfig"]["NetworkMode"] = "host" + self.sandbox.commands.run.side_effect = [ + CommandResult(stdout=json.dumps([self.inspect]), stderr="", exit_code=0) + ] + self.assertEqual(self.service.expose_tunnel(self.tunnel), self.tunnel.url) + self.assertEqual(self.sandbox.commands.run.call_count, 1) + attached = ComposeProject.attach(MagicMock(), self.project.descriptor) + self.assertEqual( + attached.service("worker").expose_tunnel(self.tunnel), self.tunnel.url + ) + + def test_stale_container_and_unknown_launch_are_never_replayed(self): + self.service.expose_tunnel(self.tunnel) + self.mock_identity.return_value = { + "container_id": "different", + "started_at": "new", + } + with self.assertRaisesRegex(BackendOperationError, "replaced"): + self.service.expose_tunnel(self.tunnel) + self.assertEqual(self.sandbox.commands.run.call_count, 2) + self.mock_identity.return_value = self.identity.copy() + record = self.project._descriptor["tunnel_endpoints"]["worker:8766"] + record.pop("url") + with self.assertRaises(BackendOperationError) as failure: + self.service.expose_tunnel(self.tunnel) + self.assertEqual(failure.exception.outcome, "unknown") + self.assertEqual(self.sandbox.commands.run.call_count, 2) + + def test_no_network_or_untrusted_listener_fails_closed(self): + self.inspect["NetworkSettings"]["Networks"] = {} + self.sandbox.commands.run.side_effect = [ + CommandResult(stdout=json.dumps([self.inspect]), stderr="", exit_code=0) + ] + with self.assertRaisesRegex(ValueError, "IPv4"): + self.service.expose_tunnel(self.tunnel) + self.assertEqual(self.sandbox.commands.run.call_count, 1) + + def test_snapshot_attachment_cannot_accidentally_bootstrap_a_missing_relay(self): + restored = MagicMock() + attached = ComposeProject.attach(restored, self.project.descriptor) + with self.assertRaisesRegex(RuntimeError, "existing tunnels"): + attached.service("worker").expose_tunnel(self.tunnel) + restored.commands.run.assert_not_called() + + +class RelayTransportTest(unittest.TestCase): + def test_http_auth_chunked_sse_and_half_close_are_preserved_byte_for_byte(self): + payload = ( + b"POST /v1/messages?trace=1 HTTP/1.1\r\n" + b"Authorization: Bearer opaque\r\nx-api-key: key\r\n\r\n" + b"x" * 200000 + ) + response = ( + b"HTTP/1.1 200 OK\r\nContent-Type: text/event-stream\r\n" + b"Transfer-Encoding: chunked\r\n\r\n" + b"data: unchanged\n\n" * 10000 + ) + upstream = socket.socket() + upstream.bind(("127.0.0.1", 0)) + upstream.listen(1) + received = [] + + def server(): + peer, _ = upstream.accept() + with peer: + chunks = [] + while chunk := peer.recv(65536): + chunks.append(chunk) + received.append(b"".join(chunks)) + peer.sendall(response) + peer.shutdown(socket.SHUT_WR) + + thread = threading.Thread(target=server, daemon=True) + thread.start() + application, relay = socket.socketpair() + forwarding = threading.Thread( + target=guest.forward, args=(relay, upstream.getsockname()), daemon=True + ) + forwarding.start() + try: + application.settimeout(5) + application.sendall(payload) + application.shutdown(socket.SHUT_WR) + result = [] + while chunk := application.recv(65536): + result.append(chunk) + self.assertEqual(b"".join(result), response) + self.assertEqual(received, [payload]) + finally: + application.close() + upstream.close() + forwarding.join(timeout=5) + thread.join(timeout=5) + self.assertFalse(forwarding.is_alive()) + self.assertFalse(thread.is_alive()) + + +class EgressRebindTest(unittest.TestCase): + def test_new_tap_replaces_only_owned_chains_without_flushing_conntrack(self): + with tempfile.TemporaryDirectory() as directory: + state = Path(directory) / "network.json" + state.write_text( + json.dumps( + { + "uplink": "tap.old", + "source": "10.88.0.2", + "iptables": "/sbin/iptables-legacy", + } + ) + ) + with ( + patch.object(guest, "STATE", state), + patch.object( + guest.shutil, "which", side_effect=lambda name: "/sbin/" + name + ), + patch.object( + guest.subprocess, + "check_output", + return_value="1.1.1.1 via 10.88.0.1 dev tap.new src 10.88.0.3", + ), + patch.object(guest.subprocess, "run") as run, + ): + record = guest.configure(reconnect=True) + self.assertEqual(record["source"], "10.88.0.3") + run.assert_called_once() + self.assertIn("--noflush", run.call_args.args[0]) + rules = run.call_args.kwargs["input"] + self.assertIn("AKERNEL_DIND_NAT -o tap.new", rules) + self.assertNotIn("tap.old", rules) + self.assertNotIn("10.88.0.2", rules) + self.assertNotIn("DOCKER", rules) + self.assertEqual(json.loads(state.read_text()), record) + run.reset_mock() + guest.configure(reconnect=True) + run.assert_not_called() diff --git a/sdk/python/tests/unit/test_docker_runtime.py b/sdk/python/tests/unit/test_docker_runtime.py new file mode 100644 index 00000000..55ccd54d --- /dev/null +++ b/sdk/python/tests/unit/test_docker_runtime.py @@ -0,0 +1,396 @@ +"""Shared lower integrity, private upper restore config and one-time startup.""" + +import hashlib +import json +import subprocess +import tempfile +import unittest +from pathlib import Path +from types import SimpleNamespace +from unittest.mock import MagicMock, patch + +import httpx + +from akernel_sdk import DockerRuntime, ImageMountPlan, Sandbox +from akernel_sdk import _image_resolver as resolver +from akernel_sdk import sandbox as public +from akernel_sdk._docker_runtime import _PREPARE_RUNTIME +from akernel_sdk._image_bridge_guest import verify_image_bridge + + +def digest(data): + return "sha256:" + hashlib.sha256(data).hexdigest() + + +class DockerRuntimeTest(unittest.TestCase): + def setUp(self): + directory = tempfile.TemporaryDirectory() + self.addCleanup(directory.cleanup) + self.directory = Path(directory.name) + + def plan(self): + blobs = [] + + def metadata(kind, value, media_type): + payload = json.dumps(value).encode() + name = kind + ".json" + (self.directory / name).write_bytes(payload) + blobs.append( + { + "file": name, + "descriptor": { + "digest": digest(payload), + "size": len(payload), + "mediaType": media_type, + }, + } + ) + return blobs[-1]["descriptor"] + + config = metadata( + "config", + { + "architecture": "amd64", + "os": "linux", + "rootfs": {"type": "layers", "diff_ids": ["sha256:" + "a" * 64]}, + }, + "application/vnd.oci.image.config.v1+json", + ) + metadata( + "manifest", + { + "schemaVersion": 2, + "mediaType": "application/vnd.oci.image.manifest.v1+json", + "config": config, + "layers": [ + { + "digest": "sha256:" + "b" * 64, + "size": 123, + "mediaType": "application/vnd.oci.image.layer.v1.tar+gzip", + } + ], + }, + "application/vnd.oci.image.manifest.v1+json", + ) + target = blobs[1]["descriptor"] + reference = "registry.example/test@" + target["digest"] + image = { + "reference": reference, + "root": "/opt/akernel-images/" + target["digest"][7:], + "target": target, + "config": blobs[0]["descriptor"], + "blobs": blobs, + "chain_id": "sha256:" + "a" * 64, + } + path = self.directory / "plan.json" + path.write_text( + json.dumps({"images": [image], "services": {"main": reference}}) + ) + return path + + def test_old_helper_is_rejected_before_project_data_is_dispatched(self): + response = SimpleNamespace(returncode=0, stdout='{"references":[]}', stderr="") + with patch.object(resolver.subprocess, "run", return_value=response) as run: + with self.assertRaisesRegex(ValueError, "schema 2"): + resolver.prepare_compose_images( + self.directory, + [], + output_dir=self.directory / "output", + bridge_binary=self.directory / "bridge", + ) + self.assertEqual(run.call_count, 1) + self.assertEqual(run.call_args.args[0][-1], "capabilities") + self.assertNotIn("input", run.call_args.kwargs) + + def test_discovery_does_not_implicitly_inherit_client_credentials(self): + response = SimpleNamespace( + returncode=0, stdout='{"references":[],"discover_schema":2}', stderr="" + ) + with ( + patch.dict("os.environ", {"AKERNEL_TOKEN": "host-only-fixture"}), + patch.object(resolver.subprocess, "run", return_value=response) as run, + ): + plan = resolver.prepare_compose_images( + self.directory, + [], + output_dir=self.directory / "output", + bridge_binary=self.directory / "bridge", + ) + self.assertTrue(plan.automatic) + self.assertTrue(all(call.kwargs["env"] == {} for call in run.call_args_list)) + + def test_integrity_and_path_escape_rejected_before_sandbox_creation(self): + for defect in ("corrupt", "escape", "root"): + with self.subTest(defect=defect): + path = self.plan() + plan = json.loads(path.read_text()) + if defect == "corrupt": + (self.directory / "config.json").write_bytes(b"changed") + elif defect == "escape": + plan["images"][0]["blobs"][0]["file"] = "../private" + else: + plan["images"][0]["root"] += "different" + path.write_text(json.dumps(plan)) + with self.assertRaises(ValueError): + ImageMountPlan.load(path) + + def test_restore_options_retain_lower_and_upper_without_startup(self): + runtime = DockerRuntime( + images=ImageMountPlan.load(self.plan()), writable_mb=2048 + ) + backend = MagicMock() + backend.from_snapshot.return_value.id = "restored" + with patch.object(public, "load_backend", return_value=backend): + for factory in (Sandbox.from_snapshot, Sandbox.create): + options = runtime.sandbox_options() + sb = factory("snapshot", **options) + spec = backend.from_snapshot.call_args.args[0] + reference = runtime.images.images[0]["reference"] + self.assertEqual(spec.mounts[0].image_url, reference) + upper = options["extra_config"]["fileBackedTmpfsMounts"][0] + self.assertEqual( + spec.extra_config["fileBackedTmpfsMounts"][0]["sizeMB"], 2048 + ) + upper["sizeMB"] = 999 + self.assertEqual( + spec.extra_config["fileBackedTmpfsMounts"][0]["sizeMB"], 2048 + ) + backend.create.assert_not_called() + backend.from_snapshot.return_value.commands.run.assert_not_called() + sb.kill() + + def test_start_checks_return_codes_and_refuses_existing_daemon(self): + sandbox = MagicMock() + sandbox.commands.run.side_effect = RuntimeError("existing daemon") + with self.assertRaisesRegex(RuntimeError, "existing daemon"): + DockerRuntime().start(sandbox) + self.assertEqual(sandbox.commands.run.call_count, 1) + self.assertTrue(sandbox.commands.run.call_args.kwargs["check"]) + sandbox.files.copy_from_local.assert_not_called() + + def test_public_runtime_bootstrap_installs_missing_tools_before_bridge(self): + sandbox = MagicMock() + runtime = DockerRuntime() + with patch.object(DockerRuntime, "_bridge", return_value=[]) as bridge: + # Refusing an existing daemon is the first operation. A failed package + # preparation on a fresh sandbox must not start any image services. + sandbox.commands.run.side_effect = [ + SimpleNamespace(), + RuntimeError("bootstrap failed"), + ] + with self.assertRaisesRegex(RuntimeError, "bootstrap failed"): + runtime.start(sandbox) + bridge.assert_not_called() + self.assertIn("apk add --no-cache", sandbox.commands.run.call_args.args[0]) + self.assertTrue(sandbox.commands.run.call_args.kwargs["check"]) + + def test_bootstrap_reuses_ready_images_and_handles_public_alpine(self): + binaries = self.directory / "bin" + binaries.mkdir() + record = self.directory / "packages" + + def executable(name, body="exit 0"): + path = binaries / name + path.write_text("#!/bin/sh\n" + body + "\n") + path.chmod(0o755) + + for name in ("docker", "dockerd", "containerd", "ctr"): + executable(name) + executable("apk", 'printf "%s\\n" "$*" > "$RECORD"') + environment = {"PATH": str(binaries), "RECORD": str(record)} + result = subprocess.run( + ["/bin/sh", "-c", _PREPARE_RUNTIME], + env=environment, + capture_output=True, + text=True, + check=False, + ) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual( + record.read_text().strip(), + "add --no-cache bash python3 coreutils util-linux shadow tar xz", + ) + record.unlink() + for name in ( + "bash", + "python3", + "findmnt", + "mount", + "tar", + "xz", + "realpath", + "usermod", + ): + executable(name) + result = subprocess.run( + ["/bin/sh", "-c", _PREPARE_RUNTIME], + env=environment, + capture_output=True, + text=True, + check=False, + ) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertFalse(record.exists()) + (binaries / "dockerd").unlink() + result = subprocess.run( + ["/bin/sh", "-c", _PREPARE_RUNTIME], + env=environment, + capture_output=True, + text=True, + check=False, + ) + self.assertEqual(result.returncode, 127) + self.assertIn("missing dockerd", result.stderr) + self.assertFalse(record.exists()) + + def test_bridge_upload_is_sdk_owned_and_precedes_dockerd(self): + sandbox = MagicMock() + sandbox.commands.run.return_value = SimpleNamespace( + stdout='["--config-file", "/bridge/docker.json"]', stderr="" + ) + binary = self.directory / "bridge" + binary.write_bytes(b"fixture") + runtime = DockerRuntime( + images=ImageMountPlan.load(self.plan()), bridge_binary=binary + ) + runtime.start(sandbox, network="host") + commands = [c.args[0] for c in sandbox.commands.run.call_args_list] + start = next(i for i, c in enumerate(commands) if "start.py --root" in c) + docker = next(i for i, c in enumerate(commands) if "dockerd --config-file" in c) + self.assertLess(start, docker) + uploads = [c.args[0] for c in sandbox.files.copy_from_local.call_args_list] + self.assertTrue( + any(p.endswith("akernel_sdk/_image_bridge_guest.py") for p in uploads) + ) + self.assertFalse(any("harbor" in p for p in uploads)) + + def test_registry_auth_pins_metadata_and_never_downloads_layers(self): + config = json.dumps( + { + "os": "linux", + "architecture": "amd64", + "rootfs": {"diff_ids": ["sha256:" + "a" * 64]}, + } + ).encode() + manifest = json.dumps( + { + "mediaType": "application/vnd.oci.image.manifest.v1+json", + "config": { + "mediaType": "application/vnd.oci.image.config.v1+json", + "digest": digest(config), + "size": len(config), + }, + "layers": [{"digest": "sha256:" + "b" * 64}], + } + ).encode() + calls = [] + + def handle(request): + calls.append(request) + if request.url.host == "auth.example": + return httpx.Response(200, json={"token": "test-only"}) + if request.headers.get("Authorization") != "Bearer test-only": + return httpx.Response( + 401, + headers={ + "www-authenticate": 'Bearer realm="https://auth.example/token",' + 'scope="repository:library/test:pull"' + }, + ) + if "/manifests/" in request.url.path: + return httpx.Response( + 200, + content=manifest, + headers={"docker-content-digest": digest(manifest)}, + ) + if request.url.path.endswith(digest(config)): + return httpx.Response(200, content=config) + raise AssertionError("must never request a layer") + + with patch.dict("os.environ", {"DOCKER_CONFIG": str(self.directory)}): + with httpx.Client(transport=httpx.MockTransport(handle)) as client: + image = resolver.RegistryMetadata(client, self.directory).prepare( + "docker.io/library/test:latest" + ) + self.assertEqual( + image["reference"], "docker.io/library/test@" + digest(manifest) + ) + self.assertEqual(len(image["blobs"]), 2) + self.assertEqual(sum(r.url.host == "auth.example" for r in calls), 1) + + def test_prepared_image_verification_checks_alias_digest_and_committed_root(self): + path = self.plan() + data = json.loads(path.read_text()) + image = data["images"][0] + alias = "registry.example/test:tag@" + image["target"]["digest"] + data["aliases"] = {alias: image["reference"]} + path.write_text(json.dumps(data)) + calls = [] + + def run(args, **kwargs): + calls.append(args) + self.assertTrue(kwargs["check"]) + if args[0] == "ctr": + payload = json.dumps( + { + "Kind": "Committed", + "Labels": { + "containerd.io/snapshot/agent-service.image-root": image[ + "root" + ] + }, + } + ) + else: + self.assertEqual(args[:3], ["docker", "image", "inspect"]) + payload = image["target"]["digest"] + "\n" + return SimpleNamespace(stdout=payload) + + with patch("akernel_sdk._image_bridge_guest.subprocess.run", side_effect=run): + verify_image_bridge(path) + self.assertEqual(calls[0][-1], image["chain_id"]) + self.assertEqual([c[-1] for c in calls[1:]], [image["reference"], alias]) + + def test_prepared_image_verification_rejects_missing_or_changed_inputs(self): + path = self.plan() + image = json.loads(path.read_text())["images"][0] + valid = json.dumps( + { + "Kind": "Committed", + "Labels": { + "containerd.io/snapshot/agent-service.image-root": image["root"] + }, + } + ) + for results, error_type in ( + ([subprocess.CalledProcessError(1, "ctr")], subprocess.CalledProcessError), + ( + [ + SimpleNamespace(stdout=valid), + subprocess.CalledProcessError(1, "docker"), + ], + subprocess.CalledProcessError, + ), + ( + [ + SimpleNamespace(stdout=valid), + SimpleNamespace(stdout="sha256:" + "0" * 64), + ], + RuntimeError, + ), + ( + [SimpleNamespace(stdout=valid.replace("Committed", "Active"))], + RuntimeError, + ), + ( + [SimpleNamespace(stdout=valid.replace(image["root"], "/other"))], + RuntimeError, + ), + ): + with self.subTest(results=results): + with patch( + "akernel_sdk._image_bridge_guest.subprocess.run", + side_effect=results, + ): + with self.assertRaises(error_type): + verify_image_bridge(path) diff --git a/sdk/python/tests/unit/test_fork.py b/sdk/python/tests/unit/test_fork.py index e5c3fd08..87713bd3 100644 --- a/sdk/python/tests/unit/test_fork.py +++ b/sdk/python/tests/unit/test_fork.py @@ -199,6 +199,9 @@ def setUp(self): self.backend = adx.AdxBackend(config) self.spec = SnapshotRestoreSpec( snapshot_id="source-origin", + extra_config={ + "fileBackedTmpfsMounts": [{"target": "/upper", "sizeMB": 64}] + }, idle_timeout=480, schedule_timeout=35, timeout=95, @@ -303,6 +306,7 @@ def test_one_capture_partial_results_inheritance_and_snapshot_cleanup(self): self.assertEqual(restored.timeout, 95) self.assertEqual(restored.command_cwd, "/work") self.assertEqual(restored.port_forwardings, (8080,)) + self.assertEqual(restored.extra_config, self.spec.extra_config) self.assertIs(restored.reverse_tunnel, self.spec.reverse_tunnel) self.assertTrue(restored.failover) self.assertFalse(restored.detached) diff --git a/sdk/python/tests/unit/test_image_plan.py b/sdk/python/tests/unit/test_image_plan.py new file mode 100644 index 00000000..445b0bab --- /dev/null +++ b/sdk/python/tests/unit/test_image_plan.py @@ -0,0 +1,139 @@ +"""OCI metadata/chain integrity and legitimate sharing of a merged root.""" + +import hashlib +import json +import tempfile +import unittest +from pathlib import Path +from types import SimpleNamespace +from unittest.mock import MagicMock, patch + +from akernel_sdk import DockerRuntime, ImageMountPlan +from akernel_sdk._image_bridge_guest import verify_image_bridge + + +class ImagePlanTest(unittest.TestCase): + def setUp(self): + temporary = tempfile.TemporaryDirectory() + self.addCleanup(temporary.cleanup) + self.directory = Path(temporary.name) + self.chain = "sha256:" + "a" * 64 + + def image(self, label, *, architecture="amd64"): + blobs = [] + + def blob(value, media): + payload = json.dumps(value).encode() + digest = "sha256:" + hashlib.sha256(payload).hexdigest() + name = digest.replace(":", "-") + ".json" + (self.directory / name).write_bytes(payload) + descriptor = {"digest": digest, "size": len(payload), "mediaType": media} + blobs.append({"file": name, "descriptor": descriptor}) + return descriptor + + config = blob( + { + "architecture": architecture, + "os": "linux", + "config": {"Env": ["LABEL=" + label]}, + "rootfs": {"type": "layers", "diff_ids": [self.chain]}, + }, + "application/vnd.oci.image.config.v1+json", + ) + target = blob( + { + "schemaVersion": 2, + "mediaType": "application/vnd.oci.image.manifest.v1+json", + "config": config, + "layers": [ + { + "digest": "sha256:" + "b" * 64, + "size": 123, + "mediaType": "application/vnd.oci.image.layer.v1.tar+gzip", + } + ], + }, + "application/vnd.oci.image.manifest.v1+json", + ) + return { + "reference": "registry.example/test@" + target["digest"], + "root": "/opt/akernel-images/" + target["digest"][7:], + "target": target, + "config": config, + "chain_id": self.chain, + "blobs": blobs, + } + + def write(self, images, **kwargs): + path = self.directory / "plan.json" + path.write_text( + json.dumps( + { + "images": images, + "services": {"main": images[0]["reference"]} if images else {}, + **kwargs, + } + ) + ) + return path + + def test_same_rootfs_with_different_config_digests_is_valid(self): + first, second = self.image("first"), self.image("second") + path = self.write([first, second]) + ImageMountPlan.load(path, bases_only=True) + + def run(args, **kwargs): + if args[0] == "ctr": + label = "containerd.io/snapshot/agent-service.image-root" + return SimpleNamespace( + stdout=json.dumps( + { + "Kind": "Committed", + "Labels": { + label: first["root"] + }, + } + ) + ) + image = next(i for i in (first, second) if i["reference"] == args[-1]) + return SimpleNamespace(stdout=image["target"]["digest"]) + + with patch("akernel_sdk._image_bridge_guest.subprocess.run", side_effect=run): + verify_image_bridge(path) + + def test_claimed_chain_must_match_the_selected_manifest_config(self): + image = self.image("first") + image["chain_id"] = "sha256:" + "c" * 64 + with self.assertRaises(ValueError): + ImageMountPlan.load(self.write([image])) + + def test_config_descriptor_must_be_reachable_from_target_manifest(self): + first, second = self.image("first"), self.image("second") + first["config"] = second["config"] + first["blobs"] += second["blobs"] + with self.assertRaises(ValueError): + ImageMountPlan.load(self.write([first])) + + def test_foreign_platform_cannot_supply_a_linux_amd64_root(self): + with self.assertRaises(ValueError): + ImageMountPlan.load(self.write([self.image("arm", architecture="arm64")])) + + def test_unused_manual_images_do_not_leave_unregistered_aliases(self): + first, second = self.image("first"), self.image("second") + aliases = { + "registry.example/first:tag": first["reference"], + "registry.example/second:tag": second["reference"], + } + plan = ImageMountPlan.load(self.write([first, second], aliases=aliases)) + self.assertEqual( + plan.aliases, {"registry.example/first:tag": first["reference"]} + ) + + def test_scratch_only_preparation_retains_empty_automatic_plan_marker(self): + path = self.write([], automatic=True) + plan = ImageMountPlan.load(path, bases_only=True) + sandbox = MagicMock() + sandbox.commands.run.return_value = SimpleNamespace(stdout="[]", stderr="") + DockerRuntime(images=plan)._bridge(sandbox) + payload = json.loads(sandbox.files.write.call_args.args[1]) + self.assertIs(payload.get("automatic"), True) diff --git a/sdk/python/tests/unit/test_service_process.py b/sdk/python/tests/unit/test_service_process.py new file mode 100644 index 00000000..4cba24fa --- /dev/null +++ b/sdk/python/tests/unit/test_service_process.py @@ -0,0 +1,191 @@ +"""Exercise the packaged service helper with real Linux processes and signals.""" + +import ctypes +import json +import os +import shutil +import signal +import subprocess +import sys +import tempfile +import unittest +from pathlib import Path +from types import SimpleNamespace + +from akernel_sdk import BackendOperationError, ServiceProcessHandle, TimeoutException +from akernel_sdk.types import CommandResult + + +class LocalService: + name = "worker" + project = SimpleNamespace(descriptor={"project_name": "sdk-test"}) + + def __init__(self): + self.current = {"container_id": "c" * 64, "started_at": "original"} + self.lose_start_reply = False + self.tokens = set() + self.identities = {} + + def identity(self): + return dict(self.current) + + def exec(self, command, **kwargs): + result = subprocess.run( + ["/bin/sh", "-c", command], + cwd=kwargs.get("cwd"), + env={**os.environ, **(kwargs.get("envs") or {})}, + text=True, + capture_output=True, + timeout=15, + ) + # Capture only this test's exclusive process directories for cleanup. + import re + + self.tokens.update(re.findall(r"\b[0-9a-f]{32}\b", command)) + for token in self.tokens: + file = Path("/tmp/akernel-service-process-" + token) / "identity.json" + if file.exists() and token not in self.identities: + self.identities[token] = json.loads(file.read_text()) + if self.lose_start_reply: + self.lose_start_reply = False + raise TimeoutException("response lost", outcome="unknown") + return CommandResult( + stdout=result.stdout, stderr=result.stderr, exit_code=result.returncode + ) + + +@unittest.skipUnless( + sys.platform.startswith("linux"), "Requires Linux /proc and subreaper" +) +class ServiceProcessTest(unittest.TestCase): + def setUp(self): + self.libc = ctypes.CDLL(None) + previous = ctypes.c_int() + self.libc.prctl(37, ctypes.byref(previous), 0, 0, 0) + self.previous_subreaper = previous.value + self.libc.prctl(36, 1, 0, 0, 0) + self.temp = tempfile.TemporaryDirectory() + self.service = LocalService() + + def tearDown(self): + for token in self.service.tokens: + root = Path("/tmp/akernel-service-process-" + token) + if not (root / "identity.json").exists(): + continue + identity = self.service.identities[token] + try: + os.kill(identity["pid"], signal.SIGKILL) + except ProcessLookupError: + pass + # The helper monitor is an adopted child, not an unrelated process. + try: + os.waitpid(identity["monitor_pid"], 0) + except ChildProcessError: + pass + shutil.rmtree(root) + self.temp.cleanup() + self.libc.prctl(36, self.previous_subreaper, 0, 0, 0) + + def start(self): + command = "exec " + sys.executable + " -c 'import time;time.sleep(60)'" + return ServiceProcessHandle.start(self.service, command, cwd=self.temp.name) + + def test_freeze_attach_and_continue_original_process(self): + handle = self.start() + pid = handle.pid + stopped = handle.signal("stop") + self.assertTrue(stopped) + self.assertTrue(all(p["state"] in ("T", "t") for p in stopped)) + attached = ServiceProcessHandle.attach(self.service, handle.descriptor) + self.assertEqual(attached.pid, pid) + self.assertEqual(attached.process_id, handle.process_id) + attached.signal("continue") + self.assertNotEqual(attached.inspect()[0]["state"], "T") + + def test_container_restart_refuses_to_signal_old_pid(self): + handle = self.start() + self.service.current["started_at"] = "restarted" + with self.assertRaisesRegex(BackendOperationError, "replaced or restarted"): + handle.signal("stop") + + def test_modified_identity_record_refuses_to_signal(self): + handle = self.start() + root = Path("/tmp/akernel-service-process-" + handle.process_id) + record = json.loads((root / "identity.json").read_text()) + record["start_time"] = "0" + (root / "identity.json").write_text(json.dumps(record)) + with self.assertRaises(BackendOperationError): + handle.signal("stop") + + def test_unknown_start_recovers_by_token_without_relaunch(self): + counter = Path(self.temp.name) / "counter" + self.service.lose_start_reply = True + command = f"echo launched >> {counter}; exec sleep 60" + with self.assertRaises(TimeoutException) as raised: + ServiceProcessHandle.start(self.service, command) + attached = ServiceProcessHandle.attach( + self.service, raised.exception.process_descriptor + ) + self.assertGreater(attached.pid, 1) + self.assertEqual(counter.read_text(), "launched\n") + attached.signal("kill") + self.assertEqual(attached.wait(timeout=3), 137) + + def test_wait_reports_exit_code_and_times_out_without_killing(self): + handle = self.start() + with self.assertRaises(TimeoutException): + handle.wait(timeout=0.05) + self.assertTrue(handle.inspect()) + handle.signal("kill") + self.assertEqual(handle.wait(timeout=3), 137) + + def test_thaw_children_before_bash_waiter_does_not_report_sigstop_as_exit(self): + import time + from unittest.mock import patch + + from akernel_sdk import _service_process_guest as guest + + root = Path(self.temp.name) + ready, result = root / "ready", root / "wait-result" + command = ( + "set -m; sleep 1 & worker=$!; " + f'echo "$worker" > {ready}; wait "$worker"; echo "$?" > {result}' + ) + handle = ServiceProcessHandle.start(self.service, command) + deadline = time.monotonic() + 3 + while not ready.exists() and time.monotonic() < deadline: + time.sleep(0.01) + self.assertTrue(ready.exists()) + child = int(ready.read_text()) + child_identity = guest.stat(child) + process_root = Path("/tmp/akernel-service-process-" + handle.process_id) + kill = os.kill + + def scheduled_kill(pid, sig): + kill(pid, sig) + if sig == signal.SIGCONT and pid == handle.pid: + # Reproduce a scheduler switch to the waiting Bash immediately + # after its thaw. A still-stopped child must not yield status 147. + time.sleep(0.08) + + try: + handle.signal("stop") + with patch.object(guest.os, "kill", side_effect=scheduled_kill): + guest.operate(process_root, "continue") + self.assertEqual(handle.wait(timeout=3), 0) + self.assertEqual(result.read_text().strip(), "0") + finally: + current = guest.stat(child) + if ( + current + and child_identity + and current["start_time"] == child_identity["start_time"] + ): + try: + kill(child, signal.SIGKILL) + except ProcessLookupError: + pass + try: + os.waitpid(child, 0) + except ChildProcessError: + pass diff --git a/sdk/python/tests/unit/test_snapshot_create.py b/sdk/python/tests/unit/test_snapshot_create.py index 5ef112db..fc1ad627 100644 --- a/sdk/python/tests/unit/test_snapshot_create.py +++ b/sdk/python/tests/unit/test_snapshot_create.py @@ -63,6 +63,9 @@ def test_options_and_facades_match_legacy_restore(self): cwd="/not-created", port_forwardings=[8080], mounts=[Mount(target="/tools", image_url="tools:latest")], + extra_config={ + "fileBackedTmpfsMounts": [{"target": "/upper", "sizeMB": 64}] + }, reverse_tunnel=HttpReverseTunnel(target="http://127.0.0.1:8123"), detached=False, failover=True, @@ -105,6 +108,8 @@ def test_invalid_options_fail_before_loading_backend(self): {"snapshot_id": "saved", "port_forwardings": [8080, 8080]}, {"snapshot_id": "saved", "mounts": [object()]}, {"snapshot_id": "saved", "network_policy": {}}, + {"snapshot_id": "saved", "extra_config": []}, + {"snapshot_id": "saved", "extra_config": {"size": float("nan")}}, ): with ( self.subTest(options=options), diff --git a/sdk/python/tests/unit/test_snapshots.py b/sdk/python/tests/unit/test_snapshots.py index c970af88..7a9d6dbd 100644 --- a/sdk/python/tests/unit/test_snapshots.py +++ b/sdk/python/tests/unit/test_snapshots.py @@ -232,7 +232,14 @@ def deny(_client, request, **kwargs): patch.object(httpx.Client, "send", autospec=True, side_effect=deny), self.assertRaises(BackendOperationError) as raised, ): - Sandbox.from_snapshot("saved", cwd="/app", node_id="node-a") + Sandbox.from_snapshot( + "saved", + cwd="/app", + node_id="node-a", + extra_config={ + "fileBackedTmpfsMounts": [{"target": "/upper", "sizeMB": 64}] + }, + ) self.assertEqual(len(requests), 1) body = json.loads(requests[0].content) self.assertEqual(body["snapshotId"], "saved") @@ -247,6 +254,10 @@ def deny(_client, request, **kwargs): self.assertNotIn(key, body) self.assertEqual(body["rootfs"], {}) self.assertEqual(body["cwd"], "/app") + self.assertEqual( + body["extra_config"], + {"fileBackedTmpfsMounts": [{"target": "/upper", "sizeMB": 64}]}, + ) self.assertEqual(raised.exception.code, "PERMISSION_DENIED") self.assertEqual(raised.exception.request_id, "restore-test") self.assertEqual(raised.exception.outcome, "not_started") diff --git a/sdk/python/tests/unit/test_types.py b/sdk/python/tests/unit/test_types.py index b1c1a8ec..d9c77d9d 100644 --- a/sdk/python/tests/unit/test_types.py +++ b/sdk/python/tests/unit/test_types.py @@ -124,6 +124,13 @@ def test_public_exports_are_minimal(self): "DockerfileBuildError", "DockerfileParseError", "BuildInstruction", + "ComposeProject", + "ComposeOperationError", + "ServiceHandle", + "ServiceProcessHandle", + "DockerRuntime", + "ImageMountPlan", + "prepare_compose_images", }, ) for removed in (