From f8eec4cf0d52e0342209be0078257ba93d50d7f8 Mon Sep 17 00:00:00 2001 From: Enrique Saurez Date: Tue, 6 Oct 2026 13:06:24 -0700 Subject: [PATCH] Route native guests through a loopback proxy and publish forwarded ports The native-host backend accepts two more provision options. A runtimeConfig.networkProxy names an HTTP or HTTPS proxy on host loopback; it must be the guest's only way out, so the network policy denies egress without rules. The guest reaches the proxy over TCP at its gateway, and every workload gets HTTP_PROXY, HTTPS_PROXY, and NO_PROXY in both cases, pointing there. microvm.provision.hostLoopbackForwards publishes guest TCP and UDP ports on host loopback; forwarded ports need hostLoopback allow and an egress default of allow, because the guest's replies pass the egress filter. Executions may now set a working directory and environment variables layered over the guest's defaults (inheritDefaultEnv). Callers can never set the proxy variables. ExecuteCommandRequest gains working_directory and environment, and the guest runtime ABI moves to microvm-abi-v2-edge-ramfs-v3. New guest tests prove that the proxy is the only way out, that forwarded TCP and UDP ports reach guest listeners, and that the largest accepted environment reaches the workload. The example gains matching options. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- aci_edge_sandboxes/README.md | 87 +++- .../examples/nvxhost_lifecycle.rs | 87 +++- aci_edge_sandboxes/src/capabilities.rs | 7 +- aci_edge_sandboxes/src/lib.rs | 8 +- aci_edge_sandboxes/src/model.rs | 87 +++- aci_edge_sandboxes/src/openvmm/native.rs | 243 +++++++++- aci_edge_sandboxes/src/testing.rs | 2 + aci_edge_sandboxes/src/validate.rs | 105 +++- aci_edge_sandboxes/tests/nvxhost_guest.rs | 449 +++++++++++++++++- 9 files changed, 999 insertions(+), 76 deletions(-) diff --git a/aci_edge_sandboxes/README.md b/aci_edge_sandboxes/README.md index dbab18594..888ce64c4 100644 --- a/aci_edge_sandboxes/README.md +++ b/aci_edge_sandboxes/README.md @@ -167,20 +167,24 @@ and `guestBuildId`; stop reports `forced` and, after a failed graceful shutdown, `consoleError`. This backend supports provision/start/exec/stop/deprovision with shell commands or argv, a -fixed caller-provided image, and the host paths and network rules described below. Positive -execution timeouts must be whole seconds, matching the guest RPC's precision; finer-grained -timeouts fail validation rather than silently extending execution. It explicitly rejects piped -stdin, execution cancellation, custom working directories and environments. Snapshot/restore, -image selection, and richer guest operations are not part of this profile. See +fixed caller-provided image, and the host paths, network rules, proxy, forwarded ports, working +directories, and environments described below. Positive execution timeouts must be whole +seconds, matching the guest RPC's precision; finer-grained timeouts fail validation rather than +silently extending execution. It explicitly rejects piped stdin and execution cancellation. +Snapshot/restore, image selection, and richer guest operations are not part of this profile. See [`examples/nvxhost_lifecycle.rs`](examples/nvxhost_lifecycle.rs) for a run requiring `--openvmm`, `--kernel`, `--initrd`, `--image`, `--host-library`, `--host-sha256`, `--state-root`, `--hypervisor`, and a command after `--`; an optional `--image-sha256` requires the image's digest when it is registered. The repeatable `--readonly`, `--readwrite`, and `--denied` options map host paths, and `--egress allow|deny` with the repeatable `--egress-allow` and `--egress-deny` options, each taking `CIDR` or `CIDR:tcp|udp:PORT`, attach a -network. The ignored `tests/nvxhost_guest.rs` exercises an actual guest when the corresponding -`NVXHOST_TEST_*` paths and approved library digest are set: under WHP on Windows, or under the -hypervisor that `NVXHOST_TEST_HYPERVISOR` names, such as `mshv` on Linux. +network. `--network-proxy URL` routes the guest's traffic through a host proxy, and +`--host-loopback allow|deny` with the repeatable `--host-loopback-forward tcp|udp:HOST:GUEST` +publishes guest ports on host loopback. The repeatable `--env KEY=VALUE`, layered over the +guest's environment, and `--cwd PATH` apply to the command. The ignored +`tests/nvxhost_guest.rs` exercises an actual guest when the corresponding `NVXHOST_TEST_*` paths +and approved library digest are set: under WHP on Windows, or under the hypervisor that +`NVXHOST_TEST_HYPERVISOR` names, such as `mshv` on Linux. For example, from `aci_edge_sandboxes` on a Windows WHP host, set the following paths to compatible, separately built artifacts and a caller-prepared GPT disk @@ -206,7 +210,10 @@ To map `C:\work\src` read-only and `C:\work\out` read-write, and let the guest r `192.0.2.10` on TCP port 443, add `--readonly C:\work\src --readwrite C:\work\out --egress deny --egress-allow 192.0.2.10:tcp:443` before `--`. The example prints where each mapped path appears in the guest, here -`/mnt/c/work/src` and `/mnt/c/work/out`. +`/mnt/c/work/src` and `/mnt/c/work/out`. To let the guest out only through a proxy that listens +on host port 3128, add `--egress deny --network-proxy http://127.0.0.1:3128` instead; to reach +a guest service on port 8080 at host `127.0.0.1:18080`, add +`--egress allow --host-loopback allow --host-loopback-forward tcp:18080:8080`. Build the native library and the static edge initramfs separately from their matching private sources; this example neither fetches nor builds them. Use the @@ -229,9 +236,11 @@ the pinned objects: if one was replaced, start fails with `backend_error`, and t must be deprovisioned and provisioned again to accept the change. - The state root holds the record, and so the plan, of every sandbox, which decides what the - next start exports and which egress it allows. Provision therefore refuses, with - `policy_validation`, a mapped path inside the state root, and one that contains it unless a - denied path inside the mapping hides it. + next start exports, which egress it allows, and which host loopback ports the guest reaches + or publishes. Start's checks catch a damaged plan, but not one changed into another plan that + planning could have produced, so only the caller may write the state root. Provision + therefore refuses, with `policy_validation`, a mapped path inside the state root, and one that + contains it unless a denied path inside the mapping hides it. - OpenVMM exports the deepest directory that contains every mapped path through one virtio-fs device, read-only unless a path is read-write, and hides the denied paths. The guest mounts the export where workloads cannot reach it and bind-mounts each mapped path @@ -246,12 +255,53 @@ must be deprovisioned and provisioned again to accept the change. runs OpenVMM, so run OpenVMM unprivileged. A host hard link that already joins a file in a read-write mapping to one in a read-only mapping stays writable through the read-write path. -- A policy that denies egress without allow rules attaches no network device. Otherwise the - guest gets `OpenVmmConfig::guest_network` (`10.0.0.2/24` by default) behind OpenVMM's NAT - gateway, the network's first address, and names that gateway as its DNS server in - `/etc/resolv.conf` when the policy allows TCP or UDP port 53 to it. Choose a guest network - that contains no address the guest must reach. Ingress and host-loopback access must be - `deny`. +- A policy that denies egress without allow rules or a proxy attaches no network device. + Otherwise the guest gets `OpenVmmConfig::guest_network` (`10.0.0.2/24` by default) behind + OpenVMM's NAT gateway, the network's first address, and names that gateway as its DNS server + in `/etc/resolv.conf` when the policy allows TCP or UDP port 53 to it. Choose a guest network + that contains no address the guest must reach. Ingress must be `deny`, and host-loopback + access must be `deny` unless ports are forwarded, as described next. + +### Native proxy and forwarded ports + +Two provision options connect the guest to host loopback without opening it in general. They +are exclusive: a proxy needs host-loopback access denied, and forwarded ports need it allowed. + +- `runtimeConfig.networkProxy` (`ProvisionRequest::with_network_proxy`) names an HTTP or HTTPS + proxy that listens on host IPv4 loopback, such as `http://127.0.0.1:3128` or + `http://localhost:3128`: an explicit port, and at most a trailing `/` after it. As in MXC's + schema, the proxy is the guest's only way out, so the network policy must deny egress without + allow or deny rules; provision rejects other policies, and IPv6 or remote proxies, with + `policy_validation`. The guest's only reachable destination is TCP to its gateway at the + proxy's port, which OpenVMM connects to the proxy; it cannot reach a DNS server, so the proxy + resolves names. Every workload gets `HTTP_PROXY`, `HTTPS_PROXY`, `http_proxy`, and + `https_proxy` naming the proxy at the gateway, such as `http://10.0.0.1:3128`, and `NO_PROXY` + and `no_proxy` set to `localhost,127.0.0.1`; an HTTPS proxy's certificate must therefore be + valid for the gateway's address. +- `microvm.provision.hostLoopbackForwards` (`ProvisionRequest::with_host_loopback_forward`), + an extension, publishes up to 64 guest ports on host loopback: OpenVMM listens on + `127.0.0.1:hostPort` and relays each TCP connection or UDP datagram to the guest's + `guestPort`. Ports are nonzero, and a host port appears at most once per protocol. Forwarded + ports need `network.ingress.hostLoopback: allow`, which also lets the guest reach every host + loopback service through its gateway; `allow` without forwarded ports is rejected. The guest + answers forwarded traffic at addresses that OpenVMM assigns inside the guest network, so the + egress default must be `allow`, which leaves the guest's other egress open, and no deny rule + may cover the guest network. + +Only the guest sets the proxy variables: an exec whose `process.env` names one of them, in any +case, fails with `policy_validation`, with or without a proxy. + +### Native working directories and environments + +`process.cwd` must be an absolute guest path without `..` of at most 4095 bytes, as with the +direct backend; `openvmm::resolve_guest_path` gives the guest path of a mapped host path. A +directory that the workload cannot enter ends the command with exit code 126 and an +`NVX-EDGE-STAGE-ERROR` line on standard error. The guest's default environment holds only +`PATH` (`/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin`) and the proxy +variables. `process.env` can only be layered over it, with `inheritDefaultEnv: true`, as at +most 256 entries of at most 32 KiB in total; replacing it fails with `policy_validation`. A +later entry for a name replaces an earlier one, and an entry replaces the default of the same +name. The caller must independently approve and protect the native asset. Checking a caller-supplied digest does not make a writable path or a self-declared digest trustworthy; use this profile @@ -633,6 +683,7 @@ An MXC `StatefulSandboxBackend` adapter maps onto this crate as follows: | `policy.network_egress` rules | `EgressPolicy` rules, field for field | | `working_directory` | `ExecRequest::with_cwd(guest_path(...))`; a directory that the workload cannot enter ends with `Failed(WorkingDirectory)` | | `process.env`, `process.inheritDefaultEnv` | `ProcessSpec::env`, `None` when omitted, and `inherit_default_env`, or `ExecRequest::with_envs` and `with_inherit_default_env`; see [Environment](#environment) | +| `runtimeConfig.networkProxy` | `ProvisionRequest::with_network_proxy` with the `nvxhost` backend; see [Native proxy and forwarded ports](#native-proxy-and-forwarded-ports) | A proof-of-concept MXC adapter, `nvx_backend`, implements this mapping. It runs each phase in its own process against a real VM and consumes exec pipes diff --git a/aci_edge_sandboxes/examples/nvxhost_lifecycle.rs b/aci_edge_sandboxes/examples/nvxhost_lifecycle.rs index 574c7d5ea..b5ea24b2d 100644 --- a/aci_edge_sandboxes/examples/nvxhost_lifecycle.rs +++ b/aci_edge_sandboxes/examples/nvxhost_lifecycle.rs @@ -10,7 +10,7 @@ use aci_edge_sandboxes::openvmm::{ }; use aci_edge_sandboxes::{ Access, AciEdgeSandbox, EgressPolicy, ExecOutcome, ExecRequest, FilesystemPolicy, - NetworkPolicy, NetworkRule, Protocol, ProvisionRequest, + ForwardProtocol, HostLoopbackForward, NetworkPolicy, NetworkRule, Protocol, ProvisionRequest, }; fn main() -> ExitCode { @@ -37,6 +37,11 @@ fn run() -> Result<(), String> { let mut egress = None; let mut allow = Vec::new(); let mut deny = Vec::new(); + let mut host_loopback = None; + let mut proxy = None; + let mut forwards = Vec::new(); + let mut environment = Vec::new(); + let mut cwd = None; let mut command = None; let mut args = std::env::args().skip(1); while let Some(option) = args.next() { @@ -62,6 +67,11 @@ fn run() -> Result<(), String> { "--egress" => egress = Some(parse_access(&option, &value()?)?), "--egress-allow" => allow.push(parse_rule(&option, &value()?)?), "--egress-deny" => deny.push(parse_rule(&option, &value()?)?), + "--host-loopback" => host_loopback = Some(parse_access(&option, &value()?)?), + "--network-proxy" => proxy = Some(value()?), + "--host-loopback-forward" => forwards.push(parse_forward(&option, &value()?)?), + "--env" => environment.push(value()?), + "--cwd" => cwd = Some(value()?), "--" => { command = Some(args.by_ref().collect::>().join(" ")); break; @@ -97,17 +107,38 @@ fn run() -> Result<(), String> { } match egress { Some(default) => { - request = request.with_network(NetworkPolicy { + let mut network = NetworkPolicy { egress: EgressPolicy { default, allow, deny, }, ..NetworkPolicy::deny_all() - }); + }; + if let Some(access) = host_loopback { + network.ingress.host_loopback = Some(access); + } + request = request.with_network(network); } - None if allow.is_empty() && deny.is_empty() => {} - None => return Err("--egress-allow and --egress-deny require --egress".to_owned()), + None if allow.is_empty() && deny.is_empty() && host_loopback.is_none() => {} + None => { + return Err( + "--egress-allow, --egress-deny, and --host-loopback require --egress".to_owned(), + ); + } + } + if let Some(proxy) = proxy { + request = request.with_network_proxy(proxy); + } + for forward in forwards { + request = request.with_host_loopback_forward(forward); + } + let mut exec = ExecRequest::command_line(command); + if let Some(cwd) = cwd { + exec = exec.with_cwd(cwd); + } + if !environment.is_empty() { + exec = exec.with_envs(environment).with_inherit_default_env(true); } let config = OpenVmmConfig::new(openvmm, kernel, initrd, hypervisor, PathBuf::from(root)); @@ -122,9 +153,7 @@ fn run() -> Result<(), String> { let started = client.start(&id); let succeeded = started.is_ok(); let executed = started.and_then(|_| { - let output = client - .exec(&id, &ExecRequest::command_line(command))? - .wait_with_output()?; + let output = client.exec(&id, &exec)?.wait_with_output()?; let logs = backend.guest_logs(&id)?; if logs.is_empty() || !logs.windows(8).any(|window| window == b"execute:") { return Err(aci_edge_sandboxes::Error::new( @@ -224,6 +253,25 @@ fn parse_rule(option: &str, text: &str) -> Result { } } +/// Parses `tcp|udp:HOST-PORT:GUEST-PORT`, such as `tcp:3000:8080`. +fn parse_forward(option: &str, text: &str) -> Result { + let malformed = || format!("{option} must be tcp|udp:HOST-PORT:GUEST-PORT"); + let parts: Vec<&str> = text.split(':').collect(); + let [protocol, host, guest] = parts.as_slice() else { + return Err(malformed()); + }; + let protocol = match *protocol { + "tcp" => ForwardProtocol::Tcp, + "udp" => ForwardProtocol::Udp, + _ => return Err(malformed()), + }; + let port = |text: &str| text.parse::().ok().filter(|port| *port != 0); + match (port(host), port(guest)) { + (Some(host), Some(guest)) => Ok(HostLoopbackForward::new(protocol, host, guest)), + _ => Err(malformed()), + } +} + fn describe(error: aci_edge_sandboxes::Error) -> String { error.to_string() } @@ -264,4 +312,27 @@ mod tests { assert_eq!(parse_access("--egress", "deny").unwrap(), Access::Deny); assert!(parse_access("--egress", "Deny").is_err()); } + + #[test] + fn forwards_name_a_protocol_and_two_ports() { + let parse = |text: &str| parse_forward("--host-loopback-forward", text); + assert_eq!( + parse("tcp:3000:8080").unwrap(), + HostLoopbackForward::new(ForwardProtocol::Tcp, 3000, 8080) + ); + assert_eq!( + parse("udp:5353:53").unwrap(), + HostLoopbackForward::new(ForwardProtocol::Udp, 5353, 53) + ); + for malformed in [ + "", + "tcp:3000", + "tcp:0:80", + "tcp:80:0", + "icmp:1:1", + "tcp:1:2:3", + ] { + assert!(parse(malformed).is_err(), "{malformed}"); + } + } } diff --git a/aci_edge_sandboxes/src/capabilities.rs b/aci_edge_sandboxes/src/capabilities.rs index 5c47f3d13..7770272ae 100644 --- a/aci_edge_sandboxes/src/capabilities.rs +++ b/aci_edge_sandboxes/src/capabilities.rs @@ -73,12 +73,17 @@ pub struct NetworkCapabilities { pub ingress_allow: bool, /// Honors `network.ingress.default: deny`. pub ingress_deny: bool, - /// Honors `network.ingress.hostLoopback: allow`. + /// Honors `network.ingress.hostLoopback: allow` without forwarded ports. pub host_loopback_allow: bool, /// Honors `network.ingress.hostLoopback: deny`. pub host_loopback_deny: bool, /// Honors `network.egress.allow` and `network.egress.deny` rules. pub egress_rules: bool, + /// Honors `runtimeConfig.networkProxy`. + pub network_proxy: bool, + /// Honors `microvm.provision.hostLoopbackForwards`, with + /// `network.ingress.hostLoopback: allow`. + pub host_loopback_forwards: bool, } /// Host filesystem mappings a backend can honor. diff --git a/aci_edge_sandboxes/src/lib.rs b/aci_edge_sandboxes/src/lib.rs index c423d6196..a4bf37199 100644 --- a/aci_edge_sandboxes/src/lib.rs +++ b/aci_edge_sandboxes/src/lib.rs @@ -82,8 +82,8 @@ pub use exec::{Canceller, ExecFailure, ExecOutcome, ExecOutput, Execution}; pub use id::SandboxId; pub use input::{InputCloser, InputSource}; pub use model::{ - Access, Command, DeprovisionResult, EgressPolicy, ExecRequest, FilesystemPolicy, IngressPolicy, - Metadata, MicrovmConfig, MicrovmProvision, NetworkPeer, NetworkPolicy, NetworkPort, - NetworkRule, ProcessSpec, Protocol, ProvisionRequest, ProvisionResult, StartResult, StdinMode, - StopResult, + Access, Command, DeprovisionResult, EgressPolicy, ExecRequest, FilesystemPolicy, + ForwardProtocol, HostLoopbackForward, IngressPolicy, Metadata, MicrovmConfig, MicrovmProvision, + NetworkPeer, NetworkPolicy, NetworkPort, NetworkRule, ProcessSpec, Protocol, ProvisionRequest, + ProvisionResult, RuntimeConfig, StartResult, StdinMode, StopResult, }; diff --git a/aci_edge_sandboxes/src/model.rs b/aci_edge_sandboxes/src/model.rs index fd37e98b3..b50a84030 100644 --- a/aci_edge_sandboxes/src/model.rs +++ b/aci_edge_sandboxes/src/model.rs @@ -14,9 +14,9 @@ pub type Metadata = serde_json::Map; /// Inputs to [`AciEdgeSandbox::provision`](crate::AciEdgeSandbox::provision). /// /// The serialized form matches the policy fields of the contract's provision request -/// (`filesystem`, `network`, and `microvm`). Envelope fields such as `version` and `phase` belong -/// to the caller's wire layer and are rejected here. Every field is optional, so `{}` provisions a -/// sandbox with the backend defaults. +/// (`filesystem`, `network`, `runtimeConfig`, and `microvm`). Envelope fields such as `version` +/// and `phase` belong to the caller's wire layer and are rejected here. Every field is optional, +/// so `{}` provisions a sandbox with the backend defaults. /// /// The sandbox runs the guest's own Alpine Linux userland directly; there are no image layers or /// scratch disks. Guest state lives in memory and lasts until the sandbox stops. @@ -29,6 +29,10 @@ pub struct ProvisionRequest { /// Network posture. `None` attaches no network device. #[serde(default, skip_serializing_if = "Option::is_none")] pub network: Option, + /// Runtime connectivity. The sandbox takes it at provision because its network is fixed + /// when the microVM starts. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub runtime_config: Option, /// MicroVM configuration. #[serde(default, skip_serializing_if = "MicrovmConfig::is_default")] pub microvm: MicrovmConfig, @@ -60,6 +64,35 @@ impl ProvisionRequest { self.microvm.provision.memory_mib = Some(memory_mib); self } + + /// Routes the guest's traffic through a proxy on host loopback, such as + /// `http://127.0.0.1:8080`; see [`RuntimeConfig::network_proxy`]. + #[must_use] + pub fn with_network_proxy(mut self, url: impl Into) -> Self { + self.runtime_config + .get_or_insert_with(RuntimeConfig::default) + .network_proxy = Some(url.into()); + self + } + + /// Publishes a guest port on a host loopback port; see + /// [`MicrovmProvision::host_loopback_forwards`]. + #[must_use] + pub fn with_host_loopback_forward(mut self, forward: HostLoopbackForward) -> Self { + self.microvm.provision.host_loopback_forwards.push(forward); + self + } +} + +/// The contract's `runtimeConfig` section. +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct RuntimeConfig { + /// An HTTP or HTTPS proxy on host loopback with an explicit port, such as + /// `http://127.0.0.1:8080`. It must be the guest's only way out: the network's egress default + /// is `deny`, without allow or deny rules. Workloads receive the proxy variables. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub network_proxy: Option, } /// The contract's `microvm` section. @@ -85,6 +118,54 @@ pub struct MicrovmProvision { /// default. #[serde(default, skip_serializing_if = "Option::is_none")] pub memory_mib: Option, + /// Host loopback ports that each reach one guest port, an extension to the contract. + /// Forwarded ports need `network.ingress.hostLoopback: allow`, which also lets the guest + /// reach host loopback services. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub host_loopback_forwards: Vec, +} + +/// A host loopback port forwarded to a guest port. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct HostLoopbackForward { + /// Transport protocol. + pub protocol: ForwardProtocol, + /// Port on host loopback (`127.0.0.1`). + pub host_port: u16, + /// Port inside the guest. + pub guest_port: u16, +} + +impl HostLoopbackForward { + /// Forwards `host_port` on host loopback to `guest_port` in the guest. + pub fn new(protocol: ForwardProtocol, host_port: u16, guest_port: u16) -> Self { + Self { + protocol, + host_port, + guest_port, + } + } +} + +/// Transport protocol of a [`HostLoopbackForward`]. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[serde(rename_all = "lowercase")] +pub enum ForwardProtocol { + /// TCP. + Tcp, + /// UDP. + Udp, +} + +impl ForwardProtocol { + /// Returns the contract spelling of this protocol. + pub const fn as_str(self) -> &'static str { + match self { + Self::Tcp => "tcp", + Self::Udp => "udp", + } + } } /// The contract's `filesystem` section. diff --git a/aci_edge_sandboxes/src/openvmm/native.rs b/aci_edge_sandboxes/src/openvmm/native.rs index e57643e4f..fc422f0b9 100644 --- a/aci_edge_sandboxes/src/openvmm/native.rs +++ b/aci_edge_sandboxes/src/openvmm/native.rs @@ -30,15 +30,20 @@ use crate::error::{Error, ErrorCode, Result}; use crate::exec::{Completion, ExecOutcome}; use crate::id::SandboxId; use crate::model::{ - Command, DeprovisionResult, ExecRequest, FilesystemPolicy, Metadata, NetworkPolicy, - ProvisionRequest, ProvisionResult, StartResult, StdinMode, StopResult, + Command, DeprovisionResult, ExecRequest, FilesystemPolicy, HostLoopbackForward, Metadata, + NetworkPolicy, ProvisionRequest, ProvisionResult, RuntimeConfig, StartResult, StdinMode, + StopResult, }; use crate::nvxhost::{HostLibrary, LaunchInputs, Session}; const BACKEND_KEY: &str = NATIVE_BACKEND_KEY; -const RUNTIME_ABI: &str = "microvm-abi-v2-edge-ramfs-v2"; +const RUNTIME_ABI: &str = "microvm-abi-v2-edge-ramfs-v3"; const MAX_EXEC_SECONDS: u64 = 3600; const MAX_OUTPUT_BYTES: usize = 1 << 20; +/// Largest number of `process.env` entries, as for the direct backend. +const MAX_ENVIRONMENT: usize = super::protocol::MAX_ENVIRONMENT; +/// Largest combined size of the `process.env` entries, which keeps exec requests small. +const MAX_ENVIRONMENT_BYTES: usize = 32 * 1024; const SHELL: &str = "/bin/sh"; const CONSOLE_POLL: Duration = Duration::from_millis(250); @@ -248,6 +253,10 @@ struct ExecuteCommandRequest { command: String, #[prost(string, repeated, tag = "2")] args: Vec, + #[prost(string, tag = "3")] + working_directory: String, + #[prost(btree_map = "string, string", tag = "4")] + environment: std::collections::BTreeMap, #[prost(int32, tag = "5")] timeout_seconds: i32, } @@ -1155,6 +1164,8 @@ fn capabilities() -> Capabilities { let mut capabilities = Capabilities::new(BACKEND_KEY); capabilities.exec.command_line = true; capabilities.exec.argv = true; + capabilities.exec.cwd = true; + capabilities.exec.env = true; capabilities.exec.max_timeout_ms = Some(MAX_EXEC_SECONDS * 1_000); capabilities.exec.max_output_bytes = Some(MAX_OUTPUT_BYTES as u64); capabilities.network.egress_allow = true; @@ -1162,28 +1173,56 @@ fn capabilities() -> Capabilities { capabilities.network.ingress_deny = true; capabilities.network.host_loopback_deny = true; capabilities.network.egress_rules = true; + capabilities.network.network_proxy = true; + capabilities.network.host_loopback_forwards = true; capabilities.filesystem.readonly_paths = true; capabilities.filesystem.readwrite_paths = true; capabilities.filesystem.denied_paths = true; capabilities } -/// The JSON of `request`'s `filesystem` and `network` sections, which nvxhost plans, or `None` -/// when the request has neither. +/// The JSON of the sections of `request` that nvxhost plans: `filesystem`, `network`, +/// `runtimeConfig`, and the forwarded ports of `microvm.provision`. `None` when the request has +/// none of them. fn device_policy(request: &ProvisionRequest) -> Result> { #[derive(serde::Serialize)] + #[serde(rename_all = "camelCase")] struct Policy<'a> { #[serde(skip_serializing_if = "Option::is_none")] filesystem: Option<&'a FilesystemPolicy>, #[serde(skip_serializing_if = "Option::is_none")] network: Option<&'a NetworkPolicy>, + #[serde(skip_serializing_if = "Option::is_none")] + runtime_config: Option<&'a RuntimeConfig>, + #[serde(skip_serializing_if = "Option::is_none")] + microvm: Option>, + } + #[derive(serde::Serialize)] + struct Microvm<'a> { + provision: Provision<'a>, } - if request.filesystem.is_none() && request.network.is_none() { + #[derive(serde::Serialize)] + #[serde(rename_all = "camelCase")] + struct Provision<'a> { + host_loopback_forwards: &'a [HostLoopbackForward], + } + let forwards = &request.microvm.provision.host_loopback_forwards; + if request.filesystem.is_none() + && request.network.is_none() + && request.runtime_config.is_none() + && forwards.is_empty() + { return Ok(None); } serde_json::to_string(&Policy { filesystem: request.filesystem.as_ref(), network: request.network.as_ref(), + runtime_config: request.runtime_config.as_ref(), + microvm: (!forwards.is_empty()).then_some(Microvm { + provision: Provision { + host_loopback_forwards: forwards, + }, + }), }) .map(Some) .map_err(|error| { @@ -1301,15 +1340,13 @@ fn copy_console( } fn prepare_exec(request: &ExecRequest) -> Result<(ExecuteCommandRequest, Option)> { - if request.stdin == StdinMode::Piped - || request.process.cwd.is_some() - || request.process.env.is_some() - { + if request.stdin == StdinMode::Piped { return Err(Error::policy_validation( - "the nvxhost backend supports no piped stdin, custom working directory, or environment", + "the nvxhost backend supports no piped stdin", )); } - let argv = match &request.process.command { + let process = &request.process; + let argv = match &process.command { Command::CommandLine(line) => vec![SHELL.to_owned(), "-c".to_owned(), line.clone()], Command::Argv(argv) => argv.clone(), }; @@ -1321,7 +1358,7 @@ fn prepare_exec(request: &ExecRequest) -> Result<(ExecuteCommandRequest, Option< "the guest command must contain non-NUL arguments totaling at most 4096 bytes", )); } - let timeout = request.process.timeout.filter(|timeout| !timeout.is_zero()); + let timeout = process.timeout.filter(|timeout| !timeout.is_zero()); if let Some(timeout) = timeout { if timeout > Duration::from_secs(MAX_EXEC_SECONDS) { return Err(Error::policy_validation( @@ -1339,12 +1376,77 @@ fn prepare_exec(request: &ExecRequest) -> Result<(ExecuteCommandRequest, Option< ExecuteCommandRequest { command: argv[0].clone(), args: argv[1..].to_vec(), + working_directory: working_directory(process.cwd.as_deref())?, + environment: environment(process.env.as_deref(), process.inherit_default_env)?, timeout_seconds: seconds, }, timeout, )) } +/// The guest working directory of `cwd`: absolute, without `..`, and bounded like the direct +/// backend's. Empty selects the guest default, `/`. +fn working_directory(cwd: Option<&str>) -> Result { + let Some(cwd) = cwd else { + return Ok(String::new()); + }; + if !cwd.starts_with('/') + || cwd.len() > super::protocol::MAX_CWD_BYTES + || cwd.split('/').any(|part| part == "..") + { + return Err(Error::policy_validation(format!( + "process.cwd {cwd:?} must be an absolute guest path without '..' of at most {} bytes; \ + map host paths with openvmm::guest_path", + super::protocol::MAX_CWD_BYTES + ))); + } + Ok(cwd.to_owned()) +} + +/// The variables that `process.env` layers over the guest's default environment, `PATH`; a +/// later entry for a name replaces an earlier one. +/// +/// The guest sets the proxy variables itself, so `HTTP_PROXY`, `HTTPS_PROXY`, and `NO_PROXY`, in +/// any case, are refused. +fn environment( + entries: Option<&[String]>, + inherit_default_env: Option, +) -> Result> { + let Some(entries) = entries else { + return Ok(std::collections::BTreeMap::new()); + }; + if inherit_default_env != Some(true) { + return Err(Error::policy_validation( + "the nvxhost backend layers process.env over the guest's default environment; set \ + inheritDefaultEnv", + )); + } + if entries.len() > MAX_ENVIRONMENT + || entries.iter().map(String::len).sum::() > MAX_ENVIRONMENT_BYTES + { + return Err(Error::policy_validation(format!( + "process.env holds at most {MAX_ENVIRONMENT} entries of at most \ + {MAX_ENVIRONMENT_BYTES} bytes in total" + ))); + } + let mut environment = std::collections::BTreeMap::new(); + for entry in entries { + let (name, value) = entry.split_once('=').ok_or_else(|| { + Error::policy_validation("process.env entries must have the form KEY=VALUE") + })?; + if ["HTTP_PROXY", "HTTPS_PROXY", "NO_PROXY"] + .iter() + .any(|reserved| name.eq_ignore_ascii_case(reserved)) + { + return Err(Error::policy_validation(format!( + "process.env may not set {name}; the guest sets the proxy variables" + ))); + } + environment.insert(name.to_owned(), value.to_owned()); + } + Ok(environment) +} + fn execute(job: ExecJob) -> Result { let ExecJob { host, @@ -1554,9 +1656,46 @@ mod tests { assert_eq!(request.timeout_seconds, 2); assert_eq!(timeout, Some(Duration::from_secs(2))); assert!(request.encode_to_vec().len() < 100); + assert!(request.working_directory.is_empty() && request.environment.is_empty()); assert!(capabilities().exec.command_line); assert!(!capabilities().exec.cancel); - assert!(!capabilities().exec.cwd); + assert!(capabilities().exec.cwd && capabilities().exec.env); + assert!(!capabilities().exec.clear_default_env); + } + + #[test] + fn working_directories_and_layered_environments_reach_the_guest() { + let (request, _) = prepare_exec( + &ExecRequest::argv(["/usr/bin/env"]) + .with_cwd("/mnt/c/work") + .with_envs(["A=1", "B=x=y", "A=2", "EMPTY="]) + .with_inherit_default_env(true), + ) + .unwrap(); + assert_eq!(request.working_directory, "/mnt/c/work"); + assert_eq!( + request.environment.into_iter().collect::>(), + [ + ("A".to_owned(), "2".to_owned()), + ("B".to_owned(), "x=y".to_owned()), + ("EMPTY".to_owned(), String::new()), + ] + ); + let decoded = ExecuteCommandRequest::decode( + prepare_exec( + &ExecRequest::command_line("pwd") + .with_cwd("/tmp") + .with_env("K=V") + .with_inherit_default_env(true), + ) + .unwrap() + .0 + .encode_to_vec() + .as_slice(), + ) + .unwrap(); + assert_eq!(decoded.working_directory, "/tmp"); + assert_eq!(decoded.environment["K"], "V"); } #[test] @@ -1567,6 +1706,7 @@ mod tests { assert!(capabilities.filesystem.denied_paths); assert!(capabilities.network.egress_allow && capabilities.network.egress_rules); assert!(!capabilities.network.ingress_allow && !capabilities.network.host_loopback_allow); + assert!(capabilities.network.network_proxy && capabilities.network.host_loopback_forwards); assert_eq!(device_policy(&ProvisionRequest::default()).unwrap(), None); // The JSON is nvxhost's planning input; its own tests parse this exact text. @@ -1601,6 +1741,33 @@ mod tests { device_policy(&request).unwrap().unwrap(), r#"{"filesystem":{"readonlyPaths":["/work/src"],"readwritePaths":["/work/out"],"deniedPaths":["/work/src/secret"]},"network":{"egress":{"default":"deny","allow":[{"to":[{"cidr":"192.0.2.0/24","except":["192.0.2.128/25"]}],"ports":[{"protocol":"tcp","port":443,"endPort":444}]}],"deny":[{"to":[{"cidr":"192.0.2.7"}]}]},"ingress":{"default":"deny","hostLoopback":"deny"}}}"# ); + + // The proxy and the forwarded ports, also in nvxhost's tests; memory stays out. + let proxied = ProvisionRequest::new() + .with_network(NetworkPolicy::deny_all()) + .with_network_proxy("http://127.0.0.1:8080") + .with_memory_mib(512); + assert_eq!( + device_policy(&proxied).unwrap().unwrap(), + r#"{"network":{"egress":{"default":"deny"},"ingress":{"default":"deny","hostLoopback":"deny"}},"runtimeConfig":{"networkProxy":"http://127.0.0.1:8080"}}"# + ); + let mut loopback = NetworkPolicy::egress(crate::model::Access::Allow); + loopback.ingress.host_loopback = Some(crate::model::Access::Allow); + let forwarded = ProvisionRequest::new() + .with_network(loopback) + .with_host_loopback_forward(crate::model::HostLoopbackForward::new( + crate::model::ForwardProtocol::Tcp, + 3000, + 8080, + )); + assert_eq!( + device_policy(&forwarded).unwrap().unwrap(), + r#"{"network":{"egress":{"default":"allow"},"ingress":{"default":"deny","hostLoopback":"allow"}},"microvm":{"provision":{"hostLoopbackForwards":[{"protocol":"tcp","hostPort":3000,"guestPort":8080}]}}}"# + ); + assert_eq!( + device_policy(&ProvisionRequest::new().with_memory_mib(512)).unwrap(), + None + ); } #[test] @@ -1661,16 +1828,44 @@ mod tests { #[test] fn unsupported_exec_options_fail_before_guest_work() { - let request = ExecRequest::command_line("echo READY").with_cwd("/tmp"); - assert_eq!( - prepare_exec(&request).unwrap_err().code(), - ErrorCode::PolicyValidation - ); - let request = ExecRequest::command_line("echo READY").with_env("K=V"); - assert_eq!( - prepare_exec(&request).unwrap_err().code(), - ErrorCode::PolicyValidation - ); + let long = format!("/{}", "d".repeat(super::super::protocol::MAX_CWD_BYTES)); + let many: Vec = (0..=MAX_ENVIRONMENT) + .map(|index| format!("V{index}=1")) + .collect(); + let large = vec![format!("BIG={}", "x".repeat(MAX_ENVIRONMENT_BYTES))]; + for request in [ + ExecRequest::command_line("cat").with_stdin(StdinMode::Piped), + ExecRequest::command_line("pwd").with_cwd("tmp"), + ExecRequest::command_line("pwd").with_cwd("/tmp/../root"), + ExecRequest::command_line("pwd").with_cwd(long.as_str()), + // The guest layers the environment over its defaults; it cannot replace them. + ExecRequest::command_line("env").with_env("K=V"), + ExecRequest::command_line("env") + .with_env("K=V") + .with_inherit_default_env(false), + ExecRequest::command_line("env") + .with_envs(many) + .with_inherit_default_env(true), + ExecRequest::command_line("env") + .with_envs(large) + .with_inherit_default_env(true), + ] { + assert_eq!( + prepare_exec(&request).unwrap_err().code(), + ErrorCode::PolicyValidation, + "{request:?}" + ); + } + for name in ["HTTP_PROXY", "https_proxy", "No_Proxy"] { + let error = prepare_exec( + &ExecRequest::command_line("env") + .with_env(format!("{name}=http://192.0.2.1:3128")) + .with_inherit_default_env(true), + ) + .unwrap_err(); + assert_eq!(error.code(), ErrorCode::PolicyValidation); + assert!(error.message().contains(name), "{error}"); + } } fn host_hypervisor() -> super::super::Hypervisor { diff --git a/aci_edge_sandboxes/src/testing.rs b/aci_edge_sandboxes/src/testing.rs index 6f672a739..237dac00a 100644 --- a/aci_edge_sandboxes/src/testing.rs +++ b/aci_edge_sandboxes/src/testing.rs @@ -91,6 +91,8 @@ impl MockBackend { network.host_loopback_allow = true; network.host_loopback_deny = true; network.egress_rules = true; + network.network_proxy = true; + network.host_loopback_forwards = true; let filesystem = &mut capabilities.filesystem; filesystem.readonly_paths = true; filesystem.readwrite_paths = true; diff --git a/aci_edge_sandboxes/src/validate.rs b/aci_edge_sandboxes/src/validate.rs index 245ebad2c..a51e9c458 100644 --- a/aci_edge_sandboxes/src/validate.rs +++ b/aci_edge_sandboxes/src/validate.rs @@ -40,6 +40,32 @@ pub(crate) fn provision_structure(request: &ProvisionRequest) -> Result<()> { } } } + if let Some(proxy) = request + .runtime_config + .as_ref() + .and_then(|config| config.network_proxy.as_deref()) + { + if proxy.trim().is_empty() { + return Err(Error::malformed_request( + "runtimeConfig.networkProxy is empty", + )); + } + no_nul(proxy, "runtimeConfig.networkProxy")?; + } + for (index, forward) in request + .microvm + .provision + .host_loopback_forwards + .iter() + .enumerate() + { + if forward.host_port == 0 || forward.guest_port == 0 { + return Err(Error::malformed_request(format!( + "microvm.provision.hostLoopbackForwards[{index}]: ports must be between 1 and \ + 65535" + ))); + } + } Ok(()) } @@ -123,11 +149,14 @@ pub(crate) fn provision_capabilities( check_access(backend, field, access, allow, deny)?; } if let Some(access) = network.ingress.host_loopback { + // Forwarded ports are the only host-loopback access a backend may honor. + let forwarded = supported.host_loopback_forwards + && !request.microvm.provision.host_loopback_forwards.is_empty(); check_access( backend, "network.ingress.hostLoopback", access, - supported.host_loopback_allow, + supported.host_loopback_allow || forwarded, supported.host_loopback_deny, )?; } @@ -138,6 +167,22 @@ pub(crate) fn provision_capabilities( ))); } } + let proxy = request + .runtime_config + .as_ref() + .is_some_and(|config| config.network_proxy.is_some()); + if proxy && !capabilities.network.network_proxy { + return Err(Error::policy_validation(format!( + "the {backend} backend cannot enforce runtimeConfig.networkProxy" + ))); + } + if !request.microvm.provision.host_loopback_forwards.is_empty() + && !capabilities.network.host_loopback_forwards + { + return Err(Error::policy_validation(format!( + "the {backend} backend cannot enforce microvm.provision.hostLoopbackForwards" + ))); + } Ok(()) } @@ -412,6 +457,64 @@ mod tests { provision_capabilities(&valid, &supported).unwrap(); } + #[test] + fn proxies_and_forwards_need_their_capabilities() { + use crate::model::{ForwardProtocol, HostLoopbackForward}; + + let proxied = request() + .with_network(NetworkPolicy::deny_all()) + .with_network_proxy("http://127.0.0.1:8080"); + provision_structure(&proxied).unwrap(); + assert_eq!( + code(provision_capabilities(&proxied, &capabilities())), + ErrorCode::PolicyValidation + ); + let mut supported = capabilities(); + supported.network.network_proxy = true; + provision_capabilities(&proxied, &supported).unwrap(); + for proxy in ["", " ", "http://127.0.0.1:80\0"] { + assert_eq!( + code(provision_structure(&request().with_network_proxy(proxy))), + ErrorCode::MalformedRequest, + "{proxy:?}" + ); + } + + let mut loopback = NetworkPolicy::egress(Access::Allow); + loopback.ingress.host_loopback = Some(Access::Allow); + let forward = HostLoopbackForward::new(ForwardProtocol::Tcp, 3000, 8080); + let forwarded = request() + .with_network(loopback.clone()) + .with_host_loopback_forward(forward); + provision_structure(&forwarded).unwrap(); + assert_eq!( + code(provision_capabilities(&forwarded, &capabilities())), + ErrorCode::PolicyValidation + ); + let mut supported = capabilities(); + supported.network.host_loopback_forwards = true; + provision_capabilities(&forwarded, &supported).unwrap(); + // Without forwarded ports, host-loopback access needs the generic capability. + assert_eq!( + code(provision_capabilities( + &request().with_network(loopback), + &supported + )), + ErrorCode::PolicyValidation + ); + for (host_port, guest_port) in [(0, 53), (53, 0)] { + let zero = request().with_host_loopback_forward(HostLoopbackForward::new( + ForwardProtocol::Udp, + host_port, + guest_port, + )); + assert_eq!( + code(provision_structure(&zero)), + ErrorCode::MalformedRequest + ); + } + } + #[test] fn exec_structure_rejects_shape_errors() { for request in [ diff --git a/aci_edge_sandboxes/tests/nvxhost_guest.rs b/aci_edge_sandboxes/tests/nvxhost_guest.rs index 9a665ccc8..982a40b48 100644 --- a/aci_edge_sandboxes/tests/nvxhost_guest.rs +++ b/aci_edge_sandboxes/tests/nvxhost_guest.rs @@ -1,12 +1,13 @@ -//! Opt-in lifecycle, robustness, host-path, and network proofs with a separately supplied native -//! library and guest. +//! Opt-in lifecycle, robustness, host-path, network, proxy, port-forwarding, and exec-environment +//! proofs with a separately supplied native library and guest. //! //! The tests need `NVXHOST_TEST_OPENVMM`, `NVXHOST_TEST_KERNEL`, `NVXHOST_TEST_INITRD`, //! `NVXHOST_TEST_IMAGE`, `NVXHOST_TEST_LIBRARY`, and the approved `NVXHOST_TEST_SHA256`. //! `NVXHOST_TEST_HYPERVISOR` selects `whp`, `mshv`, or `kvm`, and defaults to `whp` on Windows. -//! The host-path and network tests also need `python3` in the image. Run the tests one at a time -//! so that the check for leftover OpenVMM processes is exact, and optimized, since creating a -//! backend hashes the runtime files and registering an image hashes the image: +//! The host-path, network, proxy, port-forwarding, and exec-environment tests also need `python3` +//! in the image. Run the tests one at a time so that the check for leftover OpenVMM processes is +//! exact, and optimized, since creating a backend hashes the runtime files and registering an +//! image hashes the image: //! `cargo test --release --features nvxhost --test nvxhost_guest -- --ignored --test-threads=1`. #![cfg(feature = "nvxhost")] @@ -26,8 +27,8 @@ use aci_edge_sandboxes::openvmm::{ }; use aci_edge_sandboxes::{ Access, AciEdgeSandbox, EgressPolicy, Error, ErrorCode, ExecOutcome, ExecOutput, ExecRequest, - FilesystemPolicy, NetworkPolicy, NetworkRule, Protocol, ProvisionRequest, Result, SandboxId, - StdinMode, StopResult, + FilesystemPolicy, ForwardProtocol, HostLoopbackForward, NetworkPolicy, NetworkRule, Protocol, + ProvisionRequest, Result, SandboxId, StdinMode, StopResult, }; const HELPER_STATE: &str = "NVXHOST_TEST_HELPER_STATE"; @@ -546,16 +547,13 @@ fn exec_reports_exit_codes_streams_timeouts_and_output_limits() { for index in 0..20 { assert_prints(&client, id, &format!("command{index}")); } - for request in [ - ExecRequest::command_line("pwd").with_cwd("/tmp"), - ExecRequest::command_line("env").with_env("NAME=value"), - ExecRequest::command_line("cat").with_stdin(StdinMode::Piped), - ] { - assert_eq!( - failure(client.exec(id, &request)), - ErrorCode::PolicyValidation - ); - } + assert_eq!( + failure(client.exec( + id, + &ExecRequest::command_line("cat").with_stdin(StdinMode::Piped) + )), + ErrorCode::PolicyValidation + ); let logs = backend.guest_logs(id).unwrap(); assert!(logs.windows(8).any(|window| window == b"execute:")); assert!(logs.windows(13).any(|window| window == b"output-limit:")); @@ -1496,3 +1494,420 @@ fn network_policies_are_enforced() { } assert_no_new_openvmm(&before); } + +/// Reports the guest's proxy variables and whether it reaches the proxy, the proxy's port over +/// UDP, another host loopback port, and a host service. +const PROXY_PROBE: &str = r#" +import json, os, socket, sys +gateway, proxy, other, host, service = sys.argv[1:6] +def connect(address, port): + try: + with socket.create_connection((address, int(port)), timeout=3) as connection: + return "reached:" + connection.recv(16).decode() + except OSError: + return "blocked" +def datagram(address, port): + try: + with socket.socket(socket.AF_INET, socket.SOCK_DGRAM) as probe: + probe.settimeout(3) + probe.sendto(b"ping", (address, int(port))) + return "reached:" + probe.recv(16).decode() + except OSError: + return "blocked" +names = ["HTTP_PROXY", "HTTPS_PROXY", "http_proxy", "https_proxy", "NO_PROXY", "no_proxy"] +print(json.dumps({ + "variables": {name: os.environ.get(name) for name in names}, + "proxy": connect(gateway, proxy), + "proxyUdp": datagram(gateway, proxy), + "other": connect(gateway, other), + "service": connect(host, service), +})) +"#; + +/// A host UDP service that answers every datagram with `reply` until it is dropped. +struct DatagramService { + done: Arc, + thread: Option>, +} + +impl DatagramService { + fn serve(socket: UdpSocket, reply: &'static str) -> Self { + socket + .set_read_timeout(Some(Duration::from_millis(50))) + .unwrap(); + let done = Arc::new(AtomicBool::new(false)); + let stop = done.clone(); + let thread = thread::spawn(move || { + let mut buffer = [0u8; 64]; + while !stop.load(Ordering::Relaxed) { + if let Ok((_, peer)) = socket.recv_from(&mut buffer) { + let _ = socket.send_to(reply.as_bytes(), peer); + } + } + }); + Self { + done, + thread: Some(thread), + } + } +} + +impl Drop for DatagramService { + fn drop(&mut self) { + self.done.store(true, Ordering::Relaxed); + if let Some(thread) = self.thread.take() { + let _ = thread.join(); + } + } +} + +/// The guest network and gateway for `host`, chosen so that the guest reaches the host through +/// its gateway rather than consider it on-link. +fn guest_network_for(host: Ipv4Addr) -> (&'static str, &'static str) { + if host.octets()[..3] == [10, 0, 0] { + ("10.0.1.2/24", "10.0.1.1") + } else { + ("10.0.0.2/24", "10.0.0.1") + } +} + +#[test] +#[ignore = "requires an approved private library, edge initramfs, GPT image, and a hypervisor host"] +fn a_loopback_proxy_is_the_only_way_out() { + let state = tempfile::tempdir().unwrap(); + let before = openvmm_processes(); + let host = host_address(); + let (guest_network, gateway) = guest_network_for(host); + let proxy = TcpListener::bind((Ipv4Addr::LOCALHOST, 0)).unwrap(); + let port = proxy.local_addr().unwrap().port(); + let other = TcpListener::bind((Ipv4Addr::LOCALHOST, 0)).unwrap(); + let service = TcpListener::bind((host, 0)).unwrap(); + let ports = + [&other, &service].map(|listener| listener.local_addr().unwrap().port().to_string()); + let _services = Services::serve(vec![ + (proxy, "proxy"), + (other, "other"), + (service, "service"), + ]); + // A UDP service on the proxy's port proves that only TCP reaches the proxy. + let _datagrams = + DatagramService::serve(UdpSocket::bind((Ipv4Addr::LOCALHOST, port)).unwrap(), "udp"); + let backend = backend_with(state.path(), false, |config| { + config.guest_network = guest_network.to_owned(); + }); + let client = AciEdgeSandbox::from_shared(backend.clone()); + + // A proxy is a separate connectivity model from direct egress. + for network in [ + NetworkPolicy::egress(Access::Allow), + NetworkPolicy { + egress: EgressPolicy::new(Access::Deny) + .with_allow(NetworkRule::to(format!("{host}/32"))), + ..NetworkPolicy::deny_all() + }, + ] { + let request = ProvisionRequest::new() + .with_network(network) + .with_network_proxy(format!("http://127.0.0.1:{port}")); + assert_eq!( + failure(client.provision(&request)), + ErrorCode::PolicyValidation + ); + } + for url in [ + format!("http://192.0.2.1:{port}"), + format!("http://[::1]:{port}"), + "http://127.0.0.1".to_owned(), + ] { + let request = ProvisionRequest::new() + .with_network(NetworkPolicy::deny_all()) + .with_network_proxy(url); + assert_eq!( + failure(client.provision(&request)), + ErrorCode::PolicyValidation + ); + } + + let sandbox = started( + &client, + &backend, + &ProvisionRequest::new() + .with_network(NetworkPolicy::deny_all()) + .with_network_proxy(format!("http://localhost:{port}")), + ); + let host_text = host.to_string(); + let printed = python( + &client, + &sandbox.id, + PROXY_PROBE, + &[gateway, &port.to_string(), &ports[0], &host_text, &ports[1]], + ); + let url = format!("http://{gateway}:{port}"); + assert_eq!( + serde_json::from_str::(&printed).unwrap(), + serde_json::json!({ + "variables": { + "HTTP_PROXY": url, + "HTTPS_PROXY": url, + "http_proxy": url, + "https_proxy": url, + "NO_PROXY": "localhost,127.0.0.1", + "no_proxy": "localhost,127.0.0.1", + }, + "proxy": "reached:proxy", + "proxyUdp": "blocked", + "other": "blocked", + "service": "blocked", + }) + ); + // Only the guest sets the proxy variables. + for name in ["HTTP_PROXY", "https_proxy", "No_Proxy"] { + let request = ExecRequest::command_line("true") + .with_env(format!("{name}=http://192.0.2.1:3128")) + .with_inherit_default_env(true); + assert_eq!( + failure(client.exec(&sandbox.id, &request)), + ErrorCode::PolicyValidation + ); + } + assert_graceful(client.stop(&sandbox.id)); + client.deprovision(&sandbox.release()).unwrap(); + assert_no_new_openvmm(&before); +} + +/// Serves one TCP connection and one UDP datagram on the given guest ports, then exits. It gives +/// up after 30 seconds without the host, rather than wait for the command's timeout. +const FORWARD_LISTENER: &str = r#" +import socket, sys +tcp = socket.create_server(("0.0.0.0", int(sys.argv[1]))) +udp = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) +udp.bind(("0.0.0.0", int(sys.argv[2]))) +tcp.settimeout(30) +udp.settimeout(30) +connection, _ = tcp.accept() +connection.sendall(b"forwarded") +connection.close() +data, peer = udp.recvfrom(16) +udp.sendto(b"udp:" + data, peer) +print("served") +"#; + +/// Returns a host loopback port that is free now. +fn free_loopback_port(udp: bool) -> u16 { + if udp { + UdpSocket::bind((Ipv4Addr::LOCALHOST, 0)) + .unwrap() + .local_addr() + .unwrap() + .port() + } else { + TcpListener::bind((Ipv4Addr::LOCALHOST, 0)) + .unwrap() + .local_addr() + .unwrap() + .port() + } +} + +#[test] +#[ignore = "requires an approved private library, edge initramfs, GPT image, and a hypervisor host"] +fn forwarded_ports_publish_guest_listeners_on_host_loopback() { + use std::io::Read; + use std::net::TcpStream; + + let state = tempfile::tempdir().unwrap(); + let before = openvmm_processes(); + let host = host_address(); + let (guest_network, gateway) = guest_network_for(host); + let loopback = TcpListener::bind((Ipv4Addr::LOCALHOST, 0)).unwrap(); + let loopback_port = loopback.local_addr().unwrap().port().to_string(); + let _services = Services::serve(vec![(loopback, "loopback")]); + let backend = backend_with(state.path(), false, |config| { + config.guest_network = guest_network.to_owned(); + }); + let client = AciEdgeSandbox::from_shared(backend.clone()); + let mut open = NetworkPolicy::egress(Access::Allow); + open.ingress.host_loopback = Some(Access::Allow); + let forwarded = |network: NetworkPolicy, tcp_port: u16, udp_port: u16| { + ProvisionRequest::new() + .with_network(network) + .with_host_loopback_forward(HostLoopbackForward::new( + ForwardProtocol::Tcp, + tcp_port, + 8080, + )) + .with_host_loopback_forward(HostLoopbackForward::new( + ForwardProtocol::Udp, + udp_port, + 8081, + )) + }; + + // Generic host-loopback access, forwards without it, and forwards whose replies egress + // would drop all fail before any VM exists. + let mut closed = open.clone(); + closed.egress = + EgressPolicy::new(Access::Deny).with_allow(NetworkRule::to(format!("{host}/32"))); + for request in [ + ProvisionRequest::new().with_network(open.clone()), + forwarded(NetworkPolicy::egress(Access::Allow), 18080, 18081), + forwarded(closed, 18080, 18081), + ] { + assert_eq!( + failure(client.provision(&request)), + ErrorCode::PolicyValidation + ); + } + + // OpenVMM binds the host ports when it starts, and another process can take a port released + // here before then, so a failed start is retried with other ports. + let mut retries = 0; + let (sandbox, tcp_port, udp_port) = loop { + let (tcp_port, udp_port) = (free_loopback_port(false), free_loopback_port(true)); + let sandbox = provision_with( + &client, + &backend, + &forwarded(open.clone(), tcp_port, udp_port), + ); + match client.start(&sandbox.id) { + Ok(_) => break (sandbox, tcp_port, udp_port), + Err(error) if retries < 2 => { + retries += 1; + eprintln!("starting with ports {tcp_port} and {udp_port} failed: {error}"); + } + Err(error) => panic!("starting with ports {tcp_port} and {udp_port} failed: {error}"), + } + }; + let listener = thread::scope(|scope| { + let served = + scope.spawn(|| python(&client, &sandbox.id, FORWARD_LISTENER, &["8080", "8081"])); + // OpenVMM accepts on the host before the guest listens, so retry until the guest + // answers. + let deadline = Instant::now() + Duration::from_secs(30); + let greeting = loop { + let mut text = String::new(); + let answered = TcpStream::connect((Ipv4Addr::LOCALHOST, tcp_port)) + .and_then(|mut stream| { + stream.set_read_timeout(Some(Duration::from_secs(2)))?; + stream.read_to_string(&mut text) + }) + .is_ok_and(|_| !text.is_empty()); + if answered || Instant::now() > deadline { + break text; + } + thread::sleep(Duration::from_millis(200)); + }; + assert_eq!(greeting, "forwarded"); + let socket = UdpSocket::bind((Ipv4Addr::LOCALHOST, 0)).unwrap(); + socket + .set_read_timeout(Some(Duration::from_secs(1))) + .unwrap(); + let mut reply = [0u8; 16]; + let received = (0..30).find_map(|_| { + socket + .send_to(b"ping", (Ipv4Addr::LOCALHOST, udp_port)) + .ok()?; + socket.recv(&mut reply).ok() + }); + assert_eq!( + received.map(|length| &reply[..length]), + Some(&b"udp:ping"[..]) + ); + served.join().unwrap() + }); + assert_eq!(listener, "served\n"); + // Allowed host loopback also lets the guest reach host loopback services at its gateway. + let printed = python( + &client, + &sandbox.id, + "import socket, sys\nwith socket.create_connection((sys.argv[1], int(sys.argv[2])), timeout=3) as c: print(c.recv(16).decode())", + &[gateway, &loopback_port], + ); + assert_eq!(printed, "loopback\n"); + assert_graceful(client.stop(&sandbox.id)); + client.deprovision(&sandbox.release()).unwrap(); + assert_no_new_openvmm(&before); +} + +#[test] +#[ignore = "requires an approved private library, edge initramfs, GPT image, and a hypervisor host"] +fn exec_environments_layer_over_the_guest_defaults() { + let state = tempfile::tempdir().unwrap(); + let before = openvmm_processes(); + let backend = backend(state.path()); + let client = AciEdgeSandbox::from_shared(backend.clone()); + let sandbox = started(&client, &backend, &ProvisionRequest::new()); + let report = ExecRequest::argv([ + "/bin/sh", + "-c", + "pwd; printf '%s|%s|%s' \"$A\" \"$B\" \"$PATH\"", + ]); + let output = exec( + &client, + &sandbox.id, + &report + .clone() + .with_cwd("/tmp") + .with_envs(["A=1", "B=two words", "A=one"]) + .with_inherit_default_env(true), + ); + assert_eq!(output.outcome, ExecOutcome::Exited(0), "{output:?}"); + assert_eq!( + String::from_utf8_lossy(&output.stdout), + "/tmp\none|two words|/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" + ); + // The request's PATH replaces the default. + let output = exec( + &client, + &sandbox.id, + &report + .clone() + .with_env("PATH=/bin") + .with_inherit_default_env(true), + ); + assert_eq!(String::from_utf8_lossy(&output.stdout), "/\n||/bin"); + // The largest environment the backend accepts reaches the workload, also when its control + // characters take six bytes each in the guest's encoding. + let value = "\u{1}".repeat(131); + let output = exec( + &client, + &sandbox.id, + &ExecRequest::argv([ + "/bin/sh", + "-c", + "exec python3 -c \"$1\"", + "sh", + "import os; print(sum(name[0] == 'V' for name in os.environ), \ + len(os.environ['V000']), len(os.environ['V239']))", + ]) + .with_envs((0..240).map(|index| format!("V{index:03}={value}"))) + .with_inherit_default_env(true), + ); + assert_eq!(output.outcome, ExecOutcome::Exited(0), "{output:?}"); + assert_eq!(String::from_utf8_lossy(&output.stdout), "240 131 131\n"); + // A missing working directory fails the command, not the sandbox. + let output = exec( + &client, + &sandbox.id, + &report.clone().with_cwd("/nonexistent"), + ); + assert_eq!(output.outcome, ExecOutcome::Exited(126), "{output:?}"); + assert!( + String::from_utf8_lossy(&output.stderr).contains("NVX-EDGE-STAGE-ERROR"), + "{output:?}" + ); + // The guest layers the environment over its defaults; it cannot replace them. + for request in [ + report.clone().with_env("A=1"), + report.clone().with_cwd("relative"), + ] { + assert_eq!( + failure(client.exec(&sandbox.id, &request)), + ErrorCode::PolicyValidation + ); + } + assert_prints(&client, &sandbox.id, "still-running"); + assert_graceful(client.stop(&sandbox.id)); + client.deprovision(&sandbox.release()).unwrap(); + assert_no_new_openvmm(&before); +}