feat: probe nsjail with a real jail and add a gVisor compat mode

Dockerfile

@@ -18,7 +18,9 @@ RUN apt-get -y update \
     pkg-config=1.8.* \
     protobuf-compiler=3.21.*
 
-RUN git clone -b master --single-branch https://github.com/google/nsjail.git . && git checkout dccf911fd2659e7b08ce9507c25b2b38ec2c5800
+COPY docker/nsjail-patches/ /nsjail-patches/
+RUN git clone -b master --single-branch https://github.com/google/nsjail.git . && git checkout dccf911fd2659e7b08ce9507c25b2b38ec2c5800 \
+    && git apply /nsjail-patches/*.patch
 RUN make
 
 FROM ${RUST_IMAGE} AS rust_base

backend/windmill-worker/nsjail/probe.config.proto

@@ -0,0 +1,47 @@
+name: "nsjail startup probe"
+
+mode: ONCE
+hostname: "probe"
+log_level: WARNING
+time_limit: 10
+
+# Same namespaces and rlimit handling as the job profiles (nsjail's default rlimits, a cgroup
+# namespace, the user namespace unless DISABLE_NUSER): a worker whose kernel or runtime refuses
+# any of them cannot run jobs either.
+clone_newnet: false
+clone_newuser: {CLONE_NEWUSER}
+
+skip_setsid: true
+keep_caps: false
+mount_proc: true
+
+mount {
+    src: "/bin"
+    dst: "/bin"
+	is_bind: true
+}
+mount {
+    src: "/lib"
+    dst: "/lib"
+	is_bind: true
+}
+mount {
+    src: "/lib64"
+    dst: "/lib64"
+	is_bind: true
+    mandatory: false
+}
+mount {
+    src: "/usr"
+    dst: "/usr"
+	is_bind: true
+}
+mount {
+    dst: "/tmp"
+    fstype: "tmpfs"
+    rw: true
+}
+
+exec_bin {
+    path: "/bin/true"
+}

backend/windmill-worker/src/common.rs

@@ -976,12 +976,26 @@ pub fn build_command_with_isolation(program: &str, args: &[&str]) -> Command {
     }
 }
 
+/// Under `NSJAIL_GVISOR_COMPAT`, rewrites the config of an nsjail `cmd` for gVisor. A spawn of
+/// nsjail that bypasses `start_child_process` must call this itself.
+pub fn apply_nsjail_gvisor_compat(cmd: &Command) -> Result<(), Error> {
+    if *crate::NSJAIL_GVISOR_COMPAT && cmd.as_std().get_program() == crate::NSJAIL_PATH.as_str() {
+        crate::nsjail_compat::apply(cmd.as_std()).map_err(|e| {
+            Error::ExecutionErr(format!(
+                "NSJAIL_GVISOR_COMPAT could not prepare nsjail: {e}"
+            ))
+        })?;
+    }
+    Ok(())
+}
+
 pub async fn start_child_process(
     cmd: Command,
     executable: &str,
     disable_process_group: bool,
 ) -> Result<Box<dyn TokioChildWrapper>, Error> {
     use process_wrap::tokio::*;
+    apply_nsjail_gvisor_compat(&cmd)?;
     let mut cmd = TokioCommandWrap::from(cmd);
 
     // On Windows, put the child in its own console process group so a CTRL_BREAK_EVENT

backend/windmill-worker/src/dbt_executor.rs

@@ -4053,6 +4053,7 @@ pub(crate) async fn run_captured(
 ) -> error::Result<Captured> {
     use tokio::io::AsyncReadExt;
 
+    crate::common::apply_nsjail_gvisor_compat(&cmd)?;
     let mut child = cmd
         .stdout(Stdio::piped())
         .stderr(Stdio::piped())

backend/windmill-worker/src/duckdb_runner.rs

@@ -23,8 +23,7 @@ use windmill_common::error::{Error, Result};
 
 use crate::{
     common::{build_command_with_isolation, resolve_nsjail_tmp_mount_block, DEV_CONF_NSJAIL},
-    is_sandboxing_enabled, is_unshare_enabled, DISABLE_NUSER, HOME_ENV, NSJAIL_AVAILABLE,
-    NSJAIL_PATH, PATH_ENV,
+    is_sandboxing_enabled, is_unshare_enabled, DISABLE_NUSER, HOME_ENV, NSJAIL_PATH, PATH_ENV,
 };
 
 const NSJAIL_CONFIG_RUN_DUCKDB_CONTENT: &str = include_str!("../nsjail/run.duckdb.config.proto");
@@ -42,7 +41,9 @@ pub(crate) enum RunnerMode {
 
 /// Decided once per job, so every connection a job opens runs the same way.
 pub(crate) fn runner_mode() -> RunnerMode {
-    if cfg!(unix) && is_sandboxing_enabled() && NSJAIL_AVAILABLE.is_some() {
+    // Not gated on `NSJAIL_AVAILABLE`: a sandboxed worker whose nsjail cannot jail must fail
+    // the job, as every other jailed executor does, not run it outside the jail.
+    if cfg!(unix) && is_sandboxing_enabled() {
         RunnerMode::Jailed
     } else if is_unshare_enabled() {
         // `build_command_with_isolation` logs and runs unisolated when `unshare` is missing,
@@ -475,6 +476,7 @@ async fn spawn_runner(
             });
         }
     }
+    crate::common::apply_nsjail_gvisor_compat(&cmd)?;
     let mut child = cmd.spawn().map_err(|e| {
         Error::InternalErr(format!("Could not start the DuckDB runner ({mode:?}): {e}"))
     })?;

backend/windmill-worker/src/lib.rs

@@ -62,6 +62,7 @@ pub mod memory_ee;
 pub mod memory_oss;
 #[cfg(feature = "mysql")]
 mod mysql_executor;
+mod nsjail_compat;
 #[cfg(feature = "nu")]
 mod nu_executor;
 #[cfg(feature = "oracledb")]

backend/windmill-worker/src/nsjail_compat.rs

@@ -0,0 +1,49 @@
+//! `NSJAIL_GVISOR_COMPAT=true`: run nsjail inside gVisor (runsc). Each config is rewritten
+//! right before nsjail starts, with `gvisor_compat: true` (an option of the nsjail the images
+//! build, from `docker/nsjail-patches/`) and no cgroup namespace, which gVisor refuses.
+
+use std::ffi::OsStr;
+use std::io;
+use std::path::PathBuf;
+use std::process::Command;
+
+pub(crate) fn compat_config(config: &str) -> String {
+    let mut out = config
+        .lines()
+        .filter(|l| {
+            let l = l.trim_start();
+            !l.starts_with("clone_newcgroup") && !l.starts_with("gvisor_compat")
+        })
+        .collect::<Vec<_>>()
+        .join("\n");
+    out.push_str("\nclone_newcgroup: false\ngvisor_compat: true\n");
+    out
+}
+
+/// Rewrites the config `cmd` (an nsjail command line) passes with `--config`.
+pub(crate) fn apply(cmd: &Command) -> io::Result<()> {
+    let args = cmd.get_args().collect::<Vec<&OsStr>>();
+    let Some(config) = args.windows(2).find(|w| w[0] == "--config").map(|w| w[1]) else {
+        return Ok(());
+    };
+    let path = match cmd.get_current_dir() {
+        Some(dir) => dir.join(config),
+        None => PathBuf::from(config),
+    };
+    let rewritten = compat_config(&std::fs::read_to_string(&path)?);
+    std::fs::write(&path, rewritten)
+}
+
+#[cfg(test)]
+mod tests {
+    // nsjail rejects a config that sets a scalar field twice, which would fail every jailed job.
+    #[test]
+    fn compat_config_sets_each_field_once() {
+        let bun = include_str!("../nsjail/run.bun.config.proto");
+        assert!(bun.contains("clone_newcgroup: false"));
+        let twice = super::compat_config(&super::compat_config(bun));
+        assert_eq!(twice.matches("clone_newcgroup").count(), 1);
+        assert_eq!(twice.matches("gvisor_compat").count(), 1);
+        assert!(twice.ends_with("\nclone_newcgroup: false\ngvisor_compat: true\n"));
+    }
+}

backend/windmill-worker/src/worker.rs

@@ -261,6 +261,7 @@ const NUM_SECS_PING: u64 = 5;
 const NUM_SECS_READINGS: u64 = 60;
 
 const INCLUDE_DEPS_PY_SH_CONTENT: &str = include_str!("../nsjail/download_deps.py.sh");
+const NSJAIL_CONFIG_PROBE_CONTENT: &str = include_str!("../nsjail/probe.config.proto");
 
 const WORKER_SHELL_NAP_TIME_DURATION: u64 = 15;
 const WORKER_SHELL_INITIAL_NAP_TIME_DURATION: u64 = 5;
@@ -335,6 +336,14 @@ pub struct PowershellRepo {
     pub pat: String,
 }
 
+/// Runs nsjail in a form gVisor (runsc) accepts, see `nsjail_compat`.
+pub static NSJAIL_GVISOR_COMPAT: std::sync::LazyLock<bool> = std::sync::LazyLock::new(|| {
+    std::env::var("NSJAIL_GVISOR_COMPAT")
+        .ok()
+        .and_then(|x| x.parse::<bool>().ok())
+        .unwrap_or(false)
+});
+
 lazy_static::lazy_static! {
 
     static ref SLEEP_QUEUE_BASE: u64 = std::env::var("SLEEP_QUEUE")
@@ -546,29 +555,41 @@ lazy_static::lazy_static! {
     pub static ref NSJAIL_AVAILABLE: Option<String> = {
         let nsjail_path = NSJAIL_PATH.as_str();
 
-        let test_result = std::process::Command::new(nsjail_path)
-            .arg("--help")
-            .output();
-
-        match test_result {
+        match probe_nsjail(nsjail_path) {
             Ok(output) if output.status.success() => {
-                tracing::info!("nsjail available at: {}", nsjail_path);
+                tracing::info!(
+                    "nsjail available at: {}{}",
+                    nsjail_path,
+                    if *NSJAIL_GVISOR_COMPAT { " (NSJAIL_GVISOR_COMPAT)" } else { "" }
+                );
                 Some(nsjail_path.to_string())
             },
             Ok(output) => {
-                let stderr = String::from_utf8_lossy(&output.stderr);
-                tracing::error!(
-                    "nsjail test failed (exit code: {}). stderr: '{}'. path: '{}'. \
-                    Nsjail sandboxing will NOT be available. \
-                    nsjail should be included in all standard windmill images. \
-                    If job_isolation is set to 'nsjail_sandboxing' in Instance Settings, jobs will fail.",
+                let msg = format!(
+                    "nsjail could not run a test jail (exit code: {}). stderr: '{}'. path: '{}'. \
+                    Nsjail sandboxing will NOT be available: the worker reports its job isolation \
+                    without nsjail, and jobs that require nsjail fail. Under gVisor (runsc), see \
+                    NSJAIL_GVISOR_COMPAT.",
                     output.status,
-                    stderr.trim(),
+                    String::from_utf8_lossy(&output.stderr).trim(),
                     nsjail_path
                 );
+                // Most workers never jail anything, and most containers cannot.
+                if *DISABLE_NSJAIL {
+                    tracing::info!("{msg}");
+                } else {
+                    tracing::error!("{msg}");
+                }
+                None
+            },
+            Err(NsjailProbeError::Setup(e)) => {
+                tracing::error!(
+                    "Could not prepare the nsjail test jail: {}. Nsjail sandboxing will NOT be available.",
+                    e
+                );
                 None
             },
-            Err(e) => {
+            Err(NsjailProbeError::Start(e)) => {
                 if e.kind() == std::io::ErrorKind::NotFound {
                     tracing::error!(
                         "nsjail not found at '{}'. Nsjail sandboxing will NOT be available. \
@@ -1021,6 +1042,43 @@ pub(crate) fn artifact_cache_name(
     format!("mod-{}", windmill_common::utils::calculate_hash(&keyed))
 }
 
+/// Runs `/bin/true` in a jail built like a job's, through the same spawn path, so a worker
+/// whose runtime lets nsjail start but not jail (a container without the privileges, gVisor)
+/// does not report nsjail isolation.
+enum NsjailProbeError {
+    /// The probe could not be prepared: nothing is known about nsjail.
+    Setup(std::io::Error),
+    /// nsjail itself could not be executed.
+    Start(std::io::Error),
+}
+
+fn probe_nsjail(nsjail_path: &str) -> Result<std::process::Output, NsjailProbeError> {
+    // A fresh private directory: the worker runs what this config says, as itself.
+    let dir = std::env::temp_dir().join(format!("windmill-nsjail-probe-{}", uuid::Uuid::new_v4()));
+    let mut builder = std::fs::DirBuilder::new();
+    #[cfg(unix)]
+    std::os::unix::fs::DirBuilderExt::mode(&mut builder, 0o700);
+    builder.create(&dir).map_err(NsjailProbeError::Setup)?;
+    let output = (|| {
+        std::fs::write(
+            dir.join("probe.config.proto"),
+            NSJAIL_CONFIG_PROBE_CONTENT.replace("{CLONE_NEWUSER}", &(!*DISABLE_NUSER).to_string()),
+        )
+        .map_err(NsjailProbeError::Setup)?;
+        let mut cmd = std::process::Command::new(nsjail_path);
+        cmd.current_dir(&dir)
+            .env_clear()
+            .env("PATH", PATH_ENV.as_str())
+            .args(["--config", "probe.config.proto"]);
+        if *NSJAIL_GVISOR_COMPAT {
+            crate::nsjail_compat::apply(&cmd).map_err(NsjailProbeError::Setup)?;
+        }
+        cmd.output().map_err(NsjailProbeError::Start)
+    })();
+    let _ = std::fs::remove_dir_all(&dir);
+    output
+}
+
 pub fn is_sandboxing_enabled() -> bool {
     if !*DISABLE_NSJAIL {
         return true;

docker/DockerfileSlim

@@ -23,7 +23,9 @@ RUN apt-get -y update \
     && apt-get clean \
     && rm -rf /var/lib/apt/lists/*
 
-RUN git clone -b master --single-branch https://github.com/google/nsjail.git . && git checkout dccf911fd2659e7b08ce9507c25b2b38ec2c5800
+COPY docker/nsjail-patches/ /nsjail-patches/
+RUN git clone -b master --single-branch https://github.com/google/nsjail.git . && git checkout dccf911fd2659e7b08ce9507c25b2b38ec2c5800 \
+    && git apply /nsjail-patches/*.patch
 RUN make
 
 FROM ${DEBIAN_IMAGE}

docker/DockerfileSlimEe

@@ -23,7 +23,9 @@ RUN apt-get -y update \
     && apt-get clean \
     && rm -rf /var/lib/apt/lists/*
 
-RUN git clone -b master --single-branch https://github.com/google/nsjail.git . && git checkout dccf911fd2659e7b08ce9507c25b2b38ec2c5800
+COPY docker/nsjail-patches/ /nsjail-patches/
+RUN git clone -b master --single-branch https://github.com/google/nsjail.git . && git checkout dccf911fd2659e7b08ce9507c25b2b38ec2c5800 \
+    && git apply /nsjail-patches/*.patch
 RUN make
 
 FROM ${DEBIAN_IMAGE}