diff --git a/Cargo.lock b/Cargo.lock index 62214862b9..4f9be72c0e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6465,6 +6465,7 @@ dependencies = [ "pixi_build_types", "pixi_command_dispatcher", "pixi_compute_reporters", + "pixi_conda_script", "pixi_config", "pixi_consts", "pixi_core", @@ -6700,6 +6701,7 @@ dependencies = [ "thiserror 2.0.20", "tokio", "toml-span", + "toml_edit", ] [[package]] @@ -6778,6 +6780,7 @@ dependencies = [ "pixi_build_types", "pixi_command_dispatcher", "pixi_compute_reporters", + "pixi_conda_script", "pixi_config", "pixi_consts", "pixi_diff", diff --git a/Cargo.toml b/Cargo.toml index d4a84f7c92..a8210c8dca 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -109,6 +109,7 @@ pixi_compute_env_vars = { path = "crates/pixi_compute_env_vars" } pixi_compute_network = { path = "crates/pixi_compute_network" } pixi_compute_reporters = { path = "crates/pixi_compute_reporters" } pixi_compute_sources = { path = "crates/pixi_compute_sources" } +pixi_conda_script = { path = "crates/pixi_conda_script" } pixi_config = { path = "crates/pixi_config" } pixi_consts = { path = "crates/pixi_consts" } pixi_core = { path = "crates/pixi_core" } diff --git a/crates/pixi_cli/Cargo.toml b/crates/pixi_cli/Cargo.toml index 866bd2bcb3..5e9b7a560d 100644 --- a/crates/pixi_cli/Cargo.toml +++ b/crates/pixi_cli/Cargo.toml @@ -50,6 +50,7 @@ pixi_build_frontend = { workspace = true } pixi_build_types = { workspace = true } pixi_command_dispatcher = { workspace = true } pixi_compute_reporters = { workspace = true } +pixi_conda_script = { workspace = true } pixi_config = { workspace = true } pixi_consts = { workspace = true } pixi_core = { workspace = true } diff --git a/crates/pixi_cli/src/conda_script.rs b/crates/pixi_cli/src/conda_script.rs new file mode 100644 index 0000000000..88bba8e467 --- /dev/null +++ b/crates/pixi_cli/src/conda_script.rs @@ -0,0 +1,236 @@ +use std::{collections::HashMap, ffi::OsString, path::Path}; + +use miette::{IntoDiagnostic, NamedSource, Report}; +use pixi_conda_script::{ + CondaScriptError, CondaScriptManifest, + shell::{ShellContext, execute_sequence, parse_sequence}, +}; +use pixi_core::{ + Workspace, + environment::sanity_check_workspace, + lock_file::{ReinstallPackages, UpdateLockFileOptions, UpdateMode}, + workspace::virtual_packages::{ + EnvironmentRunnability, classify_environment_runnability, + verify_current_platform_can_run_environment, verify_run_platform, + }, +}; +use pixi_manifest::WithWarnings; +use pixi_task::get_task_env; +use tracing::Level; + +use crate::{process_exit, run::Args, shared::install_platform::resolve_install_platform}; + +/// Reads the `conda-script` block of a local `--script` file. +/// +/// Returns `Ok(None)` when the file has no block or when a malformed block +/// appears in a Python file, so the caller falls back to the PEP 723 path: a +/// Python script may contain an accidental line ending in the opening +/// marker, say inside an indented docstring, and must keep working as it did +/// before the conda-script format existed. When `surface_errors` is set (the +/// caller passed `--experimental`) or the file cannot be a PEP 723 script +/// anyway, a block error is reported instead. +pub(crate) fn detect_with_fallback( + path: &Path, + surface_errors: bool, +) -> miette::Result> { + match CondaScriptManifest::from_path(path) { + Ok(manifest) => Ok(manifest), + Err(error @ CondaScriptError::Io(_)) => Err(Report::new(error)), + Err(error) => { + let is_python = path + .extension() + .and_then(|extension| extension.to_str()) + .is_some_and(|extension| { + extension.eq_ignore_ascii_case("py") || extension.eq_ignore_ascii_case("pyw") + }); + if surface_errors || !is_python { + Err(Report::new(error)) + } else { + tracing::debug!( + "ignoring a malformed conda-script block in {}: {error}", + path.display() + ); + Ok(None) + } + } + } +} + +/// Whether the contents carry a conda-script block, well-formed or not. +/// +/// Transient script sources use this to explain that conda-script files only +/// run from local paths, instead of reporting a missing PEP 723 block. +pub(crate) fn looks_like_conda_script(contents: &[u8]) -> bool { + !matches!( + CondaScriptManifest::from_source("conda-script-probe", contents), + Ok(None) + ) +} + +/// Solves and installs the environment of a `conda-script` file, then runs +/// its entrypoint through the mini-shell with the CLI arguments appended. +pub(crate) async fn execute_run( + manifest: CondaScriptManifest, + args: Args, + config: pixi_config::Config, +) -> miette::Result<()> { + let script_path = manifest.path().to_owned(); + let entrypoint = manifest.metadata().entrypoint.clone(); + + let WithWarnings { + value: workspace, + warnings, + } = Workspace::from_conda_script(manifest, config)?; + for warning in warnings { + tracing::warn!("{warning}"); + } + sanity_check_workspace(&workspace).await?; + + let environment = workspace.default_environment(); + let allow_installs = args.lock_and_install_config.allow_installs(); + let user_platform = resolve_install_platform(&workspace, args.platform.as_ref())?; + let run_platform = user_platform + .clone() + .or_else(|| environment.installed_resolved_platform_name()); + let best_declared_platform = environment.named_or_best_declared_platform(run_platform.as_ref()); + if allow_installs + && best_declared_platform.is_none() + && let Some(name) = user_platform.as_ref() + { + return Err(miette::miette!( + "platform '{}' is not part of environment '{}'", + name, + environment.name(), + )); + } + if allow_installs { + environment.emit_emulation_warning(); + } + + // Select and parse the entrypoint before solving, so a syntax error or a + // missing platform key surfaces without waiting for the environment. + let activation_platform = best_declared_platform + .cloned() + .unwrap_or_else(|| environment.activation_platform()); + let subdir = activation_platform.subdir(); + let Some(command) = entrypoint.select(subdir) else { + return Err(miette::miette!( + help = "add a matching key to the `entrypoint` table, for example `unix`, `win` or the exact platform", + "the entrypoint has no command for platform '{subdir}'" + )); + }; + let sequence = parse_sequence(command).map_err(|error| { + Report::new(error).with_source_code(NamedSource::new("entrypoint", command.to_owned())) + })?; + + let progress = pixi_reporters::TopLevelProgress::from_global(); + let mut lock_file = workspace + .resolve_lock_file( + Some(progress.clone()), + UpdateLockFileOptions { + lock_file_usage: args.lock_and_install_config.lock_file_usage()?, + no_install: args.lock_and_install_config.no_install(), + max_concurrent_solves: workspace.config().max_concurrent_solves(), + ..Default::default() + }, + ) + .await? + .0; + lock_file.target_platform = user_platform.clone(); + + if allow_installs && user_platform.is_none() { + let runnability = + classify_environment_runnability(&environment, Some(lock_file.as_lock_file())); + if runnability == EnvironmentRunnability::Unsupported { + return Err( + match verify_current_platform_can_run_environment( + &environment, + Some(lock_file.as_lock_file()), + ) { + Err(err) => err.into(), + Ok(()) => environment.unsupported_platform_error().into(), + }, + ); + } + } + + if allow_installs { + lock_file + .prefix( + &environment, + UpdateMode::QuickValidate, + &ReinstallPackages::default(), + &pixi_core::environment::InstallFilter::default(), + ) + .await?; + verify_run_platform(&environment, user_platform.as_ref())?; + } + progress.on_clear(); + lock_file.command_dispatcher.clear_filesystem_caches().await; + + let command_env = get_task_env( + &environment, + &activation_platform, + args.clean_env, + Some(lock_file.as_lock_file()), + workspace.config().force_activate(), + workspace.config().experimental_activation_cache_usage(), + ) + .await?; + + if args.dry_run { + pixi_progress::println!( + "{}{}\n\n", + console::Emoji("🌵 ", ""), + console::style("Dry-run mode enabled - no tasks will be executed.") + .yellow() + .bold() + ); + } + + if tracing::enabled!(Level::WARN) { + let file_name = script_path + .file_name() + .expect("an absolute script path always has a file name") + .to_string_lossy(); + pixi_progress::println!( + "{}{}{}{}{}", + console::Emoji("✨ ", ""), + console::style("Pixi script (").bold(), + console::style(file_name).green().bold(), + console::style("): ").bold(), + command, + ); + } + + if args.dry_run { + return Ok(()); + } + + let cache_dir = workspace.pixi_dir().join("cache"); + fs_err::create_dir_all(&cache_dir).into_diagnostic()?; + let script = script_path + .into_os_string() + .into_string() + .map_err(|_| miette::miette!("the script path must contain only valid UTF-8 characters"))?; + let cache = cache_dir + .into_os_string() + .into_string() + .map_err(|_| miette::miette!("the cache path must contain only valid UTF-8 characters"))?; + + let context = ShellContext { + variables: HashMap::from([("SCRIPT".to_owned(), script), ("CACHE".to_owned(), cache)]), + env: command_env + .into_iter() + .map(|(key, value)| (OsString::from(key), OsString::from(value))) + .collect(), + cwd: std::env::current_dir().into_diagnostic()?, + }; + let code = execute_sequence(&sequence, &args.task, &context) + .await + .map_err(Report::new)?; + if code != 0 { + process_exit::exit_with_code(code); + } + Ok(()) +} diff --git a/crates/pixi_cli/src/init.rs b/crates/pixi_cli/src/init.rs index 4ad840c2ed..68f09a00d4 100644 --- a/crates/pixi_cli/src/init.rs +++ b/crates/pixi_cli/src/init.rs @@ -182,6 +182,15 @@ async fn initialize_script( channels: Option>, ) -> miette::Result<()> { let path = std::path::absolute(path).into_diagnostic()?; + // A file with a conda-script block must not get a PEP 723 block on top; + // the two kinds cannot coexist in one file. + if path.is_file() && crate::conda_script::detect_with_fallback(&path, false)?.is_some() { + return Err(miette::miette!( + help = "a file can carry either a PEP 723 block or a conda-script block, not both", + "{} is already a conda-script", + path.display() + )); + } let channels = channels .unwrap_or_default() .into_iter() diff --git a/crates/pixi_cli/src/lib.rs b/crates/pixi_cli/src/lib.rs index 355341ef7b..8ff19afe93 100644 --- a/crates/pixi_cli/src/lib.rs +++ b/crates/pixi_cli/src/lib.rs @@ -26,6 +26,7 @@ pub mod cli_config; pub mod cli_interface; pub mod command_info; pub mod completion; +mod conda_script; pub mod config; pub mod exec; pub mod global; diff --git a/crates/pixi_cli/src/lock.rs b/crates/pixi_cli/src/lock.rs index 0ab88ce789..fe8079f234 100644 --- a/crates/pixi_cli/src/lock.rs +++ b/crates/pixi_cli/src/lock.rs @@ -43,11 +43,33 @@ pub struct Args { } pub async fn execute(args: Args) -> miette::Result<()> { - let mut workspace = WorkspaceLocator::for_cli() - .with_global_config_source(args.config_source.source()) - .with_search_start(args.workspace_config.workspace_locator_start()) - .with_cli_config(args.config.clone()) - .locate()?; + let conda_script = match args.workspace_config.script.as_deref() { + Some(path) => crate::conda_script::detect_with_fallback(path, false)?, + None => None, + }; + let mut workspace = if let Some(manifest) = conda_script { + let root = manifest + .path() + .parent() + .expect("an absolute script path always has a parent") + .to_owned(); + let config = pixi_config::Config::load_with(&root, &args.config_source.source()) + .merge_config(args.config.clone().into()); + let pixi_manifest::WithWarnings { + value: workspace, + warnings, + } = pixi_core::Workspace::from_conda_script(manifest, config)?; + for warning in warnings { + tracing::warn!("{warning}"); + } + workspace + } else { + WorkspaceLocator::for_cli() + .with_global_config_source(args.config_source.source()) + .with_search_start(args.workspace_config.workspace_locator_start()) + .with_cli_config(args.config.clone()) + .locate()? + }; // Apply backend override if provided (primarily for testing) if let Some(backend_override) = args diff --git a/crates/pixi_cli/src/run.rs b/crates/pixi_cli/src/run.rs index fb1f560edf..7956bc454d 100644 --- a/crates/pixi_cli/src/run.rs +++ b/crates/pixi_cli/src/run.rs @@ -75,6 +75,14 @@ pub struct Args { #[arg(long = "executable", short = 'x')] pub executable: bool, + /// Enable experimental `--script` features; currently the `conda-script` + /// block proposed in issue #3751. + /// + /// The flag is inert for PEP 723 scripts, so a shebang line can pass it + /// unconditionally. + #[arg(long, requires = "script")] + pub experimental: bool, + #[clap(flatten)] pub workspace_config: ScriptWorkspaceConfig, @@ -168,6 +176,7 @@ pub async fn execute(mut args: Args) -> miette::Result<()> { let cli_config = args .activation_config + .clone() .merge_config(args.config.clone().into()); let is_script = args.workspace_config.script.is_some(); @@ -235,11 +244,38 @@ pub async fn execute(mut args: Args) -> miette::Result<()> { stdin_script_command = Some(prepared.command); workspace } - Some(RunScriptInput::Local(path)) => WorkspaceLocator::for_cli() - .with_global_config_source(global_config_source) - .with_search_start(pixi_core::workspace::DiscoveryStart::Script(path)) - .with_cli_config(cli_config) - .locate()?, + Some(RunScriptInput::Local(path)) => { + // A conda-script block takes this file off the PEP 723 path; a + // file with both kinds of block is rejected by the detection. + if let Some(manifest) = + crate::conda_script::detect_with_fallback(&path, args.experimental)? + { + if !args.experimental { + return Err(miette::miette!( + help = + "conda-script support is experimental; add `--experimental` to run it", + "{} contains a conda-script block", + path.display() + )); + } + let root = manifest + .path() + .parent() + .expect("an absolute script path always has a parent") + .to_owned(); + let config = pixi_config::Config::load_with(&root, &global_config_source) + .merge_config(cli_config); + if not_hidden { + global_multi_progress().set_draw_target(ProgressDrawTarget::stderr_with_hz(20)); + } + return crate::conda_script::execute_run(manifest, args, config).await; + } + WorkspaceLocator::for_cli() + .with_global_config_source(global_config_source) + .with_search_start(pixi_core::workspace::DiscoveryStart::Script(path)) + .with_cli_config(cli_config) + .locate()? + } None => WorkspaceLocator::for_cli() .with_global_config_source(global_config_source) .with_search_start(args.workspace_config.workspace_locator_start()) @@ -876,3 +912,16 @@ async fn listen_and_forward_all_signals(kill_signal: KillSignal) { } futures::future::join_all(futures).await; } + +#[cfg(test)] +mod tests { + use clap::Parser; + + use super::Args; + + #[test] + fn experimental_requires_a_script() { + assert!(Args::try_parse_from(["run", "--experimental", "--script", "main.c"]).is_ok()); + assert!(Args::try_parse_from(["run", "--experimental", "task"]).is_err()); + } +} diff --git a/crates/pixi_cli/src/run_script.rs b/crates/pixi_cli/src/run_script.rs index e994e84daa..f47d41a3fa 100644 --- a/crates/pixi_cli/src/run_script.rs +++ b/crates/pixi_cli/src/run_script.rs @@ -93,7 +93,16 @@ pub(crate) fn prepare_stdin_script( root.to_owned(), "stdin", )? - .ok_or_else(|| miette::miette!("stdin does not contain a PEP 723 metadata block"))?; + .ok_or_else(|| { + if crate::conda_script::looks_like_conda_script(&contents) { + miette::miette!( + help = "conda-script blocks run from local files: save the script and run `pixi run --experimental --script `", + "conda-script blocks are not supported on stdin" + ) + } else { + miette::miette!("stdin does not contain a PEP 723 metadata block") + } + })?; let contents = String::from_utf8(contents).expect("ScriptManifest validates the complete script as UTF-8"); Ok(PreparedStdinScript { @@ -159,11 +168,18 @@ pub(crate) async fn prepare_remote_script( cache_name.clone(), )? .ok_or_else(|| { - miette::miette!( - help = - "Download the script and initialize it locally with `pixi init --script `.", - "the remote script at {safe_original_url} does not contain a PEP 723 metadata block" - ) + if crate::conda_script::looks_like_conda_script(&contents) { + miette::miette!( + help = "conda-script blocks run from local files: download the script and run `pixi run --experimental --script `", + "the remote script at {safe_original_url} contains a conda-script block, which only runs from a local file" + ) + } else { + miette::miette!( + help = + "Download the script and initialize it locally with `pixi init --script `.", + "the remote script at {safe_original_url} does not contain a PEP 723 metadata block" + ) + } })?; Ok(PreparedRemoteScript { diff --git a/crates/pixi_conda_script/Cargo.toml b/crates/pixi_conda_script/Cargo.toml index 47ba1f9140..21281ea098 100644 --- a/crates/pixi_conda_script/Cargo.toml +++ b/crates/pixi_conda_script/Cargo.toml @@ -20,6 +20,7 @@ pixi_toml = { workspace = true } rattler_conda_types = { workspace = true } thiserror = { workspace = true } tokio = { workspace = true, features = ["io-util", "process"] } +toml_edit = { workspace = true } toml-span = { workspace = true } [dev-dependencies] diff --git a/crates/pixi_conda_script/src/manifest.rs b/crates/pixi_conda_script/src/manifest.rs index ca5fb27745..3bf937284b 100644 --- a/crates/pixi_conda_script/src/manifest.rs +++ b/crates/pixi_conda_script/src/manifest.rs @@ -94,6 +94,78 @@ impl CondaScriptManifest { pub fn toml(&self) -> &str { &self.toml } + + /// Renders the metadata as a `pixi.toml` document. + /// + /// The dependency tables are spliced in verbatim from the block, so the + /// specs reach the manifest parser exactly as written. `[dependencies]` + /// fills the default feature while `[tool.pixi.dependencies]` and + /// `[tool.pixi.pypi-dependencies]` become a separate feature of the + /// default environment, which merges the two spec sets the way pixi + /// merges features: both specs of a package reach the solver. + pub fn synthetic_manifest(&self) -> Result { + let mut block: toml_edit::DocumentMut = self.toml.parse()?; + let channels = block + .remove("channels") + .expect("`channels` is required by the metadata model"); + let dependencies = block.remove("dependencies"); + let mut pixi = block + .get_mut("tool") + .and_then(toml_edit::Item::as_table_like_mut) + .and_then(|tool| tool.get_mut("pixi")) + .and_then(toml_edit::Item::as_table_like_mut); + let pixi_dependencies = pixi.as_mut().and_then(|pixi| pixi.remove("dependencies")); + let pixi_pypi_dependencies = pixi + .as_mut() + .and_then(|pixi| pixi.remove("pypi-dependencies")); + + let name = self + .path + .file_stem() + .and_then(|stem| stem.to_str()) + .filter(|stem| !stem.is_empty()) + .unwrap_or("script"); + + let mut document = toml_edit::DocumentMut::new(); + let mut workspace = toml_edit::Table::new(); + workspace.insert("name", toml_edit::value(name)); + workspace.insert("channels", channels); + workspace.insert( + "platforms", + toml_edit::Item::Value(toml_edit::Value::Array(toml_edit::Array::new())), + ); + document.insert("workspace", toml_edit::Item::Table(workspace)); + + if let Some(dependencies) = dependencies { + document.insert("dependencies", dependencies); + } + + if pixi_dependencies.is_some() || pixi_pypi_dependencies.is_some() { + let mut tool_pixi = toml_edit::Table::new(); + tool_pixi.set_implicit(true); + if let Some(pixi_dependencies) = pixi_dependencies { + tool_pixi.insert("dependencies", pixi_dependencies); + } + if let Some(pixi_pypi_dependencies) = pixi_pypi_dependencies { + tool_pixi.insert("pypi-dependencies", pixi_pypi_dependencies); + } + let mut feature = toml_edit::Table::new(); + feature.set_implicit(true); + feature.insert("tool-pixi", toml_edit::Item::Table(tool_pixi)); + document.insert("feature", toml_edit::Item::Table(feature)); + + let mut default = toml_edit::Array::new(); + default.push("tool-pixi"); + let mut environments = toml_edit::Table::new(); + environments.insert( + "default", + toml_edit::Item::Value(toml_edit::Value::Array(default)), + ); + document.insert("environments", toml_edit::Item::Table(environments)); + } + + Ok(document.to_string()) + } } #[cfg(test)] @@ -334,6 +406,75 @@ mod tests { assert!(CondaScriptManifest::from_path(&empty).unwrap().is_none()); } + #[test] + fn renders_a_synthetic_pixi_manifest() { + let manifest = parse( + r#"// /// conda-script +// channels = ["conda-forge"] +// entrypoint = "python ${SCRIPT}" +// +// [dependencies] +// python = "3.13.*" +// simple-app = "0.1.*" +// gcc = { version = "*", when = "__unix" } +// pytorch = { +// version = ">=2.4", +// build = "*cuda*", +// } +// +// [tool.pixi.dependencies] +// simple-app = { git = "https://github.com/prefix-dev/pixi-build-testsuite.git" } +// +// [tool.pixi.pypi-dependencies] +// requests = ">=2" +// /// end-conda-script +"#, + ) + .unwrap() + .unwrap(); + + insta::assert_snapshot!(manifest.synthetic_manifest().unwrap(), @r#" + [workspace] + name = "example" + channels = ["conda-forge"] + platforms = [] + + [dependencies] + python = "3.13.*" + simple-app = "0.1.*" + gcc = { version = "*", when = "__unix" } + pytorch = { + version = ">=2.4", + build = "*cuda*", + } + + [feature.tool-pixi.dependencies] + simple-app = { git = "https://github.com/prefix-dev/pixi-build-testsuite.git" } + + [feature.tool-pixi.pypi-dependencies] + requests = ">=2" + + [environments] + default = ["tool-pixi"] + "#); + } + + #[test] + fn a_synthetic_manifest_without_tool_pixi_has_no_feature() { + let manifest = parse( + "# /// conda-script\n# channels = [\"conda-forge\"]\n# entrypoint = \"python ${SCRIPT}\"\n# /// end-conda-script\n", + ) + .unwrap() + .unwrap(); + + insta::assert_snapshot!(manifest.synthetic_manifest().unwrap(), @r#" + [workspace] + name = "example" + channels = ["conda-forge"] + platforms = [] + "#); + } + #[test] fn errors_on_an_unterminated_block() { insta::assert_snapshot!(parse_error( diff --git a/crates/pixi_core/Cargo.toml b/crates/pixi_core/Cargo.toml index 3ad44137c0..169937d996 100644 --- a/crates/pixi_core/Cargo.toml +++ b/crates/pixi_core/Cargo.toml @@ -36,6 +36,7 @@ pixi_build_frontend = { workspace = true } pixi_build_types = { workspace = true } pixi_command_dispatcher = { workspace = true } pixi_compute_reporters = { workspace = true } +pixi_conda_script = { workspace = true } pixi_config = { workspace = true } pixi_consts = { workspace = true } pixi_diff = { workspace = true } diff --git a/crates/pixi_core/src/lock_file/update.rs b/crates/pixi_core/src/lock_file/update.rs index 3751d9abe4..a6abfaff41 100644 --- a/crates/pixi_core/src/lock_file/update.rs +++ b/crates/pixi_core/src/lock_file/update.rs @@ -831,6 +831,17 @@ impl<'p> LockFileDerivedData<'p> { { return; } + // A conda-script block has no `platforms` key and no editing support, + // so the PEP 723 remedies would send its users after commands that + // reject the file. + if self.workspace.is_conda_script() { + tracing::warn!( + "a conda-script cannot declare platforms, so {} records this machine's \ + virtual packages and reproduces only on machines that provide them.", + lock_file_path.display(), + ); + return; + } let script = self.workspace.workspace.provenance.absolute_path(); tracing::warn!( "the script declares no platforms, so {} records this machine's virtual packages.\n\ diff --git a/crates/pixi_core/src/workspace/mod.rs b/crates/pixi_core/src/workspace/mod.rs index aebe722795..23106816ef 100644 --- a/crates/pixi_core/src/workspace/mod.rs +++ b/crates/pixi_core/src/workspace/mod.rs @@ -13,7 +13,7 @@ mod workspace_mut; mod workspace_script; use self::errors::VariantsError; -use self::workspace_script::WorkspaceScript; +use self::workspace_script::{ScriptSource, WorkspaceScript}; #[cfg(not(windows))] use std::os::unix::fs::symlink; use std::{ @@ -39,6 +39,7 @@ use once_cell::sync::OnceCell; use pep508_rs::Requirement; use pixi_build_frontend::BackendOverride; use pixi_command_dispatcher::{CacheDirs, CommandDispatcher, CommandDispatcherBuilder, Limits}; +use pixi_conda_script::CondaScriptManifest; use pixi_config::{CacheKind, Config, RunPostLinkScripts}; use pixi_consts::consts; use pixi_diff::LockFileDiff; @@ -46,7 +47,10 @@ use pixi_manifest::{ AssociateProvenance, BuildVariantSource, EnvironmentName, Environments, FeaturesExt, HasWorkspaceManifest, LoadManifestsError, ManifestKind, ManifestProvenance, Manifests, PackageManifest, PixiPlatform, PixiPlatformName, PrioritizedChannel, SpecType, WithProvenance, - WithWarnings, WorkspaceManifest, script::ScriptManifest, + WithWarnings, WorkspaceManifest, + script::ScriptManifest, + toml::{ExternalWorkspaceProperties, FromTomlStr, PackageDefaults, TomlManifest}, + utils::WithSourceCode, }; use pixi_path::AbsPathBuf; use pixi_pypi_spec::{PixiPypiSpec, PypiPackageName}; @@ -207,6 +211,10 @@ pub enum ScriptWorkspaceError { #[diagnostic(transparent)] Manifest(#[from] pixi_manifest::script::ScriptManifestError), + #[error(transparent)] + #[diagnostic(transparent)] + CondaScriptManifest(Box>>), + #[error("failed to resolve the script environment cache directory: {0}")] CacheDirectory(String), @@ -509,6 +517,76 @@ impl Workspace { .with_warnings(warnings)) } + /// Construct an isolated workspace for a local `conda-script` file. + /// + /// `config` must include both the selected global configuration and CLI + /// overrides, like [`Workspace::from_script`]. + pub fn from_conda_script( + script: CondaScriptManifest, + config: Config, + ) -> Result, ScriptWorkspaceError> { + let script_path = script.path().to_owned(); + let root = script_path + .parent() + .expect("an absolute script path always has a parent") + .to_owned(); + // The diagnostics of this step point into the synthesized document, + // not the script; the source name says so. + let source_name = format!("{} (synthesized manifest)", script_path.display()); + let source = script.synthetic_manifest().map_err(|error| { + ScriptWorkspaceError::CondaScriptManifest(Box::new(WithSourceCode { + error: pixi_manifest::TomlError::from(error), + source: miette::NamedSource::new(&source_name, script.toml().to_owned()), + })) + })?; + + let (mut manifest, package, warnings) = TomlManifest::from_toml_str(&source) + .and_then(|manifest| { + manifest.into_workspace_manifest( + ExternalWorkspaceProperties::default(), + PackageDefaults::default(), + &root, + ) + }) + .map_err(|error| { + ScriptWorkspaceError::CondaScriptManifest(Box::new(WithSourceCode { + error, + source: miette::NamedSource::new(&source_name, source), + })) + })?; + debug_assert!( + package.is_none(), + "a synthetic conda-script manifest never defines a package" + ); + + let cache_root = config + .cache_dir_for(CacheKind::ExecEnvironments) + .map_err(|error| ScriptWorkspaceError::CacheDirectory(error.to_string()))?; + let workspace_script = WorkspaceScript::for_local_conda_script(script, &cache_root); + let lock_file_path = workspace_script + .lock_file_path() + .expect("a local script has an adjacent lock-file path"); + set_implicit_script_platforms( + &mut manifest.workspace, + implicit_script_platforms(Some(&lock_file_path))?, + ); + + // The provenance kind only matters for flows that re-read or edit the + // manifest, which conda-script workspaces reject; PEP 723 is the + // closest embedded-metadata kind. + let workspace = + manifest.with_provenance(ManifestProvenance::new(script_path, ManifestKind::Pep723)); + + Ok(WithWarnings::from(Self::from_parsed( + workspace, + None, + root, + config, + WorkspaceStorage::Script(workspace_script), + )) + .with_warnings(warnings)) + } + /// Construct an isolated workspace for a transient PEP 723 script. pub fn from_transient_script( script: ScriptManifest, @@ -677,10 +755,22 @@ impl Workspace { let WorkspaceStorage::Script(script) = &self.storage else { return false; }; - script - .manifest() - .workspace_config() - .is_ok_and(|config| !config.platforms_explicit) + match script.source() { + ScriptSource::Pep723(manifest) => manifest + .workspace_config() + .is_ok_and(|config| !config.platforms_explicit), + // A conda-script block has no `platforms` key at all. + ScriptSource::CondaScript(_) => true, + } + } + + /// `true` when this workspace was constructed from a conda-script file. + pub fn is_conda_script(&self) -> bool { + matches!( + &self.storage, + WorkspaceStorage::Script(script) + if matches!(script.source(), ScriptSource::CondaScript(_)) + ) } /// Create the detached-environments path for this project if it is set in @@ -2134,6 +2224,71 @@ platforms = [] .value } + #[test] + fn conda_script_workspace_merges_tool_pixi_into_the_default_environment() { + let root = tempfile::tempdir().unwrap(); + let cache = tempfile::tempdir().unwrap(); + let path = root.path().join("example.c"); + fs_err::write( + &path, + r#"// /// conda-script +// channels = ["testing"] +// entrypoint = "run ${SCRIPT}" +// +// [dependencies] +// zlib = "1.3.*" +// +// [tool.pixi.pypi-dependencies] +// requests = ">=2" +// /// end-conda-script +"#, + ) + .unwrap(); + let script = CondaScriptManifest::from_path(&path).unwrap().unwrap(); + + let workspace = Workspace::from_conda_script( + script, + Config { + cache: CacheConfig { + exec_environments: Some(cache.path().to_owned()), + ..Default::default() + }, + ..Default::default() + }, + ) + .unwrap() + .value; + + let manifest = &workspace.workspace.value; + assert_eq!( + manifest + .workspace + .channels + .iter() + .map(|channel| channel.channel.to_string()) + .collect::>(), + ["testing"] + ); + assert_eq!(manifest.environments.iter().count(), 1); + assert_eq!(manifest.all_features().count(), 2); + let default_environment = workspace.default_environment(); + assert!( + default_environment + .pypi_dependencies(None) + .contains_key(&PypiPackageName::from_str("requests").unwrap()), + "the tool.pixi feature must reach the default environment" + ); + assert!( + !manifest.workspace.platforms.is_empty(), + "a conda-script workspace resolves for the machine it runs on" + ); + assert!(workspace.script_platforms_are_implicit()); + assert_eq!( + workspace.lock_file_path(), + root.path().join("example.c.pixi.lock") + ); + } + #[test] fn script_workspace_separates_source_state_and_lock_paths() { let root = tempfile::tempdir().unwrap(); diff --git a/crates/pixi_core/src/workspace/workspace_mut.rs b/crates/pixi_core/src/workspace/workspace_mut.rs index 98a5e5e296..39d234c4ff 100644 --- a/crates/pixi_core/src/workspace/workspace_mut.rs +++ b/crates/pixi_core/src/workspace/workspace_mut.rs @@ -33,7 +33,7 @@ use crate::{ lock_file::{LockFileDerivedData, ReinstallPackages, UpdateContext, UpdateMode}, workspace::{ MatchSpecs, NON_SEMVER_PACKAGES, PypiDeps, SkippedPackage, SourceSpecs, UpdateDeps, - WorkspaceStorage, grouped_environment::GroupedEnvironment, + WorkspaceStorage, grouped_environment::GroupedEnvironment, workspace_script::ScriptSource, }, }; @@ -96,10 +96,27 @@ impl WorkspaceMut { let contents = workspace.workspace.provenance.read()?.into_inner(); let workspace_manifest_document = match &workspace.storage { - WorkspaceStorage::Script(script) => { - ManifestDocument::from_script(script.manifest().clone()) - .expect("a loaded script must remain valid") - } + WorkspaceStorage::Script(script) => match script.source() { + ScriptSource::Pep723(manifest) => { + ManifestDocument::from_script((**manifest).clone()) + .expect("a loaded script must remain valid") + } + ScriptSource::CondaScript(manifest) => { + return Err(Box::new(WithSourceCode { + error: TomlError::Generic( + pixi_manifest::GenericError::new( + "conda-script files cannot be edited by pixi", + ) + .with_help("edit the `/// conda-script` block in the file directly"), + ), + source: NamedSource::new( + manifest.path().to_string_lossy(), + Arc::from(contents.as_str()), + ), + }) + .into()); + } + }, WorkspaceStorage::Project => { let toml = match DocumentMut::from_str(&contents) { Ok(document) => TomlDocument::new(document), diff --git a/crates/pixi_core/src/workspace/workspace_script.rs b/crates/pixi_core/src/workspace/workspace_script.rs index 8488b6e088..95a6987f93 100644 --- a/crates/pixi_core/src/workspace/workspace_script.rs +++ b/crates/pixi_core/src/workspace/workspace_script.rs @@ -4,6 +4,7 @@ use std::{ }; use miette::{Context, IntoDiagnostic}; +use pixi_conda_script::CondaScriptManifest; use pixi_manifest::script::ScriptManifest; use rattler_lock::LockFile; use serde::{Deserialize, Serialize}; @@ -25,6 +26,14 @@ struct CachedResolution { lock_file: String, } +/// The parsed manifest of a script workspace: a PEP 723 Python script or a +/// `conda-script` code file. +#[derive(Debug, Clone)] +pub(super) enum ScriptSource { + Pep723(Box), + CondaScript(Box), +} + /// Describes where a script reads and writes workspace state. /// /// Every script has a parsed manifest and a cache directory. A local script @@ -33,7 +42,7 @@ struct CachedResolution { /// lock file may keep its last resolution in the cache directory. #[derive(Debug, Clone)] pub(super) struct WorkspaceScript { - manifest: Box, + source: ScriptSource, pixi_dir: PathBuf, /// The adjacent lock-file path for a local script. @@ -45,7 +54,17 @@ impl WorkspaceScript { let pixi_dir = cache_root.join(local_cache_name(manifest.path())); let lock_file_path = Some(local_lock_file_path(manifest.path())); Self { - manifest: Box::new(manifest), + source: ScriptSource::Pep723(Box::new(manifest)), + pixi_dir, + lock_file_path, + } + } + + pub(super) fn for_local_conda_script(manifest: CondaScriptManifest, cache_root: &Path) -> Self { + let pixi_dir = cache_root.join(local_cache_name(manifest.path())); + let lock_file_path = Some(local_lock_file_path(manifest.path())); + Self { + source: ScriptSource::CondaScript(Box::new(manifest)), pixi_dir, lock_file_path, } @@ -59,21 +78,26 @@ impl WorkspaceScript { root: &Path, ) -> Self { Self { - manifest: Box::new(manifest), + source: ScriptSource::Pep723(Box::new(manifest)), pixi_dir: cache_root.join(transient_cache_name(cache_name, cache_key, root)), lock_file_path: None, } } - pub(super) fn manifest(&self) -> &ScriptManifest { - &self.manifest + pub(super) fn source(&self) -> &ScriptSource { + &self.source } pub(super) fn replace_manifest(&mut self, new_manifest: ScriptManifest) { if self.lock_file_path.is_some() { self.lock_file_path = Some(local_lock_file_path(new_manifest.path())); } - *self.manifest = new_manifest; + match &mut self.source { + ScriptSource::Pep723(manifest) => **manifest = new_manifest, + ScriptSource::CondaScript(_) => { + unreachable!("conda-script workspaces reject manifest edits when they are opened") + } + } } pub(super) fn pixi_dir(&self) -> &Path { diff --git a/crates/pixi_manifest/src/lib.rs b/crates/pixi_manifest/src/lib.rs index 0d3a679fa7..1452d44edd 100644 --- a/crates/pixi_manifest/src/lib.rs +++ b/crates/pixi_manifest/src/lib.rs @@ -39,7 +39,7 @@ pub use discovery::{ PixiVersionMismatchError, WorkspaceDiscoverer, WorkspaceDiscoveryError, }; pub use environment::{Environment, EnvironmentName, NewEnvironment}; -pub use error::{DependencyError, TomlError}; +pub use error::{DependencyError, GenericError, TomlError}; pub use feature::{Feature, FeatureName}; pub use features_ext::FeaturesExt; pub use has_features_iter::HasFeaturesIter; diff --git a/docs/reference/cli/pixi/run.md b/docs/reference/cli/pixi/run.md index febee258a2..f8338c07a8 100644 --- a/docs/reference/cli/pixi/run.md +++ b/docs/reference/cli/pixi/run.md @@ -21,6 +21,8 @@ pixi run [OPTIONS] [TASK]... ## Options - `--executable (-x)` : Execute the command as an executable without resolving Pixi tasks +- `--experimental` +: Enable experimental `--script` features; currently the `conda-script` block proposed in issue #3751 - `--environment (-e) ` : The environment to run the task in - `--platform (-p) ` diff --git a/tests/integration_python/test_conda_script.py b/tests/integration_python/test_conda_script.py new file mode 100644 index 0000000000..256848a2e6 --- /dev/null +++ b/tests/integration_python/test_conda_script.py @@ -0,0 +1,263 @@ +import json +import re +from pathlib import Path + +import pytest + +from .common import CONDA_FORGE_CHANNEL, CURRENT_PLATFORM, ExitCode, verify_cli_command + +PYTHON_ARGV_SCRIPT = f"""# /// conda-script +# channels = ["{CONDA_FORGE_CHANNEL}"] +# entrypoint = "python ${{SCRIPT}}" +# +# [dependencies] +# python = "3.13.*" +# /// end-conda-script +import json, os, sys + +print(json.dumps({{"argv": sys.argv[1:], "cwd": os.getcwd(), "file": __file__}})) +""" + + +def json_payload(stdout: str) -> dict[str, object]: + return json.loads(next(line for line in stdout.splitlines() if line.startswith("{"))) + + +def test_run_requires_the_experimental_flag(pixi: Path, tmp_pixi_workspace: Path) -> None: + script = tmp_pixi_workspace / "example.code" + script.write_text(PYTHON_ARGV_SCRIPT) + + verify_cli_command( + [pixi, "run", "--script", script], + ExitCode.FAILURE, + stderr_contains=["conda-script block", "--experimental"], + ) + + +def test_experimental_requires_a_script(pixi: Path) -> None: + verify_cli_command( + [pixi, "run", "--experimental", "task"], + ExitCode.INCORRECT_USAGE, + stderr_contains="--script", + ) + + +def test_rejects_a_file_with_both_block_kinds(pixi: Path, tmp_pixi_workspace: Path) -> None: + script = tmp_pixi_workspace / "example.py" + script.write_text( + "# /// script\n" + "# dependencies = []\n" + "# ///\n" + "# /// conda-script\n" + '# channels = ["conda-forge"]\n' + '# entrypoint = "python ${SCRIPT}"\n' + "# /// end-conda-script\n" + ) + + verify_cli_command( + [pixi, "run", "--experimental", "--script", script], + ExitCode.FAILURE, + stderr_contains="both a PEP 723", + ) + + +def test_pep723_routing_is_unchanged(pixi: Path, tmp_pixi_workspace: Path) -> None: + script = tmp_pixi_workspace / "example.py" + script.write_text("print('hello')\n") + + # A Python file without any block reports the PEP 723 error, with and + # without the inert flag. + for command in ( + [pixi, "run", "--script", script], + [pixi, "run", "--experimental", "--script", script], + ): + verify_cli_command( + command, + ExitCode.FAILURE, + stderr_contains="does not contain a PEP 723 metadata block", + ) + + +def test_a_stray_marker_in_a_python_file_stays_on_the_pep723_path( + pixi: Path, tmp_pixi_workspace: Path +) -> None: + script = tmp_pixi_workspace / "example.py" + script.write_text('"""\n # /// conda-script\n"""\nprint()\n') + + # Without the flag the malformed pseudo-block must not break the file. + verify_cli_command( + [pixi, "run", "--script", script], + ExitCode.FAILURE, + stderr_contains="does not contain a PEP 723 metadata block", + ) + # With the flag the block error surfaces. + verify_cli_command( + [pixi, "run", "--experimental", "--script", script], + ExitCode.FAILURE, + stderr_contains="no closing", + ) + + +def test_init_refuses_a_conda_script_file(pixi: Path, tmp_pixi_workspace: Path) -> None: + script = tmp_pixi_workspace / "example.py" + contents = ( + "# /// conda-script\n" + '# channels = ["conda-forge"]\n' + '# entrypoint = "python ${SCRIPT}"\n' + "# /// end-conda-script\n" + ) + script.write_text(contents) + + # Prepending a PEP 723 block would leave a file with both kinds that no + # command accepts anymore. + verify_cli_command( + [pixi, "init", "--script", script], + ExitCode.FAILURE, + stderr_contains="already a conda-script", + ) + assert script.read_text() == contents + + +def test_stdin_reports_conda_script_blocks(pixi: Path) -> None: + verify_cli_command( + [pixi, "run", "--script", "-"], + ExitCode.FAILURE, + stderr_contains="not supported on stdin", + stdin=( + "# /// conda-script\n" + '# channels = ["conda-forge"]\n' + '# entrypoint = "python ${SCRIPT}"\n' + "# /// end-conda-script\n" + ), + ) + + +def test_entrypoint_syntax_errors_are_reported(pixi: Path, tmp_pixi_workspace: Path) -> None: + script = tmp_pixi_workspace / "example.code" + script.write_text( + "# /// conda-script\n" + '# channels = ["conda-forge"]\n' + '# entrypoint = "python ${SCRIPT} | tee log"\n' + "# /// end-conda-script\n" + ) + + verify_cli_command( + [pixi, "run", "--experimental", "--script", script, "--dry-run"], + ExitCode.FAILURE, + stderr_contains="pipes are not supported", + ) + + +@pytest.mark.slow +def test_arguments_and_working_directory_reach_the_entrypoint( + pixi: Path, tmp_pixi_workspace: Path +) -> None: + script = tmp_pixi_workspace / "scripts" / "example.code" + script.parent.mkdir() + script.write_text(PYTHON_ARGV_SCRIPT) + invocation_dir = tmp_pixi_workspace / "elsewhere" + invocation_dir.mkdir() + + output = verify_cli_command( + [pixi, "run", "--experimental", "--script", script, "first", "--second"], + cwd=invocation_dir, + ) + + payload = json_payload(output.stdout) + assert payload["argv"] == ["first", "--second"] + assert payload["cwd"] == str(invocation_dir) + assert payload["file"] == str(script) + assert not (script.parent / "example.code.pixi.lock").exists() + + +@pytest.mark.slow +def test_the_cache_directory_persists_between_runs(pixi: Path, tmp_pixi_workspace: Path) -> None: + script = tmp_pixi_workspace / "counter.code" + script.write_text(f"""# /// conda-script +# channels = ["{CONDA_FORGE_CHANNEL}"] +# entrypoint = "python ${{SCRIPT}} ${{CACHE}}" +# +# [dependencies] +# python = "3.13.*" +# /// end-conda-script +import pathlib, sys + +counter = pathlib.Path(sys.argv[1]) / "counter" +runs = int(counter.read_text()) + 1 if counter.exists() else 1 +counter.write_text(str(runs)) +print(f"run {{runs}}") +""") + + # The cache is keyed by the absolute script path, which a reused pytest + # temp directory repeats, so assert on the increment rather than on + # absolute counts. + first = verify_cli_command([pixi, "run", "--experimental", "--script", script]) + count = re.search(r"run (\d+)", first.stdout) + assert count is not None + runs = int(count.group(1)) + verify_cli_command( + [pixi, "run", "--experimental", "--script", script], + stdout_contains=f"run {runs + 1}", + ) + + +@pytest.mark.slow +def test_a_failing_entrypoint_propagates_its_exit_code( + pixi: Path, tmp_pixi_workspace: Path +) -> None: + script = tmp_pixi_workspace / "fails.code" + script.write_text(f"""# /// conda-script +# channels = ["{CONDA_FORGE_CHANNEL}"] +# entrypoint = "python ${{SCRIPT}} && python -c 'open(\\"marker\\", \\"w\\").close()'" +# +# [dependencies] +# python = "3.13.*" +# /// end-conda-script +import sys + +print("about to fail") +sys.exit(1) +""") + + verify_cli_command( + [pixi, "run", "--experimental", "--script", script], + ExitCode.FAILURE, + stdout_contains="about to fail", + cwd=tmp_pixi_workspace, + ) + assert not (tmp_pixi_workspace / "marker").exists() + + +@pytest.mark.slow +def test_lock_writes_an_adjacent_lock_file_with_conditional_dependencies( + pixi: Path, tmp_pixi_workspace: Path +) -> None: + script = tmp_pixi_workspace / "when.code" + script.write_text(f"""# /// conda-script +# channels = ["{CONDA_FORGE_CHANNEL}"] +# entrypoint = "python -c 'print(1234)'" +# +# [dependencies] +# python = "3.13.*" +# zlib = {{ version = "*", when = "__unix" }} +# vc = {{ version = "*", when = "__win" }} +# /// end-conda-script +""") + lock_file = tmp_pixi_workspace / "when.code.pixi.lock" + + verify_cli_command([pixi, "lock", "--script", script]) + + assert lock_file.exists() + locked = lock_file.read_text() + if CURRENT_PLATFORM.startswith("win"): + assert "/vc-" in locked + assert "/zlib-" not in locked + else: + assert "/zlib-" in locked + assert "/vc-" not in locked + + # A run next to the lock file consumes it. + verify_cli_command( + [pixi, "run", "--experimental", "--script", script, "--frozen"], + stdout_contains="1234", + )