diff --git a/docs/python/scripts.md b/docs/python/scripts.md index f7485aed86..ce416c9556 100644 --- a/docs/python/scripts.md +++ b/docs/python/scripts.md @@ -10,6 +10,10 @@ Script commands use `--script `. The same `init`, `run`, `add`, `remove`, `install`, `lock`, and `update` commands used for workspaces can therefore operate on either a manifest or a standalone file. +For files in other languages there is an experimental +[`conda-script` block](../tutorials/conda_script.md) that embeds the same +kind of metadata in any code file. + ## Make a script self-contained This script downloads the USGS earthquake feed with `httpx`, then uses GDAL's diff --git a/docs/reference/pixi_configuration.md b/docs/reference/pixi_configuration.md index 3ea2ac9776..0259b8f12c 100644 --- a/docs/reference/pixi_configuration.md +++ b/docs/reference/pixi_configuration.md @@ -606,6 +606,21 @@ Set the configuration with: This feature is experimental because the cache invalidation is very tricky, and we don't want to disturb users that are not affected by activation times. +### Running conda scripts + +[Conda scripts](../tutorials/conda_script.md) normally need `--experimental` on every `pixi run`. +Turn the flag into a setting with: + +```shell +# For all of your workspaces +pixi config set experimental.conda-script true --global + +# For a specific workspace +pixi config set experimental.conda-script true --local +``` + +Running a conda script then prints a warning instead of asking for the flag, as a reminder that the format may still change. + ## Mirror configuration You can configure mirrors for conda channels. We expect that mirrors are exact diff --git a/docs/source_files/conda_scripts/main.R b/docs/source_files/conda_scripts/main.R new file mode 100644 index 0000000000..a617f98a6e --- /dev/null +++ b/docs/source_files/conda_scripts/main.R @@ -0,0 +1,16 @@ +# /// conda-script +# channels = ["https://prefix.dev/conda-forge"] +# entrypoint = "Rscript ${SCRIPT}" +# +# [dependencies] +# r-base = "*" +# r-jsonlite = "*" +# /// end-conda-script +library(jsonlite) + +document <- list( + name = "conda-script", + languages = c("r", "python"), + count = 2 +) +writeLines(toJSON(document, auto_unbox = TRUE, pretty = TRUE)) diff --git a/docs/source_files/conda_scripts/main.c b/docs/source_files/conda_scripts/main.c new file mode 100644 index 0000000000..802257e2bf --- /dev/null +++ b/docs/source_files/conda_scripts/main.c @@ -0,0 +1,17 @@ +// /// conda-script +// channels = ["https://prefix.dev/conda-forge"] +// entrypoint = "gcc -o ${CACHE}/main ${SCRIPT} $(pkg-config --cflags --libs glib-2.0) && ${CACHE}/main" +// +// [dependencies] +// gcc = "*" +// glib = "*" +// pkg-config = "*" +// /// end-conda-script +#include + +int main(void) { + gchar *digest = g_compute_checksum_for_string(G_CHECKSUM_SHA256, "conda-script", -1); + g_print("sha256(\"conda-script\") = %s\n", digest); + g_free(digest); + return 0; +} diff --git a/docs/source_files/conda_scripts/main.cpp b/docs/source_files/conda_scripts/main.cpp new file mode 100644 index 0000000000..5eb97c99bc --- /dev/null +++ b/docs/source_files/conda_scripts/main.cpp @@ -0,0 +1,17 @@ +// /// conda-script +// channels = ["https://prefix.dev/conda-forge"] +// entrypoint = "g++ -o ${CACHE}/main ${SCRIPT} -lfmt && ${CACHE}/main" +// +// [dependencies] +// gxx = "*" +// fmt = "*" +// /// end-conda-script +#include +#include +#include + +int main() { + std::vector primes{2, 3, 5, 7, 11}; + fmt::print("primes: {}\n", fmt::join(primes, ", ")); + fmt::print("pi is roughly {:.3f}\n", 3.14159); +} diff --git a/docs/source_files/conda_scripts/main.cs b/docs/source_files/conda_scripts/main.cs new file mode 100644 index 0000000000..423d5e2dfb --- /dev/null +++ b/docs/source_files/conda_scripts/main.cs @@ -0,0 +1,15 @@ +// /// conda-script +// channels = ["https://prefix.dev/conda-forge"] +// entrypoint = "dotnet run ${SCRIPT}" +// +// [dependencies] +// dotnet = "*" +// /// end-conda-script + +#:package Newtonsoft.Json@13.* + +using Newtonsoft.Json; +using Newtonsoft.Json.Linq; + +var payload = new JObject { ["item"] = "answer", ["value"] = 42 }; +Console.WriteLine(payload.ToString(Formatting.None)); diff --git a/docs/source_files/conda_scripts/main.f90 b/docs/source_files/conda_scripts/main.f90 new file mode 100644 index 0000000000..71866af8d1 --- /dev/null +++ b/docs/source_files/conda_scripts/main.f90 @@ -0,0 +1,19 @@ +! /// conda-script +! channels = ["https://prefix.dev/conda-forge"] +! entrypoint = "gfortran -o ${CACHE}/main ${SCRIPT} -llapack -lblas && ${CACHE}/main" +! +! [dependencies] +! gfortran = "*" +! liblapack = "*" +! /// end-conda-script +program solve + implicit none + real(8) :: a(2, 2), b(2) + integer :: ipiv(2), info + + a = reshape([2.0d0, 1.0d0, 1.0d0, 3.0d0], [2, 2]) + b = [5.0d0, 10.0d0] + call dgesv(2, 1, a, 2, ipiv, b, 2, info) + write (*, '(a, i0)') 'info = ', info + write (*, '(a, 2f8.3)') 'x =', b +end program solve diff --git a/docs/source_files/conda_scripts/main.main.kts b/docs/source_files/conda_scripts/main.main.kts new file mode 100644 index 0000000000..6576c016f8 --- /dev/null +++ b/docs/source_files/conda_scripts/main.main.kts @@ -0,0 +1,15 @@ +// /// conda-script +// channels = ["https://prefix.dev/conda-forge"] +// entrypoint = "kotlin ${SCRIPT}" +// +// [dependencies] +// kotlin = "*" +// /// end-conda-script +@file:DependsOn("com.google.code.gson:gson:2.13.1") + +import com.google.gson.GsonBuilder + +data class Language(val name: String, val year: Int) + +val gson = GsonBuilder().setPrettyPrinting().create() +println(gson.toJson(Language("kotlin", 2011))) diff --git a/docs/source_files/conda_scripts/main.mojo b/docs/source_files/conda_scripts/main.mojo new file mode 100644 index 0000000000..b169c42282 --- /dev/null +++ b/docs/source_files/conda_scripts/main.mojo @@ -0,0 +1,23 @@ +# /// conda-script +# channels = [ +# "https://prefix.dev/modular-community", +# "https://conda.modular.com/max", +# "https://prefix.dev/conda-forge", +# ] +# entrypoint = "mojo ${SCRIPT}" +# +# [dependencies] +# mojo = "*" +# emberjson = "*" +# /// end-conda-script + +from emberjson import parse, to_string + + +def main() raises: + var document = parse( + '{"name": "conda-script", "languages": ["mojo", "python"], "count": 2}' + ) + ref languages = document.object()["languages"].array() + print("languages:", len(languages), "first:", languages[0].string()) + print(to_string[pretty=True](document)) diff --git a/docs/source_files/conda_scripts/main.py b/docs/source_files/conda_scripts/main.py new file mode 100644 index 0000000000..dd14e4ba34 --- /dev/null +++ b/docs/source_files/conda_scripts/main.py @@ -0,0 +1,20 @@ +# /// conda-script +# channels = ["https://prefix.dev/conda-forge"] +# entrypoint = "python ${SCRIPT}" +# +# [dependencies] +# python = "*" +# pyyaml = "*" +# /// end-conda-script +import yaml + +document = yaml.safe_load( + """ +name: conda-script +languages: + - python + - c +""" +) +document["count"] = len(document["languages"]) +print(yaml.safe_dump(document, sort_keys=True), end="") diff --git a/docs/source_files/conda_scripts/main.ts b/docs/source_files/conda_scripts/main.ts new file mode 100644 index 0000000000..4540ed1ea3 --- /dev/null +++ b/docs/source_files/conda_scripts/main.ts @@ -0,0 +1,12 @@ +// /// conda-script +// channels = ["https://prefix.dev/conda-forge"] +// entrypoint = "deno run ${SCRIPT}" +// +// [dependencies] +// deno = "*" +// /// end-conda-script + +import { chunk } from "npm:lodash-es@4"; + +const pairs: string[][] = chunk(["a", "b", "c", "d"], 2); +console.log(JSON.stringify(pairs)); diff --git a/docs/source_files/pixi_config_tomls/main_config.toml b/docs/source_files/pixi_config_tomls/main_config.toml index 6f7f1efc97..5c9ecd0b44 100644 --- a/docs/source_files/pixi_config_tomls/main_config.toml +++ b/docs/source_files/pixi_config_tomls/main_config.toml @@ -130,6 +130,8 @@ solves = 2 [experimental] # Enable the use of the environment activation cache use-environment-activation-cache = true +# Run conda scripts without passing `--experimental` every time +conda-script = true # --8<-- [end:experimental] # --8<-- [start:mirrors] diff --git a/docs/tutorials/conda_script.md b/docs/tutorials/conda_script.md new file mode 100644 index 0000000000..8676ecbbd6 --- /dev/null +++ b/docs/tutorials/conda_script.md @@ -0,0 +1,232 @@ +# Standalone scripts in any language + +With conda scripts you can run a self contained script written in any language that includes dependencies and entry-point. + +!!! warning "Experimental" + This implements the draft `conda-script` proposal from [issue #3751](https://github.com/prefix-dev/pixi/issues/3751), which is on its way to becoming a CEP for the whole conda ecosystem. + Opt in with `pixi config set experimental.conda-script true --global` for now, and let us know on the issue how it goes. + +Let's take this script written in R for example. +This is the most optimal use case for `conda-script`, R is meant for scripting, `conda-forge` features a wide selection of R libraries and [unlike Python](../python/scripts.md) it doesn't have its own script syntax. + +```r title="main.R" +--8<-- "docs/source_files/conda_scripts/main.R" +``` + +Every conda script needs to declare the channels where the packages come from and the entrypoint describing how the script should be run. +Typically, you also want to add the toolchain of your language (`r-base`) and maybe a few dependencies (`r-jsonlite`) in order to make sure the script is self-contained. +In the script itself, we create a variable called `document` and then print it as JSON. + +We can then run it with by executing the following command: + +```shell +$ pixi run --script main.R +{ + "name": "conda-script", + "languages": ["r", "python"], + "count": 2 +} +``` + +Pixi solves the dependencies, installs the environment into its cache and runs the entrypoint inside it. + +## More languages + +Ideally, dependencies also come from a conda channel. +Here are examples for languages that have a great selection of libraries on conda-forge or other channels. + +=== "Python" + + ```py title="main.py" + --8<-- "docs/source_files/conda_scripts/main.py" + ``` + +=== "R" + + ```r title="main.R" + --8<-- "docs/source_files/conda_scripts/main.R" + ``` + +=== "C" + + ```c title="main.c" + --8<-- "docs/source_files/conda_scripts/main.c" + ``` + +=== "C++" + + ```cpp title="main.cpp" + --8<-- "docs/source_files/conda_scripts/main.cpp" + ``` + +=== "Fortran" + + ```fortran title="main.f90" + --8<-- "docs/source_files/conda_scripts/main.f90" + ``` + +=== "Mojo" + + ```mojo title="main.mojo" + --8<-- "docs/source_files/conda_scripts/main.mojo" + ``` + +Other languages work well with conda script, even though only the toolchain is available. +That is because they allow to specify dependencies as part of the program. + +=== "C#" + + ```csharp title="main.cs" + --8<-- "docs/source_files/conda_scripts/main.cs" + ``` + +=== "Kotlin" + + ```kotlin title="main.main.kts" + --8<-- "docs/source_files/conda_scripts/main.main.kts" + ``` + +=== "TypeScript" + + ```typescript title="main.ts" + --8<-- "docs/source_files/conda_scripts/main.ts" + ``` + +## Creating a script + +`pixi init --script ` writes a runnable starting point, choosing the comment syntax, an entrypoint and the toolchain dependency from the file extension: + +```shell +$ pixi init --script main.R +$ pixi run --script main.R +Hello from pixi! +``` + +A Python file gets a [PEP 723 block](../python/scripts.md) instead; `--format conda-script` overrides that default. +Unknown extensions error and list the supported ones. + +## The comment block + +The metadata lives in a comment block at the top of the file, written with your language's own line comments. +The file therefore stays valid source code that editors, formatters and the language's own tooling keep understanding. + +Open the block with `/// conda-script`, start every line with the same comment characters and close it with `/// end-conda-script`: + +```r +# /// conda-script +# channels = ["https://prefix.dev/conda-forge"] +# entrypoint = "Rscript ${SCRIPT}" +# /// end-conda-script +``` + +C uses `//` for the same block, Fortran `!`. +Any comment characters work as long as they contain no letters or digits, which rules out languages that spell their comments as a word like `REM`. +Block comments such as `/* */` are not supported, and a file holds at most one block. + +Inside the block you write TOML 1.1, so inline tables may span several lines. + +## Dependencies + +`[dependencies]` maps conda package names to matchspecs. +The string form is a version: + +```toml +[dependencies] +python = "3.13.*" +gcc = "*" +``` + +The table form supports `version`, `build`, `build-number`, `channel`, `subdir`, `extras`, `flags`, `md5`, `sha256`, `url` and `when`. +Platform specific dependencies use [conditional dependencies](../concepts/package_specifications.md#conditional-dependencies) with virtual packages: + +```toml +[dependencies] +gcc = { version = "*", when = "__unix" } +vs2022_win-64 = { version = "*", when = "__win" } +``` + +## The entrypoint + +`entrypoint` is the command that runs the script. +It is either a string or a table keyed by platform, where the most specific key wins: + +```toml +entrypoint = { + unix = "cc -o ${CACHE}/main ${SCRIPT} && ${CACHE}/main", + win = "cl /Fe:${CACHE}/main.exe ${SCRIPT} && ${CACHE}/main.exe", +} +``` + +The command is not passed to a system shell. +Instead, Pixi runs a built-in shell so it behaves the same on every platform. +The following syntax is supported: whitespace splitting, single and double quotes, `${VAR}` substitution, `$(command)` command substitution and `&&` sequencing. +There are no pipes, redirects, globbing, `||`, `;`, subshells or environment variable assignments. + +Two variables are defined: + +- `${SCRIPT}`: the absolute path of the script file. +- `${CACHE}`: a persistent per-script directory for build artifacts and other state that survives between runs. + +Arguments after the script path are appended to the last command: + +```shell +$ pixi run --script main.R input.txt --verbose +# runs: Rscript ${SCRIPT} input.txt --verbose +``` + +An argument list that starts with a flag needs `--` in front, so pixi does not read the flag itself: `pixi run --script main.R -- --verbose`. + +The entrypoint runs in the directory `pixi run` was invoked from, so relative paths passed to the script work. + +## Pixi-specific configuration + +Tables under `[tool.*]` belong to the named tool. Pixi reads `[tool.pixi]` +the same way as in a `pyproject.toml`, restricted to one implicit +environment: `[tool.pixi.dependencies]` for specs in pixi's native syntax, +including [source dependencies](../build/dependency_types.md), and +`[tool.pixi.pypi-dependencies]` for PyPI packages. + +`[tool.pixi.dependencies]` merges with `[dependencies]` the way pixi merges features: every spec applies. +A source dependency there composes with a version constraint in `[dependencies]`, so tools that only implement the conda-script specification still see a solvable script: + +```toml +[dependencies] +simple-app = "0.1.*" + +[tool.pixi.dependencies] +simple-app = { git = "https://github.com/prefix-dev/pixi-build-testsuite.git", subdirectory = "tests/data/pixi_build/minimal-backend-workspaces/pixi-build-python" } +``` + +## Managing dependencies + +The `--script` commands work on conda-script files the same way they work on PEP 723 scripts. +`pixi add` edits the block in place, preserving the comment prefix and the code around it: + +```shell +$ pixi add --script main.c zlib # writes [dependencies] +$ pixi add --script main.py --pypi rich # writes [tool.pixi.pypi-dependencies] +``` + +`pixi list --script`, `pixi tree --script` and `pixi update --script` read and refresh the same environment. +Since the block cannot express platform-specific tables or git specs under `[dependencies]`, `pixi add` rejects `--platform` and `--git` with a hint towards `when` conditions and `[tool.pixi.dependencies]`. + +## Locking + +Running a script does not create a lock file; the resolution is cached internally. +To pin the environment, write a lock file next to the script: + +```shell +pixi lock --script main.c +``` + +This creates `main.c.pixi.lock`, and a run uses the adjacent lock file whenever it exists, the same convention as for [PEP 723 scripts](../python/scripts.md#lock-exact-versions). + +## Shebang + +Since the shebang line lies outside the block, a Unix script can make itself executable with [`env -S`](../advanced/shebang.md): + +```sh +#!/usr/bin/env -S pixi run --script +``` + +The same shebang works for a [PEP 723 script](../python/scripts.md), so both block kinds share it. diff --git a/tests/integration_python/test_conda_script_examples.py b/tests/integration_python/test_conda_script_examples.py new file mode 100644 index 0000000000..e128975a87 --- /dev/null +++ b/tests/integration_python/test_conda_script_examples.py @@ -0,0 +1,68 @@ +"""Runs every conda-script example the documentation embeds. + +The files live in `docs/source_files/conda_scripts/` and are included into +`docs/tutorials/conda_script.md`, so a failure here means the docs show a +broken example. +""" + +from pathlib import Path + +import pytest + +from .common import CURRENT_PLATFORM, repo_root, verify_cli_command + +EXAMPLES_DIR = repo_root().joinpath("docs/source_files/conda_scripts") + +LINUX_ONLY = pytest.mark.skipif( + not CURRENT_PLATFORM.startswith("linux"), + reason="the example's toolchain packages only exist for Linux", +) +MOJO_PLATFORMS = pytest.mark.skipif( + CURRENT_PLATFORM not in ("linux-64", "osx-arm64"), + reason="the max channel does not serve mojo for this platform", +) + +EXAMPLES = [ + pytest.param( + "main.c", + [ + 'sha256("conda-script") = a733a69b1424e6d2f409c14dfb01c1c1558e0eb943786ea50eb04f55afe2226d' + ], + marks=LINUX_ONLY, + ), + pytest.param("main.py", ["count: 2", "name: conda-script"]), + pytest.param("main.R", ['"name": "conda-script"', '"count": 2']), + pytest.param( + "main.cpp", + ["primes: 2, 3, 5, 7, 11", "pi is roughly 3.142"], + marks=LINUX_ONLY, + ), + pytest.param("main.f90", ["info = 0", "x = 1.000 3.000"], marks=LINUX_ONLY), + pytest.param("main.mojo", ["languages: 2 first: mojo", '"count": 2'], marks=MOJO_PLATFORMS), + pytest.param("main.cs", ['{"item":"answer","value":42}']), + pytest.param("main.main.kts", ['"name": "kotlin"', '"year": 2011']), + pytest.param("main.ts", ['[["a","b"],["c","d"]]']), +] + + +@pytest.mark.slow +@pytest.mark.parametrize(("example", "expected"), EXAMPLES) +def test_docs_example_runs(pixi: Path, example: str, expected: list[str]) -> None: + verify_cli_command( + [pixi, "run", "--experimental", "--script", EXAMPLES_DIR / example], + stdout_contains=expected, + ) + + +def test_every_example_is_documented() -> None: + """Each example file appears as a snippet in the tutorial, and each + documented snippet has a test above.""" + tutorial = repo_root().joinpath("docs/tutorials/conda_script.md").read_text() + on_disk = {path.name for path in EXAMPLES_DIR.iterdir()} + documented = { + line.split("conda_scripts/")[1].rstrip('"').strip() + for line in tutorial.splitlines() + if "conda_scripts/" in line + } + tested = {param.values[0] for param in EXAMPLES} + assert on_disk == documented == tested diff --git a/typos.toml b/typos.toml index 7da2d82716..e998d441eb 100644 --- a/typos.toml +++ b/typos.toml @@ -13,6 +13,7 @@ ignore-hidden = false [default.extend-identifiers] _placehold = "_placehold" +arange = "arange" intoto = "intoto" plt-setp = "plt.setp" solvePnPRansac = "solvePnPRansac" diff --git a/zensical.toml b/zensical.toml index 670a85fcb0..77991e6a83 100644 --- a/zensical.toml +++ b/zensical.toml @@ -32,6 +32,7 @@ nav = [ { "Pytorch Installation" = "python/pytorch.md" }, ] }, + { "Conda Scripts" = "tutorials/conda_script.md" }, { "ROS 2" = "tutorials/ros2.md" }, { "Rust" = "tutorials/rust.md" }, {