From 334d60a4bc2b2cea5aa2e716c505a87f61a9b3f6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dani=C3=ABl=20de=20Kok?= Date: Wed, 9 Sep 2026 15:57:12 +0000 Subject: [PATCH] Rename `kernels-data` to `kernels-common` Give this a more descriptive name, since we are moving more stuff to Rust. Also rename the Rust-Python binding from `_data` to `_rust`. --- .github/workflows/lint.yml | 2 +- .github/workflows/rust.yaml | 12 ++++++------ .github/workflows/security-audit.yml | 2 +- .github/workflows/test_e2e.yaml | 2 +- .github/workflows/test_kernels.yaml | 2 +- Cargo.lock | 6 +++--- Cargo.toml | 2 +- Makefile | 2 +- kernel-builder/Cargo.toml | 2 +- kernel-builder/src/card.rs | 2 +- kernel-builder/src/hash.rs | 4 ++-- kernel-builder/src/init.rs | 2 +- kernel-builder/src/main.rs | 4 ++-- kernel-builder/src/pyproject/common.rs | 6 +++--- kernel-builder/src/pyproject/deps.rs | 2 +- kernel-builder/src/pyproject/kernel.rs | 2 +- kernel-builder/src/pyproject/mod.rs | 6 +++--- kernel-builder/src/pyproject/ops_identifier.rs | 4 ++-- kernel-builder/src/pyproject/torch/mod.rs | 4 ++-- kernel-builder/src/pyproject/torch/noarch.rs | 6 +++--- kernel-builder/src/pyproject/tvm_ffi/mod.rs | 4 ++-- kernel-builder/src/upload.rs | 4 ++-- kernel-builder/src/validate_builds.rs | 2 +- {kernels-data => kernels-common}/Cargo.toml | 4 ++-- .../src/config/compat.rs | 0 {kernels-data => kernels-common}/src/config/deps.rs | 0 .../src/config/git_url.rs | 0 .../src/config/kernel_deps.rs | 0 {kernels-data => kernels-common}/src/config/mod.rs | 0 {kernels-data => kernels-common}/src/config/name.rs | 0 {kernels-data => kernels-common}/src/config/parse.rs | 0 {kernels-data => kernels-common}/src/config/v1.rs | 0 {kernels-data => kernels-common}/src/config/v2.rs | 0 {kernels-data => kernels-common}/src/config/v3.rs | 0 {kernels-data => kernels-common}/src/config/v4.rs | 0 {kernels-data => kernels-common}/src/config/v5.rs | 0 {kernels-data => kernels-common}/src/digest.rs | 0 {kernels-data => kernels-common}/src/git.rs | 0 {kernels-data => kernels-common}/src/lib.rs | 0 {kernels-data => kernels-common}/src/lock.rs | 0 {kernels-data => kernels-common}/src/metadata.rs | 0 .../src/python_dependencies.json | 0 {kernels-data => kernels-common}/src/version.rs | 0 .../tests/data/legacy-sha-provenance.json | 0 kernels/Cargo.toml | 6 +++--- kernels/pyproject.toml | 2 +- kernels/rust/config.rs | 2 +- kernels/rust/lib.rs | 12 ++++++------ kernels/rust/lock.rs | 4 ++-- kernels/rust/version.rs | 4 ++-- kernels/src/kernels/__init__.py | 2 +- kernels/src/kernels/{_data.pyi => _rust.pyi} | 2 +- kernels/src/kernels/_versions.py | 2 +- kernels/src/kernels/archs.py | 2 +- kernels/src/kernels/cli/download.py | 2 +- kernels/src/kernels/cli/info.py | 2 +- kernels/src/kernels/cli/lock.py | 2 +- kernels/src/kernels/deps.py | 2 +- kernels/src/kernels/importer.py | 2 +- kernels/src/kernels/install.py | 2 +- kernels/src/kernels/layer/func.py | 2 +- kernels/src/kernels/layer/layer.py | 2 +- kernels/src/kernels/load.py | 2 +- kernels/src/kernels/locking.py | 2 +- kernels/src/kernels/python_deps.py | 2 +- kernels/src/kernels/resolver.py | 2 +- kernels/src/kernels/validate.py | 2 +- kernels/src/kernels/verify.py | 2 +- kernels/tests/conftest.py | 2 +- kernels/tests/test_archs.py | 2 +- kernels/tests/test_data_config.py | 2 +- kernels/tests/test_data_digest.py | 2 +- kernels/tests/test_data_kernel_lock.py | 2 +- kernels/tests/test_data_metadata.py | 2 +- kernels/tests/test_data_version.py | 2 +- kernels/tests/test_deps.py | 2 +- kernels/tests/test_resolver.py | 2 +- kernels/tests/test_validate.py | 2 +- kernels/tests/test_verify.py | 2 +- nix-builder/lib/python-deps.nix | 2 +- nix-builder/pkgs/crate-dirs.nix | 2 +- .../pkgs/get-kernel-check/get-kernel-check-hook.py | 2 +- .../pkgs/lock-kernel-deps/lock_kernel_deps.py | 2 +- nix-builder/tests/Dockerfile.test-kernel | 4 ++-- scripts/_version_common.py | 2 +- 85 files changed, 95 insertions(+), 95 deletions(-) rename {kernels-data => kernels-common}/Cargo.toml (86%) rename {kernels-data => kernels-common}/src/config/compat.rs (100%) rename {kernels-data => kernels-common}/src/config/deps.rs (100%) rename {kernels-data => kernels-common}/src/config/git_url.rs (100%) rename {kernels-data => kernels-common}/src/config/kernel_deps.rs (100%) rename {kernels-data => kernels-common}/src/config/mod.rs (100%) rename {kernels-data => kernels-common}/src/config/name.rs (100%) rename {kernels-data => kernels-common}/src/config/parse.rs (100%) rename {kernels-data => kernels-common}/src/config/v1.rs (100%) rename {kernels-data => kernels-common}/src/config/v2.rs (100%) rename {kernels-data => kernels-common}/src/config/v3.rs (100%) rename {kernels-data => kernels-common}/src/config/v4.rs (100%) rename {kernels-data => kernels-common}/src/config/v5.rs (100%) rename {kernels-data => kernels-common}/src/digest.rs (100%) rename {kernels-data => kernels-common}/src/git.rs (100%) rename {kernels-data => kernels-common}/src/lib.rs (100%) rename {kernels-data => kernels-common}/src/lock.rs (100%) rename {kernels-data => kernels-common}/src/metadata.rs (100%) rename {kernels-data => kernels-common}/src/python_dependencies.json (100%) rename {kernels-data => kernels-common}/src/version.rs (100%) rename {kernels-data => kernels-common}/tests/data/legacy-sha-provenance.json (100%) rename kernels/src/kernels/{_data.pyi => _rust.pyi} (99%) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 86af286f..7bd7962b 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -55,4 +55,4 @@ jobs: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - name: Validate python_depends.json is up-to-date - run: diff kernels-data/src/python_dependencies.json kernels/src/kernels/python_depends.json + run: diff kernels-common/src/python_dependencies.json kernels/src/kernels/python_depends.json diff --git a/.github/workflows/rust.yaml b/.github/workflows/rust.yaml index ba9326fe..02f19f22 100644 --- a/.github/workflows/rust.yaml +++ b/.github/workflows/rust.yaml @@ -23,9 +23,9 @@ jobs: - name: Cargo fmt (kernel-abi-check) run: | ( cd kernel-abi-check && cargo fmt --all -- --check ) - - name: Cargo fmt (kernels-data) + - name: Cargo fmt (kernels-common) run: | - ( cd kernels-data && cargo fmt --all -- --check ) + ( cd kernels-common && cargo fmt --all -- --check ) ( cd kernels && cargo fmt --all -- --check ) - name: Cargo fmt (kernel-builder) run: ( cd kernel-builder && cargo fmt --all -- --check ) @@ -50,9 +50,9 @@ jobs: - name: Clippy (kernel-abi-check) run: | ( cd kernel-abi-check && cargo clippy -- -D warnings ) - - name: Clippy (kernels-data) + - name: Clippy (kernels-common) run: | - ( cd kernels-data && cargo clippy -- -D warnings ) + ( cd kernels-common && cargo clippy -- -D warnings ) ( cd kernels && cargo clippy -- -D warnings ) - name: Clippy (kernel-builder) run: ( cd kernel-builder && cargo clippy -- -D warnings ) @@ -74,7 +74,7 @@ jobs: - uses: dtolnay/rust-toolchain@e081816240890017053eacbb1bdf337761dc5582 # 1.95.0 - name: Test (kernel-abi-check) run: cargo test -p kernel-abi-check - - name: Test (kernels-data) - run: cargo test -p kernels-data + - name: Test (kernels-common) + run: cargo test -p kernels-common - name: Test (kernel-builder) run: cargo test -p hf-kernel-builder diff --git a/.github/workflows/security-audit.yml b/.github/workflows/security-audit.yml index 2213e0b8..8cebfb49 100644 --- a/.github/workflows/security-audit.yml +++ b/.github/workflows/security-audit.yml @@ -97,7 +97,7 @@ jobs: * `kernel-builder`: this is the main user interface for building and uploading kernels, though it also generates CMake/pyproject files which are used by the NIx builder. * `kernels`: Python package that can download and load kernels from Hugging Face Hub. - * `kernels-data`: data structures shared by `kernel-builder` and `kernels`. + * `kernels-common`: data structures shared by `kernel-builder` and `kernels`. * `nix-builder`: Nix code that drives kernel building, also contains Nix derivations for some dependencies. diff --git a/.github/workflows/test_e2e.yaml b/.github/workflows/test_e2e.yaml index b7f61aed..dccad3db 100644 --- a/.github/workflows/test_e2e.yaml +++ b/.github/workflows/test_e2e.yaml @@ -10,7 +10,7 @@ on: - "kernels/src/**" - "kernels/rust/**" - "nix-builder/**" - - "kernels-data/**" + - "kernels-common/**" - ".github/workflows/test_e2e.yaml" workflow_dispatch: diff --git a/.github/workflows/test_kernels.yaml b/.github/workflows/test_kernels.yaml index cdc71bd9..dfd93e82 100644 --- a/.github/workflows/test_kernels.yaml +++ b/.github/workflows/test_kernels.yaml @@ -65,7 +65,7 @@ jobs: - name: Check type stubs match runtime working-directory: ./kernels - run: uv run python -m mypy.stubtest kernels._data + run: uv run python -m mypy.stubtest kernels._rust - name: Run tests working-directory: ./kernels diff --git a/Cargo.lock b/Cargo.lock index 487875eb..3f59caf7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1070,7 +1070,7 @@ dependencies = [ "indicatif", "itertools 0.13.0", "kernel-abi-check", - "kernels-data", + "kernels-common", "minijinja", "minijinja-embed", "object", @@ -1550,7 +1550,7 @@ dependencies = [ ] [[package]] -name = "kernels-data" +name = "kernels-common" version = "0.17.0-dev1" dependencies = [ "base64", @@ -1574,7 +1574,7 @@ dependencies = [ name = "kernels-python" version = "0.17.0-dev1" dependencies = [ - "kernels-data", + "kernels-common", "pyo3", "serde_json", ] diff --git a/Cargo.toml b/Cargo.toml index 780fd294..127c6a4c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -3,6 +3,6 @@ members = [ "kernel-abi-check", "kernel-builder", "kernels", - "kernels-data", + "kernels-common", ] resolver = "2" diff --git a/Makefile b/Makefile index 0c5f4bb8..a95bb60b 100644 --- a/Makefile +++ b/Makefile @@ -5,7 +5,7 @@ export check_dirs := kernels/src kernels/tests all: kernels/src/kernels/python_depends.json -kernels/src/kernels/python_depends.json: kernels-data/src/python_dependencies.json +kernels/src/kernels/python_depends.json: kernels-common/src/python_dependencies.json cp $< $@ style: diff --git a/kernel-builder/Cargo.toml b/kernel-builder/Cargo.toml index 7b354047..4aab33e1 100644 --- a/kernel-builder/Cargo.toml +++ b/kernel-builder/Cargo.toml @@ -11,7 +11,7 @@ repository = "https://github.com/huggingface/kernels" [dependencies] base32 = "0.5" kernel-abi-check = { path = "../kernel-abi-check", version = "0.17.0-dev1" } -kernels-data = { path = "../kernels-data", version = "0.17.0-dev1" } +kernels-common = { path = "../kernels-common", version = "0.17.0-dev1" } object = "0.36.7" clap = { version = "4", features = ["derive"] } clap-markdown = "0.1.5" diff --git a/kernel-builder/src/card.rs b/kernel-builder/src/card.rs index 6874b870..b304b8f1 100644 --- a/kernel-builder/src/card.rs +++ b/kernel-builder/src/card.rs @@ -5,7 +5,7 @@ use std::{ }; use eyre::{bail, Context, Result}; -use kernels_data::config::Build; +use kernels_common::config::Build; use minijinja::{context, Environment}; use rustpython_parser::{ast, Parse}; diff --git a/kernel-builder/src/hash.rs b/kernel-builder/src/hash.rs index 523d7ce5..64261056 100644 --- a/kernel-builder/src/hash.rs +++ b/kernel-builder/src/hash.rs @@ -5,8 +5,8 @@ use std::{ }; use eyre::{Context, Result}; -use kernels_data::digest::{Digest, DigestAlgorithm}; -use kernels_data::metadata::Metadata; +use kernels_common::digest::{Digest, DigestAlgorithm}; +use kernels_common::metadata::Metadata; use crate::util::{check_or_infer_kernel_dir, discover_variants}; diff --git a/kernel-builder/src/init.rs b/kernel-builder/src/init.rs index 393de2dd..f87ece1a 100644 --- a/kernel-builder/src/init.rs +++ b/kernel-builder/src/init.rs @@ -12,7 +12,7 @@ use minijinja::{context, Environment}; use crate::hf; use crate::pyproject::FileSet; -use kernels_data::config::Backend; +use kernels_common::config::Backend; fn to_camel_case(s: &str) -> String { let mut result = String::with_capacity(s.len()); diff --git a/kernel-builder/src/main.rs b/kernel-builder/src/main.rs index 6fa9db98..83a735d6 100644 --- a/kernel-builder/src/main.rs +++ b/kernel-builder/src/main.rs @@ -5,8 +5,8 @@ use std::path::PathBuf; use clap::{Args, CommandFactory, Parser, Subcommand}; use clap_complete::Shell; use eyre::{Context, Result}; -use kernels_data::config::{v5, Build, BuildCompat}; -use kernels_data::git::Oid; +use kernels_common::config::{v5, Build, BuildCompat}; +use kernels_common::git::Oid; mod card; mod check_abi; diff --git a/kernel-builder/src/pyproject/common.rs b/kernel-builder/src/pyproject/common.rs index c6ea5f7d..e38efc4f 100644 --- a/kernel-builder/src/pyproject/common.rs +++ b/kernel-builder/src/pyproject/common.rs @@ -4,9 +4,9 @@ use std::str::FromStr; use eyre::Result; use itertools::Itertools; -use kernels_data::config::{Backend, Build}; -use kernels_data::git::{GitStatus, Oid}; -use kernels_data::metadata::{KernelBuilderVersion, Metadata, Provenance}; +use kernels_common::config::{Backend, Build}; +use kernels_common::git::{GitStatus, Oid}; +use kernels_common::metadata::{KernelBuilderVersion, Metadata, Provenance}; use crate::pyproject::ops_identifier::KernelIdentifier; use crate::pyproject::FileSet; diff --git a/kernel-builder/src/pyproject/deps.rs b/kernel-builder/src/pyproject/deps.rs index 4707f695..b9acb122 100644 --- a/kernel-builder/src/pyproject/deps.rs +++ b/kernel-builder/src/pyproject/deps.rs @@ -4,7 +4,7 @@ use std::io::Write; use eyre::{Context, Result}; use minijinja::{context, Environment}; -use kernels_data::config::{Build, Dependency}; +use kernels_common::config::{Build, Dependency}; pub fn render_deps(env: &Environment, build: &Build, write: &mut impl Write) -> Result<()> { // Collect all dependencies. diff --git a/kernel-builder/src/pyproject/kernel.rs b/kernel-builder/src/pyproject/kernel.rs index 6bf78f50..d6ff15b9 100644 --- a/kernel-builder/src/pyproject/kernel.rs +++ b/kernel-builder/src/pyproject/kernel.rs @@ -2,7 +2,7 @@ use std::io::Write; use eyre::{Context, Result}; use itertools::Itertools; -use kernels_data::config::{Build, Kernel}; +use kernels_common::config::{Build, Kernel}; use minijinja::{context, Environment}; use crate::pyproject::common::prefix_and_join_includes; diff --git a/kernel-builder/src/pyproject/mod.rs b/kernel-builder/src/pyproject/mod.rs index cc8cb25f..72df0e72 100644 --- a/kernel-builder/src/pyproject/mod.rs +++ b/kernel-builder/src/pyproject/mod.rs @@ -5,9 +5,9 @@ use std::{ }; use eyre::{bail, Result}; -use kernels_data::config::{Build, Framework}; -use kernels_data::git::{GitStatus, Oid}; -use kernels_data::metadata::Provenance; +use kernels_common::config::{Build, Framework}; +use kernels_common::git::{GitStatus, Oid}; +use kernels_common::metadata::Provenance; use minijinja::Environment; use crate::{ diff --git a/kernel-builder/src/pyproject/ops_identifier.rs b/kernel-builder/src/pyproject/ops_identifier.rs index 80418e2b..f0dc40c8 100644 --- a/kernel-builder/src/pyproject/ops_identifier.rs +++ b/kernel-builder/src/pyproject/ops_identifier.rs @@ -3,8 +3,8 @@ use std::str::FromStr; use eyre::{Result, WrapErr}; use git2::Repository; -use kernels_data::config::Backend; -use kernels_data::git::{GitStatus, Oid}; +use kernels_common::config::Backend; +use kernels_common::git::{GitStatus, Oid}; use rand::Rng; pub fn random_identifier() -> String { diff --git a/kernel-builder/src/pyproject/torch/mod.rs b/kernel-builder/src/pyproject/torch/mod.rs index 8f36a8a5..1c04071c 100644 --- a/kernel-builder/src/pyproject/torch/mod.rs +++ b/kernel-builder/src/pyproject/torch/mod.rs @@ -3,8 +3,8 @@ use std::path::PathBuf; use eyre::{bail, Context, Result}; use itertools::Itertools; -use kernels_data::config::{Backend, Build, General, Torch}; -use kernels_data::metadata::Provenance; +use kernels_common::config::{Backend, Build, General, Torch}; +use kernels_common::metadata::Provenance; use minijinja::context; use crate::pyproject::common::{ diff --git a/kernel-builder/src/pyproject/torch/noarch.rs b/kernel-builder/src/pyproject/torch/noarch.rs index 763f6437..22ad7482 100644 --- a/kernel-builder/src/pyproject/torch/noarch.rs +++ b/kernel-builder/src/pyproject/torch/noarch.rs @@ -2,9 +2,9 @@ use std::path::PathBuf; use eyre::{Context, Result}; use itertools::Itertools; -use kernels_data::config::TorchNoarch; -use kernels_data::config::{Backend, Build, General}; -use kernels_data::metadata::Provenance; +use kernels_common::config::TorchNoarch; +use kernels_common::config::{Backend, Build, General}; +use kernels_common::metadata::Provenance; use minijinja::{context, Environment}; use crate::pyproject::common::write_metadata; diff --git a/kernel-builder/src/pyproject/tvm_ffi/mod.rs b/kernel-builder/src/pyproject/tvm_ffi/mod.rs index 6a487d66..a7c6ad57 100644 --- a/kernel-builder/src/pyproject/tvm_ffi/mod.rs +++ b/kernel-builder/src/pyproject/tvm_ffi/mod.rs @@ -2,8 +2,8 @@ use std::io::Write; use eyre::{bail, Context, Result}; use itertools::Itertools; -use kernels_data::config::{Backend, Build, General, TvmFfi}; -use kernels_data::metadata::Provenance; +use kernels_common::config::{Backend, Build, General, TvmFfi}; +use kernels_common::metadata::Provenance; use minijinja::{context, Environment}; use crate::pyproject::common::{ diff --git a/kernel-builder/src/upload.rs b/kernel-builder/src/upload.rs index d7e3947b..ec8a9b3f 100644 --- a/kernel-builder/src/upload.rs +++ b/kernel-builder/src/upload.rs @@ -13,8 +13,8 @@ use hf_hub::{ HFError, HFRepositorySync, RepoType, RepoTypeKernel, RepoTypeModel, }; use indicatif::{ProgressBar, ProgressStyle}; -use kernels_data::config::Build; -use kernels_data::metadata::Metadata; +use kernels_common::config::Build; +use kernels_common::metadata::Metadata; use serde::Serialize; use walkdir::WalkDir; diff --git a/kernel-builder/src/validate_builds.rs b/kernel-builder/src/validate_builds.rs index 66046cf5..3fcfbf46 100644 --- a/kernel-builder/src/validate_builds.rs +++ b/kernel-builder/src/validate_builds.rs @@ -3,7 +3,7 @@ use std::path::PathBuf; use std::{fs::File, io::BufReader}; use eyre::{Context, Result}; -use kernels_data::metadata::Metadata; +use kernels_common::metadata::Metadata; use crate::util::{check_or_infer_kernel_dir, discover_variants}; diff --git a/kernels-data/Cargo.toml b/kernels-common/Cargo.toml similarity index 86% rename from kernels-data/Cargo.toml rename to kernels-common/Cargo.toml index 51d5aacb..2709de5a 100644 --- a/kernels-data/Cargo.toml +++ b/kernels-common/Cargo.toml @@ -1,8 +1,8 @@ [package] -name = "kernels-data" +name = "kernels-common" version = "0.17.0-dev1" edition = "2024" -description = "Kernels data structures" +description = "Code shared by kernel-builder and kernels" homepage = "https://github.com/huggingface/kernels" license = "Apache-2.0" repository = "https://github.com/huggingface/kernels" diff --git a/kernels-data/src/config/compat.rs b/kernels-common/src/config/compat.rs similarity index 100% rename from kernels-data/src/config/compat.rs rename to kernels-common/src/config/compat.rs diff --git a/kernels-data/src/config/deps.rs b/kernels-common/src/config/deps.rs similarity index 100% rename from kernels-data/src/config/deps.rs rename to kernels-common/src/config/deps.rs diff --git a/kernels-data/src/config/git_url.rs b/kernels-common/src/config/git_url.rs similarity index 100% rename from kernels-data/src/config/git_url.rs rename to kernels-common/src/config/git_url.rs diff --git a/kernels-data/src/config/kernel_deps.rs b/kernels-common/src/config/kernel_deps.rs similarity index 100% rename from kernels-data/src/config/kernel_deps.rs rename to kernels-common/src/config/kernel_deps.rs diff --git a/kernels-data/src/config/mod.rs b/kernels-common/src/config/mod.rs similarity index 100% rename from kernels-data/src/config/mod.rs rename to kernels-common/src/config/mod.rs diff --git a/kernels-data/src/config/name.rs b/kernels-common/src/config/name.rs similarity index 100% rename from kernels-data/src/config/name.rs rename to kernels-common/src/config/name.rs diff --git a/kernels-data/src/config/parse.rs b/kernels-common/src/config/parse.rs similarity index 100% rename from kernels-data/src/config/parse.rs rename to kernels-common/src/config/parse.rs diff --git a/kernels-data/src/config/v1.rs b/kernels-common/src/config/v1.rs similarity index 100% rename from kernels-data/src/config/v1.rs rename to kernels-common/src/config/v1.rs diff --git a/kernels-data/src/config/v2.rs b/kernels-common/src/config/v2.rs similarity index 100% rename from kernels-data/src/config/v2.rs rename to kernels-common/src/config/v2.rs diff --git a/kernels-data/src/config/v3.rs b/kernels-common/src/config/v3.rs similarity index 100% rename from kernels-data/src/config/v3.rs rename to kernels-common/src/config/v3.rs diff --git a/kernels-data/src/config/v4.rs b/kernels-common/src/config/v4.rs similarity index 100% rename from kernels-data/src/config/v4.rs rename to kernels-common/src/config/v4.rs diff --git a/kernels-data/src/config/v5.rs b/kernels-common/src/config/v5.rs similarity index 100% rename from kernels-data/src/config/v5.rs rename to kernels-common/src/config/v5.rs diff --git a/kernels-data/src/digest.rs b/kernels-common/src/digest.rs similarity index 100% rename from kernels-data/src/digest.rs rename to kernels-common/src/digest.rs diff --git a/kernels-data/src/git.rs b/kernels-common/src/git.rs similarity index 100% rename from kernels-data/src/git.rs rename to kernels-common/src/git.rs diff --git a/kernels-data/src/lib.rs b/kernels-common/src/lib.rs similarity index 100% rename from kernels-data/src/lib.rs rename to kernels-common/src/lib.rs diff --git a/kernels-data/src/lock.rs b/kernels-common/src/lock.rs similarity index 100% rename from kernels-data/src/lock.rs rename to kernels-common/src/lock.rs diff --git a/kernels-data/src/metadata.rs b/kernels-common/src/metadata.rs similarity index 100% rename from kernels-data/src/metadata.rs rename to kernels-common/src/metadata.rs diff --git a/kernels-data/src/python_dependencies.json b/kernels-common/src/python_dependencies.json similarity index 100% rename from kernels-data/src/python_dependencies.json rename to kernels-common/src/python_dependencies.json diff --git a/kernels-data/src/version.rs b/kernels-common/src/version.rs similarity index 100% rename from kernels-data/src/version.rs rename to kernels-common/src/version.rs diff --git a/kernels-data/tests/data/legacy-sha-provenance.json b/kernels-common/tests/data/legacy-sha-provenance.json similarity index 100% rename from kernels-data/tests/data/legacy-sha-provenance.json rename to kernels-common/tests/data/legacy-sha-provenance.json diff --git a/kernels/Cargo.toml b/kernels/Cargo.toml index 6f8bcf8d..0acaff1b 100644 --- a/kernels/Cargo.toml +++ b/kernels/Cargo.toml @@ -8,7 +8,7 @@ license = "Apache-2.0" repository = "https://github.com/huggingface/kernels" [lib] -name = "_data" +name = "_rust" path = "rust/lib.rs" crate-type = ["cdylib"] @@ -16,6 +16,6 @@ crate-type = ["cdylib"] pyo3 = { version = "0.26", features = ["abi3", "abi3-py38"] } serde_json = "1" -[dependencies.kernels-data] -path = "../kernels-data" +[dependencies.kernels-common] +path = "../kernels-common" version = "0.17.0-dev1" diff --git a/kernels/pyproject.toml b/kernels/pyproject.toml index 617698bf..d55f1c1e 100644 --- a/kernels/pyproject.toml +++ b/kernels/pyproject.toml @@ -66,7 +66,7 @@ kernels = "kernels.cli:main" "kernels.lock" = "kernels.lockfile:write_egg_lockfile" [tool.maturin] -module-name = "kernels._data" +module-name = "kernels._rust" python-source = "src" bindings = 'pyo3' features = ["pyo3/extension-module"] diff --git a/kernels/rust/config.rs b/kernels/rust/config.rs index e0d23041..3ddd12fa 100644 --- a/kernels/rust/config.rs +++ b/kernels/rust/config.rs @@ -1,6 +1,6 @@ use std::path::PathBuf; -use kernels_data::config::{Build, General}; +use kernels_common::config::{Build, General}; use pyo3::exceptions::PyValueError; use pyo3::prelude::*; diff --git a/kernels/rust/lib.rs b/kernels/rust/lib.rs index 821cd247..fd336079 100644 --- a/kernels/rust/lib.rs +++ b/kernels/rust/lib.rs @@ -4,10 +4,10 @@ use std::io::BufReader; use std::path::PathBuf; use std::str::FromStr; -use kernels_data::config::{Backend, KernelDependency, KernelName, KernelVersion}; -use kernels_data::digest::{Digest, DigestAlgorithm, DigestViolation}; -use kernels_data::git::{GitStatus, Oid}; -use kernels_data::metadata::{BackendInfo, KernelBuilderVersion, Metadata, Provenance}; +use kernels_common::config::{Backend, KernelDependency, KernelName, KernelVersion}; +use kernels_common::digest::{Digest, DigestAlgorithm, DigestViolation}; +use kernels_common::git::{GitStatus, Oid}; +use kernels_common::metadata::{BackendInfo, KernelBuilderVersion, Metadata, Provenance}; use pyo3::Bound as PyBound; use pyo3::exceptions::{PyException, PyOSError, PyRuntimeError, PyValueError}; use pyo3::prelude::*; @@ -658,7 +658,7 @@ impl PyDigestViolation { } pyo3::create_exception!( - _data, + _rust, DigestValidationError, PyException, "Raised by `Digest.validate` when the actual digest does not match the \ @@ -750,7 +750,7 @@ impl PyDigest { } } -#[pyo3::pymodule(name = "_data")] +#[pyo3::pymodule(name = "_rust")] fn data_py(m: &PyBound<'_, PyModule>) -> PyResult<()> { m.add_class::()?; m.add_class::()?; diff --git a/kernels/rust/lock.rs b/kernels/rust/lock.rs index c61b2e00..2a7a803d 100644 --- a/kernels/rust/lock.rs +++ b/kernels/rust/lock.rs @@ -2,8 +2,8 @@ use std::collections::BTreeMap; use std::path::PathBuf; use std::str::FromStr; -use kernels_data::git::Oid; -use kernels_data::lock::{KernelLock, KernelLocks, KernelPaths, NixKernelLock, NixKernelLocks}; +use kernels_common::git::Oid; +use kernels_common::lock::{KernelLock, KernelLocks, KernelPaths, NixKernelLock, NixKernelLocks}; use pyo3::Bound as PyBound; use pyo3::exceptions::{PyKeyError, PyValueError}; use pyo3::prelude::*; diff --git a/kernels/rust/version.rs b/kernels/rust/version.rs index 1d3c832e..a9528965 100644 --- a/kernels/rust/version.rs +++ b/kernels/rust/version.rs @@ -1,7 +1,7 @@ use std::cmp::Ordering; use std::hash::{Hash, Hasher}; -use kernels_data::version::Version; +use kernels_common::version::Version; use pyo3::exceptions::PyValueError; use pyo3::prelude::*; @@ -10,7 +10,7 @@ use pyo3::prelude::*; #[pyclass(name = "Version", frozen, eq, ord, hash)] #[derive(Clone, Debug)] pub(crate) struct PyVersion { - // We are storing this as a `Box<[usize]>` to allow storing kernels-data + // We are storing this as a `Box<[usize]>` to allow storing kernels-common // `Version` in directly in `PyVersion` as well. inner: Box<[usize]>, } diff --git a/kernels/src/kernels/__init__.py b/kernels/src/kernels/__init__.py index f0570641..7d9bc7d0 100644 --- a/kernels/src/kernels/__init__.py +++ b/kernels/src/kernels/__init__.py @@ -2,7 +2,7 @@ __version__ = importlib.metadata.version("kernels") -from kernels._data import Metadata +from kernels._rust import Metadata from kernels._windows import _add_additional_dll_paths from kernels.benchmark import Benchmark from kernels.deps import get_kernel_dep diff --git a/kernels/src/kernels/_data.pyi b/kernels/src/kernels/_rust.pyi similarity index 99% rename from kernels/src/kernels/_data.pyi rename to kernels/src/kernels/_rust.pyi index 3c5263a3..bd63a703 100644 --- a/kernels/src/kernels/_data.pyi +++ b/kernels/src/kernels/_rust.pyi @@ -1,4 +1,4 @@ -"""Type stubs for the `kernels._data` module.""" +"""Type stubs for the `kernels._rust` module.""" import os from collections.abc import Iterator diff --git a/kernels/src/kernels/_versions.py b/kernels/src/kernels/_versions.py index 67333b27..0e997c85 100644 --- a/kernels/src/kernels/_versions.py +++ b/kernels/src/kernels/_versions.py @@ -6,7 +6,7 @@ from huggingface_hub.file_download import repo_folder_name from huggingface_hub.hf_api import GitRefInfo -from kernels._data import KernelDependency, KernelVersion +from kernels._rust import KernelDependency, KernelVersion logger = logging.getLogger(__name__) diff --git a/kernels/src/kernels/archs.py b/kernels/src/kernels/archs.py index 30d5a2ce..cfe08c8e 100644 --- a/kernels/src/kernels/archs.py +++ b/kernels/src/kernels/archs.py @@ -1,4 +1,4 @@ -from kernels._data import Backend, Metadata +from kernels._rust import Backend, Metadata from kernels.compat import has_torch diff --git a/kernels/src/kernels/cli/download.py b/kernels/src/kernels/cli/download.py index 7dcde2ac..16fc0f08 100644 --- a/kernels/src/kernels/cli/download.py +++ b/kernels/src/kernels/cli/download.py @@ -1,6 +1,6 @@ import sys -from kernels._data import KernelLocks +from kernels._rust import KernelLocks from kernels.hf_hub import CACHE_DIR, _get_hf_api from kernels.resolver import _BYTECODE_IGNORE_PATTERNS, resolve_hub_kernel diff --git a/kernels/src/kernels/cli/info.py b/kernels/src/kernels/cli/info.py index f5c3b177..a50a48ac 100644 --- a/kernels/src/kernels/cli/info.py +++ b/kernels/src/kernels/cli/info.py @@ -5,7 +5,7 @@ from huggingface_hub import constants -from kernels._data import Metadata +from kernels._rust import Metadata from kernels._versions import _get_available_versions, resolve_version_spec_as_ref from kernels.hf_hub import CACHE_DIR, _get_hf_api from kernels.variants import ( diff --git a/kernels/src/kernels/cli/lock.py b/kernels/src/kernels/cli/lock.py index 9f456d34..c536f03b 100644 --- a/kernels/src/kernels/cli/lock.py +++ b/kernels/src/kernels/cli/lock.py @@ -1,4 +1,4 @@ -from kernels._data import KernelDependency, KernelVersion +from kernels._rust import KernelDependency, KernelVersion from kernels.compat import tomllib from kernels.hf_hub import _get_hf_api from kernels.locking import extract_dependency_locks diff --git a/kernels/src/kernels/deps.py b/kernels/src/kernels/deps.py index 7b26df83..b0e008df 100644 --- a/kernels/src/kernels/deps.py +++ b/kernels/src/kernels/deps.py @@ -5,7 +5,7 @@ from huggingface_hub.hf_api import HfApi -from kernels._data import KernelDependency, KernelVersion +from kernels._rust import KernelDependency, KernelVersion from kernels.hf_hub import RepoInfo from kernels.importer import _import_from_path from kernels.resolver import ( diff --git a/kernels/src/kernels/importer.py b/kernels/src/kernels/importer.py index 80260741..02c15c4e 100644 --- a/kernels/src/kernels/importer.py +++ b/kernels/src/kernels/importer.py @@ -4,7 +4,7 @@ from pathlib import Path from types import ModuleType -from kernels._data import Metadata +from kernels._rust import Metadata from kernels.hf_hub import RepoInfo diff --git a/kernels/src/kernels/install.py b/kernels/src/kernels/install.py index 88fd8dfc..dd656cfc 100644 --- a/kernels/src/kernels/install.py +++ b/kernels/src/kernels/install.py @@ -1,6 +1,6 @@ from pathlib import Path -from kernels._data import KernelDependency +from kernels._rust import KernelDependency from kernels._versions import revision_or_version from kernels.deps import resolve_kernel_tree from kernels.hf_hub import CACHE_DIR, _get_hf_api diff --git a/kernels/src/kernels/layer/func.py b/kernels/src/kernels/layer/func.py index c8471cae..0e555d5b 100644 --- a/kernels/src/kernels/layer/func.py +++ b/kernels/src/kernels/layer/func.py @@ -6,7 +6,7 @@ from huggingface_hub import constants -from kernels._data import KernelDependency, KernelLocks +from kernels._rust import KernelDependency, KernelLocks from kernels.resolver import LockedHubCacheResolver, LockedHubResolver from .._versions import select_revision_or_version diff --git a/kernels/src/kernels/layer/layer.py b/kernels/src/kernels/layer/layer.py index 2f008a50..58cafc14 100644 --- a/kernels/src/kernels/layer/layer.py +++ b/kernels/src/kernels/layer/layer.py @@ -10,7 +10,7 @@ from huggingface_hub import constants -from kernels._data import KernelDependency, KernelLocks +from kernels._rust import KernelDependency, KernelLocks from kernels.resolver import LockedHubCacheResolver, LockedHubResolver from .._versions import select_revision_or_version diff --git a/kernels/src/kernels/load.py b/kernels/src/kernels/load.py index d96b986a..768ef82c 100644 --- a/kernels/src/kernels/load.py +++ b/kernels/src/kernels/load.py @@ -5,7 +5,7 @@ from huggingface_hub import HfApi, constants -from kernels._data import KernelDependency, KernelVersion +from kernels._rust import KernelDependency, KernelVersion from kernels._versions import revision_or_version from kernels.deps import resolve_kernel_tree from kernels.hf_hub import _get_hf_api diff --git a/kernels/src/kernels/locking.py b/kernels/src/kernels/locking.py index e821fb99..91da77ba 100644 --- a/kernels/src/kernels/locking.py +++ b/kernels/src/kernels/locking.py @@ -6,7 +6,7 @@ from huggingface_hub.hf_api import HfApi -from kernels._data import ( +from kernels._rust import ( KernelDependency, KernelLock, KernelLocks, diff --git a/kernels/src/kernels/python_deps.py b/kernels/src/kernels/python_deps.py index 182830cb..4d57cbfa 100644 --- a/kernels/src/kernels/python_deps.py +++ b/kernels/src/kernels/python_deps.py @@ -5,7 +5,7 @@ from huggingface_hub.dataclasses import strict -from kernels._data import Metadata +from kernels._rust import Metadata from kernels.backends import Backend, _backend diff --git a/kernels/src/kernels/resolver.py b/kernels/src/kernels/resolver.py index 0c9e6f24..0a21c8e7 100644 --- a/kernels/src/kernels/resolver.py +++ b/kernels/src/kernels/resolver.py @@ -5,7 +5,7 @@ from huggingface_hub.errors import LocalEntryNotFoundError from huggingface_hub.hf_api import HfApi -from kernels._data import KernelDependency, KernelLocks, KernelPaths, Metadata +from kernels._rust import KernelDependency, KernelLocks, KernelPaths, Metadata from kernels._versions import _get_available_versions, resolve_kernel_version from kernels.hf_hub import CACHE_DIR, _check_trust_remote_code from kernels.variants import ( diff --git a/kernels/src/kernels/validate.py b/kernels/src/kernels/validate.py index 4fa453db..147cf555 100644 --- a/kernels/src/kernels/validate.py +++ b/kernels/src/kernels/validate.py @@ -4,7 +4,7 @@ from packaging.version import InvalidVersion, parse -from kernels._data import Metadata, Version +from kernels._rust import Metadata, Version from kernels.archs import _check_arch_incompatibility from kernels.backends import _backend from kernels.python_deps import validate_dependencies diff --git a/kernels/src/kernels/verify.py b/kernels/src/kernels/verify.py index 12ed5b21..a0fff9b3 100644 --- a/kernels/src/kernels/verify.py +++ b/kernels/src/kernels/verify.py @@ -8,7 +8,7 @@ from sigstore.verify import Verifier, policy from sigstore.verify.policy import VerificationPolicy -from kernels._data import Digest, DigestValidationError, DigestViolation, Metadata +from kernels._rust import Digest, DigestValidationError, DigestViolation, Metadata class GitHubWorkflowPolicy(VerificationPolicy): diff --git a/kernels/tests/conftest.py b/kernels/tests/conftest.py index f72023de..9e7455aa 100644 --- a/kernels/tests/conftest.py +++ b/kernels/tests/conftest.py @@ -4,7 +4,7 @@ import pytest -from kernels._data import Metadata +from kernels._rust import Metadata from kernels.backends import _get_torch_privateuse_backend_name try: diff --git a/kernels/tests/test_archs.py b/kernels/tests/test_archs.py index 7580d8a0..8a318820 100644 --- a/kernels/tests/test_archs.py +++ b/kernels/tests/test_archs.py @@ -2,7 +2,7 @@ import torch from kernels import get_kernel, has_kernel, install_kernel -from kernels._data import Metadata +from kernels._rust import Metadata from kernels.archs import _supports_cuda_capability diff --git a/kernels/tests/test_data_config.py b/kernels/tests/test_data_config.py index e8701e1f..1030ee25 100644 --- a/kernels/tests/test_data_config.py +++ b/kernels/tests/test_data_config.py @@ -1,6 +1,6 @@ import pytest -from kernels._data import Backend, Build, KernelDependency, KernelVersion +from kernels._rust import Backend, Build, KernelDependency, KernelVersion def _write_build_toml(path, backends): diff --git a/kernels/tests/test_data_digest.py b/kernels/tests/test_data_digest.py index b5991ae6..b5ce2b74 100644 --- a/kernels/tests/test_data_digest.py +++ b/kernels/tests/test_data_digest.py @@ -1,6 +1,6 @@ import pytest -from kernels._data import Digest, DigestAlgorithm, DigestValidationError, DigestViolation +from kernels._rust import Digest, DigestAlgorithm, DigestValidationError, DigestViolation def test_hash_variant_hashes_file_with_sha256(tmp_path): diff --git a/kernels/tests/test_data_kernel_lock.py b/kernels/tests/test_data_kernel_lock.py index 3989d47c..b3db0df4 100644 --- a/kernels/tests/test_data_kernel_lock.py +++ b/kernels/tests/test_data_kernel_lock.py @@ -2,7 +2,7 @@ import pytest -from kernels._data import ( +from kernels._rust import ( KernelDependency, KernelLock, KernelLocks, diff --git a/kernels/tests/test_data_metadata.py b/kernels/tests/test_data_metadata.py index 0d7916f7..3631399e 100644 --- a/kernels/tests/test_data_metadata.py +++ b/kernels/tests/test_data_metadata.py @@ -2,7 +2,7 @@ import pytest -from kernels._data import ( +from kernels._rust import ( Backend, KernelDependency, KernelName, diff --git a/kernels/tests/test_data_version.py b/kernels/tests/test_data_version.py index 280a234a..75e8e083 100644 --- a/kernels/tests/test_data_version.py +++ b/kernels/tests/test_data_version.py @@ -1,6 +1,6 @@ import pytest -from kernels._data import Version +from kernels._rust import Version def test_version_parse(): diff --git a/kernels/tests/test_deps.py b/kernels/tests/test_deps.py index 23b95c2b..efe20733 100644 --- a/kernels/tests/test_deps.py +++ b/kernels/tests/test_deps.py @@ -5,7 +5,7 @@ import pytest -from kernels._data import KernelDependency, KernelVersion, Metadata +from kernels._rust import KernelDependency, KernelVersion, Metadata from kernels.deps import get_kernel_dep, resolve_kernel_tree, use_kernel_deps from kernels.resolver import LocalKernel diff --git a/kernels/tests/test_resolver.py b/kernels/tests/test_resolver.py index b700ee8c..16b818ac 100644 --- a/kernels/tests/test_resolver.py +++ b/kernels/tests/test_resolver.py @@ -7,7 +7,7 @@ from huggingface_hub.hf_api import GitRefInfo import kernels.resolver as resolver_module -from kernels._data import ( +from kernels._rust import ( KernelDependency, KernelLock, KernelLocks, diff --git a/kernels/tests/test_validate.py b/kernels/tests/test_validate.py index 2e23f06a..3adb7ac6 100644 --- a/kernels/tests/test_validate.py +++ b/kernels/tests/test_validate.py @@ -8,7 +8,7 @@ import kernels import kernels.validate as validate_module -from kernels._data import Metadata, Version +from kernels._rust import Metadata, Version from kernels.deps import DepTreeNode from kernels.resolver import LocalKernel from kernels.validate import ( diff --git a/kernels/tests/test_verify.py b/kernels/tests/test_verify.py index d6e7f03d..18bb6a91 100644 --- a/kernels/tests/test_verify.py +++ b/kernels/tests/test_verify.py @@ -3,7 +3,7 @@ from sigstore.verify import policy from kernels import install_kernel -from kernels._data import DigestViolation +from kernels._rust import DigestViolation from kernels._versions import select_revision_or_version from kernels.hf_hub import CACHE_DIR, _get_hf_api from kernels.resolver import _BYTECODE_IGNORE_PATTERNS diff --git a/nix-builder/lib/python-deps.nix b/nix-builder/lib/python-deps.nix index d04fb786..f7022457 100644 --- a/nix-builder/lib/python-deps.nix +++ b/nix-builder/lib/python-deps.nix @@ -8,7 +8,7 @@ let pythonDeps = let - depsJson = builtins.fromJSON (builtins.readFile ../../kernels-data/src/python_dependencies.json); + depsJson = builtins.fromJSON (builtins.readFile ../../kernels-common/src/python_dependencies.json); # Map the Nix package names to actual Nix packages. updatePackage = _name: dep: dep // { nix = map (pkg: pkgs.python3.pkgs.${pkg}) dep.nix; }; updateBackend = _backend: backendDeps: lib.mapAttrs updatePackage backendDeps; diff --git a/nix-builder/pkgs/crate-dirs.nix b/nix-builder/pkgs/crate-dirs.nix index 3a24005b..9be0102c 100644 --- a/nix-builder/pkgs/crate-dirs.nix +++ b/nix-builder/pkgs/crate-dirs.nix @@ -9,6 +9,6 @@ toSource { (fileFilter sourceFiles ../../kernel-abi-check) (fileFilter sourceFiles ../../kernel-builder) (fileFilter sourceFiles ../../kernels) - (fileFilter sourceFiles ../../kernels-data) + (fileFilter sourceFiles ../../kernels-common) ]; } diff --git a/nix-builder/pkgs/get-kernel-check/get-kernel-check-hook.py b/nix-builder/pkgs/get-kernel-check/get-kernel-check-hook.py index 7cd35150..06f182c8 100755 --- a/nix-builder/pkgs/get-kernel-check/get-kernel-check-hook.py +++ b/nix-builder/pkgs/get-kernel-check/get-kernel-check-hook.py @@ -5,7 +5,7 @@ from kernels.load import get_kernel_with_resolver from kernels.resolver import KernelPathsResolver, RepoPathsResolver, SequentialResolver from kernels.validate import AllValidator, default_metadata_validators -from kernels._data import KernelDependency, KernelPaths, KernelVersion +from kernels._rust import KernelDependency, KernelPaths, KernelVersion out = os.getenv("out") if not out: diff --git a/nix-builder/pkgs/lock-kernel-deps/lock_kernel_deps.py b/nix-builder/pkgs/lock-kernel-deps/lock_kernel_deps.py index 97b20f9e..07d6f55e 100755 --- a/nix-builder/pkgs/lock-kernel-deps/lock_kernel_deps.py +++ b/nix-builder/pkgs/lock-kernel-deps/lock_kernel_deps.py @@ -15,7 +15,7 @@ from huggingface_hub.hf_api import HfApi from kernels.locking import extract_dependency_locks -from kernels._data import Build, KernelLocks, NixKernelLock, NixKernelLocks +from kernels._rust import Build, KernelLocks, NixKernelLock, NixKernelLocks def nix_hash(repo_id: str, revision: str) -> str: diff --git a/nix-builder/tests/Dockerfile.test-kernel b/nix-builder/tests/Dockerfile.test-kernel index 3194e4a7..74f63ca1 100644 --- a/nix-builder/tests/Dockerfile.test-kernel +++ b/nix-builder/tests/Dockerfile.test-kernel @@ -62,8 +62,8 @@ RUN CUDA_MAJOR_MINOR=$(echo ${CUDA_VERSION} | cut -d'.' -f1,2) && \ # Add additional dependencies. RUN uv add "apache-tvm-ffi~=0.1.9" numpy pytest -# Copy kernels-data and kernels source to install from source. -COPY kernels-data ./kernels-data +# Copy kernels-common and kernels source to install from source. +COPY kernels-common ./kernels-common COPY kernels ./kernels-src # Install kernels (Rust/maturin package) from source. diff --git a/scripts/_version_common.py b/scripts/_version_common.py index 0f50900a..df3646f8 100644 --- a/scripts/_version_common.py +++ b/scripts/_version_common.py @@ -18,7 +18,7 @@ PYPROJECT_FILES = [PRIMARY_PYPROJECT] CARGO_FILES = [ - REPO_ROOT / "kernels-data" / "Cargo.toml", + REPO_ROOT / "kernels-common" / "Cargo.toml", REPO_ROOT / "kernels" / "Cargo.toml", REPO_ROOT / "kernel-builder" / "Cargo.toml", REPO_ROOT / "kernel-abi-check" / "Cargo.toml",