Skip to content

Commit fa01d35

Browse files
robert3005claude
andauthored
feat(ci): build and bundle Windows artifacts for the Java and Python bindings (#9739)
Add windows builds and support to python and java bindings Signed-off-by: Claude <noreply@anthropic.com> Co-authored-by: Claude <noreply@anthropic.com>
1 parent 5ea934c commit fa01d35

10 files changed

Lines changed: 132 additions & 12 deletions

File tree

‎.github/workflows/ci.yml‎

Lines changed: 63 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -141,6 +141,69 @@ jobs:
141141
bash ../scripts/bench-taskset.sh uv run --all-packages pytest --benchmark-only benchmark/
142142
working-directory: vortex-python/
143143

144+
python-windows:
145+
name: "Python (windows-x64)"
146+
timeout-minutes: 60
147+
runs-on: >-
148+
${{ github.repository == 'vortex-data/vortex'
149+
&& format('runs-on={0}/pool=windows-x64-pre/extras=s3-cache', github.run_id)
150+
|| 'windows-latest' }}
151+
steps:
152+
- uses: runs-on/action@v2
153+
if: github.repository == 'vortex-data/vortex'
154+
with:
155+
sccache: s3
156+
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
157+
- name: Setup (Windows)
158+
run: |
159+
echo "C:\rust\cargo\bin" >> $env:GITHUB_PATH
160+
- uses: ./.github/actions/setup-prebuild
161+
with:
162+
enable-sccache: "true"
163+
- uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7
164+
with:
165+
python-version: "3.11"
166+
167+
# Mirrors the wheel build in package.yml, which spells the Windows interpreter
168+
# `python` because maturin has no `python3.11` executable to resolve there.
169+
- name: Build wheel
170+
uses: PyO3/maturin-action@e83996d129638aa358a18fbd1dfb82f0b0fb5d3b # v1
171+
with:
172+
command: build
173+
maturin-version: 1.10.2
174+
target: x86_64-pc-windows-msvc
175+
args: >
176+
--profile ci
177+
--manifest-path vortex-python/Cargo.toml
178+
--out dist
179+
--interpreter python
180+
env:
181+
# Keep this constant to avoid pyo3 invalidating itself
182+
PYO3_ENVIRONMENT_SIGNATURE: "cpython3"
183+
184+
# rust-test-windows excludes vortex-python, so this is the only check that the
185+
# win_amd64 wheel imports and can round-trip a file. Runs outside the checkout so
186+
# the repo-root `vortex/` crate directory cannot shadow the installed package.
187+
- name: Import and round-trip the wheel
188+
shell: bash
189+
run: |
190+
python -m pip install dist/*.whl
191+
cd "$RUNNER_TEMP"
192+
python - <<'PY'
193+
import pyarrow as pa
194+
195+
import vortex as vx
196+
197+
table = pa.table({"a": [1, 2, 3], "b": ["x", "y", "z"]})
198+
vx.io.write(table, "smoke.vortex")
199+
# Vortex reads utf8 back as string_view, so compare column-wise rather than by table.
200+
round_tripped = vx.open("smoke.vortex").to_arrow().read_all()
201+
assert round_tripped.column_names == table.column_names
202+
assert round_tripped.column("a").to_pylist() == [1, 2, 3]
203+
assert round_tripped.column("b").to_pylist() == ["x", "y", "z"]
204+
print("vortex", vx.__version__, "round-tripped a file on Windows")
205+
PY
206+
144207
# Pins the measurement_id hash that scripts/post-ingest.py uses as the primary key of every
145208
# benchmarks-database fact row. The golden vectors are a frozen artifact (see the note inside
146209
# the JSON); this job fails if a change to scripts/_measurement_id.py drifts the hash, which

‎.github/workflows/package.yml‎

Lines changed: 49 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@ jobs:
3030
- { os: macos, runs-on: "macos-15-intel", target: x86_64-apple-darwin }
3131
- { os: ubuntu, runs-on: "ubuntu-latest", target: aarch64-unknown-linux-gnu }
3232
- { os: ubuntu, runs-on: "ubuntu-latest", target: x86_64-unknown-linux-gnu }
33+
- { os: windows, runs-on: "windows-latest", target: x86_64-pc-windows-msvc }
3334
steps:
3435
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
3536
with:
@@ -46,6 +47,7 @@ jobs:
4647
tool: cargo-edit@0.13.10
4748

4849
- name: Cargo Set Version
50+
shell: bash
4951
run: |
5052
cargo set-version --workspace ${{ inputs.version }}
5153
@@ -56,8 +58,11 @@ jobs:
5658
- name: Set Python extension version pins
5759
env:
5860
VERSION: ${{ inputs.version }}
61+
shell: bash
5962
run: |
60-
python3 - <<'PY'
63+
# Not every runner image puts a `python3` on PATH; Windows may only have `python`.
64+
PYTHON="$(command -v python3 || command -v python)"
65+
"$PYTHON" - <<'PY'
6166
import os
6267
import re
6368
from pathlib import Path
@@ -79,8 +84,9 @@ jobs:
7984
8085
- name: Set up Python
8186
uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7
82-
# Latest macOS doesn't allow maturin to install stuff into the global Python interpreter
83-
if: "${{ matrix.target.runs-on == 'macos-latest' }}"
87+
# Latest macOS doesn't allow maturin to install stuff into the global Python interpreter.
88+
# Windows has no versioned `python3.11` executable, so pin what `python` resolves to.
89+
if: "${{ matrix.target.runs-on == 'macos-latest' || matrix.target.os == 'windows' }}"
8490
with:
8591
python-version: "3.11"
8692

@@ -96,7 +102,7 @@ jobs:
96102
--release
97103
--manifest-path vortex-python/Cargo.toml
98104
--out dist
99-
--interpreter python3.11
105+
--interpreter ${{ (matrix.target.os == 'windows' && 'python') || 'python3.11' }}
100106
env:
101107
MATURIN_PEP517_USE_BASE_PYTHON: "true"
102108
# Keep this constant to avoid pyo3 invalidating itself
@@ -131,6 +137,44 @@ jobs:
131137
retention-days: 7
132138
if-no-files-found: error
133139

140+
prepare-java-windows:
141+
runs-on: >-
142+
${{ github.repository == 'vortex-data/vortex'
143+
&& format('runs-on={0}/pool=windows-x64-pre/extras=s3-cache', github.run_id)
144+
|| 'windows-latest' }}
145+
timeout-minutes: 90
146+
steps:
147+
- uses: runs-on/action@v2
148+
if: github.repository == 'vortex-data/vortex'
149+
with:
150+
sccache: s3
151+
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
152+
with:
153+
fetch-depth: 0
154+
- uses: actions/setup-java@b6effb05e454b25005698d916606bdc6ffcbf961 # v5
155+
with:
156+
distribution: "corretto"
157+
java-version: "17"
158+
- name: Setup (Windows)
159+
run: |
160+
echo "C:\rust\cargo\bin" >> $env:GITHUB_PATH
161+
- uses: ./.github/actions/setup-prebuild
162+
with:
163+
enable-sccache: ${{ github.repository == 'vortex-data/vortex' && 'true' || 'false' }}
164+
- run: cargo build --release --package vortex-jni
165+
shell: bash
166+
# Cargo names the Windows cdylib `vortex_jni.dll`; the bundled resource keeps the
167+
# `lib` prefix used on every other platform so NativeLoader builds a single name.
168+
- name: Rename JNI library
169+
shell: bash
170+
run: cp target/release/vortex_jni.dll target/release/libvortex_jni.dll
171+
- uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
172+
with:
173+
name: "libvortex_jni_x86_64-pc-windows-msvc.zip"
174+
path: "target/release/libvortex_jni.dll"
175+
retention-days: 7
176+
if-no-files-found: error
177+
134178
prepare-java-linux:
135179
timeout-minutes: 30
136180
strategy:
@@ -171,7 +215,7 @@ jobs:
171215
if-no-files-found: error
172216

173217
prepare-all:
174-
needs: [prepare-python, prepare-java-macos, prepare-java-linux]
218+
needs: [prepare-python, prepare-java-macos, prepare-java-linux, prepare-java-windows]
175219
runs-on: ubuntu-latest
176220
timeout-minutes: 10
177221
steps:

‎.github/workflows/publish.yml‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -101,9 +101,11 @@ jobs:
101101
mkdir -p vortex-jni/src/main/resources/native/darwin-aarch64
102102
mkdir -p vortex-jni/src/main/resources/native/linux-aarch64
103103
mkdir -p vortex-jni/src/main/resources/native/linux-amd64
104+
mkdir -p vortex-jni/src/main/resources/native/win-amd64
104105
cp ../libvortex_jni_aarch64-apple-darwin.zip/libvortex_jni.dylib ./vortex-jni/src/main/resources/native/darwin-aarch64
105106
cp ../libvortex_jni_aarch64-unknown-linux-gnu.zip/libvortex_jni.so ./vortex-jni/src/main/resources/native/linux-aarch64
106107
cp ../libvortex_jni_x86_64-unknown-linux-gnu.zip/libvortex_jni.so ./vortex-jni/src/main/resources/native/linux-amd64
108+
cp ../libvortex_jni_x86_64-pc-windows-msvc.zip/libvortex_jni.dll ./vortex-jni/src/main/resources/native/win-amd64
107109
- name: Build Java
108110
run: ./gradlew shadowJar
109111
- name: Publish to Maven Central

‎docs/api/java/index.rst‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@ The Java bindings are supported on the following architectures:
3232
* x86_64 Linux
3333
* ARM64 Linux
3434
* Apple Silicon macOS
35+
* x86_64 Windows
3536

3637
They support any Linux distribution with a GLIBC version >= 2.31. This includes
3738

‎docs/api/python/index.rst‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,8 @@ The Python bindings require Python 3.11 or newer. Pre-built wheels are available
2727
* x86_64 Linux
2828
* ARM64 Linux
2929
* Apple Silicon macOS
30+
* x86_64 macOS
31+
* x86_64 Windows
3032

3133
They support any Linux distribution with a GLIBC version >= 2.17. This includes
3234

‎docs/user-guide/spark.md‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -7,10 +7,10 @@ connector is published to Maven Central in two flavors:
77
- `dev.vortex:vortex-spark_2.12` for Spark 3.5.x (Scala 2.12)
88

99
Use the `all` classifier JAR (e.g. `vortex-spark_2.13-0.85.0-all.jar`). It is self-contained:
10-
it bundles the Vortex JNI bindings, native libraries for Linux (x86_64 and aarch64) and macOS
11-
(aarch64), and relocates its Arrow, Guava, and Jackson dependencies to avoid classpath
12-
conflicts with Spark. The thin (unclassified) JAR does not work on its own because it
13-
references relocated classes that only ship in the `all` JAR.
10+
it bundles the Vortex JNI bindings, native libraries for Linux (x86_64 and aarch64), macOS
11+
(aarch64), and Windows (x86_64), and relocates its Arrow, Guava, and Jackson dependencies to
12+
avoid classpath conflicts with Spark. The thin (unclassified) JAR does not work on its own
13+
because it references relocated classes that only ship in the `all` JAR.
1414

1515
## Getting Vortex into Spark
1616

‎java/README.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@ We provide two interfaces for working with Vortex from Java:
1111

1212
We publish three artifacts out of this repo at CI time to Maven Central Sonatype:
1313

14-
* `vortex-jni` JAR containing the JNI code, plus compiled native libraries for all of the following targets: `aarch64-apple-darwin`, `aarch64-unknown-linux-gnu`, `x86_64-unknown-linux-gnu`
14+
* `vortex-jni` JAR containing the JNI code, plus compiled native libraries for all of the following targets: `aarch64-apple-darwin`, `aarch64-unknown-linux-gnu`, `x86_64-unknown-linux-gnu`, `x86_64-pc-windows-msvc`
1515
* `vortex-jni-all` which is the "shadow JAR" containing all of `vortex-jni` as well as all upstream Java dependencies packaged in a single JAR.
1616
* `vortex-spark` which is the runtime JAR needed for the Vortex Spark bindings
1717

‎java/vortex-jni/.gitignore‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,2 +1,3 @@
1+
*.dll
12
*.dylib
23
*.so

‎java/vortex-jni/build.gradle.kts‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -129,7 +129,11 @@ val libExt =
129129
"win" -> ".dll"
130130
else -> throw GradleException("Unsupported OS short name: $osShortName")
131131
}
132-
val nativeLibrary = rustWorkspaceDir.resolve("target/debug/libvortex_jni$libExt")
132+
// Cargo emits `vortex_jni.dll` on Windows and `libvortex_jni.{so,dylib}` elsewhere. The
133+
// bundled resources always use the `lib` prefix so that NativeLoader can build one name.
134+
val bundledLibraryName = "libvortex_jni$libExt"
135+
val cargoLibraryName = if (osShortName == "win") "vortex_jni$libExt" else bundledLibraryName
136+
val nativeLibrary = rustWorkspaceDir.resolve("target/debug/$cargoLibraryName")
133137
val nativeLibraryDir = "src/main/resources/native/$osShortName-$osArch"
134138

135139
val buildJniLibrary =
@@ -156,7 +160,9 @@ tasks.register<Copy>("makeTestFiles") {
156160

157161
// Only populate the host-arch directory so cross-compiled libs for other
158162
// architectures (placed by the publish workflow) are preserved.
159-
from(nativeLibrary)
163+
from(nativeLibrary) {
164+
rename { bundledLibraryName }
165+
}
160166
into(nativeLibraryDir)
161167
}
162168

‎vortex-python/src/lib.rs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,7 @@ mod serde;
4545
mod session;
4646
mod store;
4747

48+
#[cfg(unix)]
4849
use vortex::io::runtime::BlockingRuntime;
4950
use vortex::io::runtime::current::CurrentThreadRuntime;
5051
use vortex::io::runtime::current::CurrentThreadWorkerPool;

0 commit comments

Comments
 (0)