Support Polars 2

.github/workflows/ci.yml

@@ -128,6 +128,15 @@ jobs:
             pytest --benchmark-disable test/test_hf_datasets.py
         working-directory: vortex-python/
 
+      # The lockfile tracks Polars 2, so also exercise the oldest supported 1.x release. The
+      # Polars integration parses Polars' serialized expression format, which differs across
+      # versions.
+      - name: Pytest - Vortex (Polars 1.x floor)
+        run: |
+          uv run --all-packages --with 'polars==1.32.0' \
+            pytest --benchmark-disable test/test_polars_.py test/test_dataset.py
+        working-directory: vortex-python/
+
       - name: Check Python documentation
         run: |
           uv run --all-packages make check

vortex-python/pyproject.toml

@@ -38,7 +38,7 @@ vx = "vortex.cli:main"
 
 [project.optional-dependencies]
 cuda = ["vortex-data-cuda==0.1.0"]
-polars = ["polars>=1.31.0"]
+polars = ["polars>=1.32.0"]
 pandas = ["pandas>=2.2.0"]
 numpy = ["numpy>=1.26.0"]
 duckdb = ["duckdb>=1.1.2"]
@@ -82,7 +82,7 @@ dev = [
     "pandas-stubs>=2.2.3.241126",
     "pandas[output-formatting]>=2.2.3",
     "pcodec>=0.3.3",
-    "polars>=1.9.0",
+    "polars>=1.32.0",
     "pyarrow-stubs>=17.16",
     "pytest-benchmark>=5.1.0",
     "pytest-xdist>=3.5.0",

vortex-python/python/vortex/file.py

@@ -353,14 +353,18 @@ def _io_source(
 
             reader = self.to_arrow(projection=with_columns, expr=vx_predicate, limit=n_rows)
 
-            for batch in reader:
-                batch = pl.DataFrame._from_arrow(batch, rechunk=False)
-                # TODO(ngates): set sortedness on DataFrame based on stats?
-                yield batch
+            try:
+                for batch in reader:
+                    # TODO(ngates): set sortedness on DataFrame based on stats?
+                    yield pl.DataFrame(batch)
+            except pa.ArrowException as e:
+                # Polars 1 wraps exceptions raised by an IO source in a ComputeError, but Polars 2
+                # re-raises them unchanged. Raise a ComputeError on both.
+                raise pl.exceptions.ComputeError(str(e)) from e
 
             # Make sure we always yield at least one empty DataFrame
-            yield pl.DataFrame._from_arrow(
-                data=pa.RecordBatch.from_arrays(
+            yield pl.DataFrame(
+                pa.RecordBatch.from_arrays(
                     [pa.array([], type=field.type) for field in reader.schema],
                     schema=reader.schema,
                 ),

vortex-python/python/vortex/polars_.py

@@ -102,7 +102,12 @@ def _polars_to_vortex(expr: dict[str, Any]) -> ve.Expr:
         elif "Time" in scalar:
             return ve.literal(_dtype.time("ns"), scalar["Time"])
         elif "Decimal" in scalar:
-            value, precision, scale = scalar["Decimal"]
+            # Polars <1.35 omits the precision: [value, scale].
+            if len(scalar["Decimal"]) == 2:
+                value, scale = scalar["Decimal"]
+                precision = 38
+            else:
+                value, precision, scale = scalar["Decimal"]
             return ve.literal(_dtype.decimal(precision=precision, scale=scale), value)
         elif "Date" in scalar:
             return ve.literal(_dtype.date("days"), scalar["Date"])