Repository navigation
Expand file tree
/
Copy pathpyproject.toml
More file actions
153 lines (141 loc) · 6.78 KB
/
Copy pathpyproject.toml
File metadata and controls
153 lines (141 loc) · 6.78 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
[project]
name = "lakebench"
version = "2.2.0"
authors = [
{ name="Miles Cole" },
]
license = {file = "LICENSE"}
description = "A multi-modal Python library for benchmarking lakehouse engines and ELT scenarios, supporting both industry-standard and novel benchmarks."
readme = "README.md"
requires-python = ">=3.8"
classifiers = [
"Development Status :: 5 - Production/Stable",
"License :: OSI Approved :: MIT License",
"Programming Language :: Python",
"Programming Language :: Python :: 3.8",
"Programming Language :: Python :: 3.9",
"Programming Language :: Python :: 3.10",
"Programming Language :: Python :: 3.11",
"Programming Language :: Python :: 3.12",
"Programming Language :: Python :: 3.13",
"Intended Audience :: Developers",
"Topic :: System :: Benchmark",
]
dependencies = [
"numpy>=1.24.4",
"sqlglot==26.30.0; python_version < '3.9'",
"sqlglot==30.18.0; python_version >= '3.9'",
"fsspec==2025.2.0",
# Keep the floor at 14 so Fabric Runtime 1.3 (pyarrow 14.0.2) and Runtime 2.0
# (24.0.0) are both already satisfied. Forcing an upgrade inside a running
# notebook session leaves the old compiled pyarrow.lib loaded and breaks with
# "cannot import name 'is_threading_enabled' from 'pyarrow.lib'".
"pyarrow>=14.0.0",
"tenacity>=8.2.3,<9; python_version < '3.9'",
"tenacity==9.1.2; python_version >= '3.9'"
]
[project.optional-dependencies]
# The core `pyarrow>=14.0.0` floor is deliberately low (see above); engines that
# need newer Arrow for table operations state it explicitly in their extras.
duckdb = ["duckdb==1.5.5; python_version >= '3.10'", "deltalake==1.6.2; python_version >= '3.10'", "pyarrow>=16.0.0; python_version >= '3.10'"]
polars = ["polars==1.44.1; python_version >= '3.10'", "deltalake==1.6.2; python_version >= '3.10'", "pyarrow>=16.0.0; python_version >= '3.10'"]
# Daft is held on deltalake 1.5.x: 1.6.x emits DataFusion `string_view` parquet
# from MERGE, and Daft's reader rejects Arrow Utf8View.
daft = ["daft[deltalake]==0.7.21; python_version >= '3.10'", "deltalake>=1.5.0,<1.6.0; python_version >= '3.10'", "pyarrow>=16.0.0; python_version >= '3.10'"]
# The vendored tpcgen-cli serves both TPC-H and TPC-DS in platform wheels.
tpcds_datagen = []
tpch_datagen = []
# Temporary fallback until the unified upstream tpcgen-cli package is released.
tpcds_duckdb_datagen = ["duckdb==1.5.5; python_version >= '3.10'"]
sparkmeasure = ["sparkmeasure==0.28.0; python_version >= '3.9'"]
spark = ["pyspark>=3.5.0,<4.0.0; python_version >= '3.9'", "delta-spark>=3.2.0,<4.0.0; python_version >= '3.9'"]
databricks_sql_warehouse = ["databricks-sql-connector>=3.0.0; python_version >= '3.9'", "pandas>=1.4.0; python_version >= '3.9'", "requests>=2.28.0; python_version >= '3.9'"]
# pysail is a floor rather than the newest release: it is a lower bound only, so
# the resolver picks the newest version available in whichever index is
# configured, and internal mirrors can lag several days behind PyPI.
sail = ["pysail>=0.7.1; python_version >= '3.10'", "pyspark[connect]>=4.0.0; python_version >= '3.10'", "deltalake>=1.6.2; python_version >= '3.10'", "pyarrow>=16.0.0; python_version >= '3.10'"]
# Fabric Data Warehouse connects over ODBC, so it also needs the Microsoft ODBC
# Driver 18 for SQL Server installed on the host.
# Result-log appends use pinned Delta and reuse the core PyArrow floor.
fabric_data_warehouse = ["sqlalchemy>=2.0.0; python_version >= '3.9'", "pyodbc>=5.0.0; python_version >= '3.9'", "pandas>=1.4.0; python_version >= '3.9'", "requests>=2.28.0", "deltalake==1.5.1; python_version >= '3.10'"]
# Backwards-compatible alias for the extra's former name.
fabric_warehouse = ["lakebench[fabric_data_warehouse]"]
fabricwarehouse = ["lakebench[fabric_data_warehouse]"]
[project.urls]
github = "https://github.com/mwc360/LakeBench"
Issues = "https://github.com/mwc360/LakeBench/issues"
[build-system]
requires = ["hatchling"]
build-backend = "hatchling.build"
[tool.hatch.build.targets.wheel]
packages = ["src/lakebench"]
# Ship the third-party attribution notice inside the distribution, not just the
# repository, so it travels with published wheels.
[tool.hatch.build.targets.wheel.force-include]
"THIRD-PARTY-NOTICES.md" = "lakebench/THIRD-PARTY-NOTICES.md"
[tool.hatch.build.hooks.custom]
[dependency-groups]
dev = [
"pytest>=7.0.0",
"pytest-cov>=4.0.0",
"ruff>=0.6.0",
"pre-commit>=3.5.0",
]
[tool.ruff]
line-length = 120
target-version = "py39"
src = ["src", "tests"]
extend-exclude = [
".venv",
"metastore_db",
"src/lakebench/benchmarks/*/resources",
]
[tool.ruff.lint]
# Conservative starter set — formatting + obvious bugs only.
# Expand later (UP, B, SIM, ANN) once the codebase is clean.
select = [
"E", # pycodestyle errors
"F", # pyflakes
"I", # isort
"W", # pycodestyle warnings
]
ignore = [
"E501", # line-too-long (line-length is advisory; many SQL strings are wide)
"E731", # lambda assignments (used intentionally in a few places)
"E741", # ambiguous variable name
]
[tool.ruff.lint.per-file-ignores]
"__init__.py" = ["F401"] # re-exports
"tests/**" = ["F401", "F811", "F841", "E712"] # fixtures + assertion patterns
"scripts/**" = ["E402", "F401", "F841"] # ad-hoc scripts
# Trailing whitespace inside multi-line SQL string literals is intentional/
# harmless and NOT touched by `ruff format` (it only formats code, not string
# contents). Keep ignoring W291/W293 globally so the embedded-SQL engines pass.
"*.py" = ["W291", "W293"]
# Engine-specific DataFrame DSLs intentionally use `col == True` to build expressions,
# and assign `result =`/`df =` to force lazy evaluation.
"src/lakebench/benchmarks/tpcdi/engine_impl/*.py" = ["E712", "F841"]
"src/lakebench/benchmarks/elt_bench/engine_impl/*.py" = ["F841"]
"src/lakebench/engines/*.py" = ["F841"]
[tool.uv]
conflicts = [
[{ extra = "spark" }, { extra = "sail" }],
[{ extra = "databricks_spark" }, { extra = "sail" }],
# Daft and Warehouse use Delta 1.5.x; DuckDB, Polars, and Sail require 1.6.x.
[{ extra = "daft" }, { extra = "duckdb" }],
[{ extra = "daft" }, { extra = "polars" }],
[{ extra = "daft" }, { extra = "sail" }],
[{ extra = "fabric_data_warehouse" }, { extra = "duckdb" }],
[{ extra = "fabric_data_warehouse" }, { extra = "polars" }],
[{ extra = "fabric_data_warehouse" }, { extra = "sail" }],
[{ extra = "fabric_warehouse" }, { extra = "duckdb" }],
[{ extra = "fabric_warehouse" }, { extra = "polars" }],
[{ extra = "fabric_warehouse" }, { extra = "sail" }],
[{ extra = "fabricwarehouse" }, { extra = "duckdb" }],
[{ extra = "fabricwarehouse" }, { extra = "polars" }],
[{ extra = "fabricwarehouse" }, { extra = "sail" }],
]
[tool.pytest.ini_options]
markers = [
"integration: integration tests that generate data and run full benchmarks (require optional extras)",
]