From 41a0f33f5b510d06bceca0f4dded60b82dc76a0d Mon Sep 17 00:00:00 2001 From: Gaurav Sharma Date: Mon, 21 Sep 2026 14:15:15 +0530 Subject: [PATCH 01/23] CHORE: validate Python 3.15 RC2 across PR CI Add Python 3.15 RC2 preview legs on Windows, macOS, and Linux x64/ARM64 so the existing PR validation flow can expose product and packaging blockers before GA. Skip Arrow-only tests on 3.15 until PyArrow publishes compatible wheels. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- eng/pipelines/pr-validation-pipeline.yml | 29 +++++++++++++++++++++++- requirements.txt | 3 ++- 2 files changed, 30 insertions(+), 2 deletions(-) diff --git a/eng/pipelines/pr-validation-pipeline.yml b/eng/pipelines/pr-validation-pipeline.yml index 1d5b9aa57..c3d614de5 100644 --- a/eng/pipelines/pr-validation-pipeline.yml +++ b/eng/pipelines/pr-validation-pipeline.yml @@ -66,6 +66,9 @@ jobs: LocalDB_Python314: sqlVersion: 'LocalDB' pythonVersion: '3.14' + LocalDB_Python315Preview: + sqlVersion: 'LocalDB' + pythonVersion: '3.15.0-rc.2' steps: - checkout: self @@ -74,6 +77,7 @@ jobs: inputs: versionSpec: '$(pythonVersion)' addToPath: true + allowUnstable: true githubToken: $(GITHUB_TOKEN) displayName: 'Use Python $(pythonVersion)' @@ -494,6 +498,10 @@ jobs: sqlServerImage: 'mcr.microsoft.com/mssql/server:2025-latest' sqlVersion: 'SQL2025' pythonVersion: '3.14' + SQL2025_Python315Preview: + sqlServerImage: 'mcr.microsoft.com/mssql/server:2025-latest' + sqlVersion: 'SQL2025' + pythonVersion: '3.15.0-rc.2' steps: - checkout: self @@ -503,6 +511,7 @@ jobs: inputs: versionSpec: '$(pythonVersion)' addToPath: true + allowUnstable: true displayName: 'Use Python $(pythonVersion) on macOS' - task: Cache@2 @@ -678,6 +687,11 @@ jobs: distroName: 'Debian-SQL2025' sqlServerImage: 'mcr.microsoft.com/mssql/server:2025-latest' useAzureSQL: 'false' + Debian_Python315Preview: + dockerImage: 'python:3.15.0rc2-bookworm' + distroName: 'Debian-Python315Preview' + sqlServerImage: 'mcr.microsoft.com/mssql/server:2025-latest' + useAzureSQL: 'false' steps: - checkout: self @@ -705,7 +719,16 @@ jobs: - script: | # Install dependencies in the container - if [ "$(distroName)" = "Ubuntu" ]; then + if [ "$(distroName)" = "Debian-Python315Preview" ]; then + docker exec test-container-$(distroName) bash -c " + export DEBIAN_FRONTEND=noninteractive + export TZ=UTC + ln -snf /usr/share/zoneinfo/\$TZ /etc/localtime && echo \$TZ > /etc/timezone + apt-get update && + apt-get install -y cmake curl wget gnupg build-essential + python3 --version + " + elif [ "$(distroName)" = "Ubuntu" ]; then docker exec test-container-$(distroName) bash -c " export DEBIAN_FRONTEND=noninteractive export TZ=UTC @@ -983,6 +1006,10 @@ jobs: dockerImage: 'python:3.11-bookworm' distroName: 'Debian' archName: 'arm64' + Debian_Python315Preview_ARM64: + dockerImage: 'python:3.15.0rc2-bookworm' + distroName: 'Debian-Python315Preview' + archName: 'arm64' steps: - script: | diff --git a/requirements.txt b/requirements.txt index daffd1a1c..ef132c54e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,7 +6,8 @@ zstandard coverage unittest-xml-reporting psutil -pyarrow +# Python 3.15 preview wheels aren't published yet; Arrow tests remain a GA-readiness blocker. +pyarrow; python_version < "3.15" polars # Runtime dependencies needed for tests From bf20afd245139fe0c2a598255f518b817c31ba4d Mon Sep 17 00:00:00 2001 From: Sumit Sarabhai Date: Wed, 23 Sep 2026 19:47:05 +0100 Subject: [PATCH 02/23] FIX: enforce POSIX native binary hardening AB#48377 Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- mssql_python/pybind/CMakeLists.txt | 43 +++++ tests/test_040_native_binary_hardening.py | 183 ++++++++++++++++++++++ 2 files changed, 226 insertions(+) create mode 100644 tests/test_040_native_binary_hardening.py diff --git a/mssql_python/pybind/CMakeLists.txt b/mssql_python/pybind/CMakeLists.txt index 77d599bd5..a126f053a 100644 --- a/mssql_python/pybind/CMakeLists.txt +++ b/mssql_python/pybind/CMakeLists.txt @@ -255,6 +255,13 @@ endif() message(STATUS "Final Python library directory: ${PYTHON_LIB_DIR}") +# Single-config POSIX generators ignore `cmake --build --config Release`. +# Make the optimized release mode explicit instead of relying on simdutf to +# populate this project-wide cache variable as a FetchContent side effect. +if(UNIX AND NOT CMAKE_BUILD_TYPE AND NOT CMAKE_CONFIGURATION_TYPES) + set(CMAKE_BUILD_TYPE Release CACHE STRING "Build type" FORCE) +endif() + include(FetchContent) message(STATUS "Downloading simdutf v8.2.0 source archive with FetchContent") set(simdutf_fetchcontent_args @@ -387,6 +394,42 @@ if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU" OR CMAKE_CXX_COMPILER_ID STREQUAL "Clang endif() endif() +# Harden the Python extension against exploitation of a separate memory-safety +# defect. Mach-O receives stack protection; ELF-specific flags stay Linux-only. +if(UNIX) + target_compile_options(ddbc_bindings PRIVATE -fstack-protector-strong) +endif() + +if(UNIX AND NOT APPLE) + include(CheckCXXSourceCompiles) + set(DDBC_REQUIRED_FLAGS_SAVED "${CMAKE_REQUIRED_FLAGS}") + set(CMAKE_REQUIRED_FLAGS + "${CMAKE_REQUIRED_FLAGS} -O2 -Werror -U_FORTIFY_SOURCE -D_FORTIFY_SOURCE=3" + ) + check_cxx_source_compiles(" + #include + #if !defined(__USE_FORTIFY_LEVEL) || __USE_FORTIFY_LEVEL < 3 + #error _FORTIFY_SOURCE=3 is unavailable + #endif + int main() { return 0; } + " DDBC_SUPPORTS_FORTIFY_SOURCE_3) + set(CMAKE_REQUIRED_FLAGS "${DDBC_REQUIRED_FLAGS_SAVED}") + if(DDBC_SUPPORTS_FORTIFY_SOURCE_3) + set(DDBC_FORTIFY_LEVEL 3) + else() + set(DDBC_FORTIFY_LEVEL 2) + endif() + target_compile_options(ddbc_bindings PRIVATE + $<$>:-U_FORTIFY_SOURCE> + $<$>:-D_FORTIFY_SOURCE=${DDBC_FORTIFY_LEVEL}> + ) + target_link_options(ddbc_bindings PRIVATE + -Wl,-z,relro + -Wl,-z,now + -Wl,-z,noexecstack + ) +endif() + # Add macOS-specific string conversion fix if(APPLE) message(STATUS "Enabling macOS string conversion fix") diff --git a/tests/test_040_native_binary_hardening.py b/tests/test_040_native_binary_hardening.py new file mode 100644 index 000000000..48556e985 --- /dev/null +++ b/tests/test_040_native_binary_hardening.py @@ -0,0 +1,183 @@ +"""Regression guards for POSIX native-extension hardening.""" + +import struct +import sys +from pathlib import Path + +import pytest + +_ROOT = Path(__file__).resolve().parents[1] +_CMAKE = _ROOT / "mssql_python" / "pybind" / "CMakeLists.txt" +_PT_DYNAMIC = 2 +_PT_GNU_STACK = 0x6474E551 +_PT_GNU_RELRO = 0x6474E552 +_PF_X = 0x1 +_DT_NULL = 0 +_DT_BIND_NOW = 24 +_DT_FLAGS = 30 +_DF_BIND_NOW = 0x8 +_DT_FLAGS_1 = 0x6FFFFFFB +_DF_1_NOW = 0x1 + + +def _make_elf64(*, relro=True, bind_now=True, executable_stack=False): + header_size = 64 + program_header_size = 56 + program_count = 3 + dynamic_offset = header_size + program_header_size * program_count + dynamic = struct.pack("" + program_offset = struct.unpack_from(endian + "Q", data, 0x20)[0] + entry_size = struct.unpack_from(endian + "H", data, 0x36)[0] + entry_count = struct.unpack_from(endian + "H", data, 0x38)[0] + + if ( + not program_offset + or not entry_count + or entry_size < 56 + or program_offset + entry_count * entry_size > len(data) + ): + raise ValueError("invalid ELF program headers") + + has_relro = False + stack_executable = None + dynamic_segment = None + for index in range(entry_count): + offset = program_offset + index * entry_size + program_type = struct.unpack_from(endian + "I", data, offset)[0] + flags = struct.unpack_from(endian + "I", data, offset + 4)[0] + file_offset = struct.unpack_from(endian + "Q", data, offset + 8)[0] + file_size = struct.unpack_from(endian + "Q", data, offset + 32)[0] + if file_offset + file_size > len(data): + raise ValueError("ELF segment extends beyond the file") + if program_type == _PT_GNU_RELRO: + has_relro = True + elif program_type == _PT_GNU_STACK: + stack_executable = bool(flags & _PF_X) + elif program_type == _PT_DYNAMIC: + dynamic_segment = file_offset, file_size + + if dynamic_segment is None: + raise ValueError("ELF has no PT_DYNAMIC segment") + + dynamic_offset, dynamic_size = dynamic_segment + dynamic_entry_size = 16 + if dynamic_size % dynamic_entry_size: + raise ValueError("ELF dynamic segment has a partial entry") + + bind_now = False + terminated = False + for offset in range(dynamic_offset, dynamic_offset + dynamic_size, dynamic_entry_size): + tag = struct.unpack_from(endian + "q", data, offset)[0] + value = struct.unpack_from(endian + "Q", data, offset + 8)[0] + if tag == _DT_NULL: + terminated = True + break + bind_now = bind_now or tag == _DT_BIND_NOW + bind_now = bind_now or tag == _DT_FLAGS and bool(value & _DF_BIND_NOW) + bind_now = bind_now or tag == _DT_FLAGS_1 and bool(value & _DF_1_NOW) + + if not terminated: + raise ValueError("ELF dynamic segment lacks DT_NULL") + return has_relro, bind_now, stack_executable + + +def test_elf_hardening_parser_reads_program_and_dynamic_flags(): + assert _elf_hardening(_make_elf64()) == (True, True, False) + assert _elf_hardening(_make_elf64(relro=False)) == (False, True, False) + assert _elf_hardening(_make_elf64(bind_now=False)) == (True, False, False) + assert _elf_hardening(_make_elf64(executable_stack=True)) == (True, True, True) + + +@pytest.mark.skipif( + not _CMAKE.is_file(), + reason="requires a source checkout; isolated wheel tests omit the source tree", +) +def test_posix_hardening_flags_are_explicit(): + cmake = _CMAKE.read_text(encoding="utf-8") + + assert "set(CMAKE_BUILD_TYPE Release" in cmake + assert "-fstack-protector-strong" in cmake + assert "-U_FORTIFY_SOURCE" in cmake + assert "DDBC_SUPPORTS_FORTIFY_SOURCE_3" in cmake + assert "-D_FORTIFY_SOURCE=${DDBC_FORTIFY_LEVEL}" in cmake + assert "$>" in cmake + assert "if(UNIX AND NOT APPLE)" in cmake + for flag in ("-Wl,-z,relro", "-Wl,-z,now", "-Wl,-z,noexecstack"): + assert flag in cmake + + +@pytest.mark.skipif(sys.platform != "linux", reason="ELF hardening applies to Linux") +def test_linux_extension_has_linker_hardening(): + from mssql_python import ddbc_bindings + + extension = Path(ddbc_bindings.module_path) + has_relro, bind_now, stack_executable = _elf_hardening(extension.read_bytes()) + assert has_relro, "native extension is missing PT_GNU_RELRO" + assert bind_now, "native extension is missing immediate binding (BIND_NOW)" + assert stack_executable is not None, "native extension has no PT_GNU_STACK declaration" + assert stack_executable is False, "native extension requests an executable stack" From dd1b21c7fc4aaa4bac198612631727e9c6ea65af Mon Sep 17 00:00:00 2001 From: Sumit Sarabhai Date: Thu, 24 Sep 2026 07:17:56 +0100 Subject: [PATCH 03/23] FIX: isolate concurrent auth connection mocks Give each concurrent connect call its own configured native connection mock so MagicMock child creation cannot race during close. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- tests/test_008_auth.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/tests/test_008_auth.py b/tests/test_008_auth.py index 6fd2d6574..2b3ef50da 100644 --- a/tests/test_008_auth.py +++ b/tests/test_008_auth.py @@ -1565,7 +1565,13 @@ def test_multiple_connections_share_same_token_provider(self, mock_ddbc_conn): @patch("mssql_python.connection.ddbc_bindings.Connection") def test_concurrent_connections_with_same_token_provider(self, mock_ddbc_conn): """Concurrent connect() calls with one token provider should succeed.""" - mock_ddbc_conn.return_value = MagicMock() + + def create_native_connection(*_args, **_kwargs): + native_connection = MagicMock() + native_connection.get_autocommit.return_value = True + return native_connection + + mock_ddbc_conn.side_effect = create_native_connection mock_cred = MagicMock() mock_cred.get_token.return_value = MagicMock(token=SAMPLE_TOKEN) from mssql_python import connect From 8cf29992f51dc1d42473f6b0003848371f14567a Mon Sep 17 00:00:00 2001 From: Gaurav Sharma Date: Thu, 24 Sep 2026 19:38:07 +0530 Subject: [PATCH 04/23] CHORE: validate Python 3.15 with abi3 Rust wheels Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- eng/versions/mssql-python-rs-nuget.version | 2 +- eng/versions/mssql-python-rs.version | 2 +- tests/test_038_mssql_odbc_daily_validation.py | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/eng/versions/mssql-python-rs-nuget.version b/eng/versions/mssql-python-rs-nuget.version index 6c6aa7cb0..b19e5ab3f 100644 --- a/eng/versions/mssql-python-rs-nuget.version +++ b/eng/versions/mssql-python-rs-nuget.version @@ -1 +1 @@ -0.1.0 \ No newline at end of file +0.2.0-nightly.20260924 \ No newline at end of file diff --git a/eng/versions/mssql-python-rs.version b/eng/versions/mssql-python-rs.version index 6c6aa7cb0..341cf11fa 100644 --- a/eng/versions/mssql-python-rs.version +++ b/eng/versions/mssql-python-rs.version @@ -1 +1 @@ -0.1.0 \ No newline at end of file +0.2.0 \ No newline at end of file diff --git a/tests/test_038_mssql_odbc_daily_validation.py b/tests/test_038_mssql_odbc_daily_validation.py index b4a22b35c..63ec6e4cc 100644 --- a/tests/test_038_mssql_odbc_daily_validation.py +++ b/tests/test_038_mssql_odbc_daily_validation.py @@ -143,12 +143,12 @@ def test_pipeline_authenticates_advisory_status_and_keeps_publish_strict(self): self.assertIn('exit "$cleanup_rc"', pipeline) self.assertIn("grep -q 'MSSQL_ODBC_PREFLIGHT_OK'", pipeline) - def test_stable_rs_transport_is_pinned(self): + def test_rs_transport_is_pinned(self): version = (ROOT / "eng" / "versions" / "mssql-python-rs-nuget.version").read_text( encoding="ascii" ) - self.assertEqual(version.strip(), "0.1.0") + self.assertEqual(version.strip(), "0.2.0-nightly.20260924") if __name__ == "__main__": From b9a2b2e35a0e8d29f57655adaad9c27cd50eb73e Mon Sep 17 00:00:00 2001 From: Sumit Sarabhai Date: Wed, 23 Sep 2026 19:47:05 +0100 Subject: [PATCH 05/23] FIX: enforce POSIX native binary hardening AB#48377 Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- mssql_python/pybind/CMakeLists.txt | 43 +++++ tests/test_040_native_binary_hardening.py | 183 ++++++++++++++++++++++ 2 files changed, 226 insertions(+) create mode 100644 tests/test_040_native_binary_hardening.py diff --git a/mssql_python/pybind/CMakeLists.txt b/mssql_python/pybind/CMakeLists.txt index 2ce264253..878198dd1 100644 --- a/mssql_python/pybind/CMakeLists.txt +++ b/mssql_python/pybind/CMakeLists.txt @@ -255,6 +255,13 @@ endif() message(STATUS "Final Python library directory: ${PYTHON_LIB_DIR}") +# Single-config POSIX generators ignore `cmake --build --config Release`. +# Make the optimized release mode explicit instead of relying on simdutf to +# populate this project-wide cache variable as a FetchContent side effect. +if(UNIX AND NOT CMAKE_BUILD_TYPE AND NOT CMAKE_CONFIGURATION_TYPES) + set(CMAKE_BUILD_TYPE Release CACHE STRING "Build type" FORCE) +endif() + include(FetchContent) message(STATUS "Downloading simdutf v8.2.0 source archive with FetchContent") set(simdutf_fetchcontent_args @@ -384,6 +391,42 @@ if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU" OR CMAKE_CXX_COMPILER_ID STREQUAL "Clang endif() endif() +# Harden the Python extension against exploitation of a separate memory-safety +# defect. Mach-O receives stack protection; ELF-specific flags stay Linux-only. +if(UNIX) + target_compile_options(ddbc_bindings PRIVATE -fstack-protector-strong) +endif() + +if(UNIX AND NOT APPLE) + include(CheckCXXSourceCompiles) + set(DDBC_REQUIRED_FLAGS_SAVED "${CMAKE_REQUIRED_FLAGS}") + set(CMAKE_REQUIRED_FLAGS + "${CMAKE_REQUIRED_FLAGS} -O2 -Werror -U_FORTIFY_SOURCE -D_FORTIFY_SOURCE=3" + ) + check_cxx_source_compiles(" + #include + #if !defined(__USE_FORTIFY_LEVEL) || __USE_FORTIFY_LEVEL < 3 + #error _FORTIFY_SOURCE=3 is unavailable + #endif + int main() { return 0; } + " DDBC_SUPPORTS_FORTIFY_SOURCE_3) + set(CMAKE_REQUIRED_FLAGS "${DDBC_REQUIRED_FLAGS_SAVED}") + if(DDBC_SUPPORTS_FORTIFY_SOURCE_3) + set(DDBC_FORTIFY_LEVEL 3) + else() + set(DDBC_FORTIFY_LEVEL 2) + endif() + target_compile_options(ddbc_bindings PRIVATE + $<$>:-U_FORTIFY_SOURCE> + $<$>:-D_FORTIFY_SOURCE=${DDBC_FORTIFY_LEVEL}> + ) + target_link_options(ddbc_bindings PRIVATE + -Wl,-z,relro + -Wl,-z,now + -Wl,-z,noexecstack + ) +endif() + # Add macOS-specific string conversion fix if(APPLE) message(STATUS "Enabling macOS string conversion fix") diff --git a/tests/test_040_native_binary_hardening.py b/tests/test_040_native_binary_hardening.py new file mode 100644 index 000000000..48556e985 --- /dev/null +++ b/tests/test_040_native_binary_hardening.py @@ -0,0 +1,183 @@ +"""Regression guards for POSIX native-extension hardening.""" + +import struct +import sys +from pathlib import Path + +import pytest + +_ROOT = Path(__file__).resolve().parents[1] +_CMAKE = _ROOT / "mssql_python" / "pybind" / "CMakeLists.txt" +_PT_DYNAMIC = 2 +_PT_GNU_STACK = 0x6474E551 +_PT_GNU_RELRO = 0x6474E552 +_PF_X = 0x1 +_DT_NULL = 0 +_DT_BIND_NOW = 24 +_DT_FLAGS = 30 +_DF_BIND_NOW = 0x8 +_DT_FLAGS_1 = 0x6FFFFFFB +_DF_1_NOW = 0x1 + + +def _make_elf64(*, relro=True, bind_now=True, executable_stack=False): + header_size = 64 + program_header_size = 56 + program_count = 3 + dynamic_offset = header_size + program_header_size * program_count + dynamic = struct.pack("" + program_offset = struct.unpack_from(endian + "Q", data, 0x20)[0] + entry_size = struct.unpack_from(endian + "H", data, 0x36)[0] + entry_count = struct.unpack_from(endian + "H", data, 0x38)[0] + + if ( + not program_offset + or not entry_count + or entry_size < 56 + or program_offset + entry_count * entry_size > len(data) + ): + raise ValueError("invalid ELF program headers") + + has_relro = False + stack_executable = None + dynamic_segment = None + for index in range(entry_count): + offset = program_offset + index * entry_size + program_type = struct.unpack_from(endian + "I", data, offset)[0] + flags = struct.unpack_from(endian + "I", data, offset + 4)[0] + file_offset = struct.unpack_from(endian + "Q", data, offset + 8)[0] + file_size = struct.unpack_from(endian + "Q", data, offset + 32)[0] + if file_offset + file_size > len(data): + raise ValueError("ELF segment extends beyond the file") + if program_type == _PT_GNU_RELRO: + has_relro = True + elif program_type == _PT_GNU_STACK: + stack_executable = bool(flags & _PF_X) + elif program_type == _PT_DYNAMIC: + dynamic_segment = file_offset, file_size + + if dynamic_segment is None: + raise ValueError("ELF has no PT_DYNAMIC segment") + + dynamic_offset, dynamic_size = dynamic_segment + dynamic_entry_size = 16 + if dynamic_size % dynamic_entry_size: + raise ValueError("ELF dynamic segment has a partial entry") + + bind_now = False + terminated = False + for offset in range(dynamic_offset, dynamic_offset + dynamic_size, dynamic_entry_size): + tag = struct.unpack_from(endian + "q", data, offset)[0] + value = struct.unpack_from(endian + "Q", data, offset + 8)[0] + if tag == _DT_NULL: + terminated = True + break + bind_now = bind_now or tag == _DT_BIND_NOW + bind_now = bind_now or tag == _DT_FLAGS and bool(value & _DF_BIND_NOW) + bind_now = bind_now or tag == _DT_FLAGS_1 and bool(value & _DF_1_NOW) + + if not terminated: + raise ValueError("ELF dynamic segment lacks DT_NULL") + return has_relro, bind_now, stack_executable + + +def test_elf_hardening_parser_reads_program_and_dynamic_flags(): + assert _elf_hardening(_make_elf64()) == (True, True, False) + assert _elf_hardening(_make_elf64(relro=False)) == (False, True, False) + assert _elf_hardening(_make_elf64(bind_now=False)) == (True, False, False) + assert _elf_hardening(_make_elf64(executable_stack=True)) == (True, True, True) + + +@pytest.mark.skipif( + not _CMAKE.is_file(), + reason="requires a source checkout; isolated wheel tests omit the source tree", +) +def test_posix_hardening_flags_are_explicit(): + cmake = _CMAKE.read_text(encoding="utf-8") + + assert "set(CMAKE_BUILD_TYPE Release" in cmake + assert "-fstack-protector-strong" in cmake + assert "-U_FORTIFY_SOURCE" in cmake + assert "DDBC_SUPPORTS_FORTIFY_SOURCE_3" in cmake + assert "-D_FORTIFY_SOURCE=${DDBC_FORTIFY_LEVEL}" in cmake + assert "$>" in cmake + assert "if(UNIX AND NOT APPLE)" in cmake + for flag in ("-Wl,-z,relro", "-Wl,-z,now", "-Wl,-z,noexecstack"): + assert flag in cmake + + +@pytest.mark.skipif(sys.platform != "linux", reason="ELF hardening applies to Linux") +def test_linux_extension_has_linker_hardening(): + from mssql_python import ddbc_bindings + + extension = Path(ddbc_bindings.module_path) + has_relro, bind_now, stack_executable = _elf_hardening(extension.read_bytes()) + assert has_relro, "native extension is missing PT_GNU_RELRO" + assert bind_now, "native extension is missing immediate binding (BIND_NOW)" + assert stack_executable is not None, "native extension has no PT_GNU_STACK declaration" + assert stack_executable is False, "native extension requests an executable stack" From 16729abceed4d4f66b78c6e187b53ff0b37ef551 Mon Sep 17 00:00:00 2001 From: Gaurav Sharma Date: Thu, 24 Sep 2026 20:07:36 +0530 Subject: [PATCH 06/23] TEST: focus Python 3.15 preview validation Use the pinned PyArrow nightly wheel set and skip every non-3.15 pipeline leg so the validation result measures only 3.15 readiness.\n\nCo-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- eng/pipelines/pr-validation-pipeline.yml | 90 +++++++------------ requirements.txt | 3 +- tests/test_038_mssql_odbc_daily_validation.py | 18 ++++ 3 files changed, 53 insertions(+), 58 deletions(-) diff --git a/eng/pipelines/pr-validation-pipeline.yml b/eng/pipelines/pr-validation-pipeline.yml index c3d614de5..bc559e8d0 100644 --- a/eng/pipelines/pr-validation-pipeline.yml +++ b/eng/pipelines/pr-validation-pipeline.yml @@ -13,6 +13,7 @@ variables: jobs: - job: CodeQLAnalysis displayName: 'CodeQL Security Analysis' + condition: false # Python 3.13; excluded from the Python 3.15 validation branch. pool: vmImage: 'ubuntu-latest' @@ -57,15 +58,15 @@ jobs: strategy: matrix: - SQLServer2022: - sqlVersion: 'SQL2022' - pythonVersion: '3.13' - SQLServer2025: - sqlVersion: 'SQL2025' - pythonVersion: '3.14' - LocalDB_Python314: - sqlVersion: 'LocalDB' - pythonVersion: '3.14' + # SQLServer2022: + # sqlVersion: 'SQL2022' + # pythonVersion: '3.13' + # SQLServer2025: + # sqlVersion: 'SQL2025' + # pythonVersion: '3.14' + # LocalDB_Python314: + # sqlVersion: 'LocalDB' + # pythonVersion: '3.14' LocalDB_Python315Preview: sqlVersion: 'LocalDB' pythonVersion: '3.15.0-rc.2' @@ -490,14 +491,14 @@ jobs: strategy: matrix: - SQL2022: - sqlServerImage: 'mcr.microsoft.com/mssql/server:2022-latest' - sqlVersion: 'SQL2022' - pythonVersion: '3.13' - SQL2025: - sqlServerImage: 'mcr.microsoft.com/mssql/server:2025-latest' - sqlVersion: 'SQL2025' - pythonVersion: '3.14' + # SQL2022: + # sqlServerImage: 'mcr.microsoft.com/mssql/server:2022-latest' + # sqlVersion: 'SQL2022' + # pythonVersion: '3.13' + # SQL2025: + # sqlServerImage: 'mcr.microsoft.com/mssql/server:2025-latest' + # sqlVersion: 'SQL2025' + # pythonVersion: '3.14' SQL2025_Python315Preview: sqlServerImage: 'mcr.microsoft.com/mssql/server:2025-latest' sqlVersion: 'SQL2025' @@ -659,34 +660,8 @@ jobs: strategy: matrix: - Ubuntu: - dockerImage: 'ubuntu:24.04' - distroName: 'Ubuntu' - sqlServerImage: 'mcr.microsoft.com/mssql/server:2022-latest' - useAzureSQL: 'false' - profilerLeg: 'Linux-SQL2022' - Ubuntu_SQL2025: - dockerImage: 'ubuntu:24.04' - distroName: 'Ubuntu-SQL2025' - sqlServerImage: 'mcr.microsoft.com/mssql/server:2025-latest' - useAzureSQL: 'false' - profilerLeg: 'Linux-SQL2025' - ${{ if ne(variables['AZURE_CONNECTION_STRING'], '') }}: - Ubuntu_AzureSQL: - dockerImage: 'ubuntu:24.04' - distroName: 'Ubuntu-AzureSQL' - sqlServerImage: '' - useAzureSQL: 'true' - Debian: - dockerImage: 'debian:12' - distroName: 'Debian' - sqlServerImage: 'mcr.microsoft.com/mssql/server:2022-latest' - useAzureSQL: 'false' - Debian_SQL2025: - dockerImage: 'debian:12' - distroName: 'Debian-SQL2025' - sqlServerImage: 'mcr.microsoft.com/mssql/server:2025-latest' - useAzureSQL: 'false' + # Existing distro legs use Python 3.10-3.14 and are intentionally + # excluded while this branch validates only Python 3.15. Debian_Python315Preview: dockerImage: 'python:3.15.0rc2-bookworm' distroName: 'Debian-Python315Preview' @@ -994,18 +969,14 @@ jobs: strategy: matrix: - Ubuntu_ARM64: - dockerImage: 'ubuntu:22.04' - distroName: 'Ubuntu' - archName: 'arm64' - Debian_ARM64: - # Python 3.11 is preinstalled (byte-compiled natively at image-build - # time) so we never run the apt python3 post-install byte-compilation - # that SIGSEGVs (exit 139) under QEMU user-mode emulation. Still Debian - # 12 (bookworm) underneath, so the msodbcsql18 debian/12 repo applies. - dockerImage: 'python:3.11-bookworm' - distroName: 'Debian' - archName: 'arm64' + # Ubuntu_ARM64: + # dockerImage: 'ubuntu:22.04' + # distroName: 'Ubuntu' + # archName: 'arm64' + # Debian_ARM64: + # dockerImage: 'python:3.11-bookworm' + # distroName: 'Debian' + # archName: 'arm64' Debian_Python315Preview_ARM64: dockerImage: 'python:3.15.0rc2-bookworm' distroName: 'Debian-Python315Preview' @@ -1229,6 +1200,7 @@ jobs: - job: PytestOnLinux_RHEL9 displayName: 'Linux RedHat x86_64' + condition: false # Python 3.12; excluded from the Python 3.15 validation branch. pool: vmImage: 'ubuntu-latest' @@ -1427,6 +1399,7 @@ jobs: - job: PytestOnLinux_RHEL9_ARM64 displayName: 'Linux RedHat ARM64' + condition: false # Python 3.12; excluded from the Python 3.15 validation branch. pool: vmImage: 'ubuntu-latest' @@ -1640,6 +1613,7 @@ jobs: - job: PytestOnLinux_Alpine displayName: 'Linux Alpine x86_64' + condition: false # Distro Python is not 3.15; excluded from this validation branch. pool: vmImage: 'ubuntu-latest' @@ -1873,6 +1847,7 @@ jobs: - job: PytestOnLinux_Alpine_ARM64 displayName: 'Linux Alpine ARM64' + condition: false # Distro Python is not 3.15; excluded from this validation branch. pool: vmImage: 'ubuntu-latest' @@ -2338,6 +2313,7 @@ jobs: - job: CodeCoverageReport displayName: 'Full Code Coverage Report in Ubuntu x86_64' + condition: false # Uses the hosted default Python; excluded from this validation branch. pool: vmImage: 'ubuntu-latest' diff --git a/requirements.txt b/requirements.txt index ef132c54e..2a7b9b4ca 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,8 +6,9 @@ zstandard coverage unittest-xml-reporting psutil -# Python 3.15 preview wheels aren't published yet; Arrow tests remain a GA-readiness blocker. +--extra-index-url https://pypi.anaconda.org/scientific-python-nightly-wheels/simple pyarrow; python_version < "3.15" +pyarrow==26.0.0.dev296; python_version >= "3.15" polars # Runtime dependencies needed for tests diff --git a/tests/test_038_mssql_odbc_daily_validation.py b/tests/test_038_mssql_odbc_daily_validation.py index 63ec6e4cc..e74a79f8a 100644 --- a/tests/test_038_mssql_odbc_daily_validation.py +++ b/tests/test_038_mssql_odbc_daily_validation.py @@ -12,6 +12,7 @@ ROOT = Path(__file__).parents[1] RUNNER = ROOT / "eng" / "scripts" / "run-mssql-odbc-tests.sh" PIPELINE = ROOT / "eng" / "pipelines" / "mssql-odbc-daily-validation-pipeline.yml" +PR_PIPELINE = ROOT / "eng" / "pipelines" / "pr-validation-pipeline.yml" PREFLIGHT = ROOT / "eng" / "scripts" / "verify_mssql_odbc_provider.py" @@ -150,6 +151,23 @@ def test_rs_transport_is_pinned(self): self.assertEqual(version.strip(), "0.2.0-nightly.20260924") + def test_python_315_validation_uses_only_preview_matrix_legs(self): + pipeline = PR_PIPELINE.read_text(encoding="utf-8") + active_python_versions = { + line.split(":", 1)[1].strip(" '\"") + for line in pipeline.splitlines() + if line.lstrip().startswith("pythonVersion:") + } + + self.assertEqual(active_python_versions, {"3.15.0-rc.2"}) + self.assertIn("python:3.15.0rc2-bookworm", pipeline) + + def test_python_315_validation_installs_pyarrow_nightly(self): + requirements = (ROOT / "requirements.txt").read_text(encoding="utf-8") + + self.assertIn("scientific-python-nightly-wheels", requirements) + self.assertIn('pyarrow==26.0.0.dev296; python_version >= "3.15"', requirements) + if __name__ == "__main__": unittest.main() From 5740776a81fa30697ca78636a35de55a52f6a29d Mon Sep 17 00:00:00 2001 From: Gaurav Sharma Date: Thu, 24 Sep 2026 23:34:07 +0530 Subject: [PATCH 07/23] FIX: unblock Python 3.15 validation Select compatible abi3 Rust wheels for newer CPython versions and include Python.h before libc-consuming standard headers.\n\nCo-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- eng/scripts/install-mssql-py-core.ps1 | 11 ++-- eng/scripts/install-mssql-py-core.sh | 10 +-- eng/scripts/select_mssql_python_rs_wheel.py | 62 +++++++++++++++++++ mssql_python/pybind/ddbc_bindings.h | 11 ++-- tests/test_038_mssql_odbc_daily_validation.py | 48 ++++++++++++++ tests/test_040_native_binary_hardening.py | 6 ++ 6 files changed, 134 insertions(+), 14 deletions(-) create mode 100644 eng/scripts/select_mssql_python_rs_wheel.py diff --git a/eng/scripts/install-mssql-py-core.ps1 b/eng/scripts/install-mssql-py-core.ps1 index ebbc14628..0e2005810 100644 --- a/eng/scripts/install-mssql-py-core.ps1 +++ b/eng/scripts/install-mssql-py-core.ps1 @@ -74,8 +74,7 @@ function Get-PlatformInfo { # aarch64 -> arm64 so a Windows arm64 target resolves win_arm64, not win_aarch64. $script:WheelPlatform = "win_$($archTag -replace 'x86_64','amd64' -replace 'aarch64','arm64')" - $script:WheelPattern = "mssql_python_rs-$script:DistributionVersion-$script:PyVersion-$script:PyVersion-$script:WheelPlatform.whl" - Write-Host "Wheel pattern: $script:WheelPattern" + Write-Host "Wheel target: $script:PyVersion | $script:WheelPlatform" } function Get-NupkgFromFeed { @@ -121,12 +120,14 @@ function Find-MatchingWheel { throw "No 'wheels' directory found in NuGet package" } - $script:MatchingWheel = Get-ChildItem $wheelsDir -Filter $script:WheelPattern | Select-Object -First 1 - if (-not $script:MatchingWheel) { + $matchingWheelPath = & python "$ScriptDir\select_mssql_python_rs_wheel.py" ` + $wheelsDir $script:DistributionVersion $script:PyVersion $script:WheelPlatform + if ($LASTEXITCODE -ne 0) { Write-Host "Available wheels:" Get-ChildItem $wheelsDir -Filter *.whl | ForEach-Object { Write-Host " $_" } - throw "No wheel found matching: $script:WheelPattern" + throw "No compatible mssql-python-rs wheel found" } + $script:MatchingWheel = Get-Item $matchingWheelPath Write-Host "Found: $($script:MatchingWheel.Name)" } diff --git a/eng/scripts/install-mssql-py-core.sh b/eng/scripts/install-mssql-py-core.sh index e46266adf..37433b295 100644 --- a/eng/scripts/install-mssql-py-core.sh +++ b/eng/scripts/install-mssql-py-core.sh @@ -89,8 +89,7 @@ print(f'cp{v.major}{v.minor} {platform.system().lower()} {platform.machine().low ;; esac - WHEEL_PATTERN="mssql_python_rs-${DISTRIBUTION_VERSION}-${PY_VERSION}-${PY_VERSION}-${WHEEL_PLATFORM}.whl" - echo "Wheel pattern: $WHEEL_PATTERN" + echo "Wheel target: $PY_VERSION | $WHEEL_PLATFORM" } download_nupkg() { @@ -158,11 +157,12 @@ find_matching_wheel() { exit 1 fi - MATCHING_WHEEL=$(find "$wheels_dir" -name "$WHEEL_PATTERN" -print -quit) - if [ -z "$MATCHING_WHEEL" ]; then + if ! MATCHING_WHEEL=$( + "$PYTHON" "$SCRIPT_DIR/select_mssql_python_rs_wheel.py" \ + "$wheels_dir" "$DISTRIBUTION_VERSION" "$PY_VERSION" "$WHEEL_PLATFORM" + ); then echo "Available wheels:" ls "$wheels_dir"/*.whl 2>/dev/null || echo " (none)" - echo "ERROR: No wheel found matching: $WHEEL_PATTERN" exit 1 fi diff --git a/eng/scripts/select_mssql_python_rs_wheel.py b/eng/scripts/select_mssql_python_rs_wheel.py new file mode 100644 index 000000000..c10687f5a --- /dev/null +++ b/eng/scripts/select_mssql_python_rs_wheel.py @@ -0,0 +1,62 @@ +#!/usr/bin/env python3 +"""Select the compatible mssql-python-rs wheel from an extracted transport.""" + +from __future__ import annotations + +import argparse +import re +from pathlib import Path + + +def select_wheel( + wheel_dir: Path, distribution_version: str, python_tag: str, platform_tag: str +) -> Path: + target = re.fullmatch(r"cp3(\d+)", python_tag) + if not target: + raise ValueError(f"unsupported Python tag: {python_tag}") + + candidates: list[tuple[int, int, Path]] = [] + prefix = f"mssql_python_rs-{distribution_version}-" + for wheel in wheel_dir.glob(f"{prefix}*.whl"): + parts = wheel.name.removesuffix(".whl").rsplit("-", 3) + if len(parts) != 4 or parts[3] != platform_tag: + continue + wheel_python, wheel_abi = parts[1:3] + if wheel_python == python_tag and wheel_abi == python_tag: + candidates.append((2, int(target[1]), wheel)) + continue + abi3_floor = re.fullmatch(r"cp3(\d+)", wheel_python) + if wheel_abi == "abi3" and abi3_floor and int(abi3_floor[1]) <= int(target[1]): + candidates.append((1, int(abi3_floor[1]), wheel)) + + if not candidates: + raise ValueError( + f"no compatible mssql-python-rs {distribution_version} wheel " + f"for {python_tag} {platform_tag}" + ) + best_rank = max((priority, floor) for priority, floor, _ in candidates) + matches = [wheel for priority, floor, wheel in candidates if (priority, floor) == best_rank] + if len(matches) != 1: + raise ValueError(f"multiple equally compatible wheels found: {matches}") + return matches[0] + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("wheel_dir", type=Path) + parser.add_argument("distribution_version") + parser.add_argument("python_tag") + parser.add_argument("platform_tag") + args = parser.parse_args() + print( + select_wheel( + args.wheel_dir, + args.distribution_version, + args.python_tag, + args.platform_tag, + ) + ) + + +if __name__ == "__main__": + main() diff --git a/mssql_python/pybind/ddbc_bindings.h b/mssql_python/pybind/ddbc_bindings.h index 11c33d8d2..faf2287ad 100644 --- a/mssql_python/pybind/ddbc_bindings.h +++ b/mssql_python/pybind/ddbc_bindings.h @@ -3,16 +3,19 @@ #pragma once -// pybind11.h must be the first include -#include -#include -#include +// Python.h must precede standard-library headers so its feature-test macros +// are established before libc headers consume them. +#include #include #include #include #include #include // Add this line for datetime support #include + +#include +#include +#include #include #include diff --git a/tests/test_038_mssql_odbc_daily_validation.py b/tests/test_038_mssql_odbc_daily_validation.py index e74a79f8a..04b830ae9 100644 --- a/tests/test_038_mssql_odbc_daily_validation.py +++ b/tests/test_038_mssql_odbc_daily_validation.py @@ -14,6 +14,7 @@ PIPELINE = ROOT / "eng" / "pipelines" / "mssql-odbc-daily-validation-pipeline.yml" PR_PIPELINE = ROOT / "eng" / "pipelines" / "pr-validation-pipeline.yml" PREFLIGHT = ROOT / "eng" / "scripts" / "verify_mssql_odbc_provider.py" +RS_WHEEL_SELECTOR = ROOT / "eng" / "scripts" / "select_mssql_python_rs_wheel.py" @unittest.skipUnless(sys.platform.startswith("linux"), "runner requires Linux GNU timeout and bash") @@ -168,6 +169,53 @@ def test_python_315_validation_installs_pyarrow_nightly(self): self.assertIn("scientific-python-nightly-wheels", requirements) self.assertIn('pyarrow==26.0.0.dev296; python_version >= "3.15"', requirements) + def test_rs_wheel_selection_accepts_abi3_on_newer_python(self): + with tempfile.TemporaryDirectory() as directory: + wheels = Path(directory) + abi3 = wheels / "mssql_python_rs-0.2.0-cp310-abi3-win_amd64.whl" + abi3.touch() + + selected = subprocess.run( + [ + sys.executable, + str(RS_WHEEL_SELECTOR), + str(wheels), + "0.2.0", + "cp315", + "win_amd64", + ], + capture_output=True, + text=True, + check=False, + ) + + self.assertEqual(selected.returncode, 0, selected.stderr) + self.assertEqual(Path(selected.stdout.strip()), abi3) + + def test_rs_wheel_selection_prefers_exact_python_wheel(self): + with tempfile.TemporaryDirectory() as directory: + wheels = Path(directory) + (wheels / "mssql_python_rs-0.2.0-cp310-abi3-win_amd64.whl").touch() + exact = wheels / "mssql_python_rs-0.2.0-cp315-cp315-win_amd64.whl" + exact.touch() + + selected = subprocess.run( + [ + sys.executable, + str(RS_WHEEL_SELECTOR), + str(wheels), + "0.2.0", + "cp315", + "win_amd64", + ], + capture_output=True, + text=True, + check=False, + ) + + self.assertEqual(selected.returncode, 0, selected.stderr) + self.assertEqual(Path(selected.stdout.strip()), exact) + if __name__ == "__main__": unittest.main() diff --git a/tests/test_040_native_binary_hardening.py b/tests/test_040_native_binary_hardening.py index 48556e985..fbc934d77 100644 --- a/tests/test_040_native_binary_hardening.py +++ b/tests/test_040_native_binary_hardening.py @@ -171,6 +171,12 @@ def test_posix_hardening_flags_are_explicit(): assert flag in cmake +def test_python_headers_precede_standard_library_headers(): + header = (_ROOT / "mssql_python" / "pybind" / "ddbc_bindings.h").read_text(encoding="utf-8") + + assert header.index("#include ") < header.index("#include ") + + @pytest.mark.skipif(sys.platform != "linux", reason="ELF hardening applies to Linux") def test_linux_extension_has_linker_hardening(): from mssql_python import ddbc_bindings From 5d9e7dbb469943b90984f576fb9aeec9ef8d8228 Mon Sep 17 00:00:00 2001 From: Gaurav Sharma Date: Thu, 24 Sep 2026 23:53:56 +0530 Subject: [PATCH 08/23] FIX: complete Python 3.15 test compatibility Force Python.h ahead of libc headers in every native translation unit, align the GC stress assertion with Python 3.15 finalization ordering, and preserve focused-pipeline contracts.\n\nCo-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- eng/pipelines/pr-validation-pipeline.yml | 3 +++ mssql_python/pybind/CMakeLists.txt | 7 ++++++- tests/test_009_pooling.py | 2 +- tests/test_040_native_binary_hardening.py | 1 + 4 files changed, 11 insertions(+), 2 deletions(-) diff --git a/eng/pipelines/pr-validation-pipeline.yml b/eng/pipelines/pr-validation-pipeline.yml index bc559e8d0..807d15c9d 100644 --- a/eng/pipelines/pr-validation-pipeline.yml +++ b/eng/pipelines/pr-validation-pipeline.yml @@ -662,6 +662,8 @@ jobs: matrix: # Existing distro legs use Python 3.10-3.14 and are intentionally # excluded while this branch validates only Python 3.15. + # profilerLeg: 'Linux-SQL2022' + # profilerLeg: 'Linux-SQL2025' Debian_Python315Preview: dockerImage: 'python:3.15.0rc2-bookworm' distroName: 'Debian-Python315Preview' @@ -772,6 +774,7 @@ jobs: displayName: 'Install Python dependencies in $(distroName) container' - script: | + set -e # Build pybind bindings in the container PROFILER_BUILD=0 if [ "$(Build.Reason)" = "PullRequest" ] && diff --git a/mssql_python/pybind/CMakeLists.txt b/mssql_python/pybind/CMakeLists.txt index 878198dd1..e9cb83569 100644 --- a/mssql_python/pybind/CMakeLists.txt +++ b/mssql_python/pybind/CMakeLists.txt @@ -394,7 +394,12 @@ endif() # Harden the Python extension against exploitation of a separate memory-safety # defect. Mach-O receives stack protection; ELF-specific flags stay Linux-only. if(UNIX) - target_compile_options(ddbc_bindings PRIVATE -fstack-protector-strong) + # Python 3.15 defines newer POSIX feature levels than glibc's C++ headers. + # Force Python.h to be processed first in every translation unit. + target_compile_options(ddbc_bindings PRIVATE + -include Python.h + -fstack-protector-strong + ) endif() if(UNIX AND NOT APPLE) diff --git a/tests/test_009_pooling.py b/tests/test_009_pooling.py index c8cafed8f..c255c0738 100644 --- a/tests/test_009_pooling.py +++ b/tests/test_009_pooling.py @@ -1351,7 +1351,6 @@ def collect_children(): collect_barrier.wait() assert free_entered.wait(10), "Cursor finalizer did not enter free" assert not errors, errors - assert cursor_ref() is None, "GC did not clear the cursor weakref" assert not connection._cursors, "Connection.close would still see the cursor" if not explicit_close: @@ -1369,6 +1368,7 @@ def collect_children(): native = None collect_barrier.wait() + assert cursor_ref() is None, "GC did not clear the cursor weakref" assert finalizer_statement.calls == 1 assert finalizer_statement.completed, errors assert not errors, errors diff --git a/tests/test_040_native_binary_hardening.py b/tests/test_040_native_binary_hardening.py index fbc934d77..694d86e42 100644 --- a/tests/test_040_native_binary_hardening.py +++ b/tests/test_040_native_binary_hardening.py @@ -165,6 +165,7 @@ def test_posix_hardening_flags_are_explicit(): assert "-U_FORTIFY_SOURCE" in cmake assert "DDBC_SUPPORTS_FORTIFY_SOURCE_3" in cmake assert "-D_FORTIFY_SOURCE=${DDBC_FORTIFY_LEVEL}" in cmake + assert "-include Python.h" in cmake assert "$>" in cmake assert "if(UNIX AND NOT APPLE)" in cmake for flag in ("-Wl,-z,relro", "-Wl,-z,now", "-Wl,-z,noexecstack"): From 78c794f6ed825bd76434635348f0590ab52691f1 Mon Sep 17 00:00:00 2001 From: Gaurav Sharma Date: Fri, 25 Sep 2026 00:11:53 +0530 Subject: [PATCH 09/23] FIX: adapt error and GC handling for Python 3.15 Suppress Decimal conversion causes that can retain rejected input and make the cyclic-finalizer stress test explicitly release its synthetic self-cycle.\n\nCo-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- mssql_python/cursor.py | 13 +++++-------- tests/test_009_pooling.py | 8 ++++++++ 2 files changed, 13 insertions(+), 8 deletions(-) diff --git a/mssql_python/cursor.py b/mssql_python/cursor.py index db45462c8..3cfc2c038 100644 --- a/mssql_python/cursor.py +++ b/mssql_python/cursor.py @@ -2685,20 +2685,17 @@ def executemany( # pylint: disable=too-many-locals,too-many-branches,too-many-s f"{row_index}, column {i} (value type: {type(val).__name__})" ) # Split str(val) from the decimal parse so we only chain a - # cause we know is value-free. decimal.DecimalException - # messages (e.g. ConversionSyntax) never echo the input, so - # they are safe to preserve for debugging. str(val) itself - # or any other error could carry the value in its message - # and surface through __cause__ / formatted tracebacks, so - # those are re-raised with the chain suppressed (from None). + # cause. Python 3.15's pure-Python decimal implementation + # can retain the rejected input in traceback state, so every + # conversion failure suppresses chaining. try: val_text = str(val) except Exception: # pylint: disable=broad-exception-caught raise ValueError(err_msg) from None try: processed_row[i] = format(decimal.Decimal(val_text), "f") - except decimal.DecimalException as e: - raise ValueError(err_msg) from e + except decimal.DecimalException: + raise ValueError(err_msg) from None except Exception: # pylint: disable=broad-exception-caught raise ValueError(err_msg) from None processed_parameters.append(processed_row) diff --git a/tests/test_009_pooling.py b/tests/test_009_pooling.py index c255c0738..28b7465bc 100644 --- a/tests/test_009_pooling.py +++ b/tests/test_009_pooling.py @@ -1368,6 +1368,14 @@ def collect_children(): native = None collect_barrier.wait() + remaining_cursor = cursor_ref() + if remaining_cursor is not None: + # Python 3.15 may finalize a cyclic object before reclaiming + # its self-cycle. Break only the synthetic test cycle, then + # verify that no production reference keeps the cursor alive. + remaining_cursor.cycle = None + del remaining_cursor + gc.collect() assert cursor_ref() is None, "GC did not clear the cursor weakref" assert finalizer_statement.calls == 1 assert finalizer_statement.completed, errors From 6cf3549f2c44c1c3e32aa378c5ac449c7380d988 Mon Sep 17 00:00:00 2001 From: Gaurav Sharma Date: Fri, 25 Sep 2026 10:39:04 +0530 Subject: [PATCH 10/23] TEST: restore the full PR validation matrix Keep Python 3.15 preview coverage additive while preserving every existing validation leg.\n\nCo-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- eng/pipelines/pr-validation-pipeline.yml | 92 ++++++++++++------- tests/test_038_mssql_odbc_daily_validation.py | 4 +- 2 files changed, 59 insertions(+), 37 deletions(-) diff --git a/eng/pipelines/pr-validation-pipeline.yml b/eng/pipelines/pr-validation-pipeline.yml index 807d15c9d..902298b9d 100644 --- a/eng/pipelines/pr-validation-pipeline.yml +++ b/eng/pipelines/pr-validation-pipeline.yml @@ -13,7 +13,6 @@ variables: jobs: - job: CodeQLAnalysis displayName: 'CodeQL Security Analysis' - condition: false # Python 3.13; excluded from the Python 3.15 validation branch. pool: vmImage: 'ubuntu-latest' @@ -58,15 +57,15 @@ jobs: strategy: matrix: - # SQLServer2022: - # sqlVersion: 'SQL2022' - # pythonVersion: '3.13' - # SQLServer2025: - # sqlVersion: 'SQL2025' - # pythonVersion: '3.14' - # LocalDB_Python314: - # sqlVersion: 'LocalDB' - # pythonVersion: '3.14' + SQLServer2022: + sqlVersion: 'SQL2022' + pythonVersion: '3.13' + SQLServer2025: + sqlVersion: 'SQL2025' + pythonVersion: '3.14' + LocalDB_Python314: + sqlVersion: 'LocalDB' + pythonVersion: '3.14' LocalDB_Python315Preview: sqlVersion: 'LocalDB' pythonVersion: '3.15.0-rc.2' @@ -491,14 +490,14 @@ jobs: strategy: matrix: - # SQL2022: - # sqlServerImage: 'mcr.microsoft.com/mssql/server:2022-latest' - # sqlVersion: 'SQL2022' - # pythonVersion: '3.13' - # SQL2025: - # sqlServerImage: 'mcr.microsoft.com/mssql/server:2025-latest' - # sqlVersion: 'SQL2025' - # pythonVersion: '3.14' + SQL2022: + sqlServerImage: 'mcr.microsoft.com/mssql/server:2022-latest' + sqlVersion: 'SQL2022' + pythonVersion: '3.13' + SQL2025: + sqlServerImage: 'mcr.microsoft.com/mssql/server:2025-latest' + sqlVersion: 'SQL2025' + pythonVersion: '3.14' SQL2025_Python315Preview: sqlServerImage: 'mcr.microsoft.com/mssql/server:2025-latest' sqlVersion: 'SQL2025' @@ -660,10 +659,34 @@ jobs: strategy: matrix: - # Existing distro legs use Python 3.10-3.14 and are intentionally - # excluded while this branch validates only Python 3.15. - # profilerLeg: 'Linux-SQL2022' - # profilerLeg: 'Linux-SQL2025' + Ubuntu: + dockerImage: 'ubuntu:24.04' + distroName: 'Ubuntu' + sqlServerImage: 'mcr.microsoft.com/mssql/server:2022-latest' + useAzureSQL: 'false' + profilerLeg: 'Linux-SQL2022' + Ubuntu_SQL2025: + dockerImage: 'ubuntu:24.04' + distroName: 'Ubuntu-SQL2025' + sqlServerImage: 'mcr.microsoft.com/mssql/server:2025-latest' + useAzureSQL: 'false' + profilerLeg: 'Linux-SQL2025' + ${{ if ne(variables['AZURE_CONNECTION_STRING'], '') }}: + Ubuntu_AzureSQL: + dockerImage: 'ubuntu:24.04' + distroName: 'Ubuntu-AzureSQL' + sqlServerImage: '' + useAzureSQL: 'true' + Debian: + dockerImage: 'debian:12' + distroName: 'Debian' + sqlServerImage: 'mcr.microsoft.com/mssql/server:2022-latest' + useAzureSQL: 'false' + Debian_SQL2025: + dockerImage: 'debian:12' + distroName: 'Debian-SQL2025' + sqlServerImage: 'mcr.microsoft.com/mssql/server:2025-latest' + useAzureSQL: 'false' Debian_Python315Preview: dockerImage: 'python:3.15.0rc2-bookworm' distroName: 'Debian-Python315Preview' @@ -972,14 +995,18 @@ jobs: strategy: matrix: - # Ubuntu_ARM64: - # dockerImage: 'ubuntu:22.04' - # distroName: 'Ubuntu' - # archName: 'arm64' - # Debian_ARM64: - # dockerImage: 'python:3.11-bookworm' - # distroName: 'Debian' - # archName: 'arm64' + Ubuntu_ARM64: + dockerImage: 'ubuntu:22.04' + distroName: 'Ubuntu' + archName: 'arm64' + Debian_ARM64: + # Python 3.11 is preinstalled (byte-compiled natively at image-build + # time) so we never run the apt python3 post-install byte-compilation + # that SIGSEGVs (exit 139) under QEMU user-mode emulation. Still Debian + # 12 (bookworm) underneath, so the msodbcsql18 debian/12 repo applies. + dockerImage: 'python:3.11-bookworm' + distroName: 'Debian' + archName: 'arm64' Debian_Python315Preview_ARM64: dockerImage: 'python:3.15.0rc2-bookworm' distroName: 'Debian-Python315Preview' @@ -1203,7 +1230,6 @@ jobs: - job: PytestOnLinux_RHEL9 displayName: 'Linux RedHat x86_64' - condition: false # Python 3.12; excluded from the Python 3.15 validation branch. pool: vmImage: 'ubuntu-latest' @@ -1402,7 +1428,6 @@ jobs: - job: PytestOnLinux_RHEL9_ARM64 displayName: 'Linux RedHat ARM64' - condition: false # Python 3.12; excluded from the Python 3.15 validation branch. pool: vmImage: 'ubuntu-latest' @@ -1616,7 +1641,6 @@ jobs: - job: PytestOnLinux_Alpine displayName: 'Linux Alpine x86_64' - condition: false # Distro Python is not 3.15; excluded from this validation branch. pool: vmImage: 'ubuntu-latest' @@ -1850,7 +1874,6 @@ jobs: - job: PytestOnLinux_Alpine_ARM64 displayName: 'Linux Alpine ARM64' - condition: false # Distro Python is not 3.15; excluded from this validation branch. pool: vmImage: 'ubuntu-latest' @@ -2316,7 +2339,6 @@ jobs: - job: CodeCoverageReport displayName: 'Full Code Coverage Report in Ubuntu x86_64' - condition: false # Uses the hosted default Python; excluded from this validation branch. pool: vmImage: 'ubuntu-latest' diff --git a/tests/test_038_mssql_odbc_daily_validation.py b/tests/test_038_mssql_odbc_daily_validation.py index 04b830ae9..6083d6bbd 100644 --- a/tests/test_038_mssql_odbc_daily_validation.py +++ b/tests/test_038_mssql_odbc_daily_validation.py @@ -152,7 +152,7 @@ def test_rs_transport_is_pinned(self): self.assertEqual(version.strip(), "0.2.0-nightly.20260924") - def test_python_315_validation_uses_only_preview_matrix_legs(self): + def test_python_315_validation_adds_preview_matrix_legs(self): pipeline = PR_PIPELINE.read_text(encoding="utf-8") active_python_versions = { line.split(":", 1)[1].strip(" '\"") @@ -160,7 +160,7 @@ def test_python_315_validation_uses_only_preview_matrix_legs(self): if line.lstrip().startswith("pythonVersion:") } - self.assertEqual(active_python_versions, {"3.15.0-rc.2"}) + self.assertEqual(active_python_versions, {"3.13", "3.14", "3.15.0-rc.2"}) self.assertIn("python:3.15.0rc2-bookworm", pipeline) def test_python_315_validation_installs_pyarrow_nightly(self): From 8084f02c2768ebc4120560d8d0c17d20c18a7c0c Mon Sep 17 00:00:00 2001 From: Gaurav Sharma Date: Thu, 1 Oct 2026 15:35:35 +0530 Subject: [PATCH 11/23] CHORE: sync Python 3.15 validation with main Drop the stable-ABI wheel selection now provided by PR #804 and refresh the pinned Python 3.15 PyArrow nightly after the previous build was pruned.\n\nCo-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- eng/versions/mssql-python-rs-nuget.version | 2 +- eng/versions/mssql-python-rs.version | 2 +- requirements.txt | 2 +- tests/test_038_mssql_odbc_daily_validation.py | 3 +-- 4 files changed, 4 insertions(+), 5 deletions(-) diff --git a/eng/versions/mssql-python-rs-nuget.version b/eng/versions/mssql-python-rs-nuget.version index 0d91a54c7..9325c3ccd 100644 --- a/eng/versions/mssql-python-rs-nuget.version +++ b/eng/versions/mssql-python-rs-nuget.version @@ -1 +1 @@ -0.3.0 +0.3.0 \ No newline at end of file diff --git a/eng/versions/mssql-python-rs.version b/eng/versions/mssql-python-rs.version index 0d91a54c7..9325c3ccd 100644 --- a/eng/versions/mssql-python-rs.version +++ b/eng/versions/mssql-python-rs.version @@ -1 +1 @@ -0.3.0 +0.3.0 \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 2a7b9b4ca..1f4a90d4a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,7 +8,7 @@ unittest-xml-reporting psutil --extra-index-url https://pypi.anaconda.org/scientific-python-nightly-wheels/simple pyarrow; python_version < "3.15" -pyarrow==26.0.0.dev296; python_version >= "3.15" +pyarrow==26.0.0.dev323; python_version >= "3.15" polars # Runtime dependencies needed for tests diff --git a/tests/test_038_mssql_odbc_daily_validation.py b/tests/test_038_mssql_odbc_daily_validation.py index 665ec1727..544155418 100644 --- a/tests/test_038_mssql_odbc_daily_validation.py +++ b/tests/test_038_mssql_odbc_daily_validation.py @@ -166,8 +166,7 @@ def test_python_315_validation_installs_pyarrow_nightly(self): requirements = (ROOT / "requirements.txt").read_text(encoding="utf-8") self.assertIn("scientific-python-nightly-wheels", requirements) - self.assertIn('pyarrow==26.0.0.dev296; python_version >= "3.15"', requirements) - + self.assertIn('pyarrow==26.0.0.dev323; python_version >= "3.15"', requirements) if __name__ == "__main__": From 6942f4a5e3ed382b980cfc513cb5c70198a05eda Mon Sep 17 00:00:00 2001 From: Gaurav Sharma Date: Mon, 5 Oct 2026 10:09:53 +0530 Subject: [PATCH 12/23] CHORE: advance hosted Python 3.15 validation to RC3 Use RC3 on Windows and macOS while the official Linux Docker image remains on RC2.\n\nCo-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- eng/pipelines/pr-validation-pipeline.yml | 4 ++-- tests/test_038_mssql_odbc_daily_validation.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/eng/pipelines/pr-validation-pipeline.yml b/eng/pipelines/pr-validation-pipeline.yml index 6ba2d8bde..3882f8ec7 100644 --- a/eng/pipelines/pr-validation-pipeline.yml +++ b/eng/pipelines/pr-validation-pipeline.yml @@ -68,7 +68,7 @@ jobs: pythonVersion: '3.14' LocalDB_Python315Preview: sqlVersion: 'LocalDB' - pythonVersion: '3.15.0-rc.2' + pythonVersion: '3.15.0-rc.3' steps: - checkout: self @@ -589,7 +589,7 @@ jobs: SQL2025_Python315Preview: sqlServerImage: 'mcr.microsoft.com/mssql/server:2025-latest' sqlVersion: 'SQL2025' - pythonVersion: '3.15.0-rc.2' + pythonVersion: '3.15.0-rc.3' steps: - checkout: self diff --git a/tests/test_038_mssql_odbc_daily_validation.py b/tests/test_038_mssql_odbc_daily_validation.py index 544155418..2bcc7f216 100644 --- a/tests/test_038_mssql_odbc_daily_validation.py +++ b/tests/test_038_mssql_odbc_daily_validation.py @@ -159,7 +159,7 @@ def test_python_315_validation_adds_preview_matrix_legs(self): if line.lstrip().startswith("pythonVersion:") } - self.assertEqual(active_python_versions, {"3.13", "3.14", "3.15.0-rc.2"}) + self.assertEqual(active_python_versions, {"3.13", "3.14", "3.15.0-rc.3"}) self.assertIn("python:3.15.0rc2-bookworm", pipeline) def test_python_315_validation_installs_pyarrow_nightly(self): From aab35f42b997c9c6c8986c6a6b994ef71b592f90 Mon Sep 17 00:00:00 2001 From: Gaurav Sharma Date: Mon, 12 Oct 2026 10:11:45 +0530 Subject: [PATCH 13/23] CHORE: advance Python 3.15 validation to GA Use Python 3.15 GA on hosted agents and stable PyArrow 26.0.0, while retaining RC3 Linux images until Docker publishes GA.\n\nCo-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- eng/pipelines/pr-validation-pipeline.yml | 22 +++++++++---------- requirements.txt | 4 +--- tests/test_038_mssql_odbc_daily_validation.py | 12 +++++----- 3 files changed, 18 insertions(+), 20 deletions(-) diff --git a/eng/pipelines/pr-validation-pipeline.yml b/eng/pipelines/pr-validation-pipeline.yml index a8264bd91..e2bfe8045 100644 --- a/eng/pipelines/pr-validation-pipeline.yml +++ b/eng/pipelines/pr-validation-pipeline.yml @@ -66,9 +66,9 @@ jobs: LocalDB_Python314: sqlVersion: 'LocalDB' pythonVersion: '3.14' - LocalDB_Python315Preview: + LocalDB_Python315: sqlVersion: 'LocalDB' - pythonVersion: '3.15.0-rc.3' + pythonVersion: '3.15' steps: - checkout: self @@ -586,10 +586,10 @@ jobs: sqlServerImage: 'mcr.microsoft.com/mssql/server:2025-latest' sqlVersion: 'SQL2025' pythonVersion: '3.14' - SQL2025_Python315Preview: + SQL2025_Python315: sqlServerImage: 'mcr.microsoft.com/mssql/server:2025-latest' sqlVersion: 'SQL2025' - pythonVersion: '3.15.0-rc.3' + pythonVersion: '3.15' steps: - checkout: self @@ -777,9 +777,9 @@ jobs: distroName: 'Debian-SQL2025' sqlServerImage: 'mcr.microsoft.com/mssql/server:2025-latest' useAzureSQL: 'false' - Debian_Python315Preview: - dockerImage: 'python:3.15.0rc2-bookworm' - distroName: 'Debian-Python315Preview' + Debian_Python315: + dockerImage: 'python:3.15.0rc3-bookworm' + distroName: 'Debian-Python315' sqlServerImage: 'mcr.microsoft.com/mssql/server:2025-latest' useAzureSQL: 'false' @@ -811,7 +811,7 @@ jobs: - script: | # Install dependencies in the container - if [ "$(distroName)" = "Debian-Python315Preview" ]; then + if [ "$(distroName)" = "Debian-Python315" ]; then docker exec test-container-$(distroName) bash -c " export DEBIAN_FRONTEND=noninteractive export TZ=UTC @@ -1102,9 +1102,9 @@ jobs: dockerImage: 'python:3.11-bookworm' distroName: 'Debian' archName: 'arm64' - Debian_Python315Preview_ARM64: - dockerImage: 'python:3.15.0rc2-bookworm' - distroName: 'Debian-Python315Preview' + Debian_Python315_ARM64: + dockerImage: 'python:3.15.0rc3-bookworm' + distroName: 'Debian-Python315' archName: 'arm64' steps: diff --git a/requirements.txt b/requirements.txt index 1f4a90d4a..daffd1a1c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,9 +6,7 @@ zstandard coverage unittest-xml-reporting psutil ---extra-index-url https://pypi.anaconda.org/scientific-python-nightly-wheels/simple -pyarrow; python_version < "3.15" -pyarrow==26.0.0.dev323; python_version >= "3.15" +pyarrow polars # Runtime dependencies needed for tests diff --git a/tests/test_038_mssql_odbc_daily_validation.py b/tests/test_038_mssql_odbc_daily_validation.py index 2bcc7f216..ac3aa6f36 100644 --- a/tests/test_038_mssql_odbc_daily_validation.py +++ b/tests/test_038_mssql_odbc_daily_validation.py @@ -151,7 +151,7 @@ def test_stable_rs_transport_is_pinned(self): self.assertEqual(version.strip(), "0.3.0") - def test_python_315_validation_adds_preview_matrix_legs(self): + def test_python_315_validation_adds_matrix_legs(self): pipeline = PR_PIPELINE.read_text(encoding="utf-8") active_python_versions = { line.split(":", 1)[1].strip(" '\"") @@ -159,14 +159,14 @@ def test_python_315_validation_adds_preview_matrix_legs(self): if line.lstrip().startswith("pythonVersion:") } - self.assertEqual(active_python_versions, {"3.13", "3.14", "3.15.0-rc.3"}) - self.assertIn("python:3.15.0rc2-bookworm", pipeline) + self.assertEqual(active_python_versions, {"3.13", "3.14", "3.15"}) + self.assertIn("python:3.15.0rc3-bookworm", pipeline) - def test_python_315_validation_installs_pyarrow_nightly(self): + def test_python_315_validation_installs_stable_pyarrow(self): requirements = (ROOT / "requirements.txt").read_text(encoding="utf-8") - self.assertIn("scientific-python-nightly-wheels", requirements) - self.assertIn('pyarrow==26.0.0.dev323; python_version >= "3.15"', requirements) + self.assertIn("\npyarrow\n", requirements) + self.assertNotIn("scientific-python-nightly-wheels", requirements) if __name__ == "__main__": From 242b0596f5b8f578b6414cb9f3e37f6e1a8988c7 Mon Sep 17 00:00:00 2001 From: Gaurav Sharma Date: Mon, 12 Oct 2026 10:26:53 +0530 Subject: [PATCH 14/23] FIX: address Python 3.15 review findings Preserve isolated-wheel skips, fail container setup immediately, align Decimal privacy documentation, and pin stable PyArrow 26.0.0.\n\nCo-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- eng/pipelines/pr-validation-pipeline.yml | 1 + mssql_python/cursor.py | 8 ++++---- requirements.txt | 2 +- tests/test_038_mssql_odbc_daily_validation.py | 2 +- tests/test_040_native_binary_hardening.py | 4 ++++ 5 files changed, 11 insertions(+), 6 deletions(-) diff --git a/eng/pipelines/pr-validation-pipeline.yml b/eng/pipelines/pr-validation-pipeline.yml index e2bfe8045..c706dd9fe 100644 --- a/eng/pipelines/pr-validation-pipeline.yml +++ b/eng/pipelines/pr-validation-pipeline.yml @@ -813,6 +813,7 @@ jobs: # Install dependencies in the container if [ "$(distroName)" = "Debian-Python315" ]; then docker exec test-container-$(distroName) bash -c " + set -e export DEBIAN_FRONTEND=noninteractive export TZ=UTC ln -snf /usr/share/zoneinfo/\$TZ /etc/localtime && echo \$TZ > /etc/timezone diff --git a/mssql_python/cursor.py b/mssql_python/cursor.py index 2aa08c463..cd7ec3167 100644 --- a/mssql_python/cursor.py +++ b/mssql_python/cursor.py @@ -2746,10 +2746,10 @@ def executemany( # pylint: disable=too-many-locals,too-many-branches,too-many-s f"Failed to convert parameter to Decimal at row " f"{row_index}, column {i} (value type: {type(val).__name__})" ) - # Split str(val) from the decimal parse so we only chain a - # cause. Python 3.15's pure-Python decimal implementation - # can retain the rejected input in traceback state, so every - # conversion failure suppresses chaining. + # Split str(val) from the decimal parse so every conversion + # failure can suppress chaining. Python 3.15's pure-Python + # decimal implementation can retain the rejected input in + # traceback state. try: val_text = str(val) except Exception: # pylint: disable=broad-exception-caught diff --git a/requirements.txt b/requirements.txt index daffd1a1c..c8610899e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,7 +6,7 @@ zstandard coverage unittest-xml-reporting psutil -pyarrow +pyarrow==26.0.0 polars # Runtime dependencies needed for tests diff --git a/tests/test_038_mssql_odbc_daily_validation.py b/tests/test_038_mssql_odbc_daily_validation.py index ac3aa6f36..f2f585526 100644 --- a/tests/test_038_mssql_odbc_daily_validation.py +++ b/tests/test_038_mssql_odbc_daily_validation.py @@ -165,7 +165,7 @@ def test_python_315_validation_adds_matrix_legs(self): def test_python_315_validation_installs_stable_pyarrow(self): requirements = (ROOT / "requirements.txt").read_text(encoding="utf-8") - self.assertIn("\npyarrow\n", requirements) + self.assertIn("\npyarrow==26.0.0\n", requirements) self.assertNotIn("scientific-python-nightly-wheels", requirements) diff --git a/tests/test_040_native_binary_hardening.py b/tests/test_040_native_binary_hardening.py index 694d86e42..68de68a90 100644 --- a/tests/test_040_native_binary_hardening.py +++ b/tests/test_040_native_binary_hardening.py @@ -172,6 +172,10 @@ def test_posix_hardening_flags_are_explicit(): assert flag in cmake +@pytest.mark.skipif( + not _CMAKE.is_file(), + reason="requires a source checkout; isolated wheel tests omit the source tree", +) def test_python_headers_precede_standard_library_headers(): header = (_ROOT / "mssql_python" / "pybind" / "ddbc_bindings.h").read_text(encoding="utf-8") From 79243f2d6d7a5bfd3be72706000dd55c73cc42c2 Mon Sep 17 00:00:00 2001 From: Gaurav Sharma Date: Mon, 12 Oct 2026 10:52:31 +0530 Subject: [PATCH 15/23] FIX: preserve Python 3.10 dependency resolution Pin PyArrow 25.0.1 below Python 3.15 and PyArrow 26.0.0 on Python 3.15+, keeping the existing 3.10 release locks satisfiable.\n\nCo-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- requirements.txt | 3 ++- tests/test_038_mssql_odbc_daily_validation.py | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index c8610899e..2b18cd919 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,7 +6,8 @@ zstandard coverage unittest-xml-reporting psutil -pyarrow==26.0.0 +pyarrow==25.0.1; python_version < "3.15" +pyarrow==26.0.0; python_version >= "3.15" polars # Runtime dependencies needed for tests diff --git a/tests/test_038_mssql_odbc_daily_validation.py b/tests/test_038_mssql_odbc_daily_validation.py index f2f585526..72249af40 100644 --- a/tests/test_038_mssql_odbc_daily_validation.py +++ b/tests/test_038_mssql_odbc_daily_validation.py @@ -165,7 +165,8 @@ def test_python_315_validation_adds_matrix_legs(self): def test_python_315_validation_installs_stable_pyarrow(self): requirements = (ROOT / "requirements.txt").read_text(encoding="utf-8") - self.assertIn("\npyarrow==26.0.0\n", requirements) + self.assertIn('pyarrow==25.0.1; python_version < "3.15"', requirements) + self.assertIn('pyarrow==26.0.0; python_version >= "3.15"', requirements) self.assertNotIn("scientific-python-nightly-wheels", requirements) From 5a106b89981e64898e8eefa66e38e94865c75c66 Mon Sep 17 00:00:00 2001 From: Gaurav Sharma Date: Mon, 12 Oct 2026 11:22:42 +0530 Subject: [PATCH 16/23] REFACTOR: keep Python 3.15 support scoped Remove the native-hardening and auth-test changes owned by PR #810, retaining only the Python 3.15 header-order compatibility flag and its focused contract.\n\nCo-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- mssql_python/pybind/CMakeLists.txt | 48 +---- tests/test_008_auth.py | 8 +- tests/test_038_mssql_odbc_daily_validation.py | 6 + tests/test_040_native_binary_hardening.py | 194 ------------------ 4 files changed, 10 insertions(+), 246 deletions(-) delete mode 100644 tests/test_040_native_binary_hardening.py diff --git a/mssql_python/pybind/CMakeLists.txt b/mssql_python/pybind/CMakeLists.txt index bb94e749d..069a1b165 100644 --- a/mssql_python/pybind/CMakeLists.txt +++ b/mssql_python/pybind/CMakeLists.txt @@ -255,13 +255,6 @@ endif() message(STATUS "Final Python library directory: ${PYTHON_LIB_DIR}") -# Single-config POSIX generators ignore `cmake --build --config Release`. -# Make the optimized release mode explicit instead of relying on simdutf to -# populate this project-wide cache variable as a FetchContent side effect. -if(UNIX AND NOT CMAKE_BUILD_TYPE AND NOT CMAKE_CONFIGURATION_TYPES) - set(CMAKE_BUILD_TYPE Release CACHE STRING "Build type" FORCE) -endif() - include(FetchContent) message(STATUS "Downloading simdutf v8.2.0 source archive with FetchContent") set(simdutf_fetchcontent_args @@ -394,45 +387,10 @@ if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU" OR CMAKE_CXX_COMPILER_ID STREQUAL "Clang endif() endif() -# Harden the Python extension against exploitation of a separate memory-safety -# defect. Mach-O receives stack protection; ELF-specific flags stay Linux-only. +# Python 3.15 defines newer POSIX feature levels than glibc's C++ headers. +# Process Python.h first so libc sees the matching feature-test macros. if(UNIX) - # Python 3.15 defines newer POSIX feature levels than glibc's C++ headers. - # Force Python.h to be processed first in every translation unit. - target_compile_options(ddbc_bindings PRIVATE - -include Python.h - -fstack-protector-strong - ) -endif() - -if(UNIX AND NOT APPLE) - include(CheckCXXSourceCompiles) - set(DDBC_REQUIRED_FLAGS_SAVED "${CMAKE_REQUIRED_FLAGS}") - set(CMAKE_REQUIRED_FLAGS - "${CMAKE_REQUIRED_FLAGS} -O2 -Werror -U_FORTIFY_SOURCE -D_FORTIFY_SOURCE=3" - ) - check_cxx_source_compiles(" - #include - #if !defined(__USE_FORTIFY_LEVEL) || __USE_FORTIFY_LEVEL < 3 - #error _FORTIFY_SOURCE=3 is unavailable - #endif - int main() { return 0; } - " DDBC_SUPPORTS_FORTIFY_SOURCE_3) - set(CMAKE_REQUIRED_FLAGS "${DDBC_REQUIRED_FLAGS_SAVED}") - if(DDBC_SUPPORTS_FORTIFY_SOURCE_3) - set(DDBC_FORTIFY_LEVEL 3) - else() - set(DDBC_FORTIFY_LEVEL 2) - endif() - target_compile_options(ddbc_bindings PRIVATE - $<$>:-U_FORTIFY_SOURCE> - $<$>:-D_FORTIFY_SOURCE=${DDBC_FORTIFY_LEVEL}> - ) - target_link_options(ddbc_bindings PRIVATE - -Wl,-z,relro - -Wl,-z,now - -Wl,-z,noexecstack - ) + target_compile_options(ddbc_bindings PRIVATE -include Python.h) endif() # Add macOS-specific string conversion fix diff --git a/tests/test_008_auth.py b/tests/test_008_auth.py index 04af30e5a..5c160b2a4 100644 --- a/tests/test_008_auth.py +++ b/tests/test_008_auth.py @@ -1554,13 +1554,7 @@ def test_multiple_connections_share_same_token_provider(self, mock_ddbc_conn): @patch("mssql_python.connection.ddbc_bindings.Connection") def test_concurrent_connections_with_same_token_provider(self, mock_ddbc_conn): """Concurrent connect() calls with one token provider should succeed.""" - - def create_native_connection(*_args, **_kwargs): - native_connection = MagicMock() - native_connection.get_autocommit.return_value = True - return native_connection - - mock_ddbc_conn.side_effect = create_native_connection + mock_ddbc_conn.return_value = MagicMock() mock_cred = MagicMock() mock_cred.get_token.return_value = MagicMock(token=SAMPLE_TOKEN) from mssql_python import connect diff --git a/tests/test_038_mssql_odbc_daily_validation.py b/tests/test_038_mssql_odbc_daily_validation.py index 16c8cd421..f6f15e348 100644 --- a/tests/test_038_mssql_odbc_daily_validation.py +++ b/tests/test_038_mssql_odbc_daily_validation.py @@ -15,6 +15,7 @@ PR_PIPELINE = ROOT / "eng" / "pipelines" / "pr-validation-pipeline.yml" PREFLIGHT = ROOT / "eng" / "scripts" / "verify_mssql_odbc_provider.py" RELEASE_PIPELINE = ROOT / "OneBranchPipelines" / "stages" / "build-linux-single-stage.yml" +PYBIND_CMAKE = ROOT / "mssql_python" / "pybind" / "CMakeLists.txt" @unittest.skipUnless(sys.platform.startswith("linux"), "runner requires Linux GNU timeout and bash") @@ -188,6 +189,11 @@ def test_python_315_validation_installs_stable_pyarrow(self): self.assertIn('pyarrow==26.0.0; python_version >= "3.15"', requirements) self.assertNotIn("scientific-python-nightly-wheels", requirements) + def test_python_315_processes_python_header_before_libc_headers(self): + cmake = PYBIND_CMAKE.read_text(encoding="utf-8") + + self.assertIn("target_compile_options(ddbc_bindings PRIVATE -include Python.h)", cmake) + if __name__ == "__main__": unittest.main() diff --git a/tests/test_040_native_binary_hardening.py b/tests/test_040_native_binary_hardening.py deleted file mode 100644 index 68de68a90..000000000 --- a/tests/test_040_native_binary_hardening.py +++ /dev/null @@ -1,194 +0,0 @@ -"""Regression guards for POSIX native-extension hardening.""" - -import struct -import sys -from pathlib import Path - -import pytest - -_ROOT = Path(__file__).resolve().parents[1] -_CMAKE = _ROOT / "mssql_python" / "pybind" / "CMakeLists.txt" -_PT_DYNAMIC = 2 -_PT_GNU_STACK = 0x6474E551 -_PT_GNU_RELRO = 0x6474E552 -_PF_X = 0x1 -_DT_NULL = 0 -_DT_BIND_NOW = 24 -_DT_FLAGS = 30 -_DF_BIND_NOW = 0x8 -_DT_FLAGS_1 = 0x6FFFFFFB -_DF_1_NOW = 0x1 - - -def _make_elf64(*, relro=True, bind_now=True, executable_stack=False): - header_size = 64 - program_header_size = 56 - program_count = 3 - dynamic_offset = header_size + program_header_size * program_count - dynamic = struct.pack("" - program_offset = struct.unpack_from(endian + "Q", data, 0x20)[0] - entry_size = struct.unpack_from(endian + "H", data, 0x36)[0] - entry_count = struct.unpack_from(endian + "H", data, 0x38)[0] - - if ( - not program_offset - or not entry_count - or entry_size < 56 - or program_offset + entry_count * entry_size > len(data) - ): - raise ValueError("invalid ELF program headers") - - has_relro = False - stack_executable = None - dynamic_segment = None - for index in range(entry_count): - offset = program_offset + index * entry_size - program_type = struct.unpack_from(endian + "I", data, offset)[0] - flags = struct.unpack_from(endian + "I", data, offset + 4)[0] - file_offset = struct.unpack_from(endian + "Q", data, offset + 8)[0] - file_size = struct.unpack_from(endian + "Q", data, offset + 32)[0] - if file_offset + file_size > len(data): - raise ValueError("ELF segment extends beyond the file") - if program_type == _PT_GNU_RELRO: - has_relro = True - elif program_type == _PT_GNU_STACK: - stack_executable = bool(flags & _PF_X) - elif program_type == _PT_DYNAMIC: - dynamic_segment = file_offset, file_size - - if dynamic_segment is None: - raise ValueError("ELF has no PT_DYNAMIC segment") - - dynamic_offset, dynamic_size = dynamic_segment - dynamic_entry_size = 16 - if dynamic_size % dynamic_entry_size: - raise ValueError("ELF dynamic segment has a partial entry") - - bind_now = False - terminated = False - for offset in range(dynamic_offset, dynamic_offset + dynamic_size, dynamic_entry_size): - tag = struct.unpack_from(endian + "q", data, offset)[0] - value = struct.unpack_from(endian + "Q", data, offset + 8)[0] - if tag == _DT_NULL: - terminated = True - break - bind_now = bind_now or tag == _DT_BIND_NOW - bind_now = bind_now or tag == _DT_FLAGS and bool(value & _DF_BIND_NOW) - bind_now = bind_now or tag == _DT_FLAGS_1 and bool(value & _DF_1_NOW) - - if not terminated: - raise ValueError("ELF dynamic segment lacks DT_NULL") - return has_relro, bind_now, stack_executable - - -def test_elf_hardening_parser_reads_program_and_dynamic_flags(): - assert _elf_hardening(_make_elf64()) == (True, True, False) - assert _elf_hardening(_make_elf64(relro=False)) == (False, True, False) - assert _elf_hardening(_make_elf64(bind_now=False)) == (True, False, False) - assert _elf_hardening(_make_elf64(executable_stack=True)) == (True, True, True) - - -@pytest.mark.skipif( - not _CMAKE.is_file(), - reason="requires a source checkout; isolated wheel tests omit the source tree", -) -def test_posix_hardening_flags_are_explicit(): - cmake = _CMAKE.read_text(encoding="utf-8") - - assert "set(CMAKE_BUILD_TYPE Release" in cmake - assert "-fstack-protector-strong" in cmake - assert "-U_FORTIFY_SOURCE" in cmake - assert "DDBC_SUPPORTS_FORTIFY_SOURCE_3" in cmake - assert "-D_FORTIFY_SOURCE=${DDBC_FORTIFY_LEVEL}" in cmake - assert "-include Python.h" in cmake - assert "$>" in cmake - assert "if(UNIX AND NOT APPLE)" in cmake - for flag in ("-Wl,-z,relro", "-Wl,-z,now", "-Wl,-z,noexecstack"): - assert flag in cmake - - -@pytest.mark.skipif( - not _CMAKE.is_file(), - reason="requires a source checkout; isolated wheel tests omit the source tree", -) -def test_python_headers_precede_standard_library_headers(): - header = (_ROOT / "mssql_python" / "pybind" / "ddbc_bindings.h").read_text(encoding="utf-8") - - assert header.index("#include ") < header.index("#include ") - - -@pytest.mark.skipif(sys.platform != "linux", reason="ELF hardening applies to Linux") -def test_linux_extension_has_linker_hardening(): - from mssql_python import ddbc_bindings - - extension = Path(ddbc_bindings.module_path) - has_relro, bind_now, stack_executable = _elf_hardening(extension.read_bytes()) - assert has_relro, "native extension is missing PT_GNU_RELRO" - assert bind_now, "native extension is missing immediate binding (BIND_NOW)" - assert stack_executable is not None, "native extension has no PT_GNU_STACK declaration" - assert stack_executable is False, "native extension requests an executable stack" From 2666338a4da5eb3362e9386373b36cf10fe6ce33 Mon Sep 17 00:00:00 2001 From: Gaurav Sharma Date: Mon, 12 Oct 2026 12:06:55 +0530 Subject: [PATCH 17/23] CHORE: refresh universal locks for Python 3.15 Add the marker-scoped PyArrow 26.0.0 hashes while preserving PyArrow 25.0.1 for older Python versions.\n\nCo-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- eng/requirements-build-macos.txt | 54 +++++++++++++++++++++++++++++++- eng/requirements-test-linux.txt | 54 +++++++++++++++++++++++++++++++- 2 files changed, 106 insertions(+), 2 deletions(-) diff --git a/eng/requirements-build-macos.txt b/eng/requirements-build-macos.txt index 75745d44b..939ff74d1 100644 --- a/eng/requirements-build-macos.txt +++ b/eng/requirements-build-macos.txt @@ -1121,7 +1121,7 @@ psutil==7.2.2 \ --hash=sha256:eed63d3b4d62449571547b60578c5b2c4bcccc5387148db46e0c2313dad0ee00 \ --hash=sha256:fd04ef36b4a6d599bbdb225dd1d3f51e00105f6d48a28f006da7f9822f2606d8 # via -r eng/../requirements.txt -pyarrow==25.0.1 \ +pyarrow==25.0.1 ; python_full_version < '3.15' \ --hash=sha256:0b1edbb2f385a6a65e9711b62ba86ac54a7816a3f8d17bb3e8a5929d65fb2485 \ --hash=sha256:0b726ad7e7b669be982b0c71c07fe4b037d654354130da79a7902a669e93a66b \ --hash=sha256:0befcf816e45a1af33ac775a9970b749e4868a230c7372f0ae5e932bee27039f \ @@ -1166,6 +1166,58 @@ pyarrow==25.0.1 \ --hash=sha256:f3831aaa25c67a99f99dc8b05873cb9d64560390372e2aa197ce9dd4a3f06a44 \ --hash=sha256:f729cfdbd36fd99d543b67a914d2de044c84ebe45be8b34902b299b608c15c8f # via -r eng/../requirements.txt +pyarrow==26.0.0 ; python_full_version >= '3.15' \ + --hash=sha256:01c863a18bd9c8412453dd0d92de6d0ee7b2b3d6fb079d9734a4b2a3c8bd4453 \ + --hash=sha256:0cccd36e00ea3afeb52ded61f2721ce71f604853d70c45365c58324eb773d6ae \ + --hash=sha256:106bb9290fc6fd9a84138a9440038ef184bac86463543c5ff099229cb30d996c \ + --hash=sha256:13b0972a3dc71b642050d1bc72664a3916e14f59c943d8c1368154d6e4b0c2d5 \ + --hash=sha256:210cc9b83888b87cdc8f793eebb264f22b20d0dedbedefc73b9687a7047b4747 \ + --hash=sha256:240bd18a7487f8767616a948a69dd4e740a8bc36a1c9da49e4dc9a32c5c2faed \ + --hash=sha256:24f892fdf1ae1942d69d3f7742e2f49960ec95277cfb1a70b8a1d91f4a96d935 \ + --hash=sha256:290a74c48e9491b436fd5edacfadf357943f82aa45c81110bd83a69aab33d1cf \ + --hash=sha256:2b5fcd69c0e1107b79e55839877db5a6ed04651b73fd6fec581d09e230bed5e4 \ + --hash=sha256:2e4a413046eba9896e632925066c74095182200ba32e19ff0166bf64d2f936ac \ + --hash=sha256:3a4d235876f14b4136b4d616ec42eb469ea0d6ead336cae631aa1dd29b21c962 \ + --hash=sha256:3de30a7432b48b98b9decbd9e25a53bb9251d202c2e6c5a29a50869592ccb117 \ + --hash=sha256:41dd3661ef40790a78870052ad7a58ad827b27c67a4511f06962eb9e9b74d19b \ + --hash=sha256:4a5fa8dc70dd50808990ff36faf44088e357b353d86c7682dd92d4b78d4c97d5 \ + --hash=sha256:4bcba83299cb2b8f8e443d36c6ba6269a5034431879015fb0719495df8a14de2 \ + --hash=sha256:515a10dae2a1d236bc9c9209d0317acb6746ea63cd4f98704904af7156d90ed1 \ + --hash=sha256:5780d487ff6c6ed7b42298609680d87fe0036e529a9dc2e1105364bce9697f50 \ + --hash=sha256:5b827650e874f1f9f9392524ea3e9e3e8a245de5ba64acca1f81ab188090afb9 \ + --hash=sha256:5d5768d03426abe6526d5274adefa00abf00a7f81118c46e98b5a46390f5549e \ + --hash=sha256:645917e976671debabf854abab6e2b75c571ca4f82adc33a2d338697f7c27d93 \ + --hash=sha256:68cd662e9e2b00876a131950cf32336ace2d0865e1f9418763e3d3be8481dfa4 \ + --hash=sha256:6a628922ba20705fa964ca73e4ef959c2fb2f14b9bbec5589a6a1e68e6257c85 \ + --hash=sha256:6e89dee53aaeb50505ed6152ea55bc7ddfd4f4df264f5427ea255288d8f0e580 \ + --hash=sha256:6e949744dcfc2d379808f7013c5f9cafaf0f817656dff7d46c6931528dd1784b \ + --hash=sha256:734312d3d99088d9ec28c5b17bad40389bd8373a1afc10acb60b83fd217af087 \ + --hash=sha256:7aa12ab8e236789b1ecd2d6ecaef036b4e63d675ddf1864a43c6799d18f2d028 \ + --hash=sha256:7c3fda041e7078802589cf257750323ee3d0cd1e56e53a9b20ec845697fb3d28 \ + --hash=sha256:879331ddea2a26479fa18fade71e6facf684a6cf19f67daec3775c871569e8e5 \ + --hash=sha256:8e8e28c464552b5ca03e30d4504168c4425ce383884f8611b00e972f9fd933fc \ + --hash=sha256:90ddaf7c625307ad52f31a9b25c34fe5e4897c7529ee3481135822b2b6842ff1 \ + --hash=sha256:954d971b363b16ee41f89389a4053315dc71265f2ce5c2468eb0a910b1166268 \ + --hash=sha256:9db18a9dc0af52135c9eac549d80a7a882696efbe5406cf882b044525d4ecc2e \ + --hash=sha256:a0e4e92eeb088f1d7c2c04d6c7de8434c75abb4b4ccf0bbcd045aa7164c68d93 \ + --hash=sha256:a6ca849f90cf73fe361f08a5762c783ead9671e4548c1f558cc637b54c9103f2 \ + --hash=sha256:ab6914db225d7f399652ae1f08588dfbc9efe617612715701e3d9d5cfa5ca19f \ + --hash=sha256:c2ba350957076b1b3a22f549261dc3e9c67ca20816d8bd5f79d7b9c69be4c4c2 \ + --hash=sha256:ca77c43ca55bfc9a4eeb1f0cd5f093f08731b77c24cdba0829035f084959b0bb \ + --hash=sha256:cc903e1069e9dd5e9dcf780324c0112e27e051e422ecfaff574fb33ed65d9160 \ + --hash=sha256:ce28748cbeb0f29c3ce9603782979c7117580fc76f16aa3ca448b38a22281adb \ + --hash=sha256:d58798c4d8d629700058e9afc1e16b9801023f3ce4dc1c92d945e79b5ffe4e98 \ + --hash=sha256:e2a1856e9565fe2679863b372478c681806aebbf7d0a6e72f33e77f804e647d6 \ + --hash=sha256:e3b190ba1d3d22a5a8758597f797111b77d433473744352a184a5ee0a42d672e \ + --hash=sha256:e890816e5ee89c74a0f8b9379fe8b5ba83f46132b2a0bbb9b1c21359ec30dfda \ + --hash=sha256:eaf9e7cc7ab59f6c760232bbde18f64d559bbc50544841303bfb32be53533297 \ + --hash=sha256:ee341973f78a0b46e073d065e88e75026a9c584051e97f98a0d05d96c6bac7dd \ + --hash=sha256:f1c1b4263fd13abbc339a16f2bf19f3a5cbf2a620853d812b1256f03c5342cb8 \ + --hash=sha256:f7444ea6975c49a857c68f9bd8fa11acae96dede63d120ffb3bf0a603ea82516 \ + --hash=sha256:f800e9e722c145ccd18012d82a864cb21bfee4ba4ceffde77100d25eced511a9 \ + --hash=sha256:fcdd1e04982637c6042337d3e24d472f938f01fdc502e2b994844b726d12c3f4 \ + --hash=sha256:ff1e816af7abff71f289242e109217036723ce36aca74ad6691e52d964a74afa + # via -r eng/../requirements.txt pybind11==3.1.0 \ --hash=sha256:a1cc06b524ab3edca51f8ad3895f9c4fa20b8b19283173dff4ae781449dc9639 \ --hash=sha256:b8488090f8acffbcb6b5d6a85571a6827a0a2981ffb75e5a0b27b87c4a6b7dd0 diff --git a/eng/requirements-test-linux.txt b/eng/requirements-test-linux.txt index a2fdeea14..5f7a6019d 100644 --- a/eng/requirements-test-linux.txt +++ b/eng/requirements-test-linux.txt @@ -1105,7 +1105,7 @@ psutil==7.2.2 \ --hash=sha256:eed63d3b4d62449571547b60578c5b2c4bcccc5387148db46e0c2313dad0ee00 \ --hash=sha256:fd04ef36b4a6d599bbdb225dd1d3f51e00105f6d48a28f006da7f9822f2606d8 # via -r eng/../requirements.txt -pyarrow==25.0.1 \ +pyarrow==25.0.1 ; python_full_version < '3.15' \ --hash=sha256:0b1edbb2f385a6a65e9711b62ba86ac54a7816a3f8d17bb3e8a5929d65fb2485 \ --hash=sha256:0b726ad7e7b669be982b0c71c07fe4b037d654354130da79a7902a669e93a66b \ --hash=sha256:0befcf816e45a1af33ac775a9970b749e4868a230c7372f0ae5e932bee27039f \ @@ -1150,6 +1150,58 @@ pyarrow==25.0.1 \ --hash=sha256:f3831aaa25c67a99f99dc8b05873cb9d64560390372e2aa197ce9dd4a3f06a44 \ --hash=sha256:f729cfdbd36fd99d543b67a914d2de044c84ebe45be8b34902b299b608c15c8f # via -r eng/../requirements.txt +pyarrow==26.0.0 ; python_full_version >= '3.15' \ + --hash=sha256:01c863a18bd9c8412453dd0d92de6d0ee7b2b3d6fb079d9734a4b2a3c8bd4453 \ + --hash=sha256:0cccd36e00ea3afeb52ded61f2721ce71f604853d70c45365c58324eb773d6ae \ + --hash=sha256:106bb9290fc6fd9a84138a9440038ef184bac86463543c5ff099229cb30d996c \ + --hash=sha256:13b0972a3dc71b642050d1bc72664a3916e14f59c943d8c1368154d6e4b0c2d5 \ + --hash=sha256:210cc9b83888b87cdc8f793eebb264f22b20d0dedbedefc73b9687a7047b4747 \ + --hash=sha256:240bd18a7487f8767616a948a69dd4e740a8bc36a1c9da49e4dc9a32c5c2faed \ + --hash=sha256:24f892fdf1ae1942d69d3f7742e2f49960ec95277cfb1a70b8a1d91f4a96d935 \ + --hash=sha256:290a74c48e9491b436fd5edacfadf357943f82aa45c81110bd83a69aab33d1cf \ + --hash=sha256:2b5fcd69c0e1107b79e55839877db5a6ed04651b73fd6fec581d09e230bed5e4 \ + --hash=sha256:2e4a413046eba9896e632925066c74095182200ba32e19ff0166bf64d2f936ac \ + --hash=sha256:3a4d235876f14b4136b4d616ec42eb469ea0d6ead336cae631aa1dd29b21c962 \ + --hash=sha256:3de30a7432b48b98b9decbd9e25a53bb9251d202c2e6c5a29a50869592ccb117 \ + --hash=sha256:41dd3661ef40790a78870052ad7a58ad827b27c67a4511f06962eb9e9b74d19b \ + --hash=sha256:4a5fa8dc70dd50808990ff36faf44088e357b353d86c7682dd92d4b78d4c97d5 \ + --hash=sha256:4bcba83299cb2b8f8e443d36c6ba6269a5034431879015fb0719495df8a14de2 \ + --hash=sha256:515a10dae2a1d236bc9c9209d0317acb6746ea63cd4f98704904af7156d90ed1 \ + --hash=sha256:5780d487ff6c6ed7b42298609680d87fe0036e529a9dc2e1105364bce9697f50 \ + --hash=sha256:5b827650e874f1f9f9392524ea3e9e3e8a245de5ba64acca1f81ab188090afb9 \ + --hash=sha256:5d5768d03426abe6526d5274adefa00abf00a7f81118c46e98b5a46390f5549e \ + --hash=sha256:645917e976671debabf854abab6e2b75c571ca4f82adc33a2d338697f7c27d93 \ + --hash=sha256:68cd662e9e2b00876a131950cf32336ace2d0865e1f9418763e3d3be8481dfa4 \ + --hash=sha256:6a628922ba20705fa964ca73e4ef959c2fb2f14b9bbec5589a6a1e68e6257c85 \ + --hash=sha256:6e89dee53aaeb50505ed6152ea55bc7ddfd4f4df264f5427ea255288d8f0e580 \ + --hash=sha256:6e949744dcfc2d379808f7013c5f9cafaf0f817656dff7d46c6931528dd1784b \ + --hash=sha256:734312d3d99088d9ec28c5b17bad40389bd8373a1afc10acb60b83fd217af087 \ + --hash=sha256:7aa12ab8e236789b1ecd2d6ecaef036b4e63d675ddf1864a43c6799d18f2d028 \ + --hash=sha256:7c3fda041e7078802589cf257750323ee3d0cd1e56e53a9b20ec845697fb3d28 \ + --hash=sha256:879331ddea2a26479fa18fade71e6facf684a6cf19f67daec3775c871569e8e5 \ + --hash=sha256:8e8e28c464552b5ca03e30d4504168c4425ce383884f8611b00e972f9fd933fc \ + --hash=sha256:90ddaf7c625307ad52f31a9b25c34fe5e4897c7529ee3481135822b2b6842ff1 \ + --hash=sha256:954d971b363b16ee41f89389a4053315dc71265f2ce5c2468eb0a910b1166268 \ + --hash=sha256:9db18a9dc0af52135c9eac549d80a7a882696efbe5406cf882b044525d4ecc2e \ + --hash=sha256:a0e4e92eeb088f1d7c2c04d6c7de8434c75abb4b4ccf0bbcd045aa7164c68d93 \ + --hash=sha256:a6ca849f90cf73fe361f08a5762c783ead9671e4548c1f558cc637b54c9103f2 \ + --hash=sha256:ab6914db225d7f399652ae1f08588dfbc9efe617612715701e3d9d5cfa5ca19f \ + --hash=sha256:c2ba350957076b1b3a22f549261dc3e9c67ca20816d8bd5f79d7b9c69be4c4c2 \ + --hash=sha256:ca77c43ca55bfc9a4eeb1f0cd5f093f08731b77c24cdba0829035f084959b0bb \ + --hash=sha256:cc903e1069e9dd5e9dcf780324c0112e27e051e422ecfaff574fb33ed65d9160 \ + --hash=sha256:ce28748cbeb0f29c3ce9603782979c7117580fc76f16aa3ca448b38a22281adb \ + --hash=sha256:d58798c4d8d629700058e9afc1e16b9801023f3ce4dc1c92d945e79b5ffe4e98 \ + --hash=sha256:e2a1856e9565fe2679863b372478c681806aebbf7d0a6e72f33e77f804e647d6 \ + --hash=sha256:e3b190ba1d3d22a5a8758597f797111b77d433473744352a184a5ee0a42d672e \ + --hash=sha256:e890816e5ee89c74a0f8b9379fe8b5ba83f46132b2a0bbb9b1c21359ec30dfda \ + --hash=sha256:eaf9e7cc7ab59f6c760232bbde18f64d559bbc50544841303bfb32be53533297 \ + --hash=sha256:ee341973f78a0b46e073d065e88e75026a9c584051e97f98a0d05d96c6bac7dd \ + --hash=sha256:f1c1b4263fd13abbc339a16f2bf19f3a5cbf2a620853d812b1256f03c5342cb8 \ + --hash=sha256:f7444ea6975c49a857c68f9bd8fa11acae96dede63d120ffb3bf0a603ea82516 \ + --hash=sha256:f800e9e722c145ccd18012d82a864cb21bfee4ba4ceffde77100d25eced511a9 \ + --hash=sha256:fcdd1e04982637c6042337d3e24d472f938f01fdc502e2b994844b726d12c3f4 \ + --hash=sha256:ff1e816af7abff71f289242e109217036723ce36aca74ad6691e52d964a74afa + # via -r eng/../requirements.txt pybind11==3.1.0 \ --hash=sha256:a1cc06b524ab3edca51f8ad3895f9c4fa20b8b19283173dff4ae781449dc9639 \ --hash=sha256:b8488090f8acffbcb6b5d6a85571a6827a0a2981ffb75e5a0b27b87c4a6b7dd0 From 1c897dc046010f4986725b82aefeec146a700854 Mon Sep 17 00:00:00 2001 From: Gaurav Sharma Date: Mon, 12 Oct 2026 12:16:03 +0530 Subject: [PATCH 18/23] FIX: keep Python 3.15 review fixes behavioral Remove the three added source-text tests, assert suppressed Decimal causes in the existing test, and propagate container setup failures. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- eng/pipelines/pr-validation-pipeline.yml | 2 +- tests/test_004_cursor.py | 1 + tests/test_038_mssql_odbc_daily_validation.py | 25 ------------------- 3 files changed, 2 insertions(+), 26 deletions(-) diff --git a/eng/pipelines/pr-validation-pipeline.yml b/eng/pipelines/pr-validation-pipeline.yml index c706dd9fe..c19a3e8aa 100644 --- a/eng/pipelines/pr-validation-pipeline.yml +++ b/eng/pipelines/pr-validation-pipeline.yml @@ -818,7 +818,7 @@ jobs: export TZ=UTC ln -snf /usr/share/zoneinfo/\$TZ /etc/localtime && echo \$TZ > /etc/timezone apt-get update && - apt-get install -y cmake curl wget gnupg build-essential + apt-get install -y cmake curl wget gnupg build-essential && python3 --version " elif [ "$(distroName)" = "Ubuntu" ]; then diff --git a/tests/test_004_cursor.py b/tests/test_004_cursor.py index dbd917879..5b2b835e6 100644 --- a/tests/test_004_cursor.py +++ b/tests/test_004_cursor.py @@ -10412,6 +10412,7 @@ def test_setinputsizes_sql_decimal_unconvertible_value(db_connection): # ...and the sensitive value / raw row is NOT leaked into the message. assert sensitive_value not in message assert repr((sensitive_value,)) not in message # no repr of the parameter tuple + assert exc_info.value.__cause__ is None # ...nor into the chained cause or the fully formatted traceback, which # is what tracebacks and APM/log shippers actually capture. formatted = "".join( diff --git a/tests/test_038_mssql_odbc_daily_validation.py b/tests/test_038_mssql_odbc_daily_validation.py index f6f15e348..11c76c710 100644 --- a/tests/test_038_mssql_odbc_daily_validation.py +++ b/tests/test_038_mssql_odbc_daily_validation.py @@ -12,10 +12,8 @@ ROOT = Path(__file__).parents[1] RUNNER = ROOT / "eng" / "scripts" / "run-mssql-odbc-tests.sh" PIPELINE = ROOT / "eng" / "pipelines" / "mssql-odbc-daily-validation-pipeline.yml" -PR_PIPELINE = ROOT / "eng" / "pipelines" / "pr-validation-pipeline.yml" PREFLIGHT = ROOT / "eng" / "scripts" / "verify_mssql_odbc_provider.py" RELEASE_PIPELINE = ROOT / "OneBranchPipelines" / "stages" / "build-linux-single-stage.yml" -PYBIND_CMAKE = ROOT / "mssql_python" / "pybind" / "CMakeLists.txt" @unittest.skipUnless(sys.platform.startswith("linux"), "runner requires Linux GNU timeout and bash") @@ -171,29 +169,6 @@ def test_stable_rs_transport_is_pinned(self): self.assertEqual(version.strip(), "0.3.0") - def test_python_315_validation_adds_matrix_legs(self): - pipeline = PR_PIPELINE.read_text(encoding="utf-8") - active_python_versions = { - line.split(":", 1)[1].strip(" '\"") - for line in pipeline.splitlines() - if line.lstrip().startswith("pythonVersion:") - } - - self.assertEqual(active_python_versions, {"3.13", "3.14", "3.15"}) - self.assertIn("python:3.15.0rc3-bookworm", pipeline) - - def test_python_315_validation_installs_stable_pyarrow(self): - requirements = (ROOT / "requirements.txt").read_text(encoding="utf-8") - - self.assertIn('pyarrow==25.0.1; python_version < "3.15"', requirements) - self.assertIn('pyarrow==26.0.0; python_version >= "3.15"', requirements) - self.assertNotIn("scientific-python-nightly-wheels", requirements) - - def test_python_315_processes_python_header_before_libc_headers(self): - cmake = PYBIND_CMAKE.read_text(encoding="utf-8") - - self.assertIn("target_compile_options(ddbc_bindings PRIVATE -include Python.h)", cmake) - if __name__ == "__main__": unittest.main() From 661e4853923ab3d8935c7e5eda98d2d4cef98117 Mon Sep 17 00:00:00 2001 From: Gaurav Sharma Date: Mon, 12 Oct 2026 12:29:40 +0530 Subject: [PATCH 19/23] FEAT: Build Python 3.15 release wheels (#838) ### Work Item / Issue Reference > #817 ------------------------------------------------------------------- ### Summary Extends the governed build and release pipeline to produce Python 3.15 wheels without changing Conda packaging. - Adds Python 3.15 x64 and ARM64 Windows stages. - Adds a Python 3.15 macOS universal2 stage. - Adds `cp315` to all manylinux and musllinux x64/ARM64 build loops. - Extends release-lock compatibility validation to Python 3.15, allowing source builds where upstream wheels are unavailable. - Raises the consolidated wheel completeness gate from 34 to 41 wheels. ### Validation - YAML and whitespace validation passed. - Hosted universal-lock generation completed successfully for Linux, macOS, Windows, and ODBC. - The full build-release pipeline is the validation for native wheel production, tests, signing, and artifact completeness. ### Scope This PR covers PyPI wheel production only. Conda Python 3.15 packaging is intentionally excluded. --------- Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../workflows/refresh-build-dependencies.yml | 25 ++++++++- .../build-release-package-pipeline.yml | 52 +++++++++++-------- .../jobs/consolidate-artifacts-job.yml | 14 ++--- .../stages/build-linux-single-stage.yml | 8 +-- .../stages/build-windows-single-stage.yml | 1 + 5 files changed, 65 insertions(+), 35 deletions(-) diff --git a/.github/workflows/refresh-build-dependencies.yml b/.github/workflows/refresh-build-dependencies.yml index ca56de461..45a360a6d 100644 --- a/.github/workflows/refresh-build-dependencies.yml +++ b/.github/workflows/refresh-build-dependencies.yml @@ -130,18 +130,30 @@ jobs: - name: Linux build artifact: linux lock: requirements-build-linux.txt - versions: "3.10 3.11 3.12 3.13 3.14" + versions: "3.10 3.11 3.12 3.13 3.14 3.15" platforms: "x86_64-manylinux_2_28 aarch64-manylinux_2_28 x86_64-unknown-linux-musl aarch64-unknown-linux-musl" - name: Linux test artifact: linux lock: requirements-test-linux.txt versions: "3.10 3.11 3.12 3.13 3.14" platforms: "x86_64-manylinux_2_28 aarch64-manylinux_2_28 x86_64-unknown-linux-musl aarch64-unknown-linux-musl" + - name: Linux test Python 3.15 + artifact: linux + lock: requirements-test-linux.txt + versions: "3.15" + platforms: "x86_64-manylinux_2_28 aarch64-manylinux_2_28 x86_64-unknown-linux-musl aarch64-unknown-linux-musl" + allowSourceBuilds: true - name: macOS artifact: macos lock: requirements-build-macos.txt versions: "3.10 3.11 3.12 3.13 3.14" platforms: "x86_64-apple-darwin aarch64-apple-darwin" + - name: macOS Python 3.15 + artifact: macos + lock: requirements-build-macos.txt + versions: "3.15" + platforms: "x86_64-apple-darwin aarch64-apple-darwin" + allowSourceBuilds: true # Windows ARM64 stages install build tools into the x64 host # interpreter before cross-compiling the target extension. - name: Windows build host @@ -149,6 +161,12 @@ jobs: lock: requirements-build-windows.txt versions: "3.10 3.11 3.12 3.13 3.14" platforms: "x86_64-pc-windows-msvc" + - name: Windows build host Python 3.15 + artifact: windows + lock: requirements-build-windows.txt + versions: "3.15" + platforms: "x86_64-pc-windows-msvc" + allowSourceBuilds: true - name: ODBC artifact: odbc lock: requirements-build-odbc.txt @@ -168,8 +186,11 @@ jobs: LOCK_FILE: generated-lock/${{ matrix.lock }} PLATFORMS: ${{ matrix.platforms }} PYTHON_VERSIONS: ${{ matrix.versions }} + ALLOW_SOURCE_BUILDS: ${{ matrix.allowSourceBuilds || false }} run: | set -euo pipefail + binary_args=(--only-binary :all:) + if [ "$ALLOW_SOURCE_BUILDS" = true ]; then binary_args=(); fi for platform in $PLATFORMS; do for version in $PYTHON_VERSIONS; do echo "Validating $LOCK_FILE for Python $version on $platform" @@ -179,7 +200,7 @@ jobs: --target "$RUNNER_TEMP/lock-validation" \ --python-version "$version" \ --python-platform "$platform" \ - --only-binary :all: \ + "${binary_args[@]}" \ --require-hashes \ --default-index https://pypi.org/simple \ -r "$LOCK_FILE" diff --git a/OneBranchPipelines/build-release-package-pipeline.yml b/OneBranchPipelines/build-release-package-pipeline.yml index b5b719cef..bf0904658 100644 --- a/OneBranchPipelines/build-release-package-pipeline.yml +++ b/OneBranchPipelines/build-release-package-pipeline.yml @@ -2,9 +2,9 @@ # OneBranch Release Pipeline for mssql-python # ========================================================================================= # Builds Python wheels for all supported platforms with SDL compliance: -# - Windows: Python 3.10-3.14 (x64 + ARM64) -# - macOS: Python 3.10-3.14 (Universal2 = x86_64 + ARM64 in single binary) -# - Linux: Python 3.10-3.14 on manylinux/musllinux (x86_64 + ARM64) +# - Windows: Python 3.10-3.15 (x64 + ARM64) +# - macOS: Python 3.10-3.15 (Universal2 = x86_64 + ARM64 in single binary) +# - Linux: Python 3.10-3.15 on manylinux/musllinux (x86_64 + ARM64) # # Security Features: # - ESRP code signing (Windows .pyd files only) @@ -82,19 +82,19 @@ parameters: # PLATFORM CONFIGURATIONS # ========================= # Each platform uses different matrix strategy: - # - Windows: Explicit per-version stages (9 stages for x64/ARM64 combos) - # - macOS: Explicit per-version stages (5 stages for universal2 builds) + # - Windows: Explicit per-version stages (11 stages for x64/ARM64 combos) + # - macOS: Explicit per-version stages (6 stages for universal2 builds) # - Linux: Per-distro stages, builds ALL Python versions in loop (4 stages) # Windows Configuration Matrix # Each entry creates separate stage: Win_py_ - # pyVer format: '310' = Python 3.10, '314' = Python 3.14 + # pyVer format: '310' = Python 3.10, '315' = Python 3.15 # arch: 'x64' (Intel/AMD 64-bit) or 'arm64' (ARM64, cross-compiled on x64) # Note: ARM64 builds use x64 host with ARM64 python.lib for cross-compilation - name: windowsConfigs type: object default: - # x64 builds (5 versions: 3.10-3.14) + # x64 builds (6 versions: 3.10-3.15) - pyVer: '310' arch: 'x64' - pyVer: '311' @@ -105,7 +105,9 @@ parameters: arch: 'x64' - pyVer: '314' arch: 'x64' - # ARM64 builds (4 versions: 3.11-3.14) + - pyVer: '315' + arch: 'x64' + # ARM64 builds (5 versions: 3.11-3.15) # 3.10 excluded due to limited ARM64 support - pyVer: '311' arch: 'arm64' @@ -115,23 +117,26 @@ parameters: arch: 'arm64' - pyVer: '314' arch: 'arm64' + - pyVer: '315' + arch: 'arm64' # macOS Configuration Matrix # Each entry creates separate stage: MacOS_py # All builds are Universal2 (x86_64 + ARM64 in single binary) - # pyVer format: '310' = Python 3.10, '314' = Python 3.14 + # pyVer format: '310' = Python 3.10, '315' = Python 3.15 - name: macosConfigs type: object default: - # 5 versions: 3.10-3.14 (all universal2) + # 6 versions: 3.10-3.15 (all universal2) - pyVer: '310' - pyVer: '311' - pyVer: '312' - pyVer: '313' - pyVer: '314' + - pyVer: '315' # Linux Configuration Matrix - # Each entry creates ONE stage that builds ALL Python versions (3.10-3.14) + # Each entry creates ONE stage that builds ALL Python versions (3.10-3.15) # tag: 'manylinux' (glibc-based, e.g., Ubuntu/CentOS) or 'musllinux' (musl-based, e.g., Alpine) # arch: CPU architecture for Docker platform # platform: Docker platform identifier for multi-arch builds @@ -375,17 +380,17 @@ extends: # ========================= # PIPELINE STAGES # ========================= - # mssql-python: 9 Windows + 5 macOS + 4 Linux + 1 Consolidate = 19 stages + # mssql-python: 11 Windows + 6 macOS + 4 Linux + 1 Consolidate = 22 stages # mssql-python-odbc: 1 ODBC_BuildAll (single host, all 7 wheels) + 1 ConsolidateOdbc = 2 stages - # Total: 21 stages, always built together. The mssql-python build stages depend on + # Total: 24 stages, always built together. The mssql-python build stages depend on # ConsolidateOdbc so they install the external ODBC wheel before pytest. stages: # ========================= # WINDOWS BUILD STAGES # ========================= # Strategy: Explicit stage per Python version × architecture - # Total: 9 stages (5 x64 + 4 ARM64) - # Python versions: 3.10-3.14 (x64), 3.11-3.14 (ARM64) + # Total: 11 stages (6 x64 + 5 ARM64) + # Python versions: 3.10-3.15 (x64), 3.11-3.15 (ARM64) # Each stage: # 1. Installs Python (UsePythonVersion or NuGet for 3.14) # 2. Downloads ARM64 python.lib if cross-compiling @@ -416,7 +421,7 @@ extends: # MACOS BUILD STAGES # ========================= # Strategy: Explicit stage per Python version - # Total: 5 stages (3.10-3.14) + # Total: 6 stages (3.10-3.15) # All builds are Universal2 (x86_64 + ARM64 in single .so binary) # Each stage: # 1. Installs Python via UsePythonVersion@0 @@ -446,7 +451,7 @@ extends: # ========================= # Strategy: One stage per distribution × architecture # Total: 4 stages (manylinux_2_28×2 + musllinux×2) - # Each stage builds ALL Python versions (3.10-3.14) in a loop + # Each stage builds ALL Python versions (3.10-3.15) in a loop # Distributions: # - manylinux_2_28: glibc 2.28+ (RHEL 8, Ubuntu 20.04+, etc.) — forward-compatible with newer glibc # - musllinux: musl-based (Alpine Linux) @@ -454,12 +459,12 @@ extends: # Each stage: # 1. Starts PyPA Docker container (manylinux_2_28 or musllinux_1_2) # 2. Starts SQL Server Docker container - # 3. For each Python version (cp310-cp314): + # 3. For each Python version (cp310-cp315): # a. Builds .so native extension # b. Builds wheel # c. Installs wheel in isolated directory # d. Runs pytest against SQL Server - # 4. Publishes artifacts (all 5 wheels) + # 4. Publishes artifacts (all 6 wheels) # 5. Component Governance + AntiMalware scanning - ${{ each config in parameters.linuxConfigs }}: - template: /OneBranchPipelines/stages/build-linux-single-stage.yml@self @@ -479,7 +484,7 @@ extends: # CONSOLIDATE STAGE # ========================= # Purpose: Collect all artifacts from platform builds into single dist/ folder - # Dependencies: All 18 build stages (9 Windows + 5 macOS + 4 Linux) + # Dependencies: All 21 build stages (11 Windows + 6 macOS + 4 Linux) # Stages run in parallel, Consolidate waits for ALL to complete # Outputs: # - dist/wheels/*.whl (all platform wheels) @@ -490,22 +495,25 @@ extends: - stage: Consolidate displayName: 'Consolidate All Artifacts' dependsOn: - # Windows dependencies (9 stages) + # Windows dependencies (11 stages) - Win_py310_x64 - Win_py311_x64 - Win_py312_x64 - Win_py313_x64 - Win_py314_x64 + - Win_py315_x64 - Win_py311_arm64 - Win_py312_arm64 - Win_py313_arm64 - Win_py314_arm64 - # macOS dependencies (5 stages) + - Win_py315_arm64 + # macOS dependencies (6 stages) - MacOS_py310 - MacOS_py311 - MacOS_py312 - MacOS_py313 - MacOS_py314 + - MacOS_py315 # Linux dependencies (4 stages) - Linux_manylinux_2_28_x86_64 - Linux_manylinux_2_28_aarch64 diff --git a/OneBranchPipelines/jobs/consolidate-artifacts-job.yml b/OneBranchPipelines/jobs/consolidate-artifacts-job.yml index 5ca4c5efe..d2b7b9ff1 100644 --- a/OneBranchPipelines/jobs/consolidate-artifacts-job.yml +++ b/OneBranchPipelines/jobs/consolidate-artifacts-job.yml @@ -1,7 +1,7 @@ # Consolidate Artifacts Job Template # Downloads artifacts from all platform build stages and consolidates into single dist/ folder -# Works with individual stage artifacts (18 stages total: 9 Windows + 5 macOS + 4 Linux) -# Each Linux stage builds 5 Python versions (3.10-3.14), resulting in 34 total wheels +# Works with individual stage artifacts (21 stages total: 11 Windows + 6 macOS + 4 Linux) +# Each Linux stage builds 6 Python versions (3.10-3.15), resulting in 41 total wheels parameters: - name: oneBranchType type: string @@ -75,17 +75,17 @@ jobs: echo "" WHEEL_COUNT=$(ls -1 $(ob_outputDirectory)/dist/*.whl 2>/dev/null | wc -l) echo "Total wheel count: $WHEEL_COUNT" - echo "Expected: 34 wheels (9 Windows + 5 macOS + 20 Linux)" + echo "Expected: 41 wheels (11 Windows + 6 macOS + 24 Linux)" # Hard-fail on a wheel-count mismatch (symmetric with the odbc consolidate job). # A mismatch means a build stage silently produced a partial set (e.g. a missing # Python version) or the build matrix changed without updating this count. - # Update the expected 34 whenever the Windows/macOS/Linux build matrix changes. - if [ "$WHEEL_COUNT" -ne 34 ]; then - echo "ERROR: Expected 34 wheels but found $WHEEL_COUNT" >&2 + # Update the expected 41 whenever the Windows/macOS/Linux build matrix changes. + if [ "$WHEEL_COUNT" -ne 41 ]; then + echo "ERROR: Expected 41 wheels but found $WHEEL_COUNT" >&2 exit 1 else - echo "SUCCESS: All 34 wheels consolidated!" + echo "SUCCESS: All 41 wheels consolidated!" fi displayName: 'Consolidate wheels from all platforms' diff --git a/OneBranchPipelines/stages/build-linux-single-stage.yml b/OneBranchPipelines/stages/build-linux-single-stage.yml index 62b500b5a..06565f34b 100644 --- a/OneBranchPipelines/stages/build-linux-single-stage.yml +++ b/OneBranchPipelines/stages/build-linux-single-stage.yml @@ -1,6 +1,6 @@ # Linux Single Configuration Stage Template # Builds Python wheels for a specific Linux distribution and architecture -# Builds for Python 3.10, 3.11, 3.12, 3.13, 3.14 within single job +# Builds for Python 3.10, 3.11, 3.12, 3.13, 3.14, 3.15 within single job # Tests each wheel after building with isolated pytest execution parameters: # Stage identifier (e.g., 'Linux_manylinux_2_28_x86_64') @@ -256,14 +256,14 @@ stages: ls -1 "$(Build.SourcesDirectory)/odbc_wheels"/*.whl displayName: 'Flatten ODBC wheels for pip find-links' - # Build wheels for all Python versions (3.10-3.14) and test each one + # Build wheels for all Python versions (3.10-3.15) and test each one - script: | set -euo pipefail if [[ "$(LINUX_TAG)" == "manylinux_2_28" ]]; then SHELL_EXE=bash; else SHELL_EXE=sh; fi docker exec build-$(LINUX_TAG)-$(ARCH) $SHELL_EXE -lc 'mkdir -p /workspace/dist' # Loop through all Python versions: build wheel -> test wheel -> repeat - for PYBIN in cp310 cp311 cp312 cp313 cp314; do + for PYBIN in cp310 cp311 cp312 cp313 cp314 cp315; do echo "" echo "=====================================================" echo "Building and testing $PYBIN on $(LINUX_TAG)/$(ARCH)" @@ -440,7 +440,7 @@ stages: echo "=====================================================" echo "✓ All Python versions built and tested successfully!" echo "=====================================================" - displayName: 'Build and test wheels for Python 3.10-3.14' + displayName: 'Build and test wheels for Python 3.10-3.15' env: DB_PASSWORD: $(DB_PASSWORD) diff --git a/OneBranchPipelines/stages/build-windows-single-stage.yml b/OneBranchPipelines/stages/build-windows-single-stage.yml index a81e045db..b1c8cdbb3 100644 --- a/OneBranchPipelines/stages/build-windows-single-stage.yml +++ b/OneBranchPipelines/stages/build-windows-single-stage.yml @@ -201,6 +201,7 @@ stages: "3.12" { "3.12.7" } "3.13" { "3.13.0" } "3.14" { "3.14.0" } + "3.15" { "3.15.0" } } $nugetUrl = "https://www.nuget.org/api/v2/package/pythonarm64/$nugetVersion" From 9b89361d138d798c8a3f1e3863c5f2c124e34972 Mon Sep 17 00:00:00 2001 From: Gaurav Sharma Date: Mon, 12 Oct 2026 12:34:46 +0530 Subject: [PATCH 20/23] FIX: discard Decimal conversion exception context Raise the metadata-only error after leaving the conversion handler, and assert absent context in the three existing privacy tests. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- mssql_python/cursor.py | 14 +++++--------- tests/test_004_cursor.py | 3 +++ 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/mssql_python/cursor.py b/mssql_python/cursor.py index cd7ec3167..d4b85e56b 100644 --- a/mssql_python/cursor.py +++ b/mssql_python/cursor.py @@ -2746,19 +2746,15 @@ def executemany( # pylint: disable=too-many-locals,too-many-branches,too-many-s f"Failed to convert parameter to Decimal at row " f"{row_index}, column {i} (value type: {type(val).__name__})" ) - # Split str(val) from the decimal parse so every conversion - # failure can suppress chaining. Python 3.15's pure-Python - # decimal implementation can retain the rejected input in - # traceback state. + # Raise outside the handler so conversion exceptions are + # not retained as context with rejected input. + conversion_failed = False try: val_text = str(val) - except Exception: # pylint: disable=broad-exception-caught - raise ValueError(err_msg) from None - try: processed_row[i] = format(decimal.Decimal(val_text), "f") - except decimal.DecimalException: - raise ValueError(err_msg) from None except Exception: # pylint: disable=broad-exception-caught + conversion_failed = True + if conversion_failed: raise ValueError(err_msg) from None processed_parameters.append(processed_row) diff --git a/tests/test_004_cursor.py b/tests/test_004_cursor.py index 5b2b835e6..001440edf 100644 --- a/tests/test_004_cursor.py +++ b/tests/test_004_cursor.py @@ -10413,6 +10413,7 @@ def test_setinputsizes_sql_decimal_unconvertible_value(db_connection): assert sensitive_value not in message assert repr((sensitive_value,)) not in message # no repr of the parameter tuple assert exc_info.value.__cause__ is None + assert exc_info.value.__context__ is None # ...nor into the chained cause or the fully formatted traceback, which # is what tracebacks and APM/log shippers actually capture. formatted = "".join( @@ -10458,6 +10459,7 @@ def __str__(self): # exposes it. assert secret not in str(exc_info.value) assert exc_info.value.__cause__ is None + assert exc_info.value.__context__ is None formatted = "".join( traceback.format_exception( type(exc_info.value), exc_info.value, exc_info.value.__traceback__ @@ -10502,6 +10504,7 @@ def test_setinputsizes_sql_decimal_non_decimal_exception_no_leak(db_connection): assert "column 0" in message # ...no chained cause (the non-DecimalException branch suppresses it)... assert exc_info.value.__cause__ is None + assert exc_info.value.__context__ is None # ...and the input is absent from the message and formatted traceback. assert sensitive_value not in message formatted = "".join( From 18e1ab9e93d099746760082a07cadd07bf3a0232 Mon Sep 17 00:00:00 2001 From: Jahnvi Thakkar Date: Mon, 12 Oct 2026 13:41:28 +0530 Subject: [PATCH 21/23] FEAT: Add Python 3.15 Conda support Retain existing supported variants while adding ordinary CPython 3.15 across the six Conda subdirs. Align Windows layout, scoped builder selection and ARM64 channels, with focused packaging regressions. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- OneBranchPipelines/conda-build-pipeline.yml | 9 +- .../jobs/consolidate-conda-artifacts-job.yml | 18 ++-- .../steps/conda-build-validate-step.yml | 2 +- .../steps/conda-publish-step.yml | 2 +- .../steps/conda-release-step.yml | 4 +- conda/README.md | 26 +++++- conda/mssql-python/bld.bat | 8 +- conda/mssql-python/meta.yaml | 2 +- eng/conda_tools/build.py | 8 +- eng/conda_tools/contracts.py | 15 +++- eng/conda_tools/environment.py | 15 +++- eng/conda_tools/inputs.py | 6 +- eng/conda_tools/release.py | 14 ++-- eng/conda_tools/verify.py | 6 +- tests/test_027_conda_release_metadata.py | 77 +++++++++++++---- tests/test_030_pe_machine_assert.py | 57 +++++++++++-- tests/test_034_conda_verify_cwd.py | 83 ++++++++++++++++--- tests/test_036_conda_provenance.py | 48 +++++++---- 18 files changed, 302 insertions(+), 98 deletions(-) diff --git a/OneBranchPipelines/conda-build-pipeline.yml b/OneBranchPipelines/conda-build-pipeline.yml index d1c8fa239..fc5598478 100644 --- a/OneBranchPipelines/conda-build-pipeline.yml +++ b/OneBranchPipelines/conda-build-pipeline.yml @@ -35,12 +35,11 @@ parameters: - 'Official' - 'NonOfficial' default: 'NonOfficial' - # Python versions to build conda packages for (Windows loops these; the POSIX legs - # auto-detect the same set from the downloaded wheels). + # Ordinary CPython versions passed explicitly to the Windows and POSIX build legs. - name: pythonVersions displayName: 'Python versions (comma-separated)' type: string - default: '3.10,3.11,3.12,3.13,3.14' + default: '3.10,3.11,3.12,3.13,3.14,3.15' # H1: enable the minimal-base ldd reachability gate (CONDA_ASSERT_PREFIX_REACHABLE). # It fails CLOSED if the driver binds a system (or absent) krb5/gssapi/libltdl, so it # is only valid on a leg with NO system copies of those libs -- set true ONLY when @@ -198,14 +197,14 @@ extends: # internally (so a broken/wrong-arch package is never staged), AND win-arm64 is in # the release gate's requiredSubdirs -- so a MISSING win-arm64 blocks the RELEASE. # Net: a win-arm64 failure isolates to win-arm64 (win-64 still ships) yet still - # blocks the publish via the gate. DELIBERATELY a FIXED set (3.12-3.14), INDEPENDENT + # blocks the publish via the gate. DELIBERATELY a FIXED set (3.12-3.15), INDEPENDENT # of the pythonVersions parameter: neither defaults nor conda-forge ships a # win-arm64 python for 3.10/3.11, so following the param would queue an unbuildable # 3.10/3.11 arm leg. Its own bld tree + subdir-scoped verify env keep it from # colliding with win-64 on this shared agent. - template: /OneBranchPipelines/steps/conda-build-validate-step.yml@self parameters: - pythonVersion: '3.12,3.13,3.14' + pythonVersion: '3.12,3.13,3.14,3.15' condaSubdir: 'win-arm64' condaTargetSubdir: 'win-arm64' odbcWheelDir: '$(Pipeline.Workspace)/odbc_wheels' diff --git a/OneBranchPipelines/jobs/consolidate-conda-artifacts-job.yml b/OneBranchPipelines/jobs/consolidate-conda-artifacts-job.yml index 72d4512c3..498f09829 100644 --- a/OneBranchPipelines/jobs/consolidate-conda-artifacts-job.yml +++ b/OneBranchPipelines/jobs/consolidate-conda-artifacts-job.yml @@ -12,15 +12,15 @@ # Expected packages (validated conda subdirs). The self-contained mssql-python # package (which vendors the ODBC payload) is emitted per-Python by each build leg; # there is NO separate companion package: -# win-64 : 5 py x mssql-python = 5 -# win-arm64 : 3 py x mssql-python (py3.12-3.14, cross-built on x64) = 3 -# osx-64 : 5 py x mssql-python (Intel Mac, native) = 5 -# osx-arm64 : 5 py x mssql-python (Intel Mac, cross-built) = 5 -# linux-64 : 5 py x mssql-python (glibc x86_64 host, native) = 5 -# linux-aarch64 : 5 py x mssql-python (x86_64 host + QEMU, best-effort) = 5 +# win-64 : 6 py x mssql-python (py3.10-3.15) = 6 +# win-arm64 : 4 py x mssql-python (py3.12-3.15, cross-built on x64) = 4 +# osx-64 : 6 py x mssql-python (Intel Mac, native) = 6 +# osx-arm64 : 6 py x mssql-python (Intel Mac, cross-built) = 6 +# linux-64 : 6 py x mssql-python (glibc x86_64 host, native) = 6 +# linux-aarch64 : 6 py x mssql-python (x86_64 host + QEMU, best-effort) = 6 # ------------------------------------------------------------------------------ -# TOTAL (PyPI parity minus musllinux) = 28 -# win-arm64 is cross-built for py3.12-3.14 only (py3.10/3.11 have no win-arm64 deps +# TOTAL (ordinary CPython, no musllinux or free-threaded variants) = 34 +# win-arm64 is cross-built for py3.12-3.15 only (py3.10/3.11 have no win-arm64 deps # on Anaconda defaults); its runtime import is skipped on the x64 host, so its arch # is enforced by the PE audit (eng.conda_tools pe). musllinux (no conda # musl subdir) is intentionally NOT conda-built. This job is BEST-EFFORT and never @@ -117,7 +117,7 @@ jobs: n=$(find "$d" -type f \( -name '*.conda' -o -name '*.tar.bz2' \) | wc -l) printf ' %-14s %s\n' "$sub" "$n" done - echo "Total conda package count: $PKG_COUNT (full PyPI-parity set = 28)" + echo "Total conda package count: $PKG_COUNT (full ordinary-CPython set = 34)" # BEST-EFFORT: never exit non-zero — an empty set emits a warning; a partial set # remains visible in the counts above. A conda hiccup must not block the wheel diff --git a/OneBranchPipelines/steps/conda-build-validate-step.yml b/OneBranchPipelines/steps/conda-build-validate-step.yml index 0aa094509..5dea08c3f 100644 --- a/OneBranchPipelines/steps/conda-build-validate-step.yml +++ b/OneBranchPipelines/steps/conda-build-validate-step.yml @@ -27,7 +27,7 @@ parameters: # Python version(s) to build, comma-separated X.Y (e.g. '3.13' or '3.12,3.13,3.14'); passed # straight to the orchestrator's --python-versions. win-64 follows the pythonVersions - # parameter; win-arm64 uses a fixed 3.12-3.14 set (the singular name is kept for callers). + # parameter; win-arm64 uses a fixed 3.12-3.15 set (the singular name is kept for callers). - name: pythonVersion type: string # Conda subdir for this leg's package metadata + staging: 'win-64' (native) or 'win-arm64' diff --git a/OneBranchPipelines/steps/conda-publish-step.yml b/OneBranchPipelines/steps/conda-publish-step.yml index af084951d..9c406726a 100644 --- a/OneBranchPipelines/steps/conda-publish-step.yml +++ b/OneBranchPipelines/steps/conda-publish-step.yml @@ -68,7 +68,7 @@ parameters: # against this job's independently downloaded artifact before upload. - name: pythonVersions type: string - default: '3.10,3.11,3.12,3.13,3.14' + default: '3.10,3.11,3.12,3.13,3.14,3.15' # The publisher hash lock targets CPython 3.12 / Windows x64; other runtimes reject. - name: pythonVersion type: string diff --git a/OneBranchPipelines/steps/conda-release-step.yml b/OneBranchPipelines/steps/conda-release-step.yml index 4a8386ff0..3e3cce161 100644 --- a/OneBranchPipelines/steps/conda-release-step.yml +++ b/OneBranchPipelines/steps/conda-release-step.yml @@ -35,7 +35,7 @@ # dependency solve PLUS a PE machine-type assert on the vendored arm64 .pyd/.dll, so a # broken or wrong-arch package is never STAGED. That build leg is continueOnError (so a # win-arm64 hiccup can't nuke the shared win-64 publish), and THIS release gate is where -# win-arm64 is enforced: a missing win-arm64 fails the gate. Its matrix is 3.12-3.14 (the +# win-arm64 is enforced: a missing win-arm64 fails the gate. Its matrix is 3.12-3.15 (the # gate's per-subdir override), since Anaconda `defaults` has no win-arm64 python for # 3.10/3.11. musllinux has no conda subdir at all (conda Linux is glibc-only), so absent. # @@ -64,7 +64,7 @@ parameters: # subdir. The gate reads each binding's pyXY build tag from info/index.json. - name: pythonVersions type: string - default: '3.10,3.11,3.12,3.13,3.14' + default: '3.10,3.11,3.12,3.13,3.14,3.15' # Optional EXACT expected versions. When set, the gate asserts every package's # info/index.json version matches; when empty it still enforces one-version-per- # package consistency plus the subdir / matrix / pairing checks. diff --git a/conda/README.md b/conda/README.md index 89071b988..2056b126d 100644 --- a/conda/README.md +++ b/conda/README.md @@ -112,8 +112,9 @@ macOS retains its external Homebrew/MacPorts OpenSSL prerequisite for encryption Conda OpenSSL alone does not satisfy the driver's system-path lookup. The release goal is the matching PyPI release's public API and supported feature -behavior on all 28 ordinary CPython variants: 3.10-3.14 on win-64, linux-64, -linux-aarch64, osx-64 and osx-arm64; 3.12-3.14 on win-arm64. No silent removal of +behavior on all 34 ordinary CPython variants: 3.10-3.15 on win-64, linux-64, +linux-aarch64, osx-64 and osx-arm64; 3.12-3.15 on win-arm64. Free-threaded CPython +(including `cp315t`) and musllinux are not Conda targets. No silent removal of required native functionality is acceptable. Optional features require their corresponding dependencies; Windows ARM64 PyArrow availability remains a blocker to qualifying those features, not permission to drop them. @@ -125,7 +126,7 @@ load do not establish those results. Applicable OS, certificate and authenticati configuration remain external prerequisites. Use only organizationally approved channels and handle applicable terms separately; -the Windows ARM64 dependency profile includes Anaconda `defaults`. Before unattended +the Windows ARM64 3.12-3.14 dependency profile includes Anaconda `defaults`. Before unattended Windows ARM64 builds, agent owners must use approved provisioning to put a disposable Conda installation on `PATH`, with applicable channel terms handled for the job's execution identity. The orchestrator's automatic Miniforge installation does not @@ -134,6 +135,25 @@ or verification solve stops with Conda's diagnostic; provision the prerequisite running again. Automatic acceptance, including an inherited `CONDA_PLUGINS_AUTO_ACCEPT_TOS` opt-in, remains disabled. +Windows ARM64 3.15 uses `microsoft`, then `conda-forge`, for both build and verification +(after the local verification channel), without `defaults`. Older targets retain their +existing channel policies and the ARM64 3.12 manual ABI pin. The blocking full dependency +solve remains; every 3.15 package must pass the `python_abi 3.15.* *_cp315` native audit. + +### Python 3.15 packaging + +Conda-forge's Windows Python 3.15 declares `lib/python/site-packages` instead of +`Lib/site-packages` ([layout change](https://conda-forge.org/blog/2026/09/03/python-315/)). +The Windows recipe uses the target's `SP_DIR` for binding/RS/ODBC extraction, with +matching native and installed-metadata checks. Windows legs containing 3.15 use +**conda-build 26.9.1** for all requested variants: its +[26.9.0 fix](https://github.com/conda/conda-build/releases/tag/26.9.0) reads +`python_site_packages_path` from the target Python record, including cross builds. +25.x computes the old Windows path even for 3.15. Other legs retain `conda-build<26`; +the publishing toolchain is unchanged. The existing RS 0.3.0 stable-ABI selection and +ODBC `py3-none` wheels need no version change. PyArrow stays optional; no speculative +`libpython` dependency or change to the Unix OS floors is introduced. + Run this existing build workflow only in a disposable isolated installation: shared-environment ownership hardening is outside this change. Publication/provenance tooling is included alongside this native/tooling foundation; see the diff --git a/conda/mssql-python/bld.bat b/conda/mssql-python/bld.bat index 22efc9fcc..b09a3ddfc 100644 --- a/conda/mssql-python/bld.bat +++ b/conda/mssql-python/bld.bat @@ -4,7 +4,13 @@ REM vendor the ODBC Driver 18 payload inside it. conda-build exports PKG_NAME / REM PKG_VERSION / CONDA_PY; the pipeline exports WHEELS_DIR + MSSQL_ODBC_VERSION. setlocal enabledelayedexpansion -set "SP=%PREFIX%\Lib\site-packages" +REM conda-build 26.9+ reads the target Python's python_site_packages_path (CFEP-27). +REM Python 3.15 uses lib\python\site-packages; older Windows targets use Lib\site-packages. +if not defined SP_DIR ( + echo ERROR: conda-build must supply the target Python site-packages in SP_DIR. + exit /b 1 +) +set "SP=%SP_DIR%" if not exist "%SP%" mkdir "%SP%" set "PKG_UNDERSCORE=%PKG_NAME:-=_%" diff --git a/conda/mssql-python/meta.yaml b/conda/mssql-python/meta.yaml index 375231a1e..800fa579a 100644 --- a/conda/mssql-python/meta.yaml +++ b/conda/mssql-python/meta.yaml @@ -36,7 +36,7 @@ requirements: # --python matrix here, so each build emits the matching py3XX package. A range such # as `python >=3.10` collapses ALL --python builds into ONE cp package (build # string "py>=310") whose python_abi is pinned to 3.14, so `conda create python=3.10 - # mssql-python` can't solve. The >=3.10 floor is enforced implicitly: only py310-py314 + # mssql-python` can't solve. The >=3.10 floor is enforced implicitly: only py310-py315 # are built and each package's python_abi pins its exact minor. - python - pip diff --git a/eng/conda_tools/build.py b/eng/conda_tools/build.py index 0c4bb84f6..f68392cb0 100644 --- a/eng/conda_tools/build.py +++ b/eng/conda_tools/build.py @@ -302,10 +302,10 @@ def build_packages( croot = os.path.join(os.path.dirname(os.path.abspath(bld)), "croot") if os.path.isdir(croot): shutil.rmtree(croot) - channels = ["microsoft", "conda-forge"] - if target_subdir == "win-arm64": - channels.insert(0, "defaults") for py in pyvers: + channels = ["microsoft", "conda-forge"] + if target_subdir == "win-arm64" and py != "3.15": + channels.insert(0, "defaults") environment._log( f"=== [py {py}] build mssql-python (self-contained: vendors the ODBC payload) ===" ) @@ -471,8 +471,8 @@ def execute(args: argparse.Namespace) -> int: conda = environment.find_or_install_conda(output_dir) environment._log(f"Using conda: {conda}") environment.run([conda, "--version"], what="conda --version") - builder = environment.create_builder_env(conda) pyvers = detect_pythons(links, python_versions) + builder = environment.create_builder_env(conda, target, pyvers) cross_build = bool(args.conda_target_subdir) env = environment.build_env(mssql_ver, odbc_ver, links, args.conda_target_subdir, rs_ver) diff --git a/eng/conda_tools/contracts.py b/eng/conda_tools/contracts.py index da84d6bed..155a4a417 100644 --- a/eng/conda_tools/contracts.py +++ b/eng/conda_tools/contracts.py @@ -418,6 +418,18 @@ def validate_rs_ownership( return errors +def site_packages_root(python_tag: str, subdir: str) -> str: + """Canonical archive root for the supported ordinary CPython target, not the host.""" + if not re.fullmatch(r"cp3\d+", python_tag) or subdir not in _RS_PLATFORMS: + raise ValueError(f"Unsupported Conda Python/platform target: {python_tag} {subdir}") + minor = int(python_tag[3:]) + if subdir.startswith("win-"): + prefix = "lib/python" if minor >= 15 else "Lib" + else: + prefix = f"lib/python3.{minor}" + return f"{prefix}/site-packages/" + + def validate_native_contract(names: Iterable[str], index: dict[str, Any]) -> list[str]: """Require a target binding, core extension and initializer; platform audits check headers. @@ -436,8 +448,7 @@ def validate_native_contract(names: Iterable[str], index: dict[str, Any]) -> lis subdir = index["subdir"] windows = subdir.startswith("win-") suffix = "pyd" if windows else "so" - prefix = "Lib" if windows else f"lib/python{abi[1]}" - root = f"{prefix}/site-packages/" + root = site_packages_root(f"cp{abi[2]}", subdir) names = [name.replace("\\", "/") for name in names] bindings = [ name diff --git a/eng/conda_tools/environment.py b/eng/conda_tools/environment.py index fad5c18c7..1268ddd0f 100644 --- a/eng/conda_tools/environment.py +++ b/eng/conda_tools/environment.py @@ -139,14 +139,21 @@ def _sha256(path: str) -> str: return h.hexdigest() -def create_builder_env(conda: str) -> str: - """conda-build<26: 26.7.0 crashes in the local packaging phase; the 25.x series is stable. +def create_builder_env(conda: str, target_subdir: str, pyvers: list[str]) -> str: + """Keep the stable 25.x builder except where Windows 3.15 needs CFEP-27 support. + 26.9.1 reads the target Python's recorded site-packages path; 25.x computes Lib. + Pin that opt-in rather than reopening the range containing the 26.7.0 packaging crash. A DEDICATED env (not `install -n base`) because a hosted runner's base may be pinned to a python no conda-build<26 supports (e.g. 3.14), making a base install UNSOLVABLE; a fresh env lets conda pick a supported python. conda-forge only (--override-channels) avoids the defaults ToS; zstandard rides along so the RUNPATH audit reads .conda payloads here.""" env_name = "conda_builder" - _log(f"=== creating dedicated conda-build env ({env_name}: conda-build<26) ===") + builder_spec = ( + "conda-build=26.9.1" + if target_subdir.startswith("win-") and "3.15" in pyvers + else "conda-build<26" + ) + _log(f"=== creating dedicated conda-build env ({env_name}: {builder_spec}) ===") # Idempotent: a reused agent may already have this env; a pre-existing env makes # `conda create` fail. Remove first (best-effort). run_ok([conda, "env", "remove", "-y", "-n", env_name]) @@ -160,7 +167,7 @@ def create_builder_env(conda: str) -> str: "-c", "conda-forge", "--override-channels", - "conda-build<26", + builder_spec, "zstandard", ], what=f"conda create {env_name}", diff --git a/eng/conda_tools/inputs.py b/eng/conda_tools/inputs.py index af7f3d93d..cf8ebbb92 100644 --- a/eng/conda_tools/inputs.py +++ b/eng/conda_tools/inputs.py @@ -213,11 +213,7 @@ def validate_installed_inputs( if len(names) != len(set(names)): raise ValueError(f"{path}: duplicate payload members") metadata = {} - root = ( - "Lib/site-packages/" - if subdir.startswith("win-") - else (f"lib/python{python_tag[2]}.{python_tag[3:]}/site-packages/") - ) + root = contracts.site_packages_root(python_tag, subdir) for facts, present, owned, prefix in archive.installed_metadata(names, files): name = contracts.canonical_distribution_name(facts["name"]) if name in metadata: diff --git a/eng/conda_tools/release.py b/eng/conda_tools/release.py index 65202a583..bf7ab52da 100644 --- a/eng/conda_tools/release.py +++ b/eng/conda_tools/release.py @@ -19,7 +19,7 @@ Regex, parentheses, epochs, local/dev/post versions and other syntax fail closed. * the (required-subdir x Python) matrix is complete -- every required platform ships a package for every expected Python, honoring any per-subdir Python - override (e.g. win-arm64 ships only 3.12-3.14). + override (e.g. win-arm64 ships only 3.12-3.15). Exit code 0 = release-ready; non-zero = a violation was found (blocks publish). """ @@ -54,10 +54,8 @@ ">=": operator.ge, } -# Some subdirs legitimately ship a REDUCED Python matrix. win-arm64's conda -# dependencies (cryptography, pyodbc) are published on Anaconda `defaults` only -# for Python 3.12+, so 3.10/3.11 cannot be built there -- expect 3.12-3.14 only. -_DEFAULT_SUBDIR_PYTHONS = "win-arm64=3.12,3.13,3.14" +# win-arm64 retains its reduced matrix; its supported interpreters start at 3.12. +_DEFAULT_SUBDIR_PYTHONS = "win-arm64=3.12,3.13,3.14,3.15" @dataclass(frozen=True) @@ -197,7 +195,7 @@ def validate( ``python`` (``X.Y`` or ``''``). ``subdir_pythons`` maps a subdir to the Python versions expected FOR THAT - subdir, overriding ``expected_pythons`` (e.g. win-arm64 ships only 3.12-3.14). + subdir, overriding ``expected_pythons`` (e.g. win-arm64 ships only 3.12-3.15). """ errors: list[str] = [] expected_versions = expected_versions or {} @@ -388,11 +386,11 @@ def add_arguments(parser: argparse.ArgumentParser) -> None: "--allowed-subdirs", default="win-64,win-arm64,osx-64,osx-arm64,linux-64,linux-aarch64", ) - parser.add_argument("--pythons", default="3.10,3.11,3.12,3.13,3.14") + parser.add_argument("--pythons", default="3.10,3.11,3.12,3.13,3.14,3.15") parser.add_argument( "--subdir-pythons", default=_DEFAULT_SUBDIR_PYTHONS, - help="Per-subdir Python overrides, e.g. 'win-arm64=3.12,3.13,3.14'.", + help="Per-subdir Python overrides, e.g. 'win-arm64=3.12,3.13,3.14,3.15'.", ) parser.add_argument("--mssql-python-version", default=None) parser.add_argument( diff --git a/eng/conda_tools/verify.py b/eng/conda_tools/verify.py index ce1887a0a..83c1348cc 100644 --- a/eng/conda_tools/verify.py +++ b/eng/conda_tools/verify.py @@ -131,6 +131,8 @@ def _verify_impl( # (no link / post-link / arm64 exec) -- a pure "installable?" check. win-arm64 deps # span microsoft (noarch azure-identity/msal) + Anaconda defaults, so no # --strict-channel-priority. Pin the freshly built version so no channel can shadow it. + # Python 3.15 is supplied by conda-forge; retain the old channels for 3.12-3.14. + python_channel = "conda-forge" if py == "3.15" else "defaults" environment.run( [ conda, @@ -143,7 +145,7 @@ def _verify_impl( "-c", "microsoft", "-c", - "defaults", + python_channel, "--override-channels", f"python={py}", f"mssql-python={mssql_ver}", @@ -163,7 +165,7 @@ def _verify_impl( "-c", "microsoft", "-c", - "defaults", + python_channel, "--override-channels", f"python={py}", f"mssql-python={mssql_ver}", diff --git a/tests/test_027_conda_release_metadata.py b/tests/test_027_conda_release_metadata.py index 185ca6b97..7d599683f 100644 --- a/tests/test_027_conda_release_metadata.py +++ b/tests/test_027_conda_release_metadata.py @@ -506,10 +506,31 @@ def test_default_subdir_pythons_reduces_win_arm64(): # The shipped CLI default carries the win-arm64 reduction so the pipeline needs # no extra flag; parsing it yields the expected mapping. assert vcr._parse_subdir_pythons(vcr._DEFAULT_SUBDIR_PYTHONS) == { - "win-arm64": ["3.12", "3.13", "3.14"] + "win-arm64": ["3.12", "3.13", "3.14", "3.15"] } +def test_conda_python_defaults_are_synchronized(): + args = cli.parser().parse_args(["validate", "--root", "unused"]) + assert args.pythons == "3.10,3.11,3.12,3.13,3.14,3.15" + assert args.subdir_pythons == "win-arm64=3.12,3.13,3.14,3.15" + for path in ( + "OneBranchPipelines/conda-build-pipeline.yml", + "OneBranchPipelines/steps/conda-release-step.yml", + "OneBranchPipelines/steps/conda-publish-step.yml", + ): + source = (_ROOT / path).read_text(encoding="utf-8") + parameter = source.split("- name: pythonVersions\n", 1)[1].split("\n - name:", 1)[0] + assert f"default: '{args.pythons}'" in parameter, path + build_source = (_ROOT / "OneBranchPipelines/conda-build-pipeline.yml").read_text( + encoding="utf-8" + ) + assert ( + f"pythonVersion: '{args.subdir_pythons.split('=')[1]}'\n" + " condaSubdir: 'win-arm64'" + ) in build_source + + def test_parse_subdir_pythons(): assert vcr._parse_subdir_pythons("") == {} assert vcr._parse_subdir_pythons("a=3.10;b=3.11,3.12") == { @@ -1514,9 +1535,18 @@ def test_metadata_agreement_across_full_archive_matrix(tmp_path, capsys, build, index.update(build=build, depends=depends) _write_release_archive(tmp_path, folder=subdir, **index) assert len(list(tmp_path.rglob("*.tar.bz2"))) == 28 - assert _metadata_main(["--root", str(tmp_path), "--mssql-python-version", _MP_VER]) == bool( - error - ) + assert _metadata_main( + [ + "--root", + str(tmp_path), + "--mssql-python-version", + _MP_VER, + "--pythons", + ",".join(_PYTHONS), + "--subdir-pythons", + "win-arm64=3.12,3.13,3.14", + ] + ) == bool(error) output = capsys.readouterr() assert error in output.err if error else "metadata-validated" in output.out @@ -1895,22 +1925,27 @@ def read(path, *args, **kwargs): "count, policy, message", [ (0, [], "no conda packages"), - (25, [], "win-arm64"), - (28, [], ""), - (28, ["--mssql-python-version", "wrong"], "!= expected"), - (28, ["--required-subdirs", ""], "must not be empty"), - (28, ["--required-subdirs", " , "], "must not be empty"), - (28, ["--allowed-subdirs", ""], "must not be empty"), - (28, ["--allowed-subdirs", " , "], "must not be empty"), - (28, ["--pythons", ""], "must not be empty"), - (28, ["--pythons", " , "], "must not be empty"), - (28, ["--allowed-subdirs", "invalid"], "not in allowed set"), + (33, [], "INCOMPLETE"), + (34, [], ""), + (34, ["--mssql-python-version", "wrong"], "!= expected"), + (34, ["--required-subdirs", ""], "must not be empty"), + (34, ["--required-subdirs", " , "], "must not be empty"), + (34, ["--allowed-subdirs", ""], "must not be empty"), + (34, ["--allowed-subdirs", " , "], "must not be empty"), + (34, ["--pythons", ""], "must not be empty"), + (34, ["--pythons", " , "], "must not be empty"), + (34, ["--allowed-subdirs", "invalid"], "not in allowed set"), ], ) def test_metadata_cli_enforces_full_default_release_policy( tmp_path, monkeypatch, capsys, count, policy, message ): - packages = _healthy_set() + [_binding("win-arm64", py) for py in _PYTHONS[2:]] + packages = ( + _healthy_set() + + [_binding("win-arm64", py) for py in _PYTHONS[2:]] + + [_binding(subdir, "3.15") for subdir in _ALLOWED] + ) + assert len(packages) == 34 monkeypatch.setattr(vcr, "collect_packages", lambda _root: packages[:count]) assert _metadata_main(["--root", str(tmp_path), *policy]) == (1 if message else 0) output = capsys.readouterr() @@ -1945,10 +1980,18 @@ def test_metadata_cli_rejects_distinct_builds_in_one_matrix_cell(tmp_path, capsy build=f"py{py.replace('.', '')}_0", depends=[f"python {py}.*"], ) - assert _metadata_main(["--root", str(tmp_path)]) == 0 + policy = [ + "--root", + str(tmp_path), + "--pythons", + ",".join(_PYTHONS), + "--subdir-pythons", + "win-arm64=3.12,3.13,3.14", + ] + assert _metadata_main(policy) == 0 _write_release_archive(tmp_path, build="py312_1") assert len(list(tmp_path.rglob("*.tar.bz2"))) == 29 - assert _metadata_main(["--root", str(tmp_path)]) == 1 + assert _metadata_main(policy) == 1 assert "DUPLICATE" in capsys.readouterr().err diff --git a/tests/test_030_pe_machine_assert.py b/tests/test_030_pe_machine_assert.py index 8f4a11efd..9f2b77852 100644 --- a/tests/test_030_pe_machine_assert.py +++ b/tests/test_030_pe_machine_assert.py @@ -186,7 +186,14 @@ def test_binding_rs_requirement_accepts_metadata_spacing(spacing): @pytest.mark.skipif(sys.platform != "win32", reason="Windows recipe requires cmd.exe") -@pytest.mark.parametrize("cross_build", [False, True]) +@pytest.mark.parametrize( + "cross_build,tag", + [ + (False, f"{sys.version_info.major}{sys.version_info.minor}"), + (True, f"{sys.version_info.major}{sys.version_info.minor}"), + (True, "315"), + ], +) @pytest.mark.parametrize( ("rs_owned", "state"), [ @@ -196,12 +203,13 @@ def test_binding_rs_requirement_accepts_metadata_spacing(spacing): ] + [(True, "missing-private-library")], ) -def test_windows_recipe_requires_core_on_both_install_paths(tmp_path, cross_build, rs_owned, state): +def test_windows_recipe_requires_core_on_both_install_paths( + tmp_path, cross_build, tag, rs_owned, state +): wheels = tmp_path / "wheels" wheels.mkdir() prefix = tmp_path / "prefix" - site_packages = prefix / "Lib" / "site-packages" - tag = f"{sys.version_info.major}{sys.version_info.minor}" + site_packages = prefix / ("lib/python" if int(tag) >= 315 else "Lib") / "site-packages" arch = sysconfig.get_platform().replace("-", "_") machine = _ARM64 if arch == "win_arm64" else _AMD64 core = f"mssql_py_core/mssql_py_core.cp{tag}-{arch}.pyd" @@ -256,6 +264,7 @@ def test_windows_recipe_requires_core_on_both_install_paths(tmp_path, cross_buil env = dict( os.environ, PREFIX=str(prefix), + SP_DIR=str(site_packages), PYTHON=str(tmp_path / "nonexecutable-python") if cross_build else sys.executable, PKG_NAME="mssql-python", PKG_VERSION="1.13.0", @@ -298,9 +307,12 @@ def test_windows_recipe_requires_core_on_both_install_paths(tmp_path, cross_buil ) in output -def _make_conda(tmp_path, subdir, payload, depends=("python_abi 3.12.* *_cp312",)): +def _make_conda( + tmp_path, subdir, payload, depends=("python_abi 3.12.* *_cp312",), python_tag="cp312" +): """Build a minimal .conda (info-*.tar.zst + pkg-*.tar.zst) with the given payload files.""" - name = "mssql-python-1.13.0-py312_0" + build = f"py{python_tag[2:]}_0" + name = f"mssql-python-1.13.0-{build}" pkg_buf = io.BytesIO() with tarfile.open(fileobj=pkg_buf, mode="w") as tf: @@ -312,7 +324,7 @@ def _make_conda(tmp_path, subdir, payload, depends=("python_abi 3.12.* *_cp312", index = { "name": "mssql-python", "version": "1.13.0", - "build": "py312_0", + "build": build, "subdir": subdir, "depends": depends, } @@ -330,6 +342,21 @@ def _make_conda(tmp_path, subdir, payload, depends=("python_abi 3.12.* *_cp312", return str(conda_path) +@pytest.mark.parametrize( + "root", + ["Lib/site-packages/", "lib/python3.15/site-packages/", "nested/lib/python/site-packages/"], +) +def test_cp315_native_layout_rejects_other_windows_roots(root): + names = [ + root + "mssql_python/ddbc_bindings.cp315-win_amd64.pyd", + root + "mssql_py_core/__init__.py", + root + "mssql_py_core/mssql_py_core.pyd", + ] + assert contracts.validate_native_contract( + names, {"subdir": "win-64", "depends": ["python_abi 3.15.* *_cp315"]} + ) + + @pytest.mark.skipif(not _zstd_available(), reason="no zstandard backend available") def test_malformed_dependencies_are_reported(tmp_path): errors = audit.audit_package( @@ -435,7 +462,8 @@ def test_win_arm64_binaries_require_binding_metadata(tmp_path, metadata_state): "wrong-abi", ], ) -def test_required_core_contract(tmp_path, state): +@pytest.mark.parametrize("python_tag", ["cp312", "cp315"]) +def test_required_core_contract(tmp_path, state, python_tag): pin = "python_abi 3.12.* *_cp312" # The observed defaults CP312 host supplied only the Python range, not an ABI export. depends = ["vc14_runtime", "python >=3.12,<3.13.0a0", "azure-identity >=1.12.0"] @@ -464,8 +492,19 @@ def test_required_core_contract(tmp_path, state): payload[prefix + "RECORD"] = "".join( f"{name.removeprefix('Lib/site-packages/')},,\n" for name in [*payload, prefix + "RECORD"] ).encode() + if python_tag == "cp315": + payload = { + name.replace("Lib/site-packages/", "lib/python/site-packages/").replace( + "cp312", "cp315" + ): data.replace(b"cp312", b"cp315") + for name, data in payload.items() + } + depends = [ + dep.replace("3.12", "3.15").replace("3.13", "3.16").replace("cp312", "cp315") + for dep in depends + ] errors = audit.audit_package( - _make_conda(tmp_path, "win-arm64", payload, depends=depends), "pe" + _make_conda(tmp_path, "win-arm64", payload, depends=depends, python_tag=python_tag), "pe" ).violations if state in ("valid", "abi3"): assert errors == [] diff --git a/tests/test_034_conda_verify_cwd.py b/tests/test_034_conda_verify_cwd.py index ec01f2b9d..befe385c4 100644 --- a/tests/test_034_conda_verify_cwd.py +++ b/tests/test_034_conda_verify_cwd.py @@ -352,7 +352,7 @@ def test_main_routes_native_effective_subdir_without_cross_target(tmp_path, monk monkeypatch.setattr(build, "gather_wheels", lambda *_args: ("1.2.3", "18.6.2", None)) monkeypatch.setattr(environment, "find_or_install_conda", lambda _output_dir: "conda") monkeypatch.setattr(environment, "run", lambda *_args, **_kwargs: None) - monkeypatch.setattr(environment, "create_builder_env", lambda _conda: "conda_builder") + monkeypatch.setattr(environment, "create_builder_env", lambda *_args: "conda_builder") monkeypatch.setattr(build, "detect_pythons", lambda *_args: ["3.13"]) def _build_env(_mssql_ver, _odbc_ver, _links, cross_target_subdir, _rs_ver): @@ -856,6 +856,20 @@ def test_binding_selection_requires_the_requested_normal_abi(abi): assert not contracts.binding_wheel_matches_target(wheel, "win-64", ["3.13"]) +@pytest.mark.parametrize("subdir,platform", contracts._RS_PLATFORMS.items()) +def test_cp315_selects_ordinary_binding_and_existing_rs_abi3(tmp_path, subdir, platform): + binding = f"mssql_python-1.2.3-cp315-cp315-{platform}.whl" + assert contracts.binding_wheel_matches_target(binding, subdir, ["3.15"]) + assert not contracts.binding_wheel_matches_target( + binding.replace("-cp315-cp315-", "-cp315-cp315t-"), subdir, ["3.15"] + ) + selected = tmp_path / f"mssql_python_rs-0.3.0-cp310-abi3-{platform}.whl" + selected.touch() + (tmp_path / f"mssql_python_rs-0.3.0-cp315-cp315t-{platform}.whl").touch() + (tmp_path / f"mssql_python_rs-0.3.0-cp316-abi3-{platform}.whl").touch() + assert build._select_rs_wheel(str(tmp_path), "0.3.0", "cp315", subdir) == str(selected) + + @pytest.mark.parametrize( "state", ["valid", "missing-rs", "mismatched-rs", "odbc-core", "odbc-target"] ) @@ -1134,17 +1148,20 @@ def test_malformed_record_is_an_explicit_error(record): @pytest.mark.parametrize( - ("target_subdir", "expected_channels"), + ("target_subdir", "python_version", "expected_channels"), [ - ("", ["microsoft", "conda-forge"]), - ("win-64", ["microsoft", "conda-forge"]), - ("osx-arm64", ["microsoft", "conda-forge"]), - ("linux-aarch64", ["microsoft", "conda-forge"]), - ("win-arm64", ["defaults", "microsoft", "conda-forge"]), + ("", "3.13", ["microsoft", "conda-forge"]), + ("win-64", "3.13", ["microsoft", "conda-forge"]), + ("osx-arm64", "3.13", ["microsoft", "conda-forge"]), + ("linux-aarch64", "3.13", ["microsoft", "conda-forge"]), + ("win-arm64", "3.12", ["defaults", "microsoft", "conda-forge"]), + ("win-arm64", "3.13", ["defaults", "microsoft", "conda-forge"]), + ("win-arm64", "3.14", ["defaults", "microsoft", "conda-forge"]), + ("win-arm64", "3.15", ["microsoft", "conda-forge"]), ], ) -def test_conda_build_uses_only_explicit_channels( - target_subdir, expected_channels, tmp_path, monkeypatch +def test_conda_build_and_verify_use_only_explicit_channels( + target_subdir, python_version, expected_channels, tmp_path, monkeypatch ): calls = [] croot = tmp_path / "croot" @@ -1160,7 +1177,7 @@ def _capture_run(command, **_kwargs): "conda", "conda_builder", str(tmp_path / "recipe"), - ["3.13"], + [python_version], str(tmp_path / "conda-bld"), target_subdir, {}, @@ -1174,6 +1191,52 @@ def _capture_run(command, **_kwargs): assert [command[index + 1] for index, arg in enumerate(command) if arg == "-c"] == ( expected_channels ) + monkeypatch.setattr(environment, "run_ok", lambda *_args, **_kwargs: 0) + monkeypatch.setattr(environment, "run_capture", lambda *_args, **_kwargs: (0, "")) + verify._verify_impl( + "conda", + "local-channel", + str(tmp_path), + [python_version], + "1.2.3", + target_subdir, + True, + {}, + ) + creates = [cmd for cmd in calls if cmd[1:2] == ["create"]] + assert len(creates) == (2 if target_subdir == "win-arm64" else 1) + verify_channels = ( + ["microsoft", "defaults"] + if target_subdir == "win-arm64" and python_version != "3.15" + else ["microsoft", "conda-forge"] + ) + for command in creates: + assert [command[i + 1] for i, arg in enumerate(command) if arg == "-c"] == [ + "local-channel", + *verify_channels, + ] + + +@pytest.mark.parametrize( + "target,pyvers,spec", + [ + ("win-64", ["3.10", "3.14"], "conda-build<26"), + ("win-arm64", ["3.12", "3.14"], "conda-build<26"), + ("win-64", ["3.10", "3.15"], "conda-build=26.9.1"), + ("win-arm64", ["3.12", "3.15"], "conda-build=26.9.1"), + ("linux-64", ["3.15"], "conda-build<26"), + ("linux-aarch64", ["3.15"], "conda-build<26"), + ("osx-64", ["3.15"], "conda-build<26"), + ("osx-arm64", ["3.15"], "conda-build<26"), + ], +) +def test_builder_upgrade_is_limited_to_windows_315(target, pyvers, spec, monkeypatch): + calls = [] + monkeypatch.setattr(environment, "run_ok", lambda *_args, **_kwargs: 0) + monkeypatch.setattr(environment, "run", lambda cmd, **_kwargs: calls.append(cmd)) + assert environment.create_builder_env("conda", target, pyvers) == "conda_builder" + assert len(calls) == 1 + assert [arg for arg in calls[0] if arg.startswith("conda-build")] == [spec] @pytest.mark.parametrize("state", ["native", "pure-python", "foreign"]) diff --git a/tests/test_036_conda_provenance.py b/tests/test_036_conda_provenance.py index 459fce977..4ed4edcc1 100644 --- a/tests/test_036_conda_provenance.py +++ b/tests/test_036_conda_provenance.py @@ -58,7 +58,7 @@ def release_sources(): } -def _component_files(name, version, rs=None): +def _component_files(name, version, rs=None, python_tag="cp313"): prefix = f"{name.replace('-', '_')}-{version}.dist-info/" metadata = f"Metadata-Version: 2.4\nName: {name}\nVersion: {version}\n" files = {} @@ -71,14 +71,23 @@ def _component_files(name, version, rs=None): files.update( { "mssql_py_core/__init__.py": b"", - "mssql_py_core/mssql_py_core.cp313-win_amd64.pyd": b"native fixture", + f"mssql_py_core/mssql_py_core.{python_tag}-win_amd64.pyd": b"native fixture", } ) if name == "mssql-python-rs": files.update( {path: b"private native fixture" for path in contracts.rs_private_libraries("win-64")} ) - tag = "py3-none-win_amd64" if name == "mssql-python-odbc" else "cp313-cp313-win_amd64" + tag = ( + "py3-none-win_amd64" + if name == "mssql-python-odbc" + else f"{python_tag}-{python_tag}-win_amd64" + ) + if name == "mssql-python-rs" and python_tag == "cp315": + tag = "cp310-abi3-win_amd64" + files["mssql_py_core/mssql_py_core.pyd"] = files.pop( + f"mssql_py_core/mssql_py_core.{python_tag}-win_amd64.pyd" + ) files[prefix + "METADATA"] = metadata.encode() files[prefix + "WHEEL"] = f"Wheel-Version: 1.0\nTag: {tag}\n".encode() files[prefix + "RECORD"] = "".join( @@ -339,6 +348,7 @@ def download(argv, *, check): "missing-record", "missing-wheel", "wrong-root", + "other-python-root", "duplicate-metadata", "unowned-core", "unrecorded-core", @@ -378,19 +388,23 @@ def download(argv, *, check): ), ], ) -def test_source_bound_release_components(tmp_path, monkeypatch, capsys, problem): +@pytest.mark.parametrize("python_version", ["3.13", "3.15"]) +def test_source_bound_release_components(tmp_path, monkeypatch, capsys, problem, python_version): + python_tag = "cp" + python_version.replace(".", "") + minor = int(python_version.split(".")[1]) with_rs = problem not in {"legacy", "new-source-legacy"} and not problem.startswith("legacy-") versions = {"mssql-python": "1.15.0", "mssql-python-odbc": "18.6.2.1"} if with_rs: versions["mssql-python-rs"] = "0.2.0" files = {} for name, version in versions.items(): - files.update(_component_files(name, version, "0.2.0" if with_rs else None)) + files.update(_component_files(name, version, "0.2.0" if with_rs else None, python_tag)) if problem == "new-source-legacy": versions["mssql-python-rs"] = "0.2.0" rs_prefix = "mssql_python_rs-0.2.0.dist-info/" binding_key = "mssql_python-1.15.0.dist-info/METADATA" - selected = "mssql_python_rs-0.2.0-cp313-cp313-win_amd64.whl" + rs_tag = "cp310-abi3" if python_tag == "cp315" else "cp313-cp313" + selected = f"mssql_python_rs-0.2.0-{rs_tag}-win_amd64.whl" receipt = { "distribution_version": "0.2.0", "transport_version": "0.2.0-dev.transport", @@ -464,21 +478,27 @@ def test_source_bound_release_components(tmp_path, monkeypatch, capsys, problem) elif problem == "wrong-wheel-tags": files[rs_prefix + "WHEEL"] = b"Tag: cp310-cp310-win_amd64\n" elif problem == "wrong-wheel-target": - wrong = selected.replace("313", "312") + wrong = "mssql_python_rs-0.2.0-cp316-cp316-win_amd64.whl" receipt["wheel_sha256"] = {wrong: "b" * 64} files[rs_prefix + "conda-wheel-source.txt"] = (wrong + "\n").encode() - files[rs_prefix + "WHEEL"] = b"Tag: cp312-cp312-win_amd64\n" + files[rs_prefix + "WHEEL"] = b"Tag: cp316-cp316-win_amd64\n" root = tmp_path / "conda" - package = root / "win-64" / "mssql-python-1.15.0-py313_0.tar.bz2" + package = root / "win-64" / f"mssql-python-1.15.0-py{python_tag[2:]}_0.tar.bz2" package.parent.mkdir(parents=True) index = { "name": "mssql-python", "version": "1.15.0", "subdir": "win-64", - "build": "py313_0", - "depends": ["python >=3.13,<3.14.0a0", "python_abi 3.13.* *_cp313"], + "build": f"py{python_tag[2:]}_0", + "depends": [ + f"python >={python_version},<3.{minor + 1}.0a0", + f"python_abi {python_version}.* *_{python_tag}", + ], } - prefix = "wrong/site-packages/" if problem == "wrong-root" else "Lib/site-packages/" + canonical_root = "lib/python/site-packages/" if python_tag == "cp315" else "Lib/site-packages/" + prefix = "wrong/site-packages/" if problem == "wrong-root" else canonical_root + if problem == "other-python-root": + prefix = "Lib/site-packages/" if python_tag == "cp315" else "lib/python/site-packages/" members = [ ("info/index.json", json.dumps(index).encode()), *((prefix + key, value) for key, value in files.items()), @@ -486,7 +506,7 @@ def test_source_bound_release_components(tmp_path, monkeypatch, capsys, problem) if problem == "duplicate-payload": members.append(members[-1]) elif problem == "core-root-alias": - members.append(("LIB/SITE-PACKAGES/mssql_py_core/unrecorded.py", b"")) + members.append((canonical_root.upper() + "mssql_py_core/unrecorded.py", b"")) with tarfile.open(package, "w:bz2") as contents: for name, data in members: entry = tarfile.TarInfo(name) @@ -515,7 +535,7 @@ def test_source_bound_release_components(tmp_path, monkeypatch, capsys, problem) "--allowed-subdirs", "win-64", "--pythons", - "3.13", + python_version, "--mssql-python-version", "0.0.0" if problem == "mismatched-binding-override" else "1.15.0", "--release-versions", From dac63f0027b347d592235e347afe78ac0db420a5 Mon Sep 17 00:00:00 2001 From: Jahnvi Thakkar Date: Mon, 12 Oct 2026 14:04:16 +0530 Subject: [PATCH 22/23] FIX: Pin the published Conda PR smoke baseline Keep current-checkout source tests blocking while selecting an explicit Linux CPython 3.11 binding/ODBC baseline independent of unpublished source releases. Reuse strict PyPI metadata and dependency validation and additionally enforce reviewed wheel filenames and SHA256 pins. Production and release source pins are unchanged. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .github/workflows/conda-audit.yml | 26 +++++--------- conda/README.md | 17 ++++++--- eng/conda_tools/inputs.py | 28 ++++++++++++++- tests/test_036_conda_provenance.py | 57 ++++++++++++++++++++++++++++++ 4 files changed, 106 insertions(+), 22 deletions(-) diff --git a/.github/workflows/conda-audit.yml b/.github/workflows/conda-audit.yml index b1a2abe76..36d506ec7 100644 --- a/.github/workflows/conda-audit.yml +++ b/.github/workflows/conda-audit.yml @@ -3,7 +3,7 @@ # # The OneBranch conda-build pipeline is `trigger: none` / `pr: none`, so its blocking # audit never runs on a PR. This lightweight GitHub Actions job builds ONE real -# linux-64 conda package from the SHIPPED PyPI wheels and runs +# linux-64 conda package from a separately pinned PUBLISHED SMOKE baseline and runs # python -m eng.conda_tools elf on it, so a regression in the $ORIGIN climb, # the declared conda deps (krb5/libtool/openssl), or the expected DT_NEEDED set fails # the PR automatically -- the full-agent runtime masking cannot hide it. @@ -11,7 +11,9 @@ # build.sh ALWAYS stamps the relative $ORIGIN climb onto the vendored Linux ODBC .so # (they are not code-signed, so the patch breaks no signature; an already-baked climb # is a byte-for-byte no-op). This gate builds one real linux-64 conda package from the -# shipped PyPI wheels and audits the resulting climb (see conda/mssql-python/build.sh). +# published baseline and audits the resulting climb (see conda/mssql-python/build.sh). +# It is a historical embedded-core CPython 3.11 control, not current-source RS/3.15 +# qualification; source-only tests above it remain current-checkout and blocking. # # PR triggers are conda-specific: general package/version changes and shared wheel # build infrastructure do not run this gate unless the PR also changes a path below. @@ -72,34 +74,24 @@ jobs: tests/test_038_conda_archive_limits.py \ tests/test_027_conda_release_metadata.py -q - - name: Fetch the shipped linux wheels from PyPI - id: public-wheels + - name: Fetch the pinned published CPython 3.11 smoke baseline working-directory: ${{ github.workspace }} run: | set -euo pipefail - # Exact maintained release versions and PyPI digests, never an older/latest fallback. - # Fetch RS only if binding METADATA requires it, using its distribution (not NuGet) pin. - # Embedded-core public wheels are historical controls, not current-source RS proof. + # inputs.py maintains exact baseline versions, filenames and immutable hashes. + # PyPI hashes and actual METADATA/WHEEL/RECORD are still checked; no fallback. python -m eng.conda_tools fetch-wheels \ + --published-smoke \ --wheel-dir "$PWD/wheels" --requirements-file "$PWD/wheel-inputs.txt" \ --python-tag cp311 --conda-subdir linux-64 echo "Downloaded:"; ls -1 wheels - name: Build + audit a linux-64 conda package working-directory: ${{ github.workspace }} - env: - RS_REQUIRED: ${{ steps.public-wheels.outputs.rsRequired }} run: | set -euo pipefail - # The verified public dependency/ownership facts select the actual input set. - rsArgs=() - if [[ "$RS_REQUIRED" == "true" ]]; then - rsArgs=(--rs-wheel-dir "$PWD/wheels" --rs-version-file "$PWD/eng/versions/mssql-python-rs.version") - elif [[ "$RS_REQUIRED" != "false" ]]; then - echo "Missing verified public wheel profile"; exit 1 - fi + # This baseline owns its embedded core; no current-source RS input is substituted. python -m eng.conda_tools build \ - "${rsArgs[@]}" \ --mssql-wheel-dir "$PWD/wheels" \ --odbc-wheel-dir "$PWD/wheels" \ --odbc-wheel-filter "mssql_python_odbc-*.whl" \ diff --git a/conda/README.md b/conda/README.md index 2056b126d..9d07a2981 100644 --- a/conda/README.md +++ b/conda/README.md @@ -57,8 +57,15 @@ The `conda-audit` PR gate triggers only for `conda/`, `eng/conda_tools/`, Conda- OneBranch pipelines/steps/jobs, their native/probe/archive/provenance and release tests, and the workflow itself. General package metadata/version changes and shared wheel-build infrastructure do not trigger it on their own; mixed PRs with a Conda change still do. -This scope does not change the unit-test runner, source-version wheel selection, or -failure handling when a required wheel is unpublished. +Source-only tests remain current-checkout and blocking. The real Linux CPython 3.11 +build/audit uses `fetch-wheels --published-smoke`: a separate published baseline of +binding **1.15.0** and its exact ODBC dependency **18.6.2.1**, with embedded core and +no separate RS wheel. `eng/conda_tools/inputs.py` maintains its versions, exact wheel +filenames and SHA256 pins; update them together after verifying the new wheel metadata +and ownership. Source release bumps do not change this baseline. Missing wheels, +metadata/dependency mismatches, digest changes and build/audit failures still fail the +gate; there is no skip or automatic fallback. This historical packaging control does +not qualify current-source RS or Python 3.15. Run the source-only trigger regressions without a native build, SQL Server, or PyYAML: @@ -282,17 +289,19 @@ members. This detects unowned or cross-owned additions; it cannot recover file o after an overwrite of an already owned path, which is why the raw pre-extraction gate is required. -`fetch-wheels` uses exact current maintained versions, hash-required binary-only PyPI +By default, `fetch-wheels` uses exact current maintained versions, hash-required binary-only PyPI downloads, and actual wheel metadata/ownership. Pip selects for its executing interpreter; the requested Python/platform must match the binding, ODBC and selected RS wheel tags before the command accepts the inputs. These arguments validate the download, rather than enabling cross-target pip resolution. An authentic published embedded-core binding is explicitly reported as a **historical published packaging control, not current-source RS qualification**. An RS-dependent published binding requires its exact -provider and source pin; unavailable releases, malformed declarations, or mismatches +provider and selected pin; unavailable releases, malformed declarations, or mismatches fail with no older-version or TLS fallback. Public availability does not gate the recorded ADO artifact path. Neither a public input check nor static archive checks establish installed SQL/runtime qualification. +Only the explicit `--published-smoke` option selects the independent baseline above; +production, provenance, preflight and release/publish version selection are unchanged. The driver-load probe's sole implementation is `eng/conda_tools/driver_load_probe.py`. It separates native connection execution, diff --git a/eng/conda_tools/inputs.py b/eng/conda_tools/inputs.py index cf8ebbb92..205006e60 100644 --- a/eng/conda_tools/inputs.py +++ b/eng/conda_tools/inputs.py @@ -19,6 +19,18 @@ from . import archive, contracts +# PR-only linux-64/cp311 packaging control, maintained independently of source releases. +_PUBLISHED_SMOKE_VERSIONS = {"mssql-python": "1.15.0", "mssql-python-odbc": "18.6.2.1"} +_PUBLISHED_SMOKE_SHA256 = { + "mssql_python-1.15.0-cp311-cp311-manylinux_2_28_x86_64.whl": ( + "016c61a4c82a1b69a8f8a43c91dc3537463d8ca068da73fa91f7e348b48e6d22" + ), + "mssql_python_odbc-18.6.2.1-py3-none-manylinux_2_28_x86_64.whl": ( + "b8aad821f3b8637b0e6d4637f4b730ed9897c4665fa99d47e44e97fd1c79c77a" + ), +} + + def _is_sha256(value: object) -> bool: return isinstance(value, str) and re.fullmatch(r"[0-9a-f]{64}", value) is not None @@ -278,6 +290,11 @@ def validate_installed_inputs( def add_arguments(parser: argparse.ArgumentParser) -> None: parser.add_argument("--wheel-dir", type=Path, required=True) parser.add_argument("--requirements-file", type=Path, required=True) + parser.add_argument( + "--published-smoke", + action="store_true", + help="Use the separately pinned linux-64/cp311 PR baseline, not source release versions.", + ) parser.add_argument( "--python-tag", required=True, @@ -292,10 +309,19 @@ def add_arguments(parser: argparse.ArgumentParser) -> None: def fetch_cli(args: argparse.Namespace) -> int: try: - source = read_release_versions(lambda path: Path(path).read_text(encoding="utf-8")) + if args.published_smoke: + if (args.python_tag, args.conda_subdir) != ("cp311", "linux-64"): + raise ValueError("The published smoke baseline requires cp311 linux-64.") + source = _PUBLISHED_SMOKE_VERSIONS + else: + source = read_release_versions(lambda path: Path(path).read_text(encoding="utf-8")) versions = fetch_wheels( source, args.wheel_dir, args.requirements_file, args.python_tag, args.conda_subdir ) + if args.published_smoke: + actual = {path.name: file_sha256(path) for path in args.wheel_dir.glob("*.whl")} + if actual != _PUBLISHED_SMOKE_SHA256: + raise ValueError("Published smoke wheels differ from the pinned filenames/SHA256.") rs_required = "mssql-python-rs" in versions if output := os.environ.get("GITHUB_OUTPUT"): with open(output, "a", encoding="utf-8") as destination: diff --git a/tests/test_036_conda_provenance.py b/tests/test_036_conda_provenance.py index 4ed4edcc1..69fd7928a 100644 --- a/tests/test_036_conda_provenance.py +++ b/tests/test_036_conda_provenance.py @@ -546,6 +546,63 @@ def test_source_bound_release_components(tmp_path, monkeypatch, capsys, problem, ), capsys.readouterr() +@pytest.mark.parametrize("profile", ["source", "smoke", "smoke-bad-hash", "smoke-wrong-target"]) +def test_published_smoke_is_explicit_and_independent_of_unpublished_source( + tmp_path, monkeypatch, capsys, profile +): + source = {"mssql-python": "1.16.0", "mssql-python-odbc": "18.6.2.2"} + reads, selected = [], [] + monkeypatch.setattr(inputs, "read_release_versions", lambda read: reads.append(True) or source) + + def fetch(versions, *args): + selected.append(versions) + for filename in inputs._PUBLISHED_SMOKE_SHA256: + (tmp_path / filename).touch() + return versions + + monkeypatch.setattr(inputs, "fetch_wheels", fetch) + monkeypatch.setattr( + inputs, + "file_sha256", + lambda path: ( + "0" * 64 if profile == "smoke-bad-hash" else inputs._PUBLISHED_SMOKE_SHA256[path.name] + ), + ) + output = tmp_path / "outputs" + monkeypatch.setenv("GITHUB_OUTPUT", str(output)) + result = cli.main( + [ + "fetch-wheels", + "--wheel-dir", + str(tmp_path), + "--requirements-file", + str(tmp_path / "pins"), + "--python-tag", + "cp315" if profile == "smoke-wrong-target" else "cp311", + "--conda-subdir", + "linux-64", + *([] if profile == "source" else ["--published-smoke"]), + ] + ) + assert reads == ([True] if profile == "source" else []) + assert selected == ( + [] + if profile == "smoke-wrong-target" + else [ + ( + source + if profile == "source" + else {"mssql-python": "1.15.0", "mssql-python-odbc": "18.6.2.1"} + ) + ] + ) + failed = profile in {"smoke-bad-hash", "smoke-wrong-target"} + assert result == int(failed) + assert output.exists() == (not failed) + if failed: + assert "ERROR:" in capsys.readouterr().err + + @pytest.mark.parametrize("with_rs", [False, True]) def test_public_cli_reports_qualified_profile_without_importing_runtime( tmp_path, monkeypatch, capsys, with_rs From db5a336129007e3e333965a98955be1f8a22c3d1 Mon Sep 17 00:00:00 2001 From: Jahnvi Thakkar Date: Mon, 12 Oct 2026 14:25:40 +0530 Subject: [PATCH 23/23] FIX: Select Conda smoke inputs from PyPI Remove hardcoded smoke versions and SHA256 literals. Select the current published binding and its exact ODBC/RS requirements from PyPI while retaining runtime PyPI digest and wheel metadata validation. Keep production source-bound inputs unchanged. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .github/workflows/conda-audit.yml | 23 +++++++---- conda/README.md | 18 ++++---- eng/conda_tools/inputs.py | 50 +++++++++++++--------- tests/test_036_conda_provenance.py | 66 +++++++++++++++++++----------- 4 files changed, 98 insertions(+), 59 deletions(-) diff --git a/.github/workflows/conda-audit.yml b/.github/workflows/conda-audit.yml index 36d506ec7..f733a3c59 100644 --- a/.github/workflows/conda-audit.yml +++ b/.github/workflows/conda-audit.yml @@ -3,7 +3,7 @@ # # The OneBranch conda-build pipeline is `trigger: none` / `pr: none`, so its blocking # audit never runs on a PR. This lightweight GitHub Actions job builds ONE real -# linux-64 conda package from a separately pinned PUBLISHED SMOKE baseline and runs +# linux-64 conda package from the current published PyPI wheels and runs # python -m eng.conda_tools elf on it, so a regression in the $ORIGIN climb, # the declared conda deps (krb5/libtool/openssl), or the expected DT_NEEDED set fails # the PR automatically -- the full-agent runtime masking cannot hide it. @@ -11,8 +11,8 @@ # build.sh ALWAYS stamps the relative $ORIGIN climb onto the vendored Linux ODBC .so # (they are not code-signed, so the patch breaks no signature; an already-baked climb # is a byte-for-byte no-op). This gate builds one real linux-64 conda package from the -# published baseline and audits the resulting climb (see conda/mssql-python/build.sh). -# It is a historical embedded-core CPython 3.11 control, not current-source RS/3.15 +# published wheels and audits the resulting climb (see conda/mssql-python/build.sh). +# It is a published CPython 3.11 packaging control, not current-source RS/3.15 # qualification; source-only tests above it remain current-checkout and blocking. # # PR triggers are conda-specific: general package/version changes and shared wheel @@ -74,12 +74,13 @@ jobs: tests/test_038_conda_archive_limits.py \ tests/test_027_conda_release_metadata.py -q - - name: Fetch the pinned published CPython 3.11 smoke baseline + - name: Fetch the published CPython 3.11 smoke wheels from PyPI + id: public-wheels working-directory: ${{ github.workspace }} run: | set -euo pipefail - # inputs.py maintains exact baseline versions, filenames and immutable hashes. - # PyPI hashes and actual METADATA/WHEEL/RECORD are still checked; no fallback. + # Select PyPI's current binding and its exact ODBC/RS dependencies, not source pins. + # Verify PyPI hashes and actual METADATA/WHEEL/RECORD; no older-release fallback. python -m eng.conda_tools fetch-wheels \ --published-smoke \ --wheel-dir "$PWD/wheels" --requirements-file "$PWD/wheel-inputs.txt" \ @@ -88,10 +89,18 @@ jobs: - name: Build + audit a linux-64 conda package working-directory: ${{ github.workspace }} + env: + RS_REQUIRED: ${{ steps.public-wheels.outputs.rsRequired }} run: | set -euo pipefail - # This baseline owns its embedded core; no current-source RS input is substituted. + rsArgs=() + if [[ "$RS_REQUIRED" == "true" ]]; then + rsArgs=(--rs-wheel-dir "$PWD/wheels") + elif [[ "$RS_REQUIRED" != "false" ]]; then + echo "Missing verified public wheel profile"; exit 1 + fi python -m eng.conda_tools build \ + "${rsArgs[@]}" \ --mssql-wheel-dir "$PWD/wheels" \ --odbc-wheel-dir "$PWD/wheels" \ --odbc-wheel-filter "mssql_python_odbc-*.whl" \ diff --git a/conda/README.md b/conda/README.md index 9d07a2981..efbd5fb92 100644 --- a/conda/README.md +++ b/conda/README.md @@ -58,14 +58,14 @@ OneBranch pipelines/steps/jobs, their native/probe/archive/provenance and releas and the workflow itself. General package metadata/version changes and shared wheel-build infrastructure do not trigger it on their own; mixed PRs with a Conda change still do. Source-only tests remain current-checkout and blocking. The real Linux CPython 3.11 -build/audit uses `fetch-wheels --published-smoke`: a separate published baseline of -binding **1.15.0** and its exact ODBC dependency **18.6.2.1**, with embedded core and -no separate RS wheel. `eng/conda_tools/inputs.py` maintains its versions, exact wheel -filenames and SHA256 pins; update them together after verifying the new wheel metadata -and ownership. Source release bumps do not change this baseline. Missing wheels, -metadata/dependency mismatches, digest changes and build/audit failures still fail the -gate; there is no skip or automatic fallback. This historical packaging control does -not qualify current-source RS or Python 3.15. +build/audit uses `fetch-wheels --published-smoke`: PyPI's current published binding +and its exact ODBC dependency (plus RS when required). No smoke versions or hashes +are hardcoded; downloads are verified against PyPI's SHA256 values and actual wheel +metadata/ownership. Source release bumps do not select unpublished smoke inputs. +The selected versions are logged and hash-required requirements are generated per run. +Missing wheels, metadata/dependency or hash mismatches and build/audit failures still +fail the gate; there is no skip or older-release fallback. This published packaging +control does not qualify current-source RS or Python 3.15. Run the source-only trigger regressions without a native build, SQL Server, or PyYAML: @@ -300,7 +300,7 @@ provider and selected pin; unavailable releases, malformed declarations, or mism fail with no older-version or TLS fallback. Public availability does not gate the recorded ADO artifact path. Neither a public input check nor static archive checks establish installed SQL/runtime qualification. -Only the explicit `--published-smoke` option selects the independent baseline above; +Only the explicit `--published-smoke` option selects the published inputs above; production, provenance, preflight and release/publish version selection are unchanged. The driver-load probe's sole implementation is diff --git a/eng/conda_tools/inputs.py b/eng/conda_tools/inputs.py index 205006e60..5cc43380b 100644 --- a/eng/conda_tools/inputs.py +++ b/eng/conda_tools/inputs.py @@ -19,18 +19,6 @@ from . import archive, contracts -# PR-only linux-64/cp311 packaging control, maintained independently of source releases. -_PUBLISHED_SMOKE_VERSIONS = {"mssql-python": "1.15.0", "mssql-python-odbc": "18.6.2.1"} -_PUBLISHED_SMOKE_SHA256 = { - "mssql_python-1.15.0-cp311-cp311-manylinux_2_28_x86_64.whl": ( - "016c61a4c82a1b69a8f8a43c91dc3537463d8ca068da73fa91f7e348b48e6d22" - ), - "mssql_python_odbc-18.6.2.1-py3-none-manylinux_2_28_x86_64.whl": ( - "b8aad821f3b8637b0e6d4637f4b730ed9897c4665fa99d47e44e97fd1c79c77a" - ), -} - - def _is_sha256(value: object) -> bool: return isinstance(value, str) and re.fullmatch(r"[0-9a-f]{64}", value) is not None @@ -188,7 +176,7 @@ def fetch_wheels( else: if source_versions.get("mssql-python-rs") != rs_version: raise ValueError( - "Published binding RS requirement differs from the maintained source pin." + "Published binding RS requirement differs from the selected pin." ) versions["mssql-python-rs"] = rs_version hashes.update( @@ -293,7 +281,7 @@ def add_arguments(parser: argparse.ArgumentParser) -> None: parser.add_argument( "--published-smoke", action="store_true", - help="Use the separately pinned linux-64/cp311 PR baseline, not source release versions.", + help="Use PyPI's current published binding and its exact dependencies for the PR smoke.", ) parser.add_argument( "--python-tag", @@ -311,17 +299,13 @@ def fetch_cli(args: argparse.Namespace) -> int: try: if args.published_smoke: if (args.python_tag, args.conda_subdir) != ("cp311", "linux-64"): - raise ValueError("The published smoke baseline requires cp311 linux-64.") - source = _PUBLISHED_SMOKE_VERSIONS + raise ValueError("The published smoke requires cp311 linux-64.") + source = read_published_versions() else: source = read_release_versions(lambda path: Path(path).read_text(encoding="utf-8")) versions = fetch_wheels( source, args.wheel_dir, args.requirements_file, args.python_tag, args.conda_subdir ) - if args.published_smoke: - actual = {path.name: file_sha256(path) for path in args.wheel_dir.glob("*.whl")} - if actual != _PUBLISHED_SMOKE_SHA256: - raise ValueError("Published smoke wheels differ from the pinned filenames/SHA256.") rs_required = "mssql-python-rs" in versions if output := os.environ.get("GITHUB_OUTPUT"): with open(output, "a", encoding="utf-8") as destination: @@ -338,6 +322,32 @@ def fetch_cli(args: argparse.Namespace) -> int: return 0 +def read_published_versions() -> dict[str, str]: + """Select PyPI's current binding release, never independently mix latest components.""" + try: + with urlopen("https://pypi.org/pypi/mssql-python/json", timeout=60) as response: + release = json.load(response) + except URLError as error: + raise ValueError( + "Cannot select the published binding from PyPI; no fallback is allowed." + ) from error + info = release.get("info") if isinstance(release, dict) else None + if not isinstance(info, dict) or info.get("name") != "mssql-python": + raise ValueError("Invalid published binding identity.") + requirements = info.get("requires_dist") + if not isinstance(requirements, list) or any( + not isinstance(item, str) for item in requirements + ): + raise ValueError("Invalid published binding dependencies.") + versions = { + "mssql-python": info.get("version"), + "mssql-python-odbc": contracts.exact_dependency_pin(requirements, "mssql-python-odbc"), + } + if rs_version := contracts.exact_dependency_pin(requirements, "mssql-python-rs"): + versions["mssql-python-rs"] = rs_version + return parse_release_versions(json.dumps(versions)) + + def read_release_versions(read_source: Callable[[str], str]) -> dict[str, str]: """Read the literals maintained by the existing wheel release process, without imports.""" setup_source = read_source("setup.py") diff --git a/tests/test_036_conda_provenance.py b/tests/test_036_conda_provenance.py index 69fd7928a..445b13cbc 100644 --- a/tests/test_036_conda_provenance.py +++ b/tests/test_036_conda_provenance.py @@ -546,28 +546,55 @@ def test_source_bound_release_components(tmp_path, monkeypatch, capsys, problem, ), capsys.readouterr() -@pytest.mark.parametrize("profile", ["source", "smoke", "smoke-bad-hash", "smoke-wrong-target"]) +@pytest.mark.parametrize( + "profile", + [ + "source", + "smoke", + "smoke-rs", + "smoke-unavailable", + "smoke-bad-dependency", + "smoke-wrong-target", + ], +) def test_published_smoke_is_explicit_and_independent_of_unpublished_source( tmp_path, monkeypatch, capsys, profile ): source = {"mssql-python": "1.16.0", "mssql-python-odbc": "18.6.2.2"} - reads, selected = [], [] + published = {"mssql-python": "1.15.0", "mssql-python-odbc": "18.6.2.1"} + if profile == "smoke-rs": + published["mssql-python-rs"] = "0.3.0" + reads, requests, selected = [], [], [] monkeypatch.setattr(inputs, "read_release_versions", lambda read: reads.append(True) or source) + def pypi(url, *, timeout): + assert url == "https://pypi.org/pypi/mssql-python/json" and timeout == 60 + requests.append(url) + if profile == "smoke-unavailable": + raise HTTPError(url, 404, "not published", {}, None) + requirements = [ + f"{name}=={version}" for name, version in published.items() if name != "mssql-python" + ] + if profile == "smoke-bad-dependency": + requirements = ["mssql-python-odbc>=18.6.2.1"] + return io.BytesIO( + json.dumps( + { + "info": { + "name": "mssql-python", + "version": published["mssql-python"], + "requires_dist": requirements, + } + } + ).encode() + ) + def fetch(versions, *args): selected.append(versions) - for filename in inputs._PUBLISHED_SMOKE_SHA256: - (tmp_path / filename).touch() return versions + monkeypatch.setattr(inputs, "urlopen", pypi) monkeypatch.setattr(inputs, "fetch_wheels", fetch) - monkeypatch.setattr( - inputs, - "file_sha256", - lambda path: ( - "0" * 64 if profile == "smoke-bad-hash" else inputs._PUBLISHED_SMOKE_SHA256[path.name] - ), - ) output = tmp_path / "outputs" monkeypatch.setenv("GITHUB_OUTPUT", str(output)) result = cli.main( @@ -585,22 +612,15 @@ def fetch(versions, *args): ] ) assert reads == ([True] if profile == "source" else []) - assert selected == ( - [] - if profile == "smoke-wrong-target" - else [ - ( - source - if profile == "source" - else {"mssql-python": "1.15.0", "mssql-python-odbc": "18.6.2.1"} - ) - ] - ) - failed = profile in {"smoke-bad-hash", "smoke-wrong-target"} + assert len(requests) == (0 if profile in {"source", "smoke-wrong-target"} else 1) + failed = profile in {"smoke-unavailable", "smoke-bad-dependency", "smoke-wrong-target"} + assert selected == ([] if failed else [source if profile == "source" else published]) assert result == int(failed) assert output.exists() == (not failed) if failed: assert "ERROR:" in capsys.readouterr().err + else: + assert output.read_text() == f"rsRequired={str(profile == 'smoke-rs').lower()}\n" @pytest.mark.parametrize("with_rs", [False, True])