From 191e2a51c2793895856286fb61465bcf8cc277a4 Mon Sep 17 00:00:00 2001 From: Jahnvi Thakkar Date: Thu, 1 Oct 2026 14:29:52 +0530 Subject: [PATCH 01/17] PERF: Cache full column counts for fetchone Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- mssql_python/pybind/ddbc_bindings.cpp | 8 +- mssql_python/pybind/result_metadata.hpp | 13 +- tests/test_fetch_settings_cache.py | 242 ++++++++++++++++++++++++ 3 files changed, 260 insertions(+), 3 deletions(-) diff --git a/mssql_python/pybind/ddbc_bindings.cpp b/mssql_python/pybind/ddbc_bindings.cpp index f410631d4..1ad82c631 100644 --- a/mssql_python/pybind/ddbc_bindings.cpp +++ b/mssql_python/pybind/ddbc_bindings.cpp @@ -6306,8 +6306,12 @@ SQLRETURN FetchOne_wrap(SqlHandlePtr StatementHandle, py::list& row, } CaptureFetchDiagnostics(hStmt, ret, messages); if (SQL_SUCCEEDED(ret)) { - // Retrieve column count - SQLSMALLINT colCount = SQLNumResultCols_wrap(StatementHandle, messages); + const auto snapshot = StatementHandle->resultMetadata.snapshot(); + SQLSMALLINT colCount = snapshot.fullColumnCount; + if (colCount < 0) { + colCount = SQLNumResultCols_wrap(StatementHandle, messages); + StatementHandle->resultMetadata.publishFullColumnCount(snapshot.generation, colCount); + } ret = SQLGetData_wrap(StatementHandle, colCount, row, charEncoding, wcharEncoding, charCtype, messages); if (!SQL_SUCCEEDED(ret)) { diff --git a/mssql_python/pybind/result_metadata.hpp b/mssql_python/pybind/result_metadata.hpp index 3a9912860..72ec0010b 100644 --- a/mssql_python/pybind/result_metadata.hpp +++ b/mssql_python/pybind/result_metadata.hpp @@ -30,11 +30,12 @@ class ResultMetadataCache { struct Snapshot { uint64_t generation; std::shared_ptr metadata; + SQLSMALLINT fullColumnCount; }; Snapshot snapshot() const { std::lock_guard lock(mutex_); - return {generation_, metadata_}; + return {generation_, metadata_, fullColumnCount_}; } void publish(uint64_t generation, std::shared_ptr metadata) { @@ -44,16 +45,26 @@ class ResultMetadataCache { } } + void publishFullColumnCount(uint64_t generation, SQLSMALLINT columnCount) { + std::lock_guard lock(mutex_); + if (generation == generation_ && columnCount >= 0) { + fullColumnCount_ = columnCount; + } + } + void clear() { std::lock_guard lock(mutex_); ++generation_; metadata_.reset(); + fullColumnCount_ = -1; } private: mutable std::mutex mutex_; uint64_t generation_ = 0; std::shared_ptr metadata_; + // Prefix GetData metadata does not establish the full result cardinality. + SQLSMALLINT fullColumnCount_ = -1; }; class ResultMetadataFailureGuard { diff --git a/tests/test_fetch_settings_cache.py b/tests/test_fetch_settings_cache.py index 78ad5229b..1aed3b857 100644 --- a/tests/test_fetch_settings_cache.py +++ b/tests/test_fetch_settings_cache.py @@ -1111,3 +1111,245 @@ def test_fetch_error_is_raised_before_wrapping_rows(connection, method, bridge_n fetch_rows(cursor, method) assert cursor._next_row_index == position assert tuple(fetch_rows(cursor, method)[0]) == (1,) + + +@pytest.mark.skipif( + sys.platform == "win32", + reason="Windows does not export the native driver function-pointer globals", +) +@pytest.mark.parametrize( + "mode", + ( + "warm", + "prefix", + "shapes", + "nextset", + "fetch-error", + "count-error", + "decode-error", + "generation", + "mixed", + ), +) +def test_native_fetchone_full_column_count_cache(conn_str, mode): + if not conn_str: + pytest.skip("DB_CONNECTION_STRING is required") + code = ( + "import runpy, sys; " + "runpy.run_path(sys.argv[1])['_check_native_fetchone_full_column_count_cache']" + "(sys.argv[2], sys.argv[3])" + ) + result = subprocess.run( + [ + sys.executable, + "-c", + code, + str(Path(__file__).resolve()), + mode, + str(Path(mssql_python.ddbc_bindings.module.__file__).resolve()), + ], + capture_output=True, + text=True, + timeout=60, + ) + assert result.returncode == 0, (result.returncode, result.stdout, result.stderr) + + +def _check_native_fetchone_full_column_count_cache(mode, expected_native): + import ctypes + import os + + native = Path(mssql_python.ddbc_bindings.module.__file__).resolve() + assert native == Path(expected_native) + library = ctypes.CDLL(str(native)) + count_pointer = ctypes.c_void_p.in_dll(library, "SQLNumResultCols_ptr") + fetch_pointer = ctypes.c_void_p.in_dll(library, "SQLFetch_ptr") + count_type = ctypes.CFUNCTYPE(ctypes.c_short, ctypes.c_void_p, ctypes.POINTER(ctypes.c_short)) + fetch_type = ctypes.CFUNCTYPE(ctypes.c_short, ctypes.c_void_p) + success = ConstantsDDBC.SQL_SUCCESS.value + error = ConstantsDDBC.SQL_ERROR.value + ddbc = mssql_python.ddbc_bindings + count_calls, callback_errors = [], [] + fail_fetch = fail_count = invalidate_count = False + + @count_type + def count_columns(handle, count): + nonlocal fail_count, invalidate_count + try: + count_calls.append(handle) + if fail_count: + fail_count = False + return error + result = original_count(handle, count) + if invalidate_count: + invalidate_count = False + status = ddbc.DDBCSQLSetStmtAttr( + cursor.hstmt, ConstantsDDBC.SQL_ATTR_QUERY_TIMEOUT.value, 0 + ) + if status != success: + callback_errors.append("Failed to invalidate the metadata generation") + return error + return result + except BaseException as failure: + callback_errors.append(type(failure).__name__) + return error + + @fetch_type + def fetch_row(handle): + nonlocal fail_fetch + try: + if fail_fetch: + fail_fetch = False + return error + return original_fetch(handle) + except BaseException as failure: + callback_errors.append(type(failure).__name__) + return error + + try: + connection = mssql_python.connect(os.environ["DB_CONNECTION_STRING"], timeout=5) + except mssql_python.Error as failure: + raise AssertionError( + f"Connection failed: {type(failure).__name__}; connection details withheld" + ) from None + with connection, connection.cursor() as cursor: + query = ( + "SELECT n AS number, CAST(N'text' AS NVARCHAR(10)) AS txt " + "FROM (VALUES (1), (2), (3), (4), (5), (6)) AS v(n) ORDER BY n" + ) + cursor.execute(query) + saved_count, saved_fetch = count_pointer.value, fetch_pointer.value + assert saved_count and saved_fetch + original_count, original_fetch = count_type(saved_count), fetch_type(saved_fetch) + try: + count_pointer.value = ctypes.cast(count_columns, ctypes.c_void_p).value + fetch_pointer.value = ctypes.cast(fetch_row, ctypes.c_void_p).value + if mode == "prefix": + assert ddbc.DDBCSQLFetch(cursor.hstmt) == success + prefix = [] + assert ( + ddbc.DDBCSQLGetData( + cursor.hstmt, + 1, + prefix, + "utf-16le", + "utf-16le", + ConstantsDDBC.SQL_C_WCHAR.value, + ) + == success + ) + assert prefix == [1] + assert count_calls == [] + assert tuple(cursor.fetchone()) == (2, "text") + assert len(count_calls) == 1 + assert tuple(cursor.fetchone()) == (3, "text") + assert len(count_calls) == 1 + elif mode == "shapes": + for sql, expected in ( + ("SELECT 7 AS number", (7,)), + ("SELECT CAST(N'new' AS NVARCHAR(10)) AS txt", ("new",)), + ("SELECT CAST(NULL AS INT) AS empty_value, 9 AS number", (None, 9)), + ): + cursor.execute(sql + " FROM (VALUES (1), (2)) AS v(n)") + count_calls.clear() + for _ in range(2): + row = tuple(cursor.fetchone()) + assert row == expected + assert tuple(map(type, row)) == tuple(map(type, expected)) + assert len(count_calls) == 1 + assert cursor.fetchone() is None + assert len(count_calls) == 1 + elif mode == "nextset": + cursor.execute( + "SELECT 1 AS number FROM (VALUES (1), (2)) AS v(n); " + "SELECT CAST(N'changed' AS NVARCHAR(10)) AS txt " + "FROM (VALUES (1), (2)) AS v(n)" + ) + count_calls.clear() + assert [cursor.fetchone()[0] for _ in range(2)] == [1, 1] + assert len(count_calls) == 1 + assert cursor.nextset() + count_calls.clear() + assert [cursor.fetchone()[0] for _ in range(2)] == ["changed", "changed"] + assert len(count_calls) == 1 + elif mode == "fetch-error": + assert tuple(cursor.fetchone()) == (1, "text") + assert len(count_calls) == 1 + fail_fetch = True + row = [] + assert ddbc.DDBCSQLFetchOne(cursor.hstmt, row) == error + assert row == [] + assert len(count_calls) == 1 + assert tuple(cursor.fetchone()) == (2, "text") + assert len(count_calls) == 2 + assert tuple(cursor.fetchone()) == (3, "text") + assert len(count_calls) == 2 + elif mode == "count-error": + fail_count = True + with pytest.raises(mssql_python.DatabaseError): + ddbc.DDBCSQLFetchOne(cursor.hstmt, []) + assert len(count_calls) == 1 + assert tuple(cursor.fetchone()) == (2, "text") + assert len(count_calls) == 2 + assert tuple(cursor.fetchone()) == (3, "text") + assert len(count_calls) == 2 + elif mode == "decode-error": + cursor.execute( + "SELECT CASE WHEN n = 2 THEN CAST(0x00D8 AS NVARCHAR(10)) " + "ELSE CAST(N'ok' AS NVARCHAR(10)) END AS txt " + "FROM (VALUES (1), (2), (3), (4)) AS v(n) ORDER BY n" + ) + count_calls.clear() + assert cursor.fetchone()[0] == "ok" + with pytest.raises(UnicodeDecodeError): + cursor.fetchone() + assert len(count_calls) == 1 + assert [cursor.fetchone()[0] for _ in range(2)] == ["ok", "ok"] + assert len(count_calls) == 2 + elif mode == "generation": + invalidate_count = True + assert tuple(cursor.fetchone()) == (1, "text") + assert len(count_calls) == 1 + assert tuple(cursor.fetchone()) == (2, "text") + assert len(count_calls) == 2 + assert tuple(cursor.fetchone()) == (3, "text") + assert len(count_calls) == 2 + elif mode == "mixed": + assert [tuple(row) for row in cursor.fetchmany(2)] == [(1, "text"), (2, "text")] + count_calls.clear() + assert tuple(cursor.fetchone()) == (3, "text") + assert len(count_calls) == 1 + assert tuple(cursor.fetchmany(1)[0]) == (4, "text") + count_calls.clear() + assert tuple(cursor.fetchone()) == (5, "text") + assert count_calls == [] + assert [tuple(row) for row in cursor.fetchall()] == [(6, "text")] + count_calls.clear() + assert cursor.fetchone() is None + assert count_calls == [] + cursor.execute(query) + with connection.cursor() as other: + other.execute("SELECT CAST(N'other' AS NVARCHAR(10))") + count_calls.clear() + assert cursor.fetchone()[0] == 1 + assert other.fetchone()[0] == "other" + assert cursor.fetchone()[0] == 2 + assert len(count_calls) == 2 + else: + assert mode == "warm" + assert [tuple(cursor.fetchone()) for _ in range(6)] == [ + (n, "text") for n in range(1, 7) + ] + assert len(count_calls) == 1 + assert cursor.fetchone() is None + assert len(count_calls) == 1 + # Direct callers still perform a real query on every invocation. + assert ddbc.DDBCSQLNumResultCols(cursor.hstmt) == 2 + assert ddbc.DDBCSQLNumResultCols(cursor.hstmt) == 2 + assert len(count_calls) == 3 + assert not callback_errors, callback_errors + assert not cursor.messages + finally: + count_pointer.value, fetch_pointer.value = saved_count, saved_fetch + cursor.execute("SELECT 42") + assert cursor.fetchone()[0] == 42 From 814de786d579f1df476720574a7e752752e43abd Mon Sep 17 00:00:00 2001 From: Jahnvi Thakkar Date: Thu, 1 Oct 2026 15:43:30 +0530 Subject: [PATCH 02/17] FIX: Isolate mixed fetch cache test connections Use a separate connection for the cross-handle assertion without requiring MARS. Preserve all fetch and native call-count assertions. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- tests/test_fetch_settings_cache.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/tests/test_fetch_settings_cache.py b/tests/test_fetch_settings_cache.py index 1aed3b857..b9e63e7ca 100644 --- a/tests/test_fetch_settings_cache.py +++ b/tests/test_fetch_settings_cache.py @@ -1328,7 +1328,15 @@ def fetch_row(handle): assert cursor.fetchone() is None assert count_calls == [] cursor.execute(query) - with connection.cursor() as other: + try: + other_connection = mssql_python.connect( + os.environ["DB_CONNECTION_STRING"], timeout=5 + ) + except mssql_python.Error as failure: + raise AssertionError( + f"Connection failed: {type(failure).__name__}; connection details withheld" + ) from None + with other_connection, other_connection.cursor() as other: other.execute("SELECT CAST(N'other' AS NVARCHAR(10))") count_calls.clear() assert cursor.fetchone()[0] == 1 From 4c43081aa94471964d16a8c047b55325865d2b2b Mon Sep 17 00:00:00 2001 From: Jahnvi Thakkar Date: Mon, 5 Oct 2026 16:23:46 +0530 Subject: [PATCH 03/17] PERF: Consolidate single-row fetch optimizations Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- mssql_python/cursor.py | 45 +- mssql_python/pybind/ddbc_bindings.cpp | 93 +++- mssql_python/pybind/ddbc_bindings.h | 4 + profiler/README.md | 31 ++ tests/test_fetch_single_row.py | 605 ++++++++++++++++++++++++++ 5 files changed, 757 insertions(+), 21 deletions(-) create mode 100644 tests/test_fetch_single_row.py diff --git a/mssql_python/cursor.py b/mssql_python/cursor.py index 79436ab2e..0b937f231 100644 --- a/mssql_python/cursor.py +++ b/mssql_python/cursor.py @@ -29,7 +29,7 @@ DatabaseError, ) from mssql_python.row import Row -from mssql_python.perf_timer import perf_phase +from mssql_python.perf_timer import perf_phase, perf_start, perf_stop from mssql_python import get_settings from mssql_python.parameter_helper import ( detect_and_convert_parameters, @@ -43,6 +43,9 @@ else: pyarrow = None +_DEFAULT_ROW_TYPE = Row +_DEFAULT_FAST_ROW_CREATE = Row._fast_create + # Constants for string handling MAX_INLINE_CHAR: int = ( 4000 # NVARCHAR/VARCHAR inline limit; this triggers NVARCHAR(MAX)/VARCHAR(MAX) + DAE @@ -2855,7 +2858,8 @@ def fetchone(self) -> Union[None, Row]: # Fetch raw data row_data = [] try: - with perf_phase("py::fetchone::cpp_call"): + started = perf_start() + try: ret = ddbc_bindings.DDBCSQLFetchOne( self.hstmt, row_data, @@ -2864,6 +2868,9 @@ def fetchone(self) -> Union[None, Row]: self._cached_char_ctype, self.messages, ) + finally: + if started: + perf_stop("py::fetchone::cpp_call", started) check_error(ddbc_sql_const.SQL_HANDLE_STMT.value, self.hstmt, ret) @@ -2884,7 +2891,8 @@ def fetchone(self) -> Union[None, Row]: # Get column and converter maps column_map, converter_map, column_map_lower = self._get_column_and_converter_maps() - with perf_phase("py::fetchone::row_wrap"): + started = perf_start() + try: if not converter_map and not self._uuid_str_indices: return Row._fast_create( row_data, column_map, self, column_map_lower, self._cached_result_columns @@ -2898,6 +2906,9 @@ def fetchone(self) -> Union[None, Row]: column_map_lower=column_map_lower, column_names=self._cached_result_columns, ) + finally: + if started: + perf_stop("py::fetchone::row_wrap", started) except Exception: # On error, don't increment rownumber - rethrow the error raise @@ -2930,7 +2941,8 @@ def fetchmany(self, size: Optional[int] = None) -> List[Row]: # Fetch raw data rows_data = [] try: - with perf_phase("py::fetchmany::cpp_call"): + started = perf_start() + try: ret = ddbc_bindings.DDBCSQLFetchMany( self.hstmt, rows_data, @@ -2940,6 +2952,9 @@ def fetchmany(self, size: Optional[int] = None) -> List[Row]: self._cached_char_ctype, self.messages, ) + finally: + if started: + perf_stop("py::fetchmany::cpp_call", started) check_error(ddbc_sql_const.SQL_HANDLE_STMT.value, self.hstmt, ret) @@ -2960,8 +2975,25 @@ def fetchmany(self, size: Optional[int] = None) -> List[Row]: # Convert raw data to Row objects uuid_idx = self._uuid_str_indices - with perf_phase("py::fetchmany::row_wrap"): + started = perf_start() + try: if not converter_map and not uuid_idx: + if ( + type(size) is int + and size == 1 + and len(rows_data) == 1 + and Row is _DEFAULT_ROW_TYPE + and Row._fast_create is _DEFAULT_FAST_ROW_CREATE + ): + return [ + Row._fast_create( + rows_data[0], + column_map, + self, + column_map_lower, + self._cached_result_columns, + ) + ] return ddbc_bindings.construct_rows( rows_data, Row, @@ -2982,6 +3014,9 @@ def fetchmany(self, size: Optional[int] = None) -> List[Row]: ) for row_data in rows_data ] + finally: + if started: + perf_stop("py::fetchmany::row_wrap", started) except Exception: # On error, don't increment rownumber - rethrow the error raise diff --git a/mssql_python/pybind/ddbc_bindings.cpp b/mssql_python/pybind/ddbc_bindings.cpp index e0c61e07e..310603eb0 100644 --- a/mssql_python/pybind/ddbc_bindings.cpp +++ b/mssql_python/pybind/ddbc_bindings.cpp @@ -4250,6 +4250,7 @@ SQLRETURN SQLFetchScroll_wrap(SqlHandlePtr StatementHandle, SQLSMALLINT FetchOri // Unbind any columns from previous fetch operations to avoid memory // corruption + StatementHandle->unboundGeneration.reset(); SQLFreeStmt_ptr(StatementHandle->get(), SQL_UNBIND); // Perform scroll operation @@ -4276,10 +4277,13 @@ SQLRETURN SQLFetchScroll_wrap(SqlHandlePtr StatementHandle, SQLSMALLINT FetchOri // For column in the result set, binds a buffer to retrieve column data // TODO: Move to anonymous namespace, since it is not used outside this file template -SQLRETURN SQLBindColums(SQLHSTMT hStmt, ColumnBuffers& buffers, const Metadata& columnNames, +SQLRETURN SQLBindColums(SqlHandlePtr handle, ColumnBuffers& buffers, const Metadata& columnNames, SQLUSMALLINT numCols, int fetchSize, int charCtype = SQL_C_WCHAR, py::handle messages = {}) { PERF_TIMER("SQLBindColums"); + // Invalidate before the first bind, including partial/failed binding. + handle->unboundGeneration.reset(); + SQLHSTMT hStmt = handle->get(); SQLRETURN ret = SQL_SUCCESS; const bool useWideChar = (charCtype == SQL_C_WCHAR); // Bind columns based on their data types @@ -4927,6 +4931,7 @@ struct FetchStateGuard { ret = SQLSetStmtAttr_ptr(handle->get(), SQL_ATTR_ROWS_FETCHED_PTR, nullptr, 0); break; default: + handle->unboundGeneration.reset(); ret = SQLFreeStmt_ptr(handle->get(), SQL_UNBIND); break; } @@ -4948,6 +4953,10 @@ struct FetchStateGuard { } }; +SQLRETURN FetchSingleRow(SqlHandlePtr handle, py::list& row, const std::string& charEncoding, + const std::string& wcharEncoding, int charCtype, py::handle messages, + SQLSMALLINT knownColumnCount = -1, bool scroll = false); + // FetchMany_wrap - Fetches multiple rows of data from the result set. // // @param StatementHandle: Handle to the statement from which data is to be @@ -5017,6 +5026,40 @@ SQLRETURN FetchMany_wrap(SqlHandlePtr StatementHandle, py::list& rows, int fetch ThrowStdException("Column metadata count does not match result column count"); } + // Only types with identical bound and SQLGetData conversions are eligible. + // Keep other types on the existing path, including their pre-fetch failures. + const bool singleNumericRow = fetchSize == 1 && numCols > 0 && + std::all_of(columnNames.begin(), columnNames.end(), [](const auto& column) { + switch (column.dataType) { + case SQL_INTEGER: + case SQL_SMALLINT: + case SQL_BIGINT: + case SQL_TINYINT: + case SQL_BIT: + case SQL_REAL: + case SQL_DOUBLE: + case SQL_FLOAT: + return true; + default: + return false; + } + }); + if (singleNumericRow) { + PERF_TIMER("FetchMany::single_numeric_row"); + FetchStateGuard fetchStateGuard(StatementHandle, messages); + // No rows-fetched pointer is needed for a single unbound row. + fetchStateGuard.configure(nullptr, 1); + py::list row; + ret = FetchSingleRow(StatementHandle, row, charEncoding, wcharEncoding, charCtype, + messages, numCols, true); + CheckFetchError(StatementHandle, ret); + if (SQL_SUCCEEDED(ret)) { + rows.append(row); + } + fetchStateGuard.close(); + return ret; + } + std::vector lobColumns; for (SQLSMALLINT i = 0; i < numCols; i++) { const auto& column = columnNames.at(i); @@ -5059,7 +5102,8 @@ SQLRETURN FetchMany_wrap(SqlHandlePtr StatementHandle, py::list& rows, int fetch FetchStateGuard fetchStateGuard(StatementHandle, messages); // Bind columns - ret = SQLBindColums(hStmt, buffers, columnNames, numCols, fetchSize, charCtype, messages); + ret = SQLBindColums(StatementHandle, buffers, columnNames, numCols, fetchSize, charCtype, + messages); if (!SQL_SUCCEEDED(ret)) { LOG("FetchMany_wrap: Error when binding columns - SQLRETURN=%d", ret); return ret; @@ -5380,7 +5424,8 @@ SQLRETURN FetchArrowBatch_wrap(SqlHandlePtr StatementHandle, py::list& capsules, FetchStateGuard fetchStateGuard(StatementHandle, messages); if (!hasLobColumns && fetchSize > 0) { - ret = SQLBindColums(hStmt, buffers, columnNames, numCols, fetchSize, charCtype, messages); + ret = SQLBindColums(StatementHandle, buffers, columnNames, numCols, fetchSize, charCtype, + messages); if (!SQL_SUCCEEDED(ret)) { LOG("Error when binding columns"); return ret; @@ -6238,7 +6283,8 @@ SQLRETURN FetchAll_wrap(SqlHandlePtr StatementHandle, py::list& rows, FetchStateGuard fetchStateGuard(StatementHandle, messages); // Bind columns - ret = SQLBindColums(hStmt, buffers, columnNames, numCols, fetchSize, charCtype, messages); + ret = SQLBindColums(StatementHandle, buffers, columnNames, numCols, fetchSize, charCtype, + messages); if (!SQL_SUCCEEDED(ret)) { LOG("FetchAll_wrap: Error when binding columns - SQLRETURN=%d", ret); return ret; @@ -6285,31 +6331,46 @@ SQLRETURN FetchOne_wrap(SqlHandlePtr StatementHandle, py::list& row, // Issue #531: upgrade SQL_C_CHAR + utf-8 to SQL_C_WCHAR on Windows so the // driver does lossless UTF-16 conversion instead of returning ACP bytes. charCtype = EffectiveCharCtypeForFetch(charCtype, charEncoding); + return FetchSingleRow(StatementHandle, row, charEncoding, wcharEncoding, charCtype, messages); +} + +SQLRETURN FetchSingleRow(SqlHandlePtr StatementHandle, py::list& row, + const std::string& charEncoding, const std::string& wcharEncoding, + int charCtype, py::handle messages, SQLSMALLINT knownColumnCount, + bool scroll) { SQLRETURN ret = SQL_ERROR; ResultMetadataFailureGuard metadataFailure(StatementHandle->resultMetadata, ret); SQLHSTMT hStmt = StatementHandle->get(); - // Unbind any columns from previous fetch operations (e.g., fetchmany) - // to avoid conflicts with SQLGetData. SQLGetData cannot be used on - // columns that are already bound. - ret = SQLFreeStmt_ptr(hStmt, SQL_UNBIND); - CaptureFetchDiagnostics(hStmt, ret, messages); - if (!SQL_SUCCEEDED(ret)) - return ret; + const auto generation = StatementHandle->resultMetadata.snapshot().generation; + if (StatementHandle->unboundGeneration != generation) { + StatementHandle->unboundGeneration.reset(); + { + PERF_TIMER("FetchSingleRow::SQL_UNBIND"); + ret = SQLFreeStmt_ptr(hStmt, SQL_UNBIND); + } + CaptureFetchDiagnostics(hStmt, ret, messages); + if (!SQL_SUCCEEDED(ret)) + return ret; + StatementHandle->unboundGeneration = generation; + } // Assume hStmt is already allocated and a query has been executed { // Release the GIL during the blocking ODBC fetch py::gil_scoped_release release; - ret = SQLFetch_ptr(hStmt); + ret = scroll ? SQLFetchScroll_ptr(hStmt, SQL_FETCH_NEXT, 0) : SQLFetch_ptr(hStmt); } CaptureFetchDiagnostics(hStmt, ret, messages); if (SQL_SUCCEEDED(ret)) { - const auto snapshot = StatementHandle->resultMetadata.snapshot(); - SQLSMALLINT colCount = snapshot.fullColumnCount; + SQLSMALLINT colCount = knownColumnCount; if (colCount < 0) { - colCount = SQLNumResultCols_wrap(StatementHandle, messages); - StatementHandle->resultMetadata.publishFullColumnCount(snapshot.generation, colCount); + const auto snapshot = StatementHandle->resultMetadata.snapshot(); + colCount = snapshot.fullColumnCount; + if (colCount < 0) { + colCount = SQLNumResultCols_wrap(StatementHandle, messages); + StatementHandle->resultMetadata.publishFullColumnCount(snapshot.generation, colCount); + } } ret = SQLGetData_wrap(StatementHandle, colCount, row, charEncoding, wcharEncoding, charCtype, messages); diff --git a/mssql_python/pybind/ddbc_bindings.h b/mssql_python/pybind/ddbc_bindings.h index 32d9f8067..3494fc833 100644 --- a/mssql_python/pybind/ddbc_bindings.h +++ b/mssql_python/pybind/ddbc_bindings.h @@ -7,6 +7,7 @@ #include #include #include +#include #include #include #include @@ -331,6 +332,9 @@ class SqlHandle { std::unordered_map describeCache; void clearDescribeCache() { describeCache.clear(); } ResultMetadataCache resultMetadata; + // Written only under the GIL. Metadata invalidation (including cancellation) + // changes the generation, so an old successful unbind cannot authorize reuse. + std::optional unboundGeneration; private: // The caller must release the GIL before waiting for native cleanup. diff --git a/profiler/README.md b/profiler/README.md index 8c0662b14..0c41faa73 100644 --- a/profiler/README.md +++ b/profiler/README.md @@ -144,6 +144,37 @@ without re-entering a held counter lock. Python samples triggered recursively du profiler bookkeeping are intentionally omitted; the cleanup itself still runs. Ordinary nested workload timers and other threads are not suppressed. +## Single-row fetch attribution + +`fetchone()` and the default iterator/`fetchval()` path share the native +single-row helper. Full result column counts are cached per metadata generation, +separately from prefix `SQLGetData` metadata. Direct `DDBCSQLNumResultCols` calls +remain uncached. `fetchval()` still calls `fetchone()` and constructs the full +row, including converters for columns beyond the first. + +`FetchSingleRow::SQL_UNBIND` counts attempted unbinds in that helper. A successful +unbind can be reused within the same generation, but does not certify row-array +attributes. Binding (including Arrow and partial binds), cleanup, and generation +changes invalidate reuse. + +`FetchMany::single_numeric_row` identifies the native `fetchmany(1)` route for +all-numeric results (integer, bit, real, float/double). It retains eager count/name +validation, row-array configuration and cleanup, and uses `SQLFetchScroll` plus +per-column `SQLGetData`. Mixed INT/NVARCHAR, text, LOB, decimal, temporal, UUID and +variant results retain their existing native routes. Numeric width can therefore +change the tradeoff; do not generalize a narrow-row result. + +The Python one-row wrapping shortcut is independent of native eligibility. It +requires a built-in `int` request of one, exactly one returned row, the canonical +`Row` and factory, and no converter or UUID work. One-row tails of larger requests +and substituted factories retain batch wrapping. The existing +`py::fetchone::{cpp_call,row_wrap}` and `py::fetchmany::{cpp_call,row_wrap}` phases +use paired start/stop calls, avoiding context-manager entry/exit when disabled. + +Use profiling-enabled builds to check operation counts and normal uninstrumented +Release builds for latency comparisons. These routes are optimization hypotheses, +not a measured speedup; source-only checks do not establish native correctness. + ## Adding a timer To time a new spot in the code: diff --git a/tests/test_fetch_single_row.py b/tests/test_fetch_single_row.py new file mode 100644 index 000000000..57f0fe0da --- /dev/null +++ b/tests/test_fetch_single_row.py @@ -0,0 +1,605 @@ +"""Shared single-row fetch contracts; native lifetime checks run in a child process.""" + +import os +import subprocess +import sys +import textwrap +from unittest.mock import patch + +import pytest + +import mssql_python +from mssql_python.cursor import Cursor + +NUMERIC_TYPES = ("INT", "SMALLINT", "BIGINT", "TINYINT", "BIT", "REAL", "FLOAT") + + +@pytest.mark.parametrize( + ("sql_type", "literal", "expected"), + ( + ("INT", "-2147483648", -2147483648), + ("INT", "2147483647", 2147483647), + ("SMALLINT", "-32768", -32768), + ("SMALLINT", "32767", 32767), + ("BIGINT", "-9223372036854775808", -9223372036854775808), + ("BIGINT", "9223372036854775807", 9223372036854775807), + ("TINYINT", "255", 255), + ("BIT", "0", False), + ("REAL", "-1.25", -1.25), + ("FLOAT", "1.7976931348623157E308", 1.7976931348623157e308), + ("FLOAT", "-2.2250738585072014E-308", -2.2250738585072014e-308), + ("FLOAT", "-0.0", 0.0), + ), + ids=( + "int_min", + "int_max", + "smallint_min", + "smallint_max", + "bigint_min", + "bigint_max", + "tinyint_max", + "bit_zero", + "real_negative", + "float_max", + "float_tiny", + "float_zero", + ), +) +def test_single_numeric_row_bound_path_parity(cursor, sql_type, literal, expected): + query = f"SELECT CAST({literal} AS {sql_type}) AS a, CAST(NULL AS {sql_type}) AS b" + cursor.execute(query) + bound = cursor.fetchmany(2)[0] + cursor.execute(query) + single = cursor.fetchmany(1)[0] + assert tuple(single) == tuple(bound) == (expected, None) + assert type(single[0]) is type(bound[0]) is type(expected) + + +@pytest.mark.parametrize("sql_type", NUMERIC_TYPES) +@pytest.mark.parametrize("method", ("fetchone", "iterator", "fetchmany", "fetchval")) +def test_numeric_single_row_values_and_eof(cursor, sql_type, method): + cursor.execute( + f"SELECT CAST(CASE WHEN n = 2 THEN NULL ELSE 1 END AS {sql_type}) AS value " + "FROM (VALUES (1), (2), (3)) AS v(n) ORDER BY n" + ) + + def fetch(): + if method == "fetchmany": + rows = cursor.fetchmany(1) + return rows[0][0] if rows else "EOF" + if method == "fetchval": + return cursor.fetchval() + row = next(cursor, None) if method == "iterator" else cursor.fetchone() + return row[0] if row is not None else "EOF" + + values = [fetch(), fetch(), fetch()] + expected = True if sql_type == "BIT" else 1.0 if sql_type in ("REAL", "FLOAT") else 1 + assert values == [expected, None, expected] + assert type(values[0]) is type(expected) + assert cursor.rowcount == 3 + assert cursor.rownumber == 2 + assert fetch() == (None if method == "fetchval" else "EOF") + assert cursor.rowcount == 3 + assert not cursor.messages + + +def test_single_row_converters_run_in_column_order(cursor): + events = [] + + def convert(value): + events.append(value) + if value == 2: + raise ValueError("keep original second column") + return value + 10 + + cursor.connection.add_output_converter(mssql_python.SQL_INTEGER, convert) + try: + cursor.execute("SELECT 1 AS a, 2 AS b, CAST(NULL AS INT) AS c") + assert cursor.fetchval() == 11 + assert events == [1, 2] + events.clear() + cursor.execute("SELECT 1 AS a, 2 AS b, CAST(NULL AS INT) AS c") + assert tuple(cursor.fetchmany(1)[0]) == (11, 2, None) + assert events == [1, 2] + finally: + cursor.connection.remove_output_converter(mssql_python.SQL_INTEGER) + + +def test_single_row_wrapper_does_not_enter_batch_factory(cursor): + from mssql_python import ddbc_bindings + + cursor.execute( + "SELECT n AS a, CAST(N'text' AS NVARCHAR(10)) AS b " + "FROM (VALUES (1),(2),(3),(4)) AS v(n) ORDER BY n" + ) + with patch.object(ddbc_bindings, "construct_rows", wraps=ddbc_bindings.construct_rows) as batch: + retained = cursor.fetchmany(1)[0] + assert tuple(retained) == (1, "text") + batch.assert_not_called() + assert [row[0] for row in cursor.fetchmany(2)] == [2, 3] + batch.assert_called_once() + assert cursor.fetchmany(2)[0][0] == 4 + assert batch.call_count == 2 + assert tuple(retained) == (1, "text") + + +@pytest.mark.parametrize("override_fast_create", (False, True)) +def test_fetchmany_preserves_substituted_row_class(cursor, override_fast_create): + import importlib + from mssql_python.row import Row + + class DerivedRow(Row): + pass + + def forbidden(*args): + raise AssertionError("batch wrapping must not call a substituted Row's factory") + + if override_fast_create: + DerivedRow._fast_create = staticmethod(forbidden) + cursor_module = importlib.import_module("mssql_python.cursor") + cursor.execute("SELECT 1 AS a") + with patch.object(cursor_module, "Row", DerivedRow): + row = cursor.fetchmany(1)[0] + assert type(row) is DerivedRow + assert row.a == 1 + + +def test_fetchmany_preserves_replaced_fast_factory(cursor): + from mssql_python.row import Row + + cursor.execute("SELECT 1 AS a") + with patch.object(Row, "_fast_create", side_effect=AssertionError("must use batch factory")): + assert cursor.fetchmany(1)[0].a == 1 + + +@pytest.mark.parametrize("via_arraysize", (False, True)) +@pytest.mark.parametrize("raises", (False, True)) +def test_fetchmany_size_subclass_equality_is_not_called(cursor, via_arraysize, raises): + from mssql_python import ddbc_bindings + + calls = [] + + class Size(int): + def __eq__(self, other): + calls.append(other) + if raises: + raise AssertionError("size equality must not run after native fetch") + return super().__eq__(other) + + cursor.execute("SELECT 1 AS a UNION ALL SELECT 2") + with patch.object(ddbc_bindings, "construct_rows", wraps=ddbc_bindings.construct_rows) as batch: + if via_arraysize: + cursor.arraysize = Size(1) + result = cursor.fetchmany() + else: + result = cursor.fetchmany(Size(1)) + assert result[0][0] == 1 + batch.assert_called_once() + assert calls == [] + assert cursor.fetchone()[0] == 2 + + +def test_real_subclass_and_instance_fetchone_overrides(cursor): + calls = [] + + class DerivedCursor(Cursor): + def fetchone(self): + calls.append("derived") + return (71, 72) + + with DerivedCursor(cursor.connection) as derived: + derived.execute("SELECT 1 AS a UNION ALL SELECT 2") + assert derived.fetchval() == 71 + assert next(derived) == (71, 72) + assert calls == ["derived", "derived"] + # fetchmany must not acquire fetchone's Python override semantics. + assert derived.fetchmany(1)[0][0] == 1 + assert calls == ["derived", "derived"] + with patch.object(derived, "fetchone", return_value=(81, 82)) as override: + assert derived.fetchval() == 81 + assert next(derived) == (81, 82) + assert derived.fetchmany(1)[0][0] == 2 + assert override.call_count == 2 + + +@pytest.mark.parametrize("enabled", (False, True)) +@pytest.mark.parametrize("method", ("fetchone", "fetchmany", "fetchval", "iterator")) +def test_single_row_python_phase_counts(cursor, enabled, method): + from mssql_python import perf_timer + + perf_timer.disable() + perf_timer.reset() + if enabled: + perf_timer.enable() + try: + cursor.execute("SELECT 1 AS a") + perf_timer.reset() + if method == "fetchmany": + assert cursor.fetchmany(1)[0][0] == 1 + elif method == "fetchval": + assert cursor.fetchval() == 1 + elif method == "iterator": + assert next(cursor)[0] == 1 + else: + assert cursor.fetchone()[0] == 1 + stats = perf_timer.get_stats() + prefix = "py::fetchmany" if method == "fetchmany" else "py::fetchone" + assert {name: item["calls"] for name, item in stats.items()} == ( + {prefix + "::cpp_call": 1, prefix + "::row_wrap": 1} if enabled else {} + ) + finally: + perf_timer.disable() + perf_timer.reset() + + +def test_single_row_native_transitions_in_subprocess(conn_str): + """Counters describe actual native work, not mocked Python bridge calls.""" + from mssql_python import ddbc_bindings + + if not hasattr(ddbc_bindings, "profiling"): + pytest.skip("requires a profiling-enabled native build") + script = textwrap.dedent(""" + import os + import mssql_python + from mssql_python import ddbc_bindings as ddbc + + p = ddbc.profiling + assert hasattr(p, "enable") + with mssql_python.connect(os.environ["DB_CONNECTION_STRING"]) as connection: + with connection.cursor() as cursor: + query = ( + "SELECT n AS a, n + 10 AS b FROM " + "(VALUES (1),(2),(3),(4),(5),(6),(7),(8)) AS v(n) ORDER BY n" + ) + cursor.execute(query) + p.reset() + p.enable() + retained = cursor.fetchone() + assert retained[0] == 1 + assert next(cursor)[0] == 2 + assert cursor.fetchval() == 3 + assert p.get_stats()["FetchSingleRow::SQL_UNBIND"]["calls"] == 1 + assert cursor.fetchmany(1)[0][0] == 4 + assert "SQLBindColums" not in p.get_stats() + assert p.get_stats()["FetchMany::single_numeric_row"]["calls"] == 1 + # Cleanup deliberately forgets the unbound marker. + assert cursor.fetchone()[0] == 5 + assert p.get_stats()["FetchSingleRow::SQL_UNBIND"]["calls"] == 2 + assert [r[0] for r in cursor.fetchmany(2)] == [6, 7] + assert p.get_stats()["SQLBindColums"]["calls"] == 1 + assert cursor.fetchone()[0] == 8 + assert cursor.fetchone() is None + assert cursor.fetchone() is None + assert p.get_stats()["FetchSingleRow::SQL_UNBIND"]["calls"] == 3 + assert tuple(retained) == (1, 11) + assert retained.a == 1 + assert not cursor.messages + + cursor.execute(query) + assert cursor.fetchone()[0] == 1 + assert p.get_stats()["FetchSingleRow::SQL_UNBIND"]["calls"] == 4 + # Public statement-attribute changes must invalidate the marker. + assert ddbc.DDBCSQLSetStmtAttr(cursor.hstmt, 0, 0) == 0 # SQL_ATTR_QUERY_TIMEOUT + assert cursor.fetchone()[0] == 2 + assert p.get_stats()["FetchSingleRow::SQL_UNBIND"]["calls"] == 5 + cursor.execute("SELECT 21 AS a; SELECT 22 AS a") + assert cursor.fetchone()[0] == 21 + assert cursor.nextset() + assert cursor.fetchone()[0] == 22 + assert p.get_stats()["FetchSingleRow::SQL_UNBIND"]["calls"] == 7 + + cursor.execute("SELECT CAST(N'text' AS NVARCHAR(10)) AS a") + assert cursor.fetchmany(1)[0][0] == "text" + assert p.get_stats()["SQLBindColums"]["calls"] == 2 + cursor.execute(query) + assert cursor.fetchone()[0] == 1 + cursor.skip(1) + before = p.get_stats()["FetchSingleRow::SQL_UNBIND"]["calls"] + assert cursor.fetchone()[0] == 3 + assert p.get_stats()["FetchSingleRow::SQL_UNBIND"]["calls"] == before + 1 + batch = cursor.arrow_batch(2) + assert batch.to_pydict() == {"a": [4, 5], "b": [14, 15]} + before = p.get_stats()["FetchSingleRow::SQL_UNBIND"]["calls"] + assert cursor.fetchone()[0] == 6 + assert p.get_stats()["FetchSingleRow::SQL_UNBIND"]["calls"] == before + 1 + assert batch.to_pydict() == {"a": [4, 5], "b": [14, 15]} + # Cancel invalidates the result generation; execute recovers the handle. + cursor.hstmt._cancel() + cursor.execute(query) + before = p.get_stats()["FetchSingleRow::SQL_UNBIND"]["calls"] + assert cursor.fetchone()[0] == 1 + assert p.get_stats()["FetchSingleRow::SQL_UNBIND"]["calls"] == before + 1 + cursor.close() + with connection.cursor() as replacement: + replacement.execute(query) + before = p.get_stats()["FetchSingleRow::SQL_UNBIND"]["calls"] + assert replacement.fetchone()[0] == 1 + assert p.get_stats()["FetchSingleRow::SQL_UNBIND"]["calls"] == before + 1 + p.disable() + """) + environment = os.environ.copy() + environment["DB_CONNECTION_STRING"] = conn_str + result = subprocess.run( + [sys.executable, "-c", script], + env=environment, + capture_output=True, + text=True, + timeout=45, + check=False, + ) + assert result.returncode == 0, result.stdout + result.stderr + + +@pytest.mark.skipif( + sys.platform == "win32", + reason="Windows does not export the native driver function-pointer globals", +) +@pytest.mark.parametrize("first_fetch", ("fetchone", "fetchmany")) +def test_count_generation_change_with_unbound_marker(conn_str, first_fetch): + """A count obtained across invalidation cannot authorize either cache.""" + script = textwrap.dedent(""" + import ctypes as c + import os + import sys + import mssql_python + from mssql_python import ddbc_bindings as ddbc + + library = c.CDLL(ddbc.module.__file__) + pointer = c.c_void_p + count_type = c.CFUNCTYPE(c.c_short, pointer, c.POINTER(c.c_short)) + unbind_type = c.CFUNCTYPE(c.c_short, pointer, c.c_ushort) + count_slot = pointer.in_dll(library, "SQLNumResultCols_ptr") + unbind_slot = pointer.in_dll(library, "SQLFreeStmt_ptr") + counts, unbinds, invalidations, callback_errors = [], [], [], [] + + @count_type + def counted(handle, value): + try: + counts.append(handle) + ret = original_count(handle, value) + if not invalidations: + invalidations.append(True) + # Change only the cache generation, not the result shape. + assert ddbc.DDBCSQLSetStmtAttr(cursor.hstmt, 0, 0) == 0 + return ret + except BaseException as error: + callback_errors.append(type(error).__name__) + return -1 + + @unbind_type + def unbound(handle, option): + try: + if option == 2: + unbinds.append(handle) + return original_unbind(handle, option) + except BaseException as error: + callback_errors.append(type(error).__name__) + return -1 + + with mssql_python.connect(os.environ["DB_CONNECTION_STRING"]) as connection: + with connection.cursor() as cursor: + cursor.execute( + "SELECT n AS a, n + 10 AS b FROM (VALUES (1),(2),(3),(4)) v(n) ORDER BY n" + ) + saved_count, saved_unbind = count_slot.value, unbind_slot.value + assert saved_count and saved_unbind + original_count = count_type(saved_count) + original_unbind = unbind_type(saved_unbind) + count_slot.value = c.cast(counted, pointer).value + unbind_slot.value = c.cast(unbound, pointer).value + try: + first = ( + cursor.fetchmany(1)[0] if sys.argv[1] == "fetchmany" + else cursor.fetchone() + ) + assert tuple(first) == (1, 11) + assert invalidations == [True] + before_count, before_unbind = len(counts), len(unbinds) + assert tuple(cursor.fetchone()) == (2, 12) + assert len(counts) == before_count + 1 + assert len(unbinds) == before_unbind + 1 + before_count, before_unbind = len(counts), len(unbinds) + assert cursor.fetchval() == 3 + assert tuple(next(cursor)) == (4, 14) + assert cursor.fetchone() is None + assert len(counts) == before_count + assert len(unbinds) == before_unbind + # Direct count calls remain uncached even after a warm fetch. + assert ddbc.DDBCSQLNumResultCols(cursor.hstmt) == 2 + assert ddbc.DDBCSQLNumResultCols(cursor.hstmt) == 2 + assert len(counts) == before_count + 2 + assert tuple(first) == (1, 11) + assert not callback_errors, callback_errors + assert not cursor.messages + finally: + count_slot.value, unbind_slot.value = saved_count, saved_unbind + cursor.execute("SELECT 42") + assert cursor.fetchval() == 42 + """) + environment = os.environ.copy() + environment["DB_CONNECTION_STRING"] = conn_str + result = subprocess.run( + [sys.executable, "-c", script, first_fetch], + env=environment, + capture_output=True, + text=True, + timeout=45, + check=False, + ) + assert result.returncode == 0, result.stdout + result.stderr + + +@pytest.mark.parametrize( + "failure", + ( + "unbind", + "unbind_warning", + "partial_bind", + "configure", + "cleanup", + "rows_fetched_cleanup", + "column_count", + "column_name", + "fetch", + "getdata", + ), +) +def test_single_row_native_failure_recovery_in_subprocess(conn_str, failure): + """Inject one native failure; restore the entry point before recovery.""" + from mssql_python import ddbc_bindings + + if sys.platform == "win32": + pytest.skip("native function-pointer globals are not exported on Windows") + if not hasattr(ddbc_bindings, "profiling"): + pytest.skip("requires a profiling-enabled native build") + script = textwrap.dedent(""" + import ctypes as c + import os + import sys + import mssql_python + from mssql_python import ddbc_bindings as ddbc + + failure = sys.argv[1] + library = c.CDLL(ddbc.module.__file__) + short, ushort, pointer, length = c.c_short, c.c_ushort, c.c_void_p, c.c_ssize_t + signatures = { + "unbind": ("SQLFreeStmt_ptr", (pointer, ushort)), + "unbind_warning": ("SQLFreeStmt_ptr", (pointer, ushort)), + "partial_bind": ("SQLBindCol_ptr", (pointer, ushort, short, pointer, length, pointer)), + "configure": ("SQLSetStmtAttr_ptr", (pointer, c.c_int, pointer, c.c_int)), + "cleanup": ("SQLSetStmtAttr_ptr", (pointer, c.c_int, pointer, c.c_int)), + "rows_fetched_cleanup": ("SQLSetStmtAttr_ptr", (pointer, c.c_int, pointer, c.c_int)), + "column_count": ("SQLNumResultCols_ptr", (pointer, pointer)), + "fetch": ("SQLFetchScroll_ptr", (pointer, short, length)), + "getdata": ("SQLGetData_ptr", (pointer, ushort, short, pointer, length, pointer)), + "column_name": ("SQLDescribeCol_ptr", + (pointer, ushort, pointer, short, pointer, pointer, + pointer, pointer, pointer)), + } + symbol, arguments = signatures[failure] + prototype = c.CFUNCTYPE(short, *arguments) + slot = pointer.in_dll(library, symbol) + calls = [] + fired = [] + callback_errors = [] + + def inject(*args): + calls.append(args) + hit = not fired + if failure == "partial_bind": + hit = hit and args[1] == 2 + elif failure in ("unbind", "unbind_warning"): + hit = hit and args[1] == 2 # SQL_UNBIND + elif failure == "rows_fetched_cleanup": + matching = [item for item in calls if item[1] == 26] + hit = hit and args[1] == 26 and len(matching) == 2 + elif failure in ("configure", "cleanup"): + matching = [item for item in calls if item[1] == 27] + hit = hit and args[1] == 27 and len(matching) == ( + 1 if failure == "configure" else 2 + ) + if hit: + fired.append(True) + if failure == "unbind_warning": + result = original(*args) + assert result == 0 + return 1 # SQL_SUCCESS_WITH_INFO after a real successful unbind + if failure == "column_name": + result = original(*args) + # Invalid UTF-16 must fail eager name validation before fetch. + c.cast(args[2], c.POINTER(ushort))[0] = 0xD800 + c.cast(args[4], c.POINTER(short))[0] = 1 + return result + return -1 # SQL_ERROR + return original(*args) + + @prototype + def injected(*args): + try: + return inject(*args) + except BaseException as error: + callback_errors.append(type(error).__name__) + return -1 + + diag_slot = pointer.in_dll(library, "SQLGetDiagRec_ptr") + diag_type = c.CFUNCTYPE( + short, short, pointer, short, pointer, pointer, pointer, short, pointer + ) + + @diag_type + def warning(handle_type, handle, record, state, native, message, capacity, size): + try: + if record > 1: + return 100 + text = "unbind warning".encode("utf-16le") + assert capacity > len(text) // 2 + c.memmove(state, "01000\\0".encode("utf-16le"), 12) + c.memmove(message, text + b"\\0\\0", len(text) + 2) + c.cast(native, c.POINTER(c.c_int))[0] = 0 + c.cast(size, c.POINTER(short))[0] = len(text) // 2 + return 0 + except BaseException as error: + callback_errors.append(type(error).__name__) + return -1 + + with mssql_python.connect(os.environ["DB_CONNECTION_STRING"]) as connection: + with connection.cursor() as cursor: + cursor.execute( + "SELECT n AS a, n + 10 AS b FROM (VALUES (1),(2),(3),(4)) v(n) ORDER BY n" + ) + # The bundled driver resolves function pointers lazily on connect. + saved, saved_diag = slot.value, diag_slot.value + assert saved and saved_diag + original = prototype(saved) + ddbc.profiling.reset() + ddbc.profiling.enable() + slot.value = c.cast(injected, pointer).value + if failure == "unbind_warning": + diag_slot.value = c.cast(warning, pointer).value + try: + if failure == "unbind_warning": + assert cursor.fetchone()[0] == 1 + assert cursor.messages == [("[01000] (0)", "unbind warning")] + assert cursor.fetchone()[0] == 2 + assert len(calls) == 1 + assert cursor.messages == [("[01000] (0)", "unbind warning")] + else: + try: + if failure == "unbind": + cursor.fetchone() + else: + cursor.fetchmany(2 if failure == "partial_bind" else 1) + except (mssql_python.Error, UnicodeError) as error: + if failure == "column_name": + assert isinstance(error, UnicodeError) + else: + assert isinstance(error, mssql_python.Error) + else: + raise AssertionError("injected failure was not propagated") + finally: + slot.value = saved + diag_slot.value = saved_diag + assert fired == [True], (failure, fired) + assert not callback_errors, callback_errors + # GetData and cleanup failures happen after consuming the row. + expected = ( + 3 if failure == "unbind_warning" else + 2 if failure in ("cleanup", "rows_fetched_cleanup", "getdata") else 1 + ) + assert cursor.fetchone()[0] == expected, failure + before = ddbc.profiling.get_stats()["FetchSingleRow::SQL_UNBIND"]["calls"] + assert cursor.fetchone()[0] == expected + 1 + assert ddbc.profiling.get_stats()["FetchSingleRow::SQL_UNBIND"]["calls"] == before + ddbc.profiling.disable() + """) + environment = os.environ.copy() + environment["DB_CONNECTION_STRING"] = conn_str + result = subprocess.run( + [sys.executable, "-c", script, failure], + env=environment, + capture_output=True, + text=True, + timeout=45, + check=False, + ) + assert result.returncode == 0, result.stdout + result.stderr From ac6e9880cd0118afa7f66cee3c45c9db91ef2fed Mon Sep 17 00:00:00 2001 From: Jahnvi Thakkar Date: Mon, 5 Oct 2026 16:38:08 +0530 Subject: [PATCH 04/17] FIX: Match single-row tests to native profiler and bind errors Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- profiler/README.md | 4 +-- tests/test_fetch_single_row.py | 49 +++++++++++++++++++--------------- 2 files changed, 29 insertions(+), 24 deletions(-) diff --git a/profiler/README.md b/profiler/README.md index 0c41faa73..1afccb9a7 100644 --- a/profiler/README.md +++ b/profiler/README.md @@ -152,12 +152,12 @@ separately from prefix `SQLGetData` metadata. Direct `DDBCSQLNumResultCols` call remain uncached. `fetchval()` still calls `fetchone()` and constructs the full row, including converters for columns beyond the first. -`FetchSingleRow::SQL_UNBIND` counts attempted unbinds in that helper. A successful +`ddbc::FetchSingleRow::SQL_UNBIND` counts attempted unbinds in that helper. A successful unbind can be reused within the same generation, but does not certify row-array attributes. Binding (including Arrow and partial binds), cleanup, and generation changes invalidate reuse. -`FetchMany::single_numeric_row` identifies the native `fetchmany(1)` route for +`ddbc::FetchMany::single_numeric_row` identifies the native `fetchmany(1)` route for all-numeric results (integer, bit, real, float/double). It retains eager count/name validation, row-array configuration and cleanup, and uses `SQLFetchScroll` plus per-column `SQLGetData`. Mixed INT/NVARCHAR, text, LOB, decimal, temporal, UUID and diff --git a/tests/test_fetch_single_row.py b/tests/test_fetch_single_row.py index 57f0fe0da..1cbd60ad3 100644 --- a/tests/test_fetch_single_row.py +++ b/tests/test_fetch_single_row.py @@ -258,63 +258,63 @@ def test_single_row_native_transitions_in_subprocess(conn_str): assert retained[0] == 1 assert next(cursor)[0] == 2 assert cursor.fetchval() == 3 - assert p.get_stats()["FetchSingleRow::SQL_UNBIND"]["calls"] == 1 + assert p.get_stats()["ddbc::FetchSingleRow::SQL_UNBIND"]["calls"] == 1 assert cursor.fetchmany(1)[0][0] == 4 - assert "SQLBindColums" not in p.get_stats() - assert p.get_stats()["FetchMany::single_numeric_row"]["calls"] == 1 + assert "ddbc::SQLBindColums" not in p.get_stats() + assert p.get_stats()["ddbc::FetchMany::single_numeric_row"]["calls"] == 1 # Cleanup deliberately forgets the unbound marker. assert cursor.fetchone()[0] == 5 - assert p.get_stats()["FetchSingleRow::SQL_UNBIND"]["calls"] == 2 + assert p.get_stats()["ddbc::FetchSingleRow::SQL_UNBIND"]["calls"] == 2 assert [r[0] for r in cursor.fetchmany(2)] == [6, 7] - assert p.get_stats()["SQLBindColums"]["calls"] == 1 + assert p.get_stats()["ddbc::SQLBindColums"]["calls"] == 1 assert cursor.fetchone()[0] == 8 assert cursor.fetchone() is None assert cursor.fetchone() is None - assert p.get_stats()["FetchSingleRow::SQL_UNBIND"]["calls"] == 3 + assert p.get_stats()["ddbc::FetchSingleRow::SQL_UNBIND"]["calls"] == 3 assert tuple(retained) == (1, 11) assert retained.a == 1 assert not cursor.messages cursor.execute(query) assert cursor.fetchone()[0] == 1 - assert p.get_stats()["FetchSingleRow::SQL_UNBIND"]["calls"] == 4 + assert p.get_stats()["ddbc::FetchSingleRow::SQL_UNBIND"]["calls"] == 4 # Public statement-attribute changes must invalidate the marker. assert ddbc.DDBCSQLSetStmtAttr(cursor.hstmt, 0, 0) == 0 # SQL_ATTR_QUERY_TIMEOUT assert cursor.fetchone()[0] == 2 - assert p.get_stats()["FetchSingleRow::SQL_UNBIND"]["calls"] == 5 + assert p.get_stats()["ddbc::FetchSingleRow::SQL_UNBIND"]["calls"] == 5 cursor.execute("SELECT 21 AS a; SELECT 22 AS a") assert cursor.fetchone()[0] == 21 assert cursor.nextset() assert cursor.fetchone()[0] == 22 - assert p.get_stats()["FetchSingleRow::SQL_UNBIND"]["calls"] == 7 + assert p.get_stats()["ddbc::FetchSingleRow::SQL_UNBIND"]["calls"] == 7 cursor.execute("SELECT CAST(N'text' AS NVARCHAR(10)) AS a") assert cursor.fetchmany(1)[0][0] == "text" - assert p.get_stats()["SQLBindColums"]["calls"] == 2 + assert p.get_stats()["ddbc::SQLBindColums"]["calls"] == 2 cursor.execute(query) assert cursor.fetchone()[0] == 1 cursor.skip(1) - before = p.get_stats()["FetchSingleRow::SQL_UNBIND"]["calls"] + before = p.get_stats()["ddbc::FetchSingleRow::SQL_UNBIND"]["calls"] assert cursor.fetchone()[0] == 3 - assert p.get_stats()["FetchSingleRow::SQL_UNBIND"]["calls"] == before + 1 + assert p.get_stats()["ddbc::FetchSingleRow::SQL_UNBIND"]["calls"] == before + 1 batch = cursor.arrow_batch(2) assert batch.to_pydict() == {"a": [4, 5], "b": [14, 15]} - before = p.get_stats()["FetchSingleRow::SQL_UNBIND"]["calls"] + before = p.get_stats()["ddbc::FetchSingleRow::SQL_UNBIND"]["calls"] assert cursor.fetchone()[0] == 6 - assert p.get_stats()["FetchSingleRow::SQL_UNBIND"]["calls"] == before + 1 + assert p.get_stats()["ddbc::FetchSingleRow::SQL_UNBIND"]["calls"] == before + 1 assert batch.to_pydict() == {"a": [4, 5], "b": [14, 15]} # Cancel invalidates the result generation; execute recovers the handle. cursor.hstmt._cancel() cursor.execute(query) - before = p.get_stats()["FetchSingleRow::SQL_UNBIND"]["calls"] + before = p.get_stats()["ddbc::FetchSingleRow::SQL_UNBIND"]["calls"] assert cursor.fetchone()[0] == 1 - assert p.get_stats()["FetchSingleRow::SQL_UNBIND"]["calls"] == before + 1 + assert p.get_stats()["ddbc::FetchSingleRow::SQL_UNBIND"]["calls"] == before + 1 cursor.close() with connection.cursor() as replacement: replacement.execute(query) - before = p.get_stats()["FetchSingleRow::SQL_UNBIND"]["calls"] + before = p.get_stats()["ddbc::FetchSingleRow::SQL_UNBIND"]["calls"] assert replacement.fetchone()[0] == 1 - assert p.get_stats()["FetchSingleRow::SQL_UNBIND"]["calls"] == before + 1 + assert p.get_stats()["ddbc::FetchSingleRow::SQL_UNBIND"]["calls"] == before + 1 p.disable() """) environment = os.environ.copy() @@ -569,8 +569,13 @@ def warning(handle_type, handle, record, state, native, message, capacity, size) cursor.fetchone() else: cursor.fetchmany(2 if failure == "partial_bind" else 1) - except (mssql_python.Error, UnicodeError) as error: - if failure == "column_name": + except (mssql_python.Error, UnicodeError, RuntimeError) as error: + if failure == "partial_bind": + assert type(error) is RuntimeError + assert str(error) == ( + "Failed to bind column - b, Type - 4, column ID - 2" + ) + elif failure == "column_name": assert isinstance(error, UnicodeError) else: assert isinstance(error, mssql_python.Error) @@ -587,9 +592,9 @@ def warning(handle_type, handle, record, state, native, message, capacity, size) 2 if failure in ("cleanup", "rows_fetched_cleanup", "getdata") else 1 ) assert cursor.fetchone()[0] == expected, failure - before = ddbc.profiling.get_stats()["FetchSingleRow::SQL_UNBIND"]["calls"] + before = ddbc.profiling.get_stats()["ddbc::FetchSingleRow::SQL_UNBIND"]["calls"] assert cursor.fetchone()[0] == expected + 1 - assert ddbc.profiling.get_stats()["FetchSingleRow::SQL_UNBIND"]["calls"] == before + assert ddbc.profiling.get_stats()["ddbc::FetchSingleRow::SQL_UNBIND"]["calls"] == before ddbc.profiling.disable() """) environment = os.environ.copy() From cc75f5010d2add29d33a11c0e8e19d3e06b1c964 Mon Sep 17 00:00:00 2001 From: Jahnvi Thakkar Date: Tue, 6 Oct 2026 11:22:03 +0530 Subject: [PATCH 05/17] PERF: Reduce single-row count and value construction overhead Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- mssql_python/pybind/ddbc_bindings.cpp | 73 +++- profiler/README.md | 31 +- tests/test_fetch_single_row_construction.py | 444 ++++++++++++++++++++ 3 files changed, 524 insertions(+), 24 deletions(-) create mode 100644 tests/test_fetch_single_row_construction.py diff --git a/mssql_python/pybind/ddbc_bindings.cpp b/mssql_python/pybind/ddbc_bindings.cpp index 310603eb0..b6d4c5ac3 100644 --- a/mssql_python/pybind/ddbc_bindings.cpp +++ b/mssql_python/pybind/ddbc_bindings.cpp @@ -3111,6 +3111,24 @@ SQLSMALLINT SQLNumResultCols_wrap(SqlHandlePtr statementHandle, py::handle messa namespace { +SQLSMALLINT CachedResultColumnCount(const SqlHandlePtr& handle, py::handle messages) { + const auto snapshot = handle->resultMetadata.snapshot(); + if (snapshot.fullColumnCount >= 0) { + return snapshot.fullColumnCount; + } + const auto count = SQLNumResultCols_wrap(handle, messages); + handle->resultMetadata.publishFullColumnCount(snapshot.generation, count); + return count; +} + +// Adopt a new cell reference even if appending to the caller's list fails. +void AppendFetchedCell(py::list& row, PyObject* value) { + auto owned = steal(value); + if (!owned || PyList_Append(row.ptr(), owned.ptr()) < 0) { + throw py::error_already_set(); + } +} + py::dict GetFetchColumnMetadata(const py::list& columns, size_t index) { return columns[index].cast(); } @@ -3696,22 +3714,36 @@ SQLRETURN SQLGetData_wrap(SqlHandlePtr StatementHandle, SQLUSMALLINT colCount, p // Use Python's codec system to decode bytes. const std::string decodeEncoding = GetEffectiveCharDecoding(effectiveCharEnc); - py::bytes raw_bytes(reinterpret_cast(dataBuffer.data()), - static_cast(dataLen)); + py::object decoded; try { - py::object decoded = - raw_bytes.attr("decode")(decodeEncoding, "strict"); - row.append(decoded); + // bytes.decode rejects embedded NULs; the C codec API does not. + if (decodeEncoding.find('\0') != std::string::npos) { + PyErr_SetString(PyExc_ValueError, "embedded null character"); + throw py::error_already_set(); + } + decoded = steal(PyUnicode_Decode( + reinterpret_cast(dataBuffer.data()), + static_cast(dataLen), + decodeEncoding.c_str(), "strict")); + if (!decoded) throw py::error_already_set(); + if (PyList_Append(row.ptr(), decoded.ptr()) < 0) + throw py::error_already_set(); LOG("SQLGetData: CHAR column %d decoded with '%s', %zu bytes " "-> %zu chars", i, decodeEncoding.c_str(), (size_t)dataLen, py::len(decoded)); } catch (const py::error_already_set& e) { + if (e.matches(PyExc_MemoryError)) throw; LOG_ERROR( "SQLGetData: Failed to decode CHAR column %d with '%s': %s", i, decodeEncoding.c_str(), e.what()); - // Return raw bytes as fallback - row.append(raw_bytes); + // Preserve the existing codec-error bytes fallback. + decoded = steal(PyBytes_FromStringAndSize( + reinterpret_cast(dataBuffer.data()), + static_cast(dataLen))); + if (!decoded) throw py::error_already_set(); + if (PyList_Append(row.ptr(), decoded.ptr()) < 0) + throw py::error_already_set(); } } else { // Buffer too small, fallback to streaming @@ -3847,7 +3879,7 @@ SQLRETURN SQLGetData_wrap(SqlHandlePtr StatementHandle, SQLUSMALLINT colCount, p ret = SQLGetData_ptr(hStmt, i, SQL_C_LONG, &intValue, 0, &indicator); CaptureFetchDiagnostics(hStmt, ret, messages); if (SQL_SUCCEEDED(ret) && indicator != SQL_NULL_DATA) { - row.append(static_cast(intValue)); + AppendFetchedCell(row, PyLong_FromLong(intValue)); } else { row.append(py::none()); } @@ -3863,7 +3895,7 @@ SQLRETURN SQLGetData_wrap(SqlHandlePtr StatementHandle, SQLUSMALLINT colCount, p break; } if (SQL_SUCCEEDED(ret)) { - row.append(static_cast(smallIntValue)); + AppendFetchedCell(row, PyLong_FromLong(smallIntValue)); } else { LOG("SQLGetData: Error retrieving SQL_SMALLINT for column " "%d - SQLRETURN=%d", @@ -3882,7 +3914,7 @@ SQLRETURN SQLGetData_wrap(SqlHandlePtr StatementHandle, SQLUSMALLINT colCount, p break; } if (SQL_SUCCEEDED(ret)) { - row.append(realValue); + AppendFetchedCell(row, PyFloat_FromDouble(realValue)); } else { LOG("SQLGetData: Error retrieving SQL_REAL for column %d - " "SQLRETURN=%d", @@ -3962,7 +3994,7 @@ SQLRETURN SQLGetData_wrap(SqlHandlePtr StatementHandle, SQLUSMALLINT colCount, p break; } if (SQL_SUCCEEDED(ret)) { - row.append(doubleValue); + AppendFetchedCell(row, PyFloat_FromDouble(doubleValue)); } else { LOG("SQLGetData: Error retrieving SQL_DOUBLE/FLOAT for " "column %d - SQLRETURN=%d", @@ -3981,7 +4013,7 @@ SQLRETURN SQLGetData_wrap(SqlHandlePtr StatementHandle, SQLUSMALLINT colCount, p break; } if (SQL_SUCCEEDED(ret)) { - row.append(static_cast(bigintValue)); + AppendFetchedCell(row, PyLong_FromLongLong(bigintValue)); } else { LOG("SQLGetData: Error retrieving SQL_BIGINT for column %d " "- SQLRETURN=%d", @@ -4152,7 +4184,7 @@ SQLRETURN SQLGetData_wrap(SqlHandlePtr StatementHandle, SQLUSMALLINT colCount, p break; } if (SQL_SUCCEEDED(ret)) { - row.append(static_cast(tinyIntValue)); + AppendFetchedCell(row, PyLong_FromLong(tinyIntValue)); } else { LOG("SQLGetData: Error retrieving SQL_TINYINT for column " "%d - SQLRETURN=%d", @@ -4171,7 +4203,7 @@ SQLRETURN SQLGetData_wrap(SqlHandlePtr StatementHandle, SQLUSMALLINT colCount, p break; } if (SQL_SUCCEEDED(ret)) { - row.append(static_cast(bitValue)); + AppendFetchedCell(row, PyBool_FromLong(bitValue != 0)); } else { LOG("SQLGetData: Error retrieving SQL_BIT for column %d - " "SQLRETURN=%d", @@ -4985,8 +5017,10 @@ SQLRETURN FetchMany_wrap(SqlHandlePtr StatementHandle, py::list& rows, int fetch SQLRETURN ret = SQL_ERROR; ResultMetadataFailureGuard metadataFailure(StatementHandle->resultMetadata, ret); SQLHSTMT hStmt = StatementHandle->get(); - // Retrieve column count - SQLSMALLINT numCols = SQLNumResultCols_wrap(StatementHandle, messages); + // Keep count/name validation before advancing, including the size-one route. + SQLSMALLINT numCols = fetchSize == 1 + ? CachedResultColumnCount(StatementHandle, messages) + : SQLNumResultCols_wrap(StatementHandle, messages); // Retrieve column metadata auto snapshot = StatementHandle->resultMetadata.snapshot(); @@ -6365,12 +6399,7 @@ SQLRETURN FetchSingleRow(SqlHandlePtr StatementHandle, py::list& row, if (SQL_SUCCEEDED(ret)) { SQLSMALLINT colCount = knownColumnCount; if (colCount < 0) { - const auto snapshot = StatementHandle->resultMetadata.snapshot(); - colCount = snapshot.fullColumnCount; - if (colCount < 0) { - colCount = SQLNumResultCols_wrap(StatementHandle, messages); - StatementHandle->resultMetadata.publishFullColumnCount(snapshot.generation, colCount); - } + colCount = CachedResultColumnCount(StatementHandle, messages); } ret = SQLGetData_wrap(StatementHandle, colCount, row, charEncoding, wcharEncoding, charCtype, messages); diff --git a/profiler/README.md b/profiler/README.md index 1afccb9a7..c09ad5496 100644 --- a/profiler/README.md +++ b/profiler/README.md @@ -148,8 +148,12 @@ Ordinary nested workload timers and other threads are not suppressed. `fetchone()` and the default iterator/`fetchval()` path share the native single-row helper. Full result column counts are cached per metadata generation, -separately from prefix `SQLGetData` metadata. Direct `DDBCSQLNumResultCols` calls -remain uncached. `fetchval()` still calls `fetchone()` and constructs the full +separately from prefix `SQLGetData` metadata. `fetchmany(1)` reuses that count for +all result shapes, while still validating count and names before fetching. Cold +`fetchmany(1)` can make two count calls (eager validation and `DescribeColumns`); +subsequent size-one calls, including EOF, do not reacquire it within the same +generation. Larger batches and direct `DDBCSQLNumResultCols` calls retain their +uncached count behavior. `fetchval()` still calls `fetchone()` and constructs the full row, including converters for columns beyond the first. `ddbc::FetchSingleRow::SQL_UNBIND` counts attempted unbinds in that helper. A successful @@ -171,6 +175,29 @@ and substituted factories retain batch wrapping. The existing `py::fetchone::{cpp_call,row_wrap}` and `py::fetchmany::{cpp_call,row_wrap}` phases use paired start/stop calls, avoiding context-manager entry/exit when disabled. +Row-wise integer, bit and floating-point values use checked CPython constructors +and append operations. This removes generic scalar marshalling, not the scalar +allocations themselves. The public `DDBCSQLGetData` destination still retains its +identity, preexisting elements and completed cells on a later-column error; it +is not replaced with a presized, partially initialized list. + +Bounded narrow-character GetData decodes the driver buffer directly through +Python's codec API. Successful decoding avoids an intermediate bytes object, +bound `decode` method and Python call arguments. Codec failures retain the logged +bytes fallback, including codec names containing an embedded NUL rather than +silently truncating them for the C API. Allocation failures now propagate instead +of being mistaken for codec failures. The decoded value is still appended before +debug logging; if a custom string's length raises during logging, the existing +decoded cell and subsequent bytes fallback are retained. Wide text and LOB +streaming are unchanged. + +Mixed INT/NVARCHAR is intentionally not routed through the numeric specialization: +bound and GetData paths differ on malformed UTF-16, `SQL_NO_TOTAL`, truncation +continuation and diagnostic timing. The unbind witness does not justify removing +row-array configuration/cleanup. Row user attributes, weakrefs and substituted +factories remain supported, as do dynamic `fetchone` overrides. No +first-column-only `fetchval` route is introduced. + Use profiling-enabled builds to check operation counts and normal uninstrumented Release builds for latency comparisons. These routes are optimization hypotheses, not a measured speedup; source-only checks do not establish native correctness. diff --git a/tests/test_fetch_single_row_construction.py b/tests/test_fetch_single_row_construction.py new file mode 100644 index 000000000..7386c9992 --- /dev/null +++ b/tests/test_fetch_single_row_construction.py @@ -0,0 +1,444 @@ +"""Count reuse and checked row-wise construction; driver faults stay in subprocesses.""" + +import datetime +import decimal +import gc +import os +import subprocess +import sys +import textwrap +import weakref + +import pytest + +from mssql_python import ddbc_bindings as ddbc + + +def test_getdata_appends_to_existing_list_without_python_append(cursor): + class Destination(list): + def append(self, value): + raise AssertionError("native append must not dispatch to list overrides") + + class Marker: + pass + + marker = Marker() + reference = weakref.ref(marker) + destination = Destination([marker]) + alias = destination + cursor.execute( + "SELECT CAST(-2147483648 AS INT), CAST(-32768 AS SMALLINT), " + "CAST(-9223372036854775808 AS BIGINT), CAST(255 AS TINYINT), " + "CAST(1 AS BIT), CAST(-1.25 AS REAL), CAST(1.5 AS FLOAT), " + "CAST(NULL AS INT), CAST(NULL AS SMALLINT), CAST(NULL AS BIGINT), " + "CAST(NULL AS TINYINT), CAST(NULL AS BIT), CAST(NULL AS REAL), " + "CAST(NULL AS FLOAT), CAST(12.50 AS DECIMAL(5,2)), " + "CAST('2024-02-29' AS DATE), CAST(0x0001FF AS VARBINARY(3))" + ) + assert ddbc.DDBCSQLFetch(cursor.hstmt) == 0 + assert ddbc.DDBCSQLGetData(cursor.hstmt, 17, destination, "utf-16le", "utf-16le", -8) == 0 + expected = [ + -2147483648, + -32768, + -9223372036854775808, + 255, + True, + -1.25, + 1.5, + *([None] * 7), + decimal.Decimal("12.50"), + datetime.date(2024, 2, 29), + b"\x00\x01\xff", + ] + assert destination is alias and destination[0] is marker + assert destination[1:] == expected + assert list(map(type, destination[1:])) == list(map(type, expected)) + del marker, destination, alias + gc.collect() + assert reference() is None + cursor.execute("SELECT 42") + assert cursor.fetchval() == 42 + + +@pytest.mark.parametrize("size", (1, 2)) +def test_getdata_decode_error_keeps_preexisting_and_completed_cells(cursor, size): + cursor.execute("SELECT 7, CAST(0x00D8 AS NVARCHAR(1))") + assert ddbc.DDBCSQLFetch(cursor.hstmt) == 0 + marker = object() + row = [marker] + if size == 1: + assert ddbc.DDBCSQLGetData(cursor.hstmt, size, row, "utf-16le", "utf-16le", -8) == 0 + else: + with pytest.raises(UnicodeDecodeError): + ddbc.DDBCSQLGetData(cursor.hstmt, size, row, "utf-16le", "utf-16le", -8) + assert row == [marker, 7] + cursor.execute("SELECT 42") + assert cursor.fetchval() == 42 + + +@pytest.mark.skipif( + sys.platform == "win32", + reason="Windows does not export the native driver function-pointer globals", +) +@pytest.mark.parametrize( + "mode", + ( + "numeric", + "mixed", + "lob", + "prefix", + "generation", + "count_error", + "nextset", + "warning", + "decode_error", + ), +) +def test_fetchmany_one_full_count_calls_in_subprocess(conn_str, mode): + script = textwrap.dedent(""" + import ctypes as c + import os + import sys + import mssql_python + from mssql_python import ddbc_bindings as ddbc + + mode = sys.argv[1] + pointer, short, ushort = c.c_void_p, c.c_short, c.c_ushort + count_type = c.CFUNCTYPE(short, pointer, c.POINTER(short)) + diag_type = c.CFUNCTYPE( + short, short, pointer, short, pointer, pointer, pointer, short, pointer + ) + library = c.CDLL(ddbc.module.__file__) + count_slot = pointer.in_dll(library, "SQLNumResultCols_ptr") + diag_slot = pointer.in_dll(library, "SQLGetDiagRec_ptr") + calls, errors, injected = [], [], [] + warning_pending = False + + @count_type + def counted(handle, count): + global warning_pending + try: + calls.append(handle) + if mode == "count_error" and not injected: + injected.append(True) + return -1 + result = original_count(handle, count) + if mode == "generation" and not injected: + injected.append(True) + assert ddbc.DDBCSQLSetStmtAttr(cursor.hstmt, 0, 0) == 0 + if mode == "warning" and not injected: + injected.append(True) + warning_pending = True + return 1 + return result + except BaseException as error: + errors.append(type(error).__name__) + return -1 + + @diag_type + def diagnostic(handle_type, handle, record, state, native, message, capacity, size): + global warning_pending + try: + if not warning_pending: + return original_diag( + handle_type, handle, record, state, native, message, capacity, size + ) + if record > 1: + warning_pending = False + return 100 + text = "count warning".encode("utf-16le") + assert capacity > len(text) // 2 + c.memmove(state, "01000\\0".encode("utf-16le"), 12) + c.memmove(message, text + b"\\0\\0", len(text) + 2) + c.cast(native, c.POINTER(c.c_int))[0] = 0 + c.cast(size, c.POINTER(short))[0] = len(text) // 2 + return 0 + except BaseException as error: + errors.append(type(error).__name__) + return -1 + + with mssql_python.connect(os.environ["DB_CONNECTION_STRING"]) as connection: + with connection.cursor() as cursor: + value_sql = ( + "CASE WHEN n = 2 THEN CAST(0x00D8 AS NVARCHAR(10)) " + "ELSE CAST(N'text' AS NVARCHAR(10)) END" if mode == "decode_error" else + "CAST(N'text' AS NVARCHAR(MAX))" if mode == "lob" else + "CAST(N'text' AS NVARCHAR(10))" if mode in ("mixed", "prefix") else + "n + 10" + ) + query = ( + f"SELECT n AS a, {value_sql} AS b FROM " + "(VALUES (1),(2),(3),(4),(5),(6)) v(n) ORDER BY n" + ) + cursor.execute(query) + saved_count, saved_diag = count_slot.value, diag_slot.value + assert saved_count and saved_diag + original_count, original_diag = count_type(saved_count), diag_type(saved_diag) + count_slot.value = c.cast(counted, pointer).value + diag_slot.value = c.cast(diagnostic, pointer).value + try: + expected = 1 + if mode == "prefix": + assert ddbc.DDBCSQLFetch(cursor.hstmt) == 0 + prefix = [] + assert ddbc.DDBCSQLGetData( + cursor.hstmt, 1, prefix, "utf-16le", "utf-16le", -8 + ) == 0 + assert prefix == [1] and calls == [] + expected = 2 + if mode == "count_error": + try: + cursor.fetchmany(1) + except mssql_python.DatabaseError: + pass + else: + raise AssertionError("count failure was not propagated") + assert len(calls) == 1 + calls.clear() + first = cursor.fetchmany(1)[0] + assert first[0] == expected and len(first) == 2 + assert first[1] == ( + "text" if mode in ("mixed", "prefix", "lob", "decode_error") + else expected + 10 + ) + # Cold eager count and DescribeColumns' independent count. + assert len(calls) == 2, (mode, calls) + if mode == "decode_error": + try: + cursor.fetchone() + except UnicodeDecodeError: + pass + else: + raise AssertionError("invalid UTF-16 must fail GetData decoding") + assert len(calls) == 2 + assert tuple(cursor.fetchmany(1)[0]) == (3, "text") + assert len(calls) == 4 # failure invalidated count and metadata + expected = 3 + cold = len(calls) + assert cursor.fetchmany(1)[0][0] == expected + 1 + assert len(calls) == cold + (1 if mode == "generation" else 0) + warm = len(calls) + assert cursor.fetchval() == expected + 2 + assert cursor.fetchmany(1)[0][0] == expected + 3 + assert len(calls) == warm + assert ddbc.DDBCSQLNumResultCols(cursor.hstmt) == 2 + assert ddbc.DDBCSQLNumResultCols(cursor.hstmt) == 2 + assert len(calls) == warm + 2 # public API is still uncached + assert not errors, errors + if mode == "warning": + assert cursor.messages == [("[01000] (0)", "count warning")] + else: + assert not cursor.messages + while cursor.fetchmany(1): + pass + assert len(calls) == warm + 2 # EOF does not reacquire the count + if mode == "nextset": + cursor.execute("SELECT 7 AS a; SELECT 8 AS a, 9 AS b") + calls.clear() + assert tuple(cursor.fetchmany(1)[0]) == (7,) + assert len(calls) == 2 + assert cursor.nextset() + calls.clear() + assert tuple(cursor.fetchmany(1)[0]) == (8, 9) + assert len(calls) == 2 + assert cursor.fetchmany(1) == [] + assert len(calls) == 2 + finally: + count_slot.value, diag_slot.value = saved_count, saved_diag + assert not errors, errors + """) + result = subprocess.run( + [sys.executable, "-c", script, mode], + env={**os.environ, "DB_CONNECTION_STRING": conn_str}, + capture_output=True, + text=True, + timeout=60, + check=False, + ) + assert result.returncode == 0, result.stdout + result.stderr + + +@pytest.mark.skipif( + sys.platform == "win32", + reason="Windows does not export the native driver function-pointer globals", +) +@pytest.mark.parametrize("payload", ("utf8", "empty", "null", "invalid", "warning")) +def test_narrow_getdata_decoding_in_subprocess(conn_str, payload): + script = textwrap.dedent(""" + import ctypes as c + import os + import sys + import mssql_python + from mssql_python import ddbc_bindings as ddbc + + mode = sys.argv[1] + data = b"\\xff" if mode == "invalid" else ( + b"" if mode == "empty" else "\\ufeffA\\x00\\U0001f600".encode("utf-8") + ) + pointer, short, length = c.c_void_p, c.c_short, c.c_ssize_t + get_type = c.CFUNCTYPE(short, pointer, c.c_ushort, short, pointer, length, pointer) + diag_type = c.CFUNCTYPE( + short, short, pointer, short, pointer, pointer, pointer, short, pointer + ) + library = c.CDLL(ddbc.module.__file__) + slot = pointer.in_dll(library, "SQLGetData_ptr") + diag_slot = pointer.in_dll(library, "SQLGetDiagRec_ptr") + calls, errors = [], [] + + @diag_type + def diagnostic(handle_type, handle, record, state, native, message, capacity, size): + try: + if record > 1: + return 100 + text = "getdata warning".encode("utf-16le") + assert capacity > len(text) // 2 + c.memmove(state, "01000\\0".encode("utf-16le"), 12) + c.memmove(message, text + b"\\0\\0", len(text) + 2) + c.cast(native, c.POINTER(c.c_int))[0] = 0 + c.cast(size, c.POINTER(short))[0] = len(text) // 2 + return 0 + except BaseException as error: + errors.append(type(error).__name__) + return -1 + + @get_type + def getdata(handle, column, ctype, buffer, capacity, indicator): + try: + assert ctype == 1 # SQL_C_CHAR + calls.append(column) + result = original(handle, column, ctype, buffer, capacity, indicator) + assert result == 0 and capacity > len(data) + c.memmove(buffer, data + b"\\0", len(data) + 1) + c.cast(indicator, c.POINTER(length))[0] = -1 if mode == "null" else len(data) + return 1 if mode == "warning" else result + except BaseException as error: + errors.append(type(error).__name__) + return -1 + + with mssql_python.connect(os.environ["DB_CONNECTION_STRING"]) as connection: + with connection.cursor() as cursor: + cursor.execute("SELECT CAST('text' AS VARCHAR(30))") + saved, saved_diag = slot.value, diag_slot.value + assert saved and saved_diag + original = get_type(saved) + assert ddbc.DDBCSQLFetch(cursor.hstmt) == 0 + marker = object() + row = [marker] + slot.value = c.cast(getdata, pointer).value + if mode == "warning": + diag_slot.value = c.cast(diagnostic, pointer).value + try: + ret = ddbc.DDBCSQLGetData( + cursor.hstmt, 1, row, "utf-8", "utf-16le", 1, cursor.messages + ) + assert ret == (1 if mode == "warning" else 0) + finally: + slot.value, diag_slot.value = saved, saved_diag + expected = None if mode == "null" else ( + data if mode == "invalid" else data.decode("utf-8") + ) + assert row == [marker, expected] + assert type(row[1]) is type(expected) + assert calls == [1] and not errors, (calls, errors) + assert cursor.messages == ( + [("[01000] (0)", "getdata warning")] if mode == "warning" else [] + ) + cursor.execute("SELECT 42") + assert cursor.fetchval() == 42 + """) + result = subprocess.run( + [sys.executable, "-c", script, payload], + env={**os.environ, "DB_CONNECTION_STRING": conn_str}, + capture_output=True, + text=True, + timeout=45, + check=False, + ) + assert result.returncode == 0, result.stdout + result.stderr + + +@pytest.mark.skipif( + sys.platform != "win32", + reason="Unix narrow fetches intentionally decode the driver's UTF-8, not a custom codec", +) +@pytest.mark.parametrize("failure", ("memory", "codec", "embedded_nul", "len_error", "len_quiet")) +def test_narrow_custom_codec_failure_in_subprocess(conn_str, failure): + script = textwrap.dedent(""" + import codecs + import os + import sys + import mssql_python + from mssql_python import ddbc_bindings as ddbc + from mssql_python.logging import logger + + calls = [] + length_calls = [] + saved_level = logger._cached_level + + class Decoded(str): + def __len__(self): + length_calls.append(True) + raise ValueError("injected decoded length failure") + + def decode(data, errors="strict"): + calls.append(bytes(data)) + if sys.argv[1] == "memory": + raise MemoryError("injected codec allocation failure") + if sys.argv[1] in ("len_error", "len_quiet"): + return Decoded("text"), len(data) + raise ValueError("injected codec failure") + + def search(name): + if name == "mssql_fetch_test_codec": + return codecs.CodecInfo(name=name, encode=None, decode=decode) + return None + + codecs.register(search) + try: + with mssql_python.connect(os.environ["DB_CONNECTION_STRING"]) as connection: + with connection.cursor() as cursor: + cursor.execute("SELECT 7, CAST('text' AS VARCHAR(10))") + assert ddbc.DDBCSQLFetch(cursor.hstmt) == 0 + marker = object() + row = [marker] + encoding = ( + "utf-8\\0suffix" if sys.argv[1] == "embedded_nul" + else "mssql_fetch_test_codec" + ) + if sys.argv[1] in ("len_error", "len_quiet"): + # Native LOG arguments are evaluated only at DEBUG. + ddbc.update_log_level(10 if sys.argv[1] == "len_error" else 50) + try: + result = ddbc.DDBCSQLGetData( + cursor.hstmt, 2, row, encoding, "utf-16le", 1 + ) + except MemoryError: + assert sys.argv[1] == "memory" + assert row == [marker, 7] + else: + assert result == 0 + if sys.argv[1] in ("len_error", "len_quiet"): + assert type(row[2]) is Decoded + assert row[:3] == [marker, 7, "text"] + assert row[3:] == ([b"text"] if sys.argv[1] == "len_error" else []) + assert length_calls == ([True] if sys.argv[1] == "len_error" else []) + else: + assert sys.argv[1] in ("codec", "embedded_nul") + assert row == [marker, 7, b"text"] + assert type(row[2]) is bytes + assert calls == ([] if sys.argv[1] == "embedded_nul" else [b"text"]) + cursor.execute("SELECT 42") + assert cursor.fetchval() == 42 + finally: + ddbc.update_log_level(saved_level) + codecs.unregister(search) + """) + result = subprocess.run( + [sys.executable, "-c", script, failure], + env={**os.environ, "DB_CONNECTION_STRING": conn_str}, + capture_output=True, + text=True, + timeout=45, + check=False, + ) + assert result.returncode == 0, result.stdout + result.stderr From 80487e1fb0da590281d9093732ed834156982ad2 Mon Sep 17 00:00:00 2001 From: Jahnvi Thakkar Date: Tue, 6 Oct 2026 13:05:32 +0530 Subject: [PATCH 06/17] PERF: Add guarded shared fetch and Row construction Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- mssql_python/cursor.py | 281 +++++++++++++------- mssql_python/pybind/ddbc_bindings.cpp | 66 +++++ mssql_python/pybind/row_factory.hpp | 84 +++++- profiler/README.md | 49 ++++ tests/test_fetch_settings_cache.py | 44 +++- tests/test_fetch_single_row.py | 356 +++++++++++++++++++++++++- 6 files changed, 764 insertions(+), 116 deletions(-) diff --git a/mssql_python/cursor.py b/mssql_python/cursor.py index 0b937f231..996961bbf 100644 --- a/mssql_python/cursor.py +++ b/mssql_python/cursor.py @@ -45,6 +45,29 @@ _DEFAULT_ROW_TYPE = Row _DEFAULT_FAST_ROW_CREATE = Row._fast_create +_DEFAULT_FAST_ROW_CODE = _DEFAULT_FAST_ROW_CREATE.__code__ +_DEFAULT_FAST_ROW_DESCRIPTOR = vars(Row)["_fast_create"] +_DEFAULT_NATIVE_FETCH_ONE = ddbc_bindings.DDBCSQLFetchOne +_DEFAULT_NATIVE_FETCH_MANY = ddbc_bindings.DDBCSQLFetchMany +_NATIVE_ROW_PLAN = object() + + +def _native_row_eligible(row_type): + # Inspect raw class entries: getattr would execute user descriptors before the factory. + if row_type is not _DEFAULT_ROW_TYPE or type(row_type) is not type: + return False + namespace = type.__getattribute__(row_type, "__dict__") + bases = type.__getattribute__(row_type, "__bases__") + return ( + len(bases) == 1 + and bases[0] is object + and "__new__" not in namespace + and "__setattr__" not in namespace + and namespace.get("_fast_create") is _DEFAULT_FAST_ROW_DESCRIPTOR + and _DEFAULT_FAST_ROW_CREATE.__code__ is _DEFAULT_FAST_ROW_CODE + and _DEFAULT_FAST_ROW_CREATE.__globals__.get("Row") is row_type + ) + # Constants for string handling MAX_INLINE_CHAR: int = ( @@ -2860,7 +2883,20 @@ def fetchone(self) -> Union[None, Row]: try: started = perf_start() try: - ret = ddbc_bindings.DDBCSQLFetchOne( + fetch = ddbc_bindings.DDBCSQLFetchOne + if not started and fetch is _DEFAULT_NATIVE_FETCH_ONE and _native_row_eligible(Row): + return ddbc_bindings.DDBCSQLFetchRow( + self.hstmt, + row_data, + char_enc, + wchar_enc, + self._cached_char_ctype, + self.messages, + False, + self._finish_fetchone, + _NATIVE_ROW_PLAN, + ) + ret = fetch( self.hstmt, row_data, char_enc, @@ -2872,46 +2908,63 @@ def fetchone(self) -> Union[None, Row]: if started: perf_stop("py::fetchone::cpp_call", started) - check_error(ddbc_sql_const.SQL_HANDLE_STMT.value, self.hstmt, ret) + return self._finish_fetchone(ret, row_data) + except Exception: + # On error, don't increment rownumber - rethrow the error + raise - if ret == ddbc_sql_const.SQL_NO_DATA.value: - # No more data available - if self._next_row_index == 0 and self.description is not None: - self.rowcount = 0 - return None + def _finish_fetchone(self, ret, row_data, native=False): + check_error(ddbc_sql_const.SQL_HANDLE_STMT.value, self.hstmt, ret) - # Update internal position after successful fetch - if self._skip_increment_for_next_fetch: - self._skip_increment_for_next_fetch = False - self._next_row_index += 1 - else: - self._increment_rownumber() + if ret == ddbc_sql_const.SQL_NO_DATA.value: + # No more data available + if self._next_row_index == 0 and self.description is not None: + self.rowcount = 0 + return None - self.rowcount = self._next_row_index + # Update internal position after successful fetch + if self._skip_increment_for_next_fetch: + self._skip_increment_for_next_fetch = False + self._next_row_index += 1 + else: + self._increment_rownumber() - # Get column and converter maps - column_map, converter_map, column_map_lower = self._get_column_and_converter_maps() - started = perf_start() - try: - if not converter_map and not self._uuid_str_indices: - return Row._fast_create( - row_data, column_map, self, column_map_lower, self._cached_result_columns + self.rowcount = self._next_row_index + + # Get column and converter maps + column_map, converter_map, column_map_lower = self._get_column_and_converter_maps() + started = perf_start() + try: + if not converter_map and not self._uuid_str_indices: + if native and not started and _native_row_eligible(Row): + row_type = Row + factory = row_type._fast_create + column_names = self._cached_result_columns + return ( + _NATIVE_ROW_PLAN, + row_type, + column_map, + self, + column_map_lower, + column_names, + factory, + _DEFAULT_FAST_ROW_CODE, ) - return Row( - row_data, - column_map, - cursor=self, - converter_map=converter_map, - uuid_str_indices=self._uuid_str_indices, - column_map_lower=column_map_lower, - column_names=self._cached_result_columns, + return Row._fast_create( + row_data, column_map, self, column_map_lower, self._cached_result_columns ) - finally: - if started: - perf_stop("py::fetchone::row_wrap", started) - except Exception: - # On error, don't increment rownumber - rethrow the error - raise + return Row( + row_data, + column_map, + cursor=self, + converter_map=converter_map, + uuid_str_indices=self._uuid_str_indices, + column_map_lower=column_map_lower, + column_names=self._cached_result_columns, + ) + finally: + if started: + perf_stop("py::fetchone::row_wrap", started) def fetchmany(self, size: Optional[int] = None) -> List[Row]: """ @@ -2943,7 +2996,26 @@ def fetchmany(self, size: Optional[int] = None) -> List[Row]: try: started = perf_start() try: - ret = ddbc_bindings.DDBCSQLFetchMany( + fetch = ddbc_bindings.DDBCSQLFetchMany + if ( + not started + and type(size) is int + and size == 1 + and fetch is _DEFAULT_NATIVE_FETCH_MANY + and _native_row_eligible(Row) + ): + return ddbc_bindings.DDBCSQLFetchRow( + self.hstmt, + rows_data, + char_enc, + wchar_enc, + self._cached_char_ctype, + self.messages, + True, + self._finish_fetchmany, + _NATIVE_ROW_PLAN, + ) + ret = fetch( self.hstmt, rows_data, size, @@ -2956,70 +3028,87 @@ def fetchmany(self, size: Optional[int] = None) -> List[Row]: if started: perf_stop("py::fetchmany::cpp_call", started) - check_error(ddbc_sql_const.SQL_HANDLE_STMT.value, self.hstmt, ret) + return self._finish_fetchmany(ret, rows_data, size=size) + except Exception: + # On error, don't increment rownumber - rethrow the error + raise - # Update rownumber for the number of rows actually fetched - if rows_data and self._has_result_set: - # advance counters by number of rows actually returned - self._next_row_index += len(rows_data) - self._rownumber = self._next_row_index - 1 + def _finish_fetchmany(self, ret, rows_data, native=False, size=1): + check_error(ddbc_sql_const.SQL_HANDLE_STMT.value, self.hstmt, ret) - # Centralize rowcount assignment after fetch - if len(rows_data) == 0 and self._next_row_index == 0: - self.rowcount = 0 - else: - self.rowcount = self._next_row_index + # Update rownumber for the number of rows actually fetched + if rows_data and self._has_result_set: + # advance counters by number of rows actually returned + self._next_row_index += len(rows_data) + self._rownumber = self._next_row_index - 1 - # Get column and converter maps - column_map, converter_map, column_map_lower = self._get_column_and_converter_maps() + # Centralize rowcount assignment after fetch + if len(rows_data) == 0 and self._next_row_index == 0: + self.rowcount = 0 + else: + self.rowcount = self._next_row_index - # Convert raw data to Row objects - uuid_idx = self._uuid_str_indices - started = perf_start() - try: - if not converter_map and not uuid_idx: - if ( - type(size) is int - and size == 1 - and len(rows_data) == 1 - and Row is _DEFAULT_ROW_TYPE - and Row._fast_create is _DEFAULT_FAST_ROW_CREATE - ): - return [ - Row._fast_create( - rows_data[0], - column_map, - self, - column_map_lower, - self._cached_result_columns, - ) - ] - return ddbc_bindings.construct_rows( - rows_data, - Row, - column_map, - self, - column_map_lower, - self._cached_result_columns, - ) - return [ - Row( - row_data, - column_map, - cursor=self, - converter_map=converter_map, - uuid_str_indices=uuid_idx, - column_map_lower=column_map_lower, - column_names=self._cached_result_columns, - ) - for row_data in rows_data - ] - finally: - if started: - perf_stop("py::fetchmany::row_wrap", started) - except Exception: - # On error, don't increment rownumber - rethrow the error - raise + # Get column and converter maps + column_map, converter_map, column_map_lower = self._get_column_and_converter_maps() + + # Convert raw data to Row objects + uuid_idx = self._uuid_str_indices + started = perf_start() + try: + if not converter_map and not uuid_idx: + if ( + type(size) is int + and size == 1 + and len(rows_data) == 1 + and Row is _DEFAULT_ROW_TYPE + and Row._fast_create is _DEFAULT_FAST_ROW_CREATE + ): + if native and not started and _native_row_eligible(Row): + row_type = Row + factory = row_type._fast_create + column_names = self._cached_result_columns + return ( + _NATIVE_ROW_PLAN, + row_type, + column_map, + self, + column_map_lower, + column_names, + factory, + _DEFAULT_FAST_ROW_CODE, + ) + return [ + Row._fast_create( + rows_data[0], + column_map, + self, + column_map_lower, + self._cached_result_columns, + ) + ] + return ddbc_bindings.construct_rows( + rows_data, + Row, + column_map, + self, + column_map_lower, + self._cached_result_columns, + ) + return [ + Row( + row_data, + column_map, + cursor=self, + converter_map=converter_map, + uuid_str_indices=uuid_idx, + column_map_lower=column_map_lower, + column_names=self._cached_result_columns, + ) + for row_data in rows_data + ] + finally: + if started: + perf_stop("py::fetchmany::row_wrap", started) def fetchall(self) -> List[Row]: """ diff --git a/mssql_python/pybind/ddbc_bindings.cpp b/mssql_python/pybind/ddbc_bindings.cpp index b6d4c5ac3..4e7554ea1 100644 --- a/mssql_python/pybind/ddbc_bindings.cpp +++ b/mssql_python/pybind/ddbc_bindings.cpp @@ -6413,6 +6413,55 @@ SQLRETURN FetchSingleRow(SqlHandlePtr StatementHandle, py::list& row, return ret; } +// Python completion retains return validation, counters, maps and custom factory ordering. +py::object FetchRow_wrap(SqlHandlePtr statement, py::list& data, + const std::string& charEncoding, const std::string& wcharEncoding, + int charCtype, py::handle messages, bool many, + const py::function& complete, const py::object& planToken, + const py::tuple& attributes, const py::str& rowGlobalName, + const py::str& newName, const py::str& setattrName) { + SQLRETURN ret = many + ? FetchMany_wrap(statement, data, 1, charEncoding, wcharEncoding, charCtype, messages) + : FetchOne_wrap(statement, data, charEncoding, wcharEncoding, charCtype, messages); + py::object result = complete(ret, data, true); + if (!PyTuple_CheckExact(result.ptr()) || PyTuple_GET_SIZE(result.ptr()) != 8 || + PyTuple_GET_ITEM(result.ptr(), 0) != planToken.ptr()) { + return result; + } + if (many && PyList_GET_SIZE(data.ptr()) != 1) { + throw py::value_error("A single-row construction plan requires exactly one fetched row"); + } + py::tuple plan = borrow(result.ptr()); + py::object values = many ? borrow(PyList_GET_ITEM(data.ptr(), 0)) : borrow(data.ptr()); + py::object factory = borrow(PyTuple_GET_ITEM(plan.ptr(), 6)); + py::object row; + PyObject* factoryRowType = nullptr; + if (PyFunction_Check(factory.ptr())) { + factoryRowType = PyDict_GetItemWithError(PyFunction_GetGlobals(factory.ptr()), + rowGlobalName.ptr()); + if (!factoryRowType && PyErr_Occurred()) { + throw py::error_already_set(); + } + } + // Final argument evaluation or plan allocation can change the captured factory in place. + if (PyFunction_Check(factory.ptr()) && + PyFunction_GetCode(factory.ptr()) == PyTuple_GET_ITEM(plan.ptr(), 7) && + factoryRowType == PyTuple_GET_ITEM(plan.ptr(), 1) && + RowFactory::has_default_row_allocation(factoryRowType, newName, setattrName)) { + PERF_TIMER("FetchRow::construct_row"); + row = RowFactory::construct_row(values, plan[1].cast(), plan[2], + plan[3], plan[4], plan[5], attributes); + } else { + row = factory(values, plan[2], plan[3], plan[4], plan[5]); + } + if (!many) { + return row; + } + py::list rows(1); + PyList_SET_ITEM(rows.ptr(), 0, row.release().ptr()); + return rows; +} + // Wrap SQLMoreResults SQLRETURN SQLMoreResults_wrap(SqlHandlePtr StatementHandle) { PERF_TIMER("SQLMoreResults_wrap"); @@ -6727,6 +6776,23 @@ PYBIND11_MODULE(ddbc_bindings, m) { py::arg("column_map_lower") = py::none(), py::arg("column_names") = py::none()); + // Owned by binding defaults and released with the module; no static Python handles. + const py::tuple rowAttributes = py::make_tuple( + "_values", "_column_map", "_cursor", "_column_map_lower", "_column_names"); + m.def("construct_row", &RowFactory::construct_row, + "Build one compatible Row with checked ownership", + py::arg("values"), py::arg("row_class"), py::arg("column_map"), py::arg("cursor"), + py::arg("column_map_lower") = py::none(), py::arg("column_names") = py::none(), + py::arg("attributes") = rowAttributes); + m.def("DDBCSQLFetchRow", &FetchRow_wrap, + "Fetch and construct one compatible Row with ordered Python completion", + py::arg("statement"), py::arg("data"), py::arg("char_encoding"), + py::arg("wchar_encoding"), py::arg("char_ctype"), py::arg("messages"), + py::arg("many"), py::arg("complete"), py::arg("plan_token"), + py::arg("attributes") = rowAttributes, py::arg("row_global_name") = py::str("Row"), + py::arg("new_name") = py::str("__new__"), + py::arg("setattr_name") = py::str("__setattr__")); + // Expose logger bridge function to Python m.def("update_log_level", &mssql_python::logging::LoggerBridge::updateLevel, "Update the cached log level in C++ bridge"); diff --git a/mssql_python/pybind/row_factory.hpp b/mssql_python/pybind/row_factory.hpp index 85d271128..6ef1a7298 100644 --- a/mssql_python/pybind/row_factory.hpp +++ b/mssql_python/pybind/row_factory.hpp @@ -7,6 +7,25 @@ namespace RowFactory { +inline void initialize_row( + const py::object& row, PyObject* row_data, const py::object& column_map, + const py::object& cursor_obj, const py::object& column_map_lower, + const py::object& column_names, const py::handle& attr_values, const py::handle& attr_column_map, + const py::handle& attr_cursor, const py::handle& attr_column_map_lower, + const py::handle& attr_column_names, + int (*set_attr)(PyObject*, PyObject*, PyObject*) = PyObject_GenericSetAttr) { + if (!row) + throw py::error_already_set(); + + if (set_attr(row.ptr(), attr_values.ptr(), row_data) < 0 || + set_attr(row.ptr(), attr_column_map.ptr(), column_map.ptr()) < 0 || + set_attr(row.ptr(), attr_cursor.ptr(), cursor_obj.ptr()) < 0 || + set_attr(row.ptr(), attr_column_map_lower.ptr(), column_map_lower.ptr()) < 0 || + set_attr(row.ptr(), attr_column_names.ptr(), column_names.ptr()) < 0) { + throw py::error_already_set(); + } +} + // Wrap fetched values without converter or UUID processing. // Accepts Row and its subclasses, bypassing __init__. inline py::list construct_rows(const py::list& rows_data, const py::object& row_class, @@ -36,22 +55,67 @@ inline py::list construct_rows(const py::list& rows_data, const py::object& row_ py::object row = steal(row_type->tp_alloc(row_type, 0)); if (!row) throw py::error_already_set(); - PyObject* row_data = PyList_GET_ITEM(rows_data.ptr(), i); + initialize_row(row, row_data, column_map, cursor_obj, column_map_lower, column_names, + attr_values, attr_column_map, attr_cursor, attr_column_map_lower, + attr_column_names); + PyList_SET_ITEM(result.ptr(), i, row.release().ptr()); + } - if (PyObject_GenericSetAttr(row.ptr(), attr_values.ptr(), row_data) < 0 || - PyObject_GenericSetAttr(row.ptr(), attr_column_map.ptr(), column_map.ptr()) < 0 || - PyObject_GenericSetAttr(row.ptr(), attr_cursor.ptr(), cursor_obj.ptr()) < 0 || - PyObject_GenericSetAttr(row.ptr(), attr_column_map_lower.ptr(), - column_map_lower.ptr()) < 0 || - PyObject_GenericSetAttr(row.ptr(), attr_column_names.ptr(), column_names.ptr()) < 0) { + return result; +} + +// Passive final eligibility check: do not execute newly installed class descriptors. +inline bool has_default_row_allocation(PyObject* row_class, const py::str& new_name, + const py::str& setattr_name) { + if (!PyUnicode_CheckExact(new_name.ptr()) || !PyUnicode_CheckExact(setattr_name.ptr())) { + throw py::type_error("Row allocation guard names must be exact strings"); + } + if (!row_class || Py_TYPE(row_class) != &PyType_Type) { + return false; + } + auto* row_type = reinterpret_cast(row_class); + if (!row_type->tp_bases || !PyTuple_CheckExact(row_type->tp_bases) || + PyTuple_GET_SIZE(row_type->tp_bases) != 1 || + PyTuple_GET_ITEM(row_type->tp_bases, 0) != reinterpret_cast(&PyBaseObject_Type) || + !row_type->tp_dict) { + return false; + } + PyObject* names[] = {new_name.ptr(), setattr_name.ptr()}; + for (PyObject* name : names) { + PyObject* member = PyDict_GetItemWithError(row_type->tp_dict, name); + if (member) { + return false; + } + if (PyErr_Occurred()) { throw py::error_already_set(); } - - PyList_SET_ITEM(result.ptr(), i, row.release().ptr()); } + return true; +} - return result; +// Attribute names are prepared once as binding defaults, not allocated for each row. +inline py::object construct_row(const py::object& values, const py::type& row_class, + const py::object& column_map, const py::object& cursor_obj, + const py::object& column_map_lower, const py::object& column_names, + const py::tuple& attributes) { + if (attributes.size() != 5) { + throw py::value_error("Row construction requires five attribute names"); + } + for (py::handle name : attributes) { + if (!PyUnicode_Check(name.ptr())) { + throw py::type_error("Row attribute names must be strings"); + } + } + const py::object new_method = row_class.attr("__new__"); + py::object row = steal(PyObject_CallOneArg(new_method.ptr(), row_class.ptr())); + initialize_row(row, values.ptr(), column_map, cursor_obj, column_map_lower, column_names, + py::handle(PyTuple_GET_ITEM(attributes.ptr(), 0)), + py::handle(PyTuple_GET_ITEM(attributes.ptr(), 1)), + py::handle(PyTuple_GET_ITEM(attributes.ptr(), 2)), + py::handle(PyTuple_GET_ITEM(attributes.ptr(), 3)), + py::handle(PyTuple_GET_ITEM(attributes.ptr(), 4)), PyObject_SetAttr); + return row; } } // namespace RowFactory diff --git a/profiler/README.md b/profiler/README.md index c09ad5496..d6c2608f1 100644 --- a/profiler/README.md +++ b/profiler/README.md @@ -198,6 +198,55 @@ row-array configuration/cleanup. Row user attributes, weakrefs and substituted factories remain supported, as do dynamic `fetchone` overrides. No first-column-only `fetchval` route is introduced. +The experimental shared `DDBCSQLFetchRow` entry performs the original native fetch, +then calls the extracted Python completion body for return checking, row counters +and map acquisition. An eligible completion returns a private construction plan; +the same native call builds the Row and returns it (or a one-element list for +`fetchmany(1)`). Unsupported/customized completion constructs the already-fetched +values in Python. It never refetches. `fetchval()` still dispatches through +`self.fetchone()` and all-column conversions remain in the completion body. + +Eligibility inspects raw class dictionaries and the original factory code without +executing allocation/assignment descriptors or metaclass hooks. It is checked +before fetching and again after map acquisition. The captured factory code and +Row global are checked again in native code after final argument evaluation; +late changes call that captured factory on the fetched values, without refetching. +A final native raw-type/dictionary guard also rejects newly installed allocation or +assignment hooks without invoking them. The captured Python factory then preserves +`Row.__new__(Row)` lookup order, including a descriptor changing the second `Row`. +In-place factory changes, custom +allocation/assignment, converters and UUID policies retain their fallback behavior. +The original `Row._fast_create` body is unchanged. Constructor attribute names are +an immutable tuple owned by binding defaults, prepared once per module rather +than five Python string allocations per row; no static owning Python handles or +Row layout offsets are used. Native construction still uses `__new__` validation +and normal checked attribute assignment, preserving descriptor callbacks. + +This is a shared fetch-and-construction call, **not** an all-native cursor: +completion still re-enters Python, and eligible rows allocate a small plan tuple. +The ordinary real-source frame trace removes one `_fast_create` frame but adds +two `_native_row_eligible` frames and one `_finish_fetch*` frame: net **two more +Python frames**. The outer Python-to-native call count is unchanged, and completion +adds a native-to-Python crossing. This is not reduced Python dispatch or fewer +crossings. These costs may outweigh native assignment work; no speedup or slowdown +is established without measurement. Larger requests, integer subclasses and substituted low-level +fetch bindings retain the original entry. Native numeric/non-numeric routing and +cleanup are unchanged. + +When Python phase profiling is active, the original split entry is used so +`cpp_call` and `row_wrap` retain their existing boundaries. With Python phases off, +`ddbc::FetchRow::construct_row` attributes native construction when native profiling +is available. Timing the split Python-profiled path is not evidence of fused-path +latency; comparisons must identify which route actually ran. + +The isolated `test_single_row_fusion_native_counters_in_subprocess` cases enable +native counters with Python phases disabled. For each API they require two native +Row constructions for two rows, none at EOF, and none for converter fallback while +all-column callbacks still run. These three cases skip on native profiling-OFF +builds; the constructor-frame tests still run there. Split-route profiling alone +cannot qualify fusion. These are future exact-source CI contracts, not local +runtime measurements. + Use profiling-enabled builds to check operation counts and normal uninstrumented Release builds for latency comparisons. These routes are optimization hypotheses, not a measured speedup; source-only checks do not establish native correctness. diff --git a/tests/test_fetch_settings_cache.py b/tests/test_fetch_settings_cache.py index b9e63e7ca..828283b0c 100644 --- a/tests/test_fetch_settings_cache.py +++ b/tests/test_fetch_settings_cache.py @@ -617,7 +617,7 @@ def assert_row_instance_capabilities(row): return reference -@pytest.mark.parametrize("construction", ("direct", "python_fast", "native")) +@pytest.mark.parametrize("construction", ("direct", "python_fast", "native", "native_single")) def test_constructed_row_preserves_instance_capabilities(construction): values = [42] column_map = {"number": 0} @@ -625,6 +625,8 @@ def test_constructed_row_preserves_instance_capabilities(construction): row = Row(values, column_map) elif construction == "python_fast": row = Row._fast_create(values, column_map, None) + elif construction == "native_single": + row = mssql_python.ddbc_bindings.construct_row(values, Row, column_map, None) else: row = mssql_python.ddbc_bindings.construct_rows([values], Row, column_map, None)[0] assert row._values is values @@ -643,8 +645,12 @@ def test_fetched_row_preserves_instance_capabilities(connection, method): assert reference() is None -@pytest.mark.parametrize("size", (0, 1, 3)) -def test_construct_rows_repeated_calls_release_references(size): +@pytest.mark.parametrize( + ("size", "single"), + ((0, False), (1, False), (3, False), (1, True)), + ids=("0", "1", "3", "native-single"), +) +def test_construct_rows_repeated_calls_release_references(size, single): values = [[index] for index in range(size)] column_map = {"Number": 0} column_map_lower = {"number": 0} @@ -653,9 +659,16 @@ def test_construct_rows_repeated_calls_release_references(size): tracked = (values, column_map, column_map_lower, column_names, cursor, *values) references = [sys.getrefcount(value) for value in tracked] for _ in range(10): - rows = mssql_python.ddbc_bindings.construct_rows( - values, Row, column_map, cursor, column_map_lower, column_names - ) + if single: + rows = [ + mssql_python.ddbc_bindings.construct_row( + values[0], Row, column_map, cursor, column_map_lower, column_names + ) + ] + else: + rows = mssql_python.ddbc_bindings.construct_rows( + values, Row, column_map, cursor, column_map_lower, column_names + ) assert len(rows) == size assert all(row._values is values[index] for index, row in enumerate(rows)) assert all(row._column_map is column_map for row in rows) @@ -667,6 +680,14 @@ def test_construct_rows_repeated_calls_release_references(size): def test_construct_rows_releases_partial_batch_on_attribute_error(): + _assert_construct_row_attribute_failure(single=False) + + +def test_construct_single_row_releases_references_on_attribute_error(): + _assert_construct_row_attribute_failure(single=True) + + +def _assert_construct_row_attribute_failure(single): class FailingRow(Row): __slots__ = () @@ -687,9 +708,14 @@ def _column_names(self, names): references = [sys.getrefcount(value) for value in tracked] for _ in range(10): with pytest.raises(RuntimeError, match="injected slot assignment failure"): - mssql_python.ddbc_bindings.construct_rows( - values, FailingRow, column_map, cursor, None, column_names - ) + if single: + mssql_python.ddbc_bindings.construct_row( + values[-1], FailingRow, column_map, cursor, None, column_names + ) + else: + mssql_python.ddbc_bindings.construct_rows( + values, FailingRow, column_map, cursor, None, column_names + ) assert [sys.getrefcount(value) for value in tracked] == references diff --git a/tests/test_fetch_single_row.py b/tests/test_fetch_single_row.py index 1cbd60ad3..d157eb11c 100644 --- a/tests/test_fetch_single_row.py +++ b/tests/test_fetch_single_row.py @@ -4,7 +4,7 @@ import subprocess import sys import textwrap -from unittest.mock import patch +from unittest.mock import Mock, patch import pytest @@ -608,3 +608,357 @@ def warning(handle_type, handle, record, state, native, message, capacity, size) check=False, ) assert result.returncode == 0, result.stdout + result.stderr + + +@pytest.mark.parametrize("mutation", ("new", "setattr", "code", "abstract", "descriptor")) +def test_fast_row_inplace_customization(monkeypatch, mutation): + import weakref + from mssql_python import ddbc_bindings + from mssql_python.row import Row + + factory = Row._fast_create + events = [] + + def custom_new(cls): + events.append("new") + return object.__new__(cls) + + def custom_setattr(self, name, value): + events.append(name) + object.__setattr__(self, name, value) + + def custom_factory(values, column_map, cursor, column_map_lower=None, column_names=None): + raise RuntimeError("customized factory code") + + def fail_descriptor(self, value): + events.append(weakref.ref(self)) + raise RuntimeError("customized descriptor") + + if mutation == "new": + monkeypatch.setattr(Row, "__new__", staticmethod(custom_new)) + elif mutation == "setattr": + monkeypatch.setattr(Row, "__setattr__", custom_setattr) + elif mutation == "code": + monkeypatch.setattr(factory, "__code__", custom_factory.__code__) + elif mutation == "abstract": + monkeypatch.setattr(Row, "__abstractmethods__", frozenset({"required"}), raising=False) + else: + monkeypatch.setattr(Row, "_column_names", property(fset=fail_descriptor)) + + assert Row._fast_create is factory + with patch.object(ddbc_bindings, "construct_row", wraps=ddbc_bindings.construct_row) as native: + if mutation in ("code", "descriptor"): + with pytest.raises(RuntimeError, match="customized"): + factory([42], {"number": 0}, None) + elif mutation == "abstract": + with pytest.raises(TypeError, match="abstract"): + factory([42], {"number": 0}, None) + else: + assert factory([42], {"number": 0}, None).number == 42 + native.assert_not_called() + if mutation == "new": + assert events == ["new"] + elif mutation == "setattr": + assert events == ["_values", "_column_map", "_cursor", "_column_map_lower", "_column_names"] + elif mutation == "descriptor": + assert len(events) == 1 and events[0]() is None + + +@pytest.mark.parametrize("method", ("fetchone", "fetchmany", "fetchval")) +@pytest.mark.parametrize("failure", ("maps", "factory")) +def test_single_row_construction_failure_keeps_fetch_position(cursor, method, failure): + from mssql_python import ddbc_bindings + + cursor.execute("SELECT n AS number FROM (VALUES (1), (2)) AS v(n) ORDER BY n") + from mssql_python.row import Row + + bridge_name = "DDBCSQLFetchRow" + bridge = ddbc_bindings.DDBCSQLFetchRow + + def fetch(): + value = cursor.fetchmany(1) if method == "fetchmany" else getattr(cursor, method)() + return value[0][0] if method == "fetchmany" else value if method == "fetchval" else value[0] + + def fail(*args): + assert cursor.rowcount == 1 + assert cursor.rownumber == 0 + assert cursor._next_row_index == 1 + raise RuntimeError("injected post-fetch failure") + + failed_stage = Mock(side_effect=fail) + failure_patch = ( + patch.object(cursor, "_get_column_and_converter_maps", failed_stage) + if failure == "maps" + else patch.object(Row, "_column_names", property(fset=failed_stage)) + ) + with patch.object(ddbc_bindings, bridge_name, wraps=bridge) as native_fetch, failure_patch: + with pytest.raises(RuntimeError, match="injected post-fetch failure"): + fetch() + native_fetch.assert_called_once() + failed_stage.assert_called_once() + assert fetch() == 2 + assert cursor.rowcount == 2 + assert cursor.rownumber == 1 + + +@pytest.mark.parametrize("target", ("__new__", "__setattr__")) +def test_native_row_guard_does_not_invoke_descriptors(monkeypatch, target): + from mssql_python.cursor import _native_row_eligible + from mssql_python.row import Row + + events = [] + + class Descriptor: + def __get__(self, instance, owner): + events.append("lookup") + if target == "__new__": + if len(events) > 1: + raise RuntimeError("duplicate allocator lookup") + + def allocate(cls): + events.append("allocate") + return object.__new__(cls) + + return allocate + return lambda name, value: object.__setattr__(instance, name, value) + + monkeypatch.setattr(Row, target, Descriptor()) + assert not _native_row_eligible(Row) + assert events == [] + row = Row._fast_create([42], {"number": 0}, None) + assert row.number == 42 + assert events == (["lookup", "allocate"] if target == "__new__" else ["lookup"] * 5) + + +def test_native_row_guard_does_not_invoke_metaclass_hooks(): + from mssql_python.cursor import _native_row_eligible + from mssql_python.row import Row + + class Meta(type): + def __getattribute__(cls, name): + raise AssertionError("guard must not inspect substituted class through metaclass") + + class CustomRow(Row, metaclass=Meta): + pass + + assert not _native_row_eligible(CustomRow) + + +@pytest.mark.parametrize("method", ("fetchone", "fetchmany", "fetchval")) +@pytest.mark.parametrize("when", ("maps", "final_argument")) +def test_single_row_fusion_handles_post_fetch_factory_change(cursor, method, when="maps"): + from mssql_python import ddbc_bindings + from mssql_python.row import Row + + original_maps = cursor._get_column_and_converter_maps + factory = Row._fast_create + + def replacement(values, column_map, cursor, column_map_lower=None, column_names=None): + raise RuntimeError("factory changed after native advancement") + + def maps(): + factory.__code__ = replacement.__code__ + return original_maps() + + def names(self): + factory.__code__ = replacement.__code__ + return self.__dict__["_cached_result_columns"] + + cursor.execute("SELECT n AS number FROM (VALUES (1), (2)) AS v(n) ORDER BY n") + cursor._get_column_and_converter_maps() + change = ( + patch.object(cursor, "_get_column_and_converter_maps", side_effect=maps) + if when == "maps" + else patch.object(type(cursor), "_cached_result_columns", property(names), create=True) + ) + old_code = factory.__code__ + try: + with ( + change, + patch.object( + ddbc_bindings, "DDBCSQLFetchRow", wraps=ddbc_bindings.DDBCSQLFetchRow + ) as fused, + ): + with pytest.raises(RuntimeError, match="factory changed after native advancement"): + cursor.fetchmany(1) if method == "fetchmany" else getattr(cursor, method)() + fused.assert_called_once() + assert Row._fast_create is factory + assert cursor.rowcount == 1 and cursor.rownumber == 0 + finally: + factory.__code__ = old_code + assert cursor.fetchone().number == 2 + + +@pytest.mark.parametrize("method", ("fetchone", "fetchmany", "fetchval")) +def test_single_row_fusion_uses_native_constructor(cursor, method): + from mssql_python import ddbc_bindings, perf_timer + from mssql_python.row import Row + + cursor.execute("SELECT 42 AS number, CAST(N'text' AS NVARCHAR(10)) AS label") + factory_code = Row._fast_create.__code__ + calls = [] + previous_profile = sys.getprofile() + phases_enabled = perf_timer.is_enabled() + + def profile(frame, event, arg): + if event == "call" and frame.f_code is factory_code: + calls.append(event) + + perf_timer.disable() + try: + with patch.object( + ddbc_bindings, "DDBCSQLFetchRow", wraps=ddbc_bindings.DDBCSQLFetchRow + ) as fused: + sys.setprofile(profile) + try: + result = cursor.fetchmany(1) if method == "fetchmany" else getattr(cursor, method)() + finally: + sys.setprofile(previous_profile) + fused.assert_called_once() + if method == "fetchval": + assert result == 42 + else: + row = result[0] if method == "fetchmany" else result + assert type(row) is Row + assert tuple(row) == (42, "text") + assert calls == [] + assert cursor.rowcount == 1 and cursor.rownumber == 0 + finally: + if phases_enabled: + perf_timer.enable() + + +@pytest.mark.parametrize("method", ("fetchone", "fetchmany", "fetchval")) +def test_single_row_late_allocator_preserves_factory_global_lookup(cursor, method): + from mssql_python import ddbc_bindings + from mssql_python.row import Row + + original_row = Row + + class ChangedRow(original_row): + pass + + events = [] + factory = original_row._fast_create + assert "__new__" not in vars(original_row) + + class Allocator: + def __get__(self, instance, owner): + events.append("new_lookup") + factory.__globals__["Row"] = ChangedRow + + def allocate(cls): + events.append("allocate:" + cls.__name__) + return object.__new__(cls) + + return allocate + + def names(self): + events.append("names_lookup") + original_row.__new__ = Allocator() + return self.__dict__["_cached_result_columns"] + + cursor.execute("SELECT n AS number FROM (VALUES (1), (2)) AS v(n) ORDER BY n") + cursor._get_column_and_converter_maps() + try: + with ( + patch.object(type(cursor), "_cached_result_columns", property(names), create=True), + patch.object( + ddbc_bindings, "DDBCSQLFetchRow", wraps=ddbc_bindings.DDBCSQLFetchRow + ) as fused, + ): + result = cursor.fetchmany(1) if method == "fetchmany" else getattr(cursor, method)() + fused.assert_called_once() + if method == "fetchval": + assert result == 1 + else: + row = result[0] if method == "fetchmany" else result + assert type(row) is ChangedRow + assert row.number == 1 + assert events == ["names_lookup", "new_lookup", "allocate:ChangedRow"] + assert cursor.rowcount == 1 and cursor.rownumber == 0 + finally: + factory.__globals__["Row"] = original_row + if "__new__" in vars(original_row): + delattr(original_row, "__new__") + assert cursor.fetchone().number == 2 + + +@pytest.mark.parametrize("method", ("fetchone", "fetchmany", "fetchval")) +def test_single_row_fusion_native_counters_in_subprocess(conn_str, method): + from mssql_python import ddbc_bindings + + if not hasattr(ddbc_bindings, "profiling"): + pytest.skip("requires a profiling-enabled native build; Python phases remain disabled") + script = textwrap.dedent(""" + import os + import sys + import mssql_python + from mssql_python import ddbc_bindings as ddbc, perf_timer + + method = sys.argv[1] + p = ddbc.profiling + perf_timer.disable() + perf_timer.reset() + p.disable() + query = ( + "SELECT n AS a, n + 10 AS b, CAST(N'text' AS NVARCHAR(10)) AS label " + "FROM (VALUES (1), (2)) AS v(n) ORDER BY n" + ) + fetch_timer = "ddbc::FetchMany_wrap" if method == "fetchmany" else "ddbc::FetchOne_wrap" + + def calls(name): + return p.get_stats().get(name, {}).get("calls", 0) + + with mssql_python.connect(os.environ["DB_CONNECTION_STRING"]) as connection: + with connection.cursor() as cursor: + def fetch(): + if method == "fetchval": + return cursor.fetchval() + result = cursor.fetchmany(1) if method == "fetchmany" else cursor.fetchone() + row = result[0] if result and method == "fetchmany" else result + return tuple(row) if row else None + + try: + cursor.execute(query) + p.reset() + p.enable() + assert fetch() == (1 if method == "fetchval" else (1, 11, "text")) + assert calls("ddbc::FetchRow::construct_row") == 1 + assert calls(fetch_timer) == 1 + assert fetch() == (2 if method == "fetchval" else (2, 12, "text")) + assert fetch() is None + assert fetch() is None + assert calls("ddbc::FetchRow::construct_row") == 2 + assert calls(fetch_timer) == 4 + assert perf_timer.get_stats() == {} + assert cursor.rowcount == 2 and cursor.rownumber == 1 + + p.disable() + converted = [] + def convert(value): + converted.append(value) + return value + 100 + connection.add_output_converter(mssql_python.SQL_INTEGER, convert) + cursor.execute(query) + p.reset() + p.enable() + assert fetch() == (101 if method == "fetchval" else (101, 111, "text")) + assert converted == [1, 11] + assert calls("ddbc::FetchRow::construct_row") == 0 + assert calls(fetch_timer) == 1 + assert perf_timer.get_stats() == {} + finally: + p.disable() + """) + environment = os.environ.copy() + environment["DB_CONNECTION_STRING"] = conn_str + result = subprocess.run( + [sys.executable, "-c", script, method], + env=environment, + capture_output=True, + text=True, + timeout=45, + check=False, + ) + assert result.returncode == 0, result.stdout + result.stderr From 08d57010a4d38b6b9064f7eec99cbdc4147b19a8 Mon Sep 17 00:00:00 2001 From: Jahnvi Thakkar Date: Tue, 6 Oct 2026 13:37:06 +0530 Subject: [PATCH 07/17] FIX: Correct parametrized fetch regression test signature Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- tests/test_fetch_single_row.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_fetch_single_row.py b/tests/test_fetch_single_row.py index d157eb11c..501ba8d50 100644 --- a/tests/test_fetch_single_row.py +++ b/tests/test_fetch_single_row.py @@ -746,7 +746,7 @@ class CustomRow(Row, metaclass=Meta): @pytest.mark.parametrize("method", ("fetchone", "fetchmany", "fetchval")) @pytest.mark.parametrize("when", ("maps", "final_argument")) -def test_single_row_fusion_handles_post_fetch_factory_change(cursor, method, when="maps"): +def test_single_row_fusion_handles_post_fetch_factory_change(cursor, method, when): from mssql_python import ddbc_bindings from mssql_python.row import Row From f539cafa4bc616a8c8a3fd1bd9b1c93c67957bda Mon Sep 17 00:00:00 2001 From: Jahnvi Thakkar Date: Tue, 6 Oct 2026 20:05:13 +0530 Subject: [PATCH 08/17] PERF: Wire latency-first single-row PR performance reports Preserve bounded CI budgets and report unavailable modes explicitly. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- eng/pipelines/pr-validation-pipeline.yml | 2 +- eng/profiler_benchmarks/controller.py | 475 ++++++++- eng/profiler_benchmarks/report.py | 409 +++++++- eng/profiler_benchmarks/workloads.py | 66 ++ profiler/README.md | 108 ++ tests/test_036_profiler_ci.py | 1169 ++++++++++++++++++++++ 6 files changed, 2176 insertions(+), 53 deletions(-) diff --git a/eng/pipelines/pr-validation-pipeline.yml b/eng/pipelines/pr-validation-pipeline.yml index 2107a5bfe..d6cdcad81 100644 --- a/eng/pipelines/pr-validation-pipeline.yml +++ b/eng/pipelines/pr-validation-pipeline.yml @@ -1009,7 +1009,7 @@ jobs: ACCEPT_EULA=Y apt-get install -y --no-install-recommends git unixodbc unixodbc-dev libodbc2 libodbcinst2 odbcinst msodbcsql18 git config --global --add safe.directory /workspace odbcinst -q -d -n "ODBC Driver 18 for SQL Server" - python -m eng.profiler_benchmarks.controller --reuse-candidate --leg "$(profilerLeg)" --output profiler-results + python -m eng.profiler_benchmarks.controller --reuse-candidate --ci-report --leg "$(profilerLeg)" --output profiler-results ' else echo "Skipping performance benchmarks on $(distroName) (only runs on Ubuntu with local SQL Server)" diff --git a/eng/profiler_benchmarks/controller.py b/eng/profiler_benchmarks/controller.py index 143164e21..5c79e0122 100644 --- a/eng/profiler_benchmarks/controller.py +++ b/eng/profiler_benchmarks/controller.py @@ -3,6 +3,7 @@ import argparse import contextlib import faulthandler +import hashlib import importlib.util import io import json @@ -11,13 +12,14 @@ import platform import re import signal +import shutil import subprocess import sys import tarfile import tempfile import time -from .report import LEGS +from .report import LEGS, MODES, validate, validate_ci_mode, validate_ci_header, ci_mode_reports from . import workloads ROOT = Path(__file__).resolve().parents[2] @@ -28,18 +30,33 @@ LOCAL_BENCHMARK_TIMEOUT = 105 * 60 WORKER_TIMEOUT = 6 * 60 WINDOWS = os.name == "nt" +FETCH_WORKER_TIMEOUT = 30 +CI_FINISH_RESERVE = 180 -def git(*args): - return subprocess.check_output(["git", "-C", str(ROOT), *args], text=True).strip() +class ProcessCleanupError(RuntimeError): + """Further work is unsafe until the previous process tree is reaped.""" -def resolve_revisions(base, candidate): - candidate = git("rev-parse", "--verify", "--end-of-options", f"{candidate}^{{commit}}") +def git(*args, timeout=None): + return subprocess.check_output( + ["git", "-C", str(ROOT), *args], text=True, timeout=timeout + ).strip() + + +def resolve_revisions(base, candidate, timeout=None): + options = {"timeout": timeout} if timeout is not None else {} + candidate = git( + "rev-parse", "--verify", "--end-of-options", f"{candidate}^{{commit}}", **options + ) # ADO validates refs/pull/N/merge. Its first parent is the exact target snapshot, # not whichever main build happened to finish most recently. base = git( - "rev-parse", "--verify", "--end-of-options", f"{base or candidate + '^1'}^{{commit}}" + "rev-parse", + "--verify", + "--end-of-options", + f"{base or candidate + '^1'}^{{commit}}", + **options, ) return base, candidate @@ -90,16 +107,19 @@ def terminate_process_tree(process): process.wait(timeout=5) -def build(path, log, timeout=900): - env = dict(os.environ, ENABLE_PROFILING="1") +def build(path, log, timeout=900, profiling=True): + env = dict(os.environ, ENABLE_PROFILING="1" if profiling else "0") # build scripts find Python via PATH; keep the controller's interpreter. env["PATH"] = str(Path(sys.executable).parent) + os.pathsep + env["PATH"] command = ["cmd", "/c", "build.bat"] if os.name == "nt" else ["bash", "build.sh"] + run_process(command, log, timeout, cwd=path / "mssql_python/pybind", env=env) + + +def run_process(command, log, timeout, **options): with log.open("w", encoding="utf-8") as output: process = subprocess.Popen( command, - cwd=path / "mssql_python/pybind", - env=env, + **options, stdout=output, stderr=subprocess.STDOUT, start_new_session=not WINDOWS, @@ -108,7 +128,10 @@ def build(path, log, timeout=900): try: returncode = process.wait(timeout=timeout) except subprocess.TimeoutExpired: - terminate_process_tree(process) + try: + terminate_process_tree(process) + except (OSError, RuntimeError, subprocess.TimeoutExpired) as error: + raise ProcessCleanupError("Could not reap the timed-out process tree") from error raise if returncode: raise subprocess.CalledProcessError(returncode, command) @@ -145,10 +168,150 @@ def load_suite(): return core, workloads +class _FetchContext: + """Controller-only recording policy; the interactive Profiler stays unchanged.""" + + def __init__(self, mode, native, python): + if mode not in ("latency", "route") or (native is not None) != (mode == "route"): + raise ValueError("Invalid fetch recording configuration") + self.native, self.python = native, python + + def disable(self): + self.python.disable() + self.python.disable_timeline() + if self.native is not None: + self.native.disable() + self.native.disable_timeline() + + def enable(self): + self.disable() + self.python.reset() + if self.native is not None: + self.native.reset() + self.native.enable() + + def collect(self): + if self.python.is_enabled(): + raise RuntimeError("Python phases became enabled during a fetch measurement") + self.disable() + py = self.python.get_stats() + if py: + raise RuntimeError("Python phase samples contaminate the fetch route") + return self.native.get_stats() if self.native is not None else {}, py + + +def verify_reused_source(source_root, revision): + if source_root.resolve() == ROOT.resolve(): + if git("rev-parse", "HEAD", timeout=5) != revision: + raise ValueError("Reused checkout revision changed") + git( + "diff", + "--exit-code", + "--quiet", + revision, + "--", + "mssql_python", + "mssql_python_odbc", + timeout=5, + ) + + +def native_identity(source_root, revision, profiling): + from mssql_python import ddbc_bindings + + native_file = Path(ddbc_bindings.module.__file__).resolve() + if not native_file.is_relative_to(source_root.resolve()): + raise RuntimeError("Native binary is outside the selected checkout") + return dict( + source_commit=revision, + native_file=str(native_file), + native_sha256=hashlib.sha256(native_file.read_bytes()).hexdigest(), + native_profiling=profiling, + guarded_row=hasattr(ddbc_bindings, "DDBCSQLFetchRow"), + ) + + +def fetch_worker(args, mode): + if mode not in ("latency", "route") or not SHA.fullmatch(args.revision or ""): + raise ValueError("Fetch measurement requires a mode and exact revision") + verify_reused_source(args.source_root, args.revision) + check_build(args.source_root, profiling=mode == "route") + import mssql_python + from mssql_python import ddbc_bindings, perf_timer + + provenance = native_identity(args.source_root, args.revision, mode == "route") + cases = workloads.single_row_registry() + chosen = args.scenarios if args.scenarios is not None else list(cases) + if not chosen or set(chosen) - set(cases): + raise ValueError("Unknown or empty single-row workload selection") + ctx = _FetchContext(mode, ddbc_bindings.profiling if mode == "route" else None, perf_timer) + output = {} + + def checkpoint(active=None, environment=None): + args.output.write_text( + json.dumps( + dict( + status="running" if environment is None else "complete", + active_scenario=active, + mode=mode, + provenance=provenance, + scenarios=output, + environment=environment, + ), + allow_nan=False, + ), + encoding="utf-8", + ) + + checkpoint() + try: + with mssql_python.connect(os.environ["DB_CONNECTION_STRING"]) as conn: + for name in chosen: + checkpoint(name) + result = cases[name](conn, ctx) + if mode == "route": + timer = ( + "ddbc::FetchMany_wrap" + if name.endswith("fetchmany") + else "ddbc::FetchOne_wrap" + ) + if result["cpp"].get(timer, {}).get("calls") != workloads.SINGLE_ROW_COUNT + 1: + raise RuntimeError("Native fetch call count does not match the workload") + constructors = ( + result["cpp"].get("ddbc::FetchRow::construct_row", {}).get("calls", 0) + ) + if constructors != ( + workloads.SINGLE_ROW_COUNT if provenance["guarded_row"] else 0 + ): + raise RuntimeError("Native Row construction route was not established") + output[name] = {key: result[key] for key in ("wall_ms", "cpp", "py")} + output[name]["work"] = result["detail"] + checkpoint() + with conn.cursor() as cursor: + cursor.execute("SELECT CAST(SERVERPROPERTY('ProductVersion') AS VARCHAR(80))") + sql_version = cursor.fetchone()[0] + environment = dict( + os=platform.system(), + architecture=platform.machine().lower(), + python=platform.python_version(), + sql_version=sql_version, + ) + finally: + ctx.disable() + checkpoint(environment=environment) + + def worker(args): + mode = getattr(args, "mode", "diagnostic") + if mode != "diagnostic": + return fetch_worker(args, mode) # Import the chosen driver FIRST, then the SAME workload/controller for both # revisions. Never mix two native extensions into one interpreter. + revision = getattr(args, "revision", None) + if revision: + verify_reused_source(args.source_root, revision) check_build(args.source_root, profiling=True) + provenance = native_identity(args.source_root, revision, True) if revision else None core, workloads = load_suite() cases = workloads.registry() @@ -192,13 +355,21 @@ def worker(args): python=platform.python_version(), sql_version=sql_version, ) - args.output.write_text( - json.dumps(dict(environment=environment, scenarios=output), allow_nan=False), - encoding="utf-8", - ) - - -def measure(path, output, scenarios, timeout=WORKER_TIMEOUT): + result = dict(environment=environment, scenarios=output) + if provenance is not None: + result.update(status="complete", mode="diagnostic", provenance=provenance) + args.output.write_text(json.dumps(result, allow_nan=False), encoding="utf-8") + + +def measure( + path, + output, + scenarios, + timeout=WORKER_TIMEOUT, + mode="diagnostic", + revision=None, + isolated=False, +): command = [ sys.executable, "-u", @@ -210,11 +381,18 @@ def measure(path, output, scenarios, timeout=WORKER_TIMEOUT): "--output", str(output), ] + if mode != "diagnostic" or revision is not None: + command += ["--mode", mode, "--revision", revision] if scenarios: command += ["--scenarios", *scenarios] output.unlink(missing_ok=True) - with output.with_suffix(".log").open("w", encoding="utf-8") as log: - subprocess.run(command, stdout=log, stderr=subprocess.STDOUT, timeout=timeout, check=True) + if isolated: + run_process(command, output.with_suffix(".log"), timeout) + else: + with output.with_suffix(".log").open("w", encoding="utf-8") as log: + subprocess.run( + command, stdout=log, stderr=subprocess.STDOUT, timeout=timeout, check=True + ) return json.loads(output.read_text(encoding="utf-8")) @@ -225,7 +403,215 @@ def remaining(deadline, limit): return min(seconds, limit) +def write_report(path, report): + temporary = path.with_suffix(".tmp") + temporary.write_text(json.dumps(report, allow_nan=False), encoding="utf-8") + temporary.replace(path) + + +def run_ci_report(args): + if not args.reuse_candidate or args.mode != "diagnostic" or args.scenarios is not None: + raise ValueError( + "--ci-report requires --reuse-candidate and the complete default workload sets" + ) + if args.samples != 5 or args.warmups != 1: + raise ValueError("--ci-report requires five pairs and one warmup pair") + finish_deadline = time.monotonic() + BENCHMARK_TIMEOUT + deadline = finish_deadline - CI_FINISH_RESERVE + base, candidate = resolve_revisions(args.base, args.candidate, timeout=remaining(deadline, 30)) + if candidate != git("rev-parse", "HEAD", timeout=remaining(deadline, 30)): + raise ValueError("--ci-report must reuse checkout HEAD") + head = os.environ.get("SYSTEM_PULLREQUEST_SOURCECOMMITID", candidate) + if not SHA.fullmatch(head): + raise ValueError("Invalid PR head identity") + args.output.mkdir(parents=True, exist_ok=True) + report_path = args.output / "report.json" + common = dict( + status="incomplete", + leg=args.leg, + base_commit=base, + source_commit=candidate, + head_commit=head, + build_id=int(os.environ.get("BUILD_BUILDID", "0")), + samples=5, + warmups=1, + ) + modes = { + mode: dict( + common, + schema_version=1 if mode == "diagnostic" else 2, + pairs=[], + unavailable_reason="Not started: waiting for earlier modes", + ) + for mode in MODES + } + for mode in ("latency", "route"): + modes[mode]["mode"] = mode + bundle = modes["diagnostic"] + bundle.update( + measurement_bundle_version=1, + fetch_measurements={mode: modes[mode] for mode in ("latency", "route")}, + ) + validate_ci_header(bundle) + write_report(report_path, bundle) + on_identities = {} + failures = ( + subprocess.CalledProcessError, + subprocess.TimeoutExpired, + TimeoutError, + ValueError, + KeyError, + TypeError, + OSError, + ) + directory = tempfile.mkdtemp(prefix="profiler-ci-bundle-") + safe_to_clean = True + try: + roots = {name: Path(directory) / name for name in ("base-off", "candidate-off", "base-on")} + on_ready = False + for mode in ("latency", "route", "diagnostic"): + report = modes[mode] + stage = "admission" + try: + remaining(deadline, 1) + if mode == "diagnostic" and not on_ready: + raise ValueError("Shared ON build unavailable") + builds = ( + (("base-off", base), ("candidate-off", candidate)) + if mode == "latency" + else (("base-on", base),) if mode == "route" else () + ) + for name, revision in builds: + stage = "archive " + name + report["unavailable_reason"] = "Incomplete: " + stage + write_report(report_path, bundle) + run_process( + [ + sys.executable, + "-m", + "eng.profiler_benchmarks.controller", + "--archive-source", + revision, + "--source-root", + str(roots[name]), + ], + args.output / ("archive-" + name + ".log"), + remaining(deadline, 60), + ) + stage = "build " + name + report["unavailable_reason"] = "Incomplete: " + stage + write_report(report_path, bundle) + build( + roots[name], + args.output / ("build-" + name + ".log"), + remaining(deadline, 900), + profiling=mode != "latency", + ) + if mode == "route": + on_ready = True + paths = { + "base": roots["base-off" if mode == "latency" else "base-on"], + "candidate": roots["candidate-off"] if mode == "latency" else ROOT, + } + identity = None + for sample in range(6): + pair = {} + for side in ( + ("base", "candidate") if sample % 2 == 0 else ("candidate", "base") + ): + stage = f"{mode} pair {sample} {side}" + report["unavailable_reason"] = "Incomplete: " + stage + write_report(report_path, bundle) + pair[side] = measure( + paths[side], + args.output / f"{mode}-{side}-{sample}.json", + None, + remaining( + deadline, + WORKER_TIMEOUT if mode == "diagnostic" else FETCH_WORKER_TIMEOUT, + ), + mode=mode, + revision=base if side == "base" else candidate, + isolated=True, + ) + probe = dict(report, pairs=[*report["pairs"], pair]) + validate_ci_mode(probe, mode) + observed = { + side: (pair[side]["provenance"], pair[side]["environment"]) for side in pair + } + if identity is not None and observed != identity: + raise ValueError("Worker identity changed after warmup") + identity = observed + if mode != "latency": + for side in pair: + if side in on_identities and on_identities[side] != observed[side]: + raise ValueError("Shared ON worker identity mismatch") + on_identities[side] = observed[side] + if sample: + report["pairs"].append(pair) + write_report(report_path, bundle) + remaining(deadline, 1) + report["status"] = "complete" + validate_ci_mode(report, mode) + report.pop("unavailable_reason", None) + except ProcessCleanupError: + safe_to_clean = False + bundle["cleanup_required"] = directory + for pending in modes.values(): + if pending["status"] != "complete": + pending["unavailable_reason"] = ( + "Not completed: prior process cleanup failed" + ) + report["status"] = "incomplete" + report["unavailable_reason"] = ( + "Process cleanup failed; further modes were not started" + ) + write_report(report_path, bundle) + raise + except failures as error: + report["status"] = "incomplete" + detail = ( + str(error)[:140] + if isinstance(error, ValueError) + else "see raw worker/build evidence" + ) + report["unavailable_reason"] = f"{stage}: {type(error).__name__}: {detail}" + print(report["unavailable_reason"], file=sys.stderr, flush=True) + finally: + write_report(report_path, bundle) + finally: + if safe_to_clean: + try: + shutil.rmtree(directory) + except OSError as error: + bundle["status"] = "incomplete" + bundle["cleanup_required"] = directory + bundle["unavailable_reason"] = ( + "Build-directory cleanup failed; retained evidence requires cleanup" + ) + write_report(report_path, bundle) + raise ProcessCleanupError("CI build-directory cleanup failed") from error + _, errors = ci_mode_reports(bundle) + for mode, reason in errors.items(): + modes[mode]["status"] = "incomplete" + modes[mode]["unavailable_reason"] = reason + write_report(report_path, bundle) + if time.monotonic() > finish_deadline: + bundle["status"] = "incomplete" + bundle["unavailable_reason"] = "Aggregate finish deadline exceeded during finalization" + write_report(report_path, bundle) + if any(report["status"] != "complete" for report in modes.values()): + raise RuntimeError("CI performance report is incomplete; see per-mode reasons and raw logs") + + def run(args): + mode = getattr(args, "mode", "diagnostic") + if mode not in MODES: + raise ValueError("Unknown measurement mode") + if mode == "latency" and args.reuse_candidate: + raise ValueError( + "Latency requires fresh native-OFF builds; cannot reuse the CI profiling build" + ) base, candidate = resolve_revisions(args.base, args.candidate) args.output.mkdir(parents=True, exist_ok=True) report_path = args.output / "report.json" @@ -234,7 +620,7 @@ def run(args): if not SHA.fullmatch(head): head = candidate report = dict( - schema_version=1, + schema_version=1 if mode == "diagnostic" else 2, status="incomplete", leg=args.leg, base_commit=base, @@ -245,6 +631,8 @@ def run(args): warmups=args.warmups, pairs=[], ) + if mode != "diagnostic": + report["mode"] = mode report_path.write_text(json.dumps(report), encoding="utf-8") timeout = BENCHMARK_TIMEOUT if args.reuse_candidate else LOCAL_BENCHMARK_TIMEOUT deadline = time.monotonic() + timeout @@ -270,8 +658,14 @@ def run(args): ) continue checkout(revision, paths[side]) - print(f"Building profiling {side}: {revision}", flush=True) - build(paths[side], args.output / f"build-{side}.log", remaining(deadline, 900)) + print(f"Building {mode} {side}: {revision}", flush=True) + build_options = {"profiling": mode != "latency"} if mode != "diagnostic" else {} + build( + paths[side], + args.output / f"build-{side}.log", + remaining(deadline, 900), + **build_options, + ) for sample in range(args.warmups + args.samples): pair = {} order = ("base", "candidate") if sample % 2 == 0 else ("candidate", "base") @@ -282,6 +676,11 @@ def run(args): args.output / f"{side}-{sample}.json", args.scenarios, remaining(deadline, WORKER_TIMEOUT), + **( + {"mode": mode, "revision": base if side == "base" else candidate} + if mode != "diagnostic" + else {} + ), ) if pair["base"]["environment"] != pair["candidate"]["environment"]: raise RuntimeError("Base and candidate environments differ") @@ -290,6 +689,8 @@ def run(args): report_path.write_text(json.dumps(report, allow_nan=False), encoding="utf-8") if args.scenarios is None: report["status"] = "complete" + if mode != "diagnostic": + validate(report) report_path.write_text(json.dumps(report, allow_nan=False), encoding="utf-8") print(f"Paired profiler report: {report_path}", flush=True) @@ -303,6 +704,19 @@ def main(): parser.add_argument("--samples", type=int, default=5) parser.add_argument("--warmups", type=int, default=1) parser.add_argument("--scenarios", nargs="+", help="Local subset; CI runs the full registry") + parser.add_argument( + "--mode", + choices=MODES, + default="diagnostic", + help="diagnostic: both recorders; latency: native OFF/Python OFF; route: native ON/Python OFF", + ) + parser.add_argument("--revision", help=argparse.SUPPRESS) + parser.add_argument( + "--ci-report", + action="store_true", + help="Bounded latency-first CI report with route and legacy diagnostics", + ) + parser.add_argument("--archive-source", help=argparse.SUPPRESS) parser.add_argument("--worker", action="store_true", help=argparse.SUPPRESS) parser.add_argument("--source-root", type=Path, help=argparse.SUPPRESS) parser.add_argument( @@ -316,11 +730,19 @@ def main(): help="Verify native compile configuration and recording OFF, then exit", ) args = parser.parse_args() - if args.check_build: + if args.ci_report and (args.worker or args.archive_source or args.check_build): + parser.error("--ci-report cannot be combined with worker, archive or build-check modes") + if args.archive_source: + if not SHA.fullmatch(args.archive_source) or args.source_root is None: + parser.error("--archive-source requires an exact revision and --source-root") + checkout(args.archive_source, args.source_root) + elif args.check_build: check_build(ROOT, profiling=args.check_build == "on") elif args.worker: if args.source_root is None or args.output is None: parser.error("--worker requires --source-root and --output") + if args.mode != "diagnostic" and not SHA.fullmatch(args.revision or ""): + parser.error("Fetch measurement workers require an exact --revision") # Dumps contain stack locations, not locals or connection strings. The # parent still kills/reaps the worker at its deadline if it cannot finish. faulthandler.enable() @@ -337,7 +759,10 @@ def main(): or not 1 <= args.warmups <= 3 ): parser.error("Choose a leg, 3-15 measured pairs and 1-3 warmup pairs") - run(args) + if args.ci_report: + run_ci_report(args) + else: + run(args) if __name__ == "__main__": diff --git a/eng/profiler_benchmarks/report.py b/eng/profiler_benchmarks/report.py index 07f176f18..4f714fb63 100644 --- a/eng/profiler_benchmarks/report.py +++ b/eng/profiler_benchmarks/report.py @@ -41,6 +41,34 @@ "scalar_fetchval": "10,000 scalar values / fetchval() (debug disabled)", } CASES = tuple(TASK_NAMES) +MODES = ("diagnostic", "latency", "route") +FETCH_CASES = tuple( + f"{shape}_{method}" + for shape in ("numeric", "mixed") + for method in ("fetchone", "fetchmany", "fetchval") +) +TASK_NAMES.update( + { + name: f"1,000 {name.split('_')[0]} rows / " + + ("fetchmany(1)" if name.endswith("fetchmany") else name.split("_")[1] + "()") + for name in FETCH_CASES + } +) + + +def measurement_mode(report): + version = report.get("schema_version") + if version == 1 and "mode" not in report: + return "diagnostic" + if version == 2 and report.get("mode") in ("latency", "route"): + return report["mode"] + raise ValueError("Unsupported measurement schema or mode") + + +def cases_for(report): + return CASES if measurement_mode(report) == "diagnostic" else FETCH_CASES + + MAX_BYTES = 8 * 1024 * 1024 MAX_COMMENT_CHARS = 60000 MAX_DIAGNOSTIC_ROWS = 20 @@ -114,8 +142,9 @@ def validate(report, build_id=None, head=None, source=None, base=None): def _validate(report, build_id=None, head=None, source=None, base=None): - if not isinstance(report, dict) or report.get("schema_version") != 1: + if not isinstance(report, dict): raise ValueError("Unsupported report schema") + mode = measurement_mode(report) if report.get("leg") not in LEGS or report.get("status") not in ("complete", "incomplete"): raise ValueError("Invalid report status or leg") if type(report.get("build_id")) is not int or report["build_id"] < 0: @@ -143,8 +172,15 @@ def _validate(report, build_id=None, head=None, source=None, base=None): return report if len(pairs) != samples: raise ValueError("Incomplete sample pairs") + return validate_samples(report) + + +def validate_samples(report): + mode = measurement_mode(report) + pairs = report["pairs"] environment = None work = {} + provenance = {} for pair in pairs: if not isinstance(pair, dict) or set(pair) != {"base", "candidate"}: raise ValueError("Invalid paired sample") @@ -152,6 +188,13 @@ def _validate(report, build_id=None, head=None, source=None, base=None): sample = pair[side] if not isinstance(sample, dict): raise ValueError("Invalid sample") + if mode != "diagnostic": + if sample.get("mode") != mode or sample.get("status") != "complete": + raise ValueError("Incomplete or mismatched measurement mode") + native_identity = validate_native_identity(sample, report, side, mode) + if side in provenance and provenance[side] != native_identity: + raise ValueError("Native identity changed between samples") + provenance[side] = native_identity env = sample["environment"] if not isinstance(env, dict) or set(env) != { "os", @@ -173,7 +216,7 @@ def _validate(report, build_id=None, head=None, source=None, base=None): scenarios = sample["scenarios"] if not isinstance(scenarios, dict): raise ValueError("Invalid scenarios object") - if set(scenarios) != set(CASES): + if set(scenarios) != set(cases_for(report)): raise ValueError("Scenario set incomplete or changed") for name, scenario in scenarios.items(): if not isinstance(scenario, dict): @@ -185,12 +228,34 @@ def _validate(report, build_id=None, head=None, source=None, base=None): if name in work and work[name] != identity: raise ValueError(f"Workload changed for {name}") work[name] = identity + if mode != "diagnostic": + if scenario["py"] or (mode == "latency" and scenario["cpp"]): + raise ValueError("Recording contaminated the measurement mode") + shape, method = name.split("_") + if scenario["work"] != f"Rows: 1000; shape: {shape}; API: {method}; EOF: 1": + raise ValueError("Single-row workload identity mismatch") + if mode == "route": + timer = ( + "ddbc::FetchMany_wrap" + if method == "fetchmany" + else "ddbc::FetchOne_wrap" + ) + stats = scenario["cpp"] + if not isinstance(stats, dict) or not isinstance(stats.get(timer), dict): + raise ValueError("Missing native fetch route evidence") + if stats[timer].get("calls") != 1001: + raise ValueError("Native fetch route count mismatch") + constructor = stats.get("ddbc::FetchRow::construct_row", {}) + if not isinstance(constructor, dict) or constructor.get("calls", 0) != ( + 1000 if native_identity["guarded_row"] else 0 + ): + raise ValueError("Native constructor route count mismatch") for layer in ("cpp", "py"): stats = scenario[layer] if ( not isinstance(stats, dict) or len(stats) > 300 - or (layer == "cpp" and not stats) + or (layer == "cpp" and mode != "latency" and not stats) ): raise ValueError("Missing or oversized profiling data") for label, counter in stats.items(): @@ -209,6 +274,236 @@ def _validate(report, build_id=None, head=None, source=None, base=None): return report +def validate_native_identity(sample, report, side, mode): + identity = sample.get("provenance") + if not isinstance(identity, dict) or set(identity) != { + "source_commit", + "native_file", + "native_sha256", + "native_profiling", + "guarded_row", + }: + raise ValueError("Missing native measurement identity") + if identity["source_commit"] != report["base_commit" if side == "base" else "source_commit"]: + raise ValueError("Worker revision mismatch") + text(identity["native_file"], limit=4096) + if not isinstance(identity["native_sha256"], str) or not re.fullmatch( + r"[0-9a-f]{64}", identity["native_sha256"] + ): + raise ValueError("Invalid native binary digest") + if ( + identity["native_profiling"] is not (mode != "latency") + or type(identity["guarded_row"]) is not bool + ): + raise ValueError("Invalid native measurement configuration") + return identity + + +def validate_ci_mode(report, mode): + if measurement_mode(report) != mode: + raise ValueError("CI measurement mode mismatch") + validate(report) + if report["status"] == "incomplete": + text(report["unavailable_reason"], limit=240) + validate_samples(report) + identities = {} + for pair in report["pairs"]: + for side, sample in pair.items(): + if sample.get("status") != "complete" or sample.get("mode") != mode: + raise ValueError("Incomplete or mismatched worker mode") + identity = validate_native_identity(sample, report, side, mode) + if side in identities and identities[side] != identity: + raise ValueError("Native identity changed between samples") + identities[side] = identity + return report + + +def has_ci_bundle(report): + return isinstance(report, dict) and ( + "measurement_bundle_version" in report or "fetch_measurements" in report + ) + + +def validate_ci_header(report, build_id=None, head=None, source=None, base=None): + if ( + type(report.get("measurement_bundle_version")) is not int + or report["measurement_bundle_version"] != 1 + ): + raise ValueError("Unsupported CI measurement bundle version") + if measurement_mode(report) != "diagnostic": + raise ValueError("CI bundle root must be the diagnostic report") + validate(dict(report, status="incomplete", pairs=[]), build_id, head, source, base) + if report["samples"] != 5 or report["warmups"] != 1: + raise ValueError("CI bundle requires five pairs and one warmup") + children = report.get("fetch_measurements") + if not isinstance(children, dict) or set(children) - {"latency", "route"}: + raise ValueError("Invalid CI measurement children") + return report + + +def ci_mode_reports(report): + validate_ci_header(report) + valid, errors = {}, {} + for mode in ("latency", "route", "diagnostic"): + item = report if mode == "diagnostic" else report["fetch_measurements"].get(mode) + try: + if not isinstance(item, dict): + raise ValueError("Missing mode") + for key in ( + "build_id", + "head_commit", + "source_commit", + "base_commit", + "leg", + "samples", + "warmups", + ): + if item.get(key) != report[key]: + raise ValueError("Mode provenance mismatch: " + key) + validate_ci_mode(item, mode) + valid[mode] = item + if item["status"] != "complete": + errors[mode] = item["unavailable_reason"] + except (KeyError, TypeError, ValueError) as error: + errors[mode] = "Invalid mode data: " + str(error)[:180] + on = [valid.get(mode) for mode in ("route", "diagnostic")] + if all(item is not None and item["status"] == "complete" for item in on): + if any( + on[0]["pairs"][0][side][key] != on[1]["pairs"][0][side][key] + for side in ("base", "candidate") + for key in ("provenance", "environment") + ): + for mode in ("route", "diagnostic"): + valid.pop(mode) + errors[mode] = "Shared ON binary/environment identity mismatch" + latency = valid.get("latency") + if latency is not None and latency["status"] == "complete": + for mode in ("route", "diagnostic"): + item = valid.get(mode) + if ( + item is not None + and item["status"] == "complete" + and item["pairs"][0]["base"]["environment"] + != latency["pairs"][0]["base"]["environment"] + ): + valid.pop(mode) + errors[mode] = "Environment differs from the latency comparison" + return valid, errors + + +def render_ci_reports(reports, head, build_id, issues=()): + modes = {mode: [] for mode in MODES} + reasons = {} + seen = set() + for report in reports: + leg = report["leg"] + if leg in seen: + raise ValueError("Duplicate performance report leg") + seen.add(leg) + if has_ci_bundle(report): + valid, errors = ci_mode_reports(report) + else: + validate(report) + valid = {measurement_mode(report): report} + errors = { + mode: "Mode not supplied by this legacy artifact" + for mode in MODES + if mode not in valid + } + for mode, item in valid.items(): + if item["status"] == "complete": + modes[mode].append(item) + for mode, reason in errors.items(): + reasons[leg, mode] = reason + latency_issues = list(issues) + [ + leg + " (" + reasons.get((leg, "latency"), "missing latency measurement") + ")" + for leg in LEGS + if not any(r["leg"] == leg for r in modes["latency"]) + ] + primary = render(modes["latency"], head, build_id, latency_issues, default_mode="latency") + primary = primary.replace( + "## PR Performance Report", + "## PR Performance Report\n\n**Primary verdict: native OFF / Python phases OFF latency only.**", + 1, + ) + lines = [ + "", + "### Measurement availability", + "", + "| Environment | Latency OFF/OFF | Route ON/OFF | Diagnostics ON/ON |", + "|---|---|---|---|", + ] + for leg in LEGS: + cells = [] + for mode in ("latency", "route", "diagnostic"): + complete = any(item["leg"] == leg for item in modes[mode]) + cells.append( + "Complete" + if complete + else "Unavailable: " + escape(reasons.get((leg, mode), "artifact missing")) + ) + lines.append("| " + environment_name(leg) + " | " + " | ".join(cells) + " |") + for mode, label in ( + ( + "route", + "Native route attribution (instrumented, Python phases OFF; not production latency)", + ), + ("diagnostic", "Legacy diagnostics (native and Python phases ON; not production latency)"), + ): + lines += ["", "
", "" + label + "", ""] + if not modes[mode]: + lines.append("Unavailable. No latency inference is made from this section.") + for report in modes[mode]: + lines += [ + "", + "#### " + environment_name(report["leg"]), + "", + "| Task | Before ms | After ms | Paired change | Ratio median [min, max] |", + "|---|---:|---:|---:|---:|", + ] + notes = [] + for row in comparisons(report): + lines.append( + f"| {TASK_NAMES[row['name']]} | {row['base_ms']:.3f} | {row['candidate_ms']:.3f} | {row['change_pct']:+.1f}% | {row['ratio']:.3f} [{row['ratio_min']:.3f}, {row['ratio_max']:.3f}] |" + ) + if mode == "diagnostic" and (row["phases"] or row["counts"]): + phases = "; ".join( + f"{escape(label)} {delta:+.3f} ms" for delta, label in row["phases"] + ) + counts_text = "; ".join(escape(count) for count in row["counts"]) + notes.append( + "Inclusive phase changes: " + + (phases or "none recorded") + + ". Call changes: " + + (counts_text or "none recorded") + + "." + ) + if mode == "route": + counts = [ + report["pairs"][0][side]["scenarios"][row["name"]]["cpp"] + .get("ddbc::FetchRow::construct_row", {}) + .get("calls", 0) + for side in ("base", "candidate") + ] + notes.append( + f"Native proof for {row['name']}: 1001 fetch calls per side/sample; Row constructions base/candidate {counts[0]}/{counts[1]}." + ) + lines += [""] + notes + lines.append( + "Raw paired counters and worker/native identities are retained in the artifact; inclusive phases must not be summed." + ) + lines += ["", "
"] + appendix = "\n".join(lines) + if len(primary) + len(appendix) + 1 > MAX_COMMENT_CHARS: + appendix = "\n".join( + line for line in lines if not line.startswith("Inclusive phase changes:") + ) + appendix += "\nExpanded phase details exceed the comment budget; full counters remain in the raw artifact." + if len(primary) + len(appendix) + 1 > MAX_COMMENT_CHARS: + raise ValueError("CI performance comment exceeds its bounded size") + return primary + "\n" + appendix + + def assess(evidence, artifact_urls, load_artifact, issues=()): issues = list(issues) try: @@ -238,15 +533,13 @@ def assess(evidence, artifact_urls, load_artifact, issues=()): # Match coverage's trust boundary: select the exact PR-head build and treat # its bounded artifacts as data without requiring an identical producer tree. reports = [] + ci_requested = False for leg, url in artifact_urls.items(): try: - report = validate( - artifact_report(load_artifact(url)), - build_id, - evidence.head, - source, - evidence.base, - ) + report = artifact_report(load_artifact(url)) + ci_requested = ci_requested or has_ci_bundle(report) + validator = validate_ci_header if has_ci_bundle(report) else validate + validator(report, build_id, evidence.head, source, evidence.base) if report["leg"] != leg: raise ValueError("Artifact leg mismatch") reports.append(report) @@ -261,7 +554,8 @@ def assess(evidence, artifact_urls, load_artifact, issues=()): issues.append(leg + " (invalid artifact)") try: - return render(reports, evidence.head, build_id, issues) + renderer = render_ci_reports if ci_requested else render + return renderer(reports, evidence.head, build_id, issues) except ValueError: return unavailable("Performance report rendering failed.") @@ -269,7 +563,7 @@ def assess(evidence, artifact_urls, load_artifact, issues=()): def comparisons(report): """Do not add inclusive phase totals together or treat them as wall-clock time.""" output = [] - for name in CASES: + for name in cases_for(report): base = [pair["base"]["scenarios"][name] for pair in report["pairs"]] candidate = [pair["candidate"]["scenarios"][name] for pair in report["pairs"]] ratios = [new["wall_ms"] / old["wall_ms"] for old, new in zip(base, candidate)] @@ -319,6 +613,9 @@ def comparisons(report): base_ms=old, candidate_ms=new, change_pct=(ratio - 1) * 100, + ratio=ratio, + ratio_min=min(ratios), + ratio_max=max(ratios), status=status, phases=phases, counts=sorted(changed_counts)[:3], @@ -349,8 +646,12 @@ def issue_reason(leg, issues): return global_issues[0] if global_issues else "incomplete benchmark" -def render(reports, head, build_id, issues=()): +def render(reports, head, build_id, issues=(), default_mode="diagnostic"): url = f"https://dev.azure.com/sqlclientdrivers/public/_build/results?buildId={build_id}" + modes = {measurement_mode(r) for r in reports} + if len(modes) > 1: + raise ValueError("Cannot combine different measurement modes") + mode = next(iter(modes), default_mode) by_leg = {r["leg"]: r for r in reports} if len(by_leg) != len(reports): raise ValueError("Duplicate performance report leg") @@ -425,6 +726,12 @@ def render(reports, head, build_id, issues=()): ) verdict = "✅ No regression detected" + if mode == "route" and completed: + verdict = "Native route attribution (instrumented)" + opening = ( + "Instrumented paired timings below describe the verified native route, not production latency. " + + opening + ) improvement_tasks = len({row["name"] for _, row in improvements}) regression_tasks = len({row["name"] for _, row in regressions}) lines = [ @@ -442,13 +749,25 @@ def render(reports, head, build_id, issues=()): f"{len(completed)}/{len(LEGS)} ENVIRONMENTS", "", ] + if mode != "diagnostic": + lines += [ + "**Measurement:** " + + ( + "native instrumentation OFF / Python phases OFF; controlled fetch-loop latency." + if mode == "latency" + else "native instrumentation ON / Python phases OFF; route attribution, not production latency." + ), + "", + ] if noisy: noisy_tasks = len({row["name"] for _, row in noisy}) lines += [ f"{noisy_tasks} INCONSISTENT SLOWDOWN" f"{'S' if noisy_tasks != 1 else ''}", "", ] - affected_tasks = [name for name in CASES if any(row["name"] == name for _, row in highlighted)] + affected_tasks = [ + name for name in TASK_NAMES if any(row["name"] == name for _, row in highlighted) + ] if highlighted and len(affected_tasks) <= MAX_FINGERPRINT_TASKS: affected_legs = [leg for leg in LEGS if any(item_leg == leg for item_leg, _ in highlighted)] by_signal = {(leg, row["name"]): row for leg, row in highlighted} @@ -476,7 +795,9 @@ def render(reports, head, build_id, issues=()): lines.append("") if regressions: lines.append( - "The largest recorded phase increases for these tasks are shown below. " + "Paired fetch-loop timings are shown below. Phase attribution is unavailable in latency mode." + if mode == "latency" + else "The largest recorded phase increases for these tasks are shown below. " "Phase timings are supporting evidence, not root-cause proof." ) if noisy: @@ -537,7 +858,11 @@ def render(reports, head, build_id, issues=()): diagnostics += 1 phases = "; ".join(f"{escape(label)} {delta:+.3f} ms" for delta, label in row["phases"]) counts = "; ".join(escape(label) for label in row["counts"]) - detail = phases or "no measured phase delta" + detail = phases or ( + "phase attribution unavailable (recording OFF)" + if mode == "latency" + else "no measured phase delta" + ) if counts: detail += f". Call changes: {counts}" lines.append(f"**{TASK_NAMES[row['name']]}:** {detail}.") @@ -564,8 +889,12 @@ def render(reports, head, build_id, issues=()): lines += [ "", f"### {environment_name(leg)}", - "| Database task | Before | After | Paired change | Result |", - "|---|---:|---:|---:|---|", + ( + "| Database task | Before | After | Paired change | Result |" + if mode == "diagnostic" + else "| Database task | Before | After | Paired change | Ratio median [min, max] | Result |" + ), + "|---|---:|---:|---:|---|" if mode == "diagnostic" else "|---|---:|---:|---:|---:|---|", ] for row in rows: result = { @@ -576,7 +905,13 @@ def render(reports, head, build_id, issues=()): }[row["status"]] lines.append( f"| {TASK_NAMES[row['name']]} | {row['base_ms']:.3f} ms | " - f"{row['candidate_ms']:.3f} ms | {row['change_pct']:+.1f}% | {result} |" + f"{row['candidate_ms']:.3f} ms | {row['change_pct']:+.1f}% | " + + ( + f"{row['ratio']:.3f} [{row['ratio_min']:.3f}, {row['ratio_max']:.3f}] | " + if mode != "diagnostic" + else "" + ) + + f"{result} |" ) lines += [ "", @@ -587,7 +922,11 @@ def render(reports, head, build_id, issues=()): "", ] lines += [ - f"[ADO build {build_id}]({url})", + ( + f"[ADO build {build_id}]({url})" + if build_id or mode == "diagnostic" + else "Local paired comparison (no ADO build)" + ), "", f"PR head: `{head}`", ] @@ -595,7 +934,7 @@ def render(reports, head, build_id, issues=()): first = next(iter(completed.values()))[0] lines += [ f"Base: `{first['base_commit']}`", - f"Measured merge: `{first['source_commit']}`", + f"Measured {'merge' if mode == 'diagnostic' else 'source'}: `{first['source_commit']}`", "", ] for leg, (report, _) in completed.items(): @@ -605,6 +944,13 @@ def render(reports, head, build_id, issues=()): f"{escape(env['architecture'])}, SQL {escape(env['sql_version'])}; " f"{report['samples']} paired comparisons and {report['warmups']} warmup." ) + if mode != "diagnostic": + for side in ("base", "candidate"): + identity = report["pairs"][0][side]["provenance"] + lines.append( + f"- {environment_name(leg)} {side} native SHA256: `{identity['native_sha256']}`; " + f"guarded Row entry available: {identity['guarded_row']}." + ) lines += [ "", "A consistent change requires more than 20% median paired movement, at least " @@ -619,9 +965,15 @@ def render(reports, head, build_id, issues=()): lines += ["", "Unavailable or rejected data: " + ", ".join(escape(x) for x in issues)] lines += [ "", - "Both revisions use profiling-enabled builds on the same agent and database, " - "with alternating order and discarded warmups. Results are diagnostic and do " - "not represent production-wheel latency.", + ( + "Both revisions use native-instrumentation-OFF builds with Python phases OFF on " + "the same agent and database, alternating order and discarded warmups. This is " + "controlled fetch-loop latency, not a customer-production or pyodbc comparison." + if mode == "latency" + else "Both revisions use profiling-enabled builds on the same agent and database, " + "with alternating order and discarded warmups. Results are diagnostic and do " + "not represent production-wheel latency." + ), "", "Raw samples and logs are attached to the ADO run as `profiler-*` artifacts.", "", @@ -648,17 +1000,20 @@ def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("reports", nargs="+", type=Path) args = parser.parse_args() - reports = [validate(json.loads(path.read_text(encoding="utf-8"))) for path in args.reports] + reports = [json.loads(path.read_text(encoding="utf-8")) for path in args.reports] + for report in reports: + (validate_ci_header if has_ci_bundle(report) else validate)(report) first = reports[0] for report in reports[1:]: - validate( + (validate_ci_header if has_ci_bundle(report) else validate)( report, build_id=first["build_id"], head=first["head_commit"], source=first["source_commit"], base=first["base_commit"], ) - print(render(reports, first["head_commit"], first["build_id"])) + renderer = render_ci_reports if any(has_ci_bundle(report) for report in reports) else render + print(renderer(reports, first["head_commit"], first["build_id"])) if __name__ == "__main__": diff --git a/eng/profiler_benchmarks/workloads.py b/eng/profiler_benchmarks/workloads.py index 9059ab6c9..bb1096bdd 100644 --- a/eng/profiler_benchmarks/workloads.py +++ b/eng/profiler_benchmarks/workloads.py @@ -209,3 +209,69 @@ def registry(): result["lob_varchar_256k_fetchall"] = (lob_fetch, False) result["scalar_fetchval"] = (scalar_fetchval, True) return result + + +SINGLE_ROW_COUNT = 1000 + + +def single_row_fetch(conn, ctx, method, shape): + """Time one API through EOF; validate every column outside the timed window.""" + from mssql_python.logging import logger + + if method not in ("fetchone", "fetchmany", "fetchval") or shape not in ("numeric", "mixed"): + raise ValueError("Unknown single-row workload") + if logger.is_debug_enabled: + raise ValueError("Single-row measurements require debug logging OFF") + second = "n + 10" if shape == "numeric" else "CAST(N'text' AS NVARCHAR(10))" + query = ( + "WITH digits(n) AS (SELECT n FROM (VALUES (0),(1),(2),(3),(4),(5),(6),(7),(8),(9)) d(n)), " + "numbers(n) AS (SELECT a.n + 10*b.n + 100*c.n FROM digits a CROSS JOIN digits b " + "CROSS JOIN digits c) " + f"SELECT n AS number, {second} AS value FROM numbers ORDER BY n" + ) + with conn.cursor() as cursor: + cursor.execute(query) + try: + ctx.enable() + start = time.perf_counter() + if method == "fetchmany": + values = [cursor.fetchmany(1) for _ in range(SINGLE_ROW_COUNT)] + eof = cursor.fetchmany(1) + elif method == "fetchval": + values = [cursor.fetchval() for _ in range(SINGLE_ROW_COUNT)] + eof = cursor.fetchval() + else: + values = [cursor.fetchone() for _ in range(SINGLE_ROW_COUNT)] + eof = cursor.fetchone() + wall_ms = (time.perf_counter() - start) * 1000 + cpp, py = ctx.collect() + if method == "fetchmany": + assert eof == [] and all(len(batch) == 1 for batch in values) + values = [batch[0] for batch in values] + else: + assert eof is None, "Single-row fetch did not reach EOF" + for n, value in enumerate(values): + if method == "fetchval": + assert type(value) is int and value == n + else: + expected = (n, n + 10 if shape == "numeric" else "text") + assert tuple(value) == expected + assert tuple(map(type, value)) == tuple(map(type, expected)) + assert not cursor.messages, "Clean single-row fetch produced diagnostics" + return dict( + title=f"{shape} / {method}", + wall_ms=wall_ms, + cpp=cpp, + py=py, + detail=f"Rows: {SINGLE_ROW_COUNT}; shape: {shape}; API: {method}; EOF: 1", + ) + finally: + ctx.disable() + + +def single_row_registry(): + return { + f"{shape}_{method}": partial(single_row_fetch, method=method, shape=shape) + for shape in ("numeric", "mixed") + for method in ("fetchone", "fetchmany", "fetchval") + } diff --git a/profiler/README.md b/profiler/README.md index d6c2608f1..119e78082 100644 --- a/profiler/README.md +++ b/profiler/README.md @@ -275,3 +275,111 @@ void MyFunction(...) { `PERF_TIMER` compiles to nothing unless the build has `ENABLE_PROFILING`, so adding timers costs nothing in released builds. + + +### Explicit single-row comparison modes + +The paired controller retains its existing `diagnostic` default: native and Python +recording are enabled, so the guarded Row path deliberately uses the split route. +That report is not a latency measurement of ordinary fused fetching. + +Two opt-in modes use the same six read-only workloads: `numeric_fetchone`, +`numeric_fetchmany`, `numeric_fetchval`, and their `mixed_` equivalents. Each fetches +1,000 ordered rows plus EOF; numeric rows contain two INT columns, mixed rows contain +INT and NVARCHAR. `fetchmany` always requests the built-in integer `1`. `fetchval` +uses its public API, not a first-column-only native shortcut. + +- `--mode latency` builds both revisions with native instrumentation OFF and keeps + Python phases OFF. It rejects `--reuse-candidate` rather than silently timing the + profiling-enabled CI extension. Setup/execute and validation are outside the + timed window; the API loop, result retention and EOF call are inside it. +- `--mode route` uses native instrumentation ON with Python phases OFF. Each case + must record 1,001 native fetch calls and, when the guarded Row entry exists, + exactly 1,000 native Row constructions. An older base without that entry must + record no such constructions. This is route attribution, not production latency. + Existing native regression tests separately cover converter fallback, all-column + callbacks, repeated EOF, and customization failures. + +For example, after obtaining approval for the required builds and database work: + +```console +python -m eng.profiler_benchmarks.controller --mode latency --base --candidate --leg Linux-SQL2022 --output +python -m eng.profiler_benchmarks.controller --mode route --base --candidate --leg Linux-SQL2022 --output +``` + +Run these separately, not concurrently. No extra CI arm or automatic execution is +introduced. Each invocation retains the controller's existing bounded sample, +worker and overall time limits. Use distinct output directories; subset runs remain +incomplete and cannot produce a full report. The standalone interactive Profiler's +recording and timeline defaults are unchanged. + +These modes emit schema version 2 with explicit mode, worker revision, actual native +binary path/SHA256 and compile-capability identity. The validator rejects mixed modes, +changed binaries, missing/failed samples, unexpected recording, and wrong route counts. +Schema version 1 remains the existing 22-workload diagnostic report. Failed runs stay +incomplete/unavailable; missing timings are never replaced with zero. Raw samples +retain every paired observation. Tables show signed subthreshold changes and median +candidate/base ratios with their observed min/max range, not confidence intervals. +The classification policy remains **more than 20% median paired change, at least +1 ms between median times, and at least 80% of pairs beyond the relative threshold**. +A subthreshold change is not a proven win or proof of no effect. Neither mode adds a +pyodbc comparison or changes production fetch behavior. + + +### Latency-first PR CI report + +The existing Ubuntu/SQL Server 2022 and Ubuntu/SQL Server 2025 PR legs invoke +`--reuse-candidate --ci-report`. Windows profiling remains disabled. The aggregate +runs OFF/OFF latency first, ON/OFF native-route attribution second, and the original +22-workload ON/ON diagnostics last. The six numeric/mixed workloads in each fetch +mode and all legacy workloads retain **five measured pairs and one warmup pair**. +Each mode alternates base/candidate order independently, using separate workers. + +The aggregate builds isolated base-OFF, candidate-OFF and base-ON directories and +reuses the already tested candidate-ON build. Route and diagnostic workers must +attest the same ON binary paths, SHA256 digests, source revisions and environments. +Each reused-checkout worker also checks HEAD and tracked driver/provider source +cleanliness before importing the driver. The reused build is never rebuilt or +toggled in place. Per leg this permits at most +three performance-step builds, 36 measurement workers and 408 workload executions +(including warmups), rather than one build, 12 workers and 264 executions. Both +active legs together add at most four builds, 48 workers and 288 executions. No new +matrix arm, pyodbc comparison or database setup is introduced. + +The controller retains a **90-minute aggregate budget**, including a three-minute +finish reserve; the pipeline step remains 100 minutes and its job 160 minutes. +Archives/preflight, builds and workers receive bounded timeouts clipped to the +remaining work budget. Fetch workers receive at most 30 seconds; diagnostic workers +retain their six-minute cap. The fetch allowance is not yet runtime-qualified. +The proposal already required 132 minutes if its build/worker caps and planning +overhead were all consumed. Metadata operations also consume the shared deadline; +they never extend it. Completing every mode is **not guaranteed**. Unused early time flows to later modes. Exhaustion or +failure leaves that mode unavailable with a stage/reason and its raw checkpoints; +there are no retries, trimmed workload sets, reduced pair counts or zero substitutes. +Unsafe process cleanup aborts subsequent work and reports the retained temporary +root rather than removing files beneath an unreaped worker. + +There is still exactly one `report.json` per existing artifact. Its root remains the +schema-1 diagnostic report and its `status` describes diagnostics only. The additive +`measurement_bundle_version: 1` extension contains `fetch_measurements.latency` and +`fetch_measurements.route` (schema 2). Each mode has an independent complete/incomplete +status. Root and child checkpoints are written atomically; an incomplete aggregate +exits nonzero, while the pipeline's existing failure-tolerant artifact publication +retains independently valid modes. The finish deadline is checked after final +validation and the atomic write. An overrun gets one corrective incomplete checkpoint +and a nonzero exit, preserving completed latency/route measurements without a recheck +loop. Raw build logs and mode-prefixed worker JSON/log files stay in the same artifact, +without nested files named `report.json`. + +The updated collector validates shared PR/build/base/merge identity, then each mode +independently. The primary performance verdict comes **only from OFF/OFF latency**. +Native route counts/timings and both-ON diagnostics appear in separately labeled +sections; neither replaces missing latency. Missing, malformed and unstarted modes +are explicitly unavailable. Legacy readers can still read the diagnostic root; +standalone schema-1/schema-2 reports and interactive Profiler defaults are unchanged. +Fork reporting continues to use trusted base code. Existing artifact, download, +comment-size and publisher time limits are not expanded. + +This wiring is source-only until approved exact-head CI runs it. Source/fake-boundary +controls and prior-head CI do not establish native correctness, completion within +these allowances, or a speedup. diff --git a/tests/test_036_profiler_ci.py b/tests/test_036_profiler_ci.py index 8f5e24eed..37cb3243d 100644 --- a/tests/test_036_profiler_ci.py +++ b/tests/test_036_profiler_ci.py @@ -13,6 +13,8 @@ import sys import tarfile import time +import tempfile +import unittest from types import SimpleNamespace from unittest.mock import MagicMock from urllib.error import URLError @@ -1859,3 +1861,1170 @@ def test_comment_workflow_separates_same_repo_and_fork_trust(): "head.ref" not in workflow and "head.sha }}" not in workflow.split("ref:", 1)[1].split("persist", 1)[0] ) + + +class TestSingleRowMeasurementModes(unittest.TestCase): + """Pure fake-boundary contracts; these do not qualify native timing.""" + + def sample_report(self, mode="latency", ratio=0.9): + def counter(calls): + return dict(calls=calls, total_us=10000, min_us=1, max_us=100) + + def sample(side): + guarded = side == "candidate" + cases = {} + for name in reporting.FETCH_CASES: + shape, method = name.split("_") + cpp = {} + if mode == "route": + timer = ( + "ddbc::FetchMany_wrap" if method == "fetchmany" else "ddbc::FetchOne_wrap" + ) + cpp[timer] = counter(1001) + if guarded: + cpp["ddbc::FetchRow::construct_row"] = counter(1000) + cases[name] = dict( + wall_ms=100 * (ratio if guarded else 1), + cpp=cpp, + py={}, + work=f"Rows: 1000; shape: {shape}; API: {method}; EOF: 1", + ) + return dict( + status="complete", + mode=mode, + provenance=dict( + source_commit=("b" if guarded else "a") * 40, + native_file="/source/native.so", + native_sha256=("d" if guarded else "e") * 64, + native_profiling=mode == "route", + guarded_row=guarded, + ), + environment=dict( + os="Linux", architecture="x86_64", python="3.13.7", sql_version="16.0" + ), + scenarios=cases, + ) + + return dict( + schema_version=2, + mode=mode, + status="complete", + leg="Linux-SQL2022", + base_commit="a" * 40, + source_commit="b" * 40, + head_commit="c" * 40, + build_id=42, + samples=5, + warmups=1, + pairs=[dict(base=sample("base"), candidate=sample("candidate")) for _ in range(5)], + ) + + def test_modes_validate_and_keep_subthreshold_changes_visible(self): + self.assertEqual(tuple(benchmark_workloads.single_row_registry()), reporting.FETCH_CASES) + self.assertEqual(len(reporting.CASES), 22) + for mode in ("latency", "route"): + for ratio in (0.9, 1.1): + with self.subTest(mode=mode, ratio=ratio): + report = self.sample_report(mode, ratio) + reporting.validate(report) + self.assertTrue( + all(row["status"] == "ok" for row in reporting.comparisons(report)) + ) + body = reporting.render([report], "c" * 40, 42) + self.assertIn(f"{(ratio - 1) * 100:+.1f}%", body) + self.assertIn("Python phases OFF", body) + self.assertIn("All database tasks and timings", body) + self.assertIn(f"{ratio:.3f} [{ratio:.3f}, {ratio:.3f}]", body) + self.assertIn("1,000 mixed rows / fetchmany(1)", body) + if mode == "route": + self.assertIn("not production latency", body) + + def test_threshold_policy_is_unchanged(self): + self.assertEqual((reporting.THRESHOLD, reporting.MIN_DELTA_MS), (0.20, 1.0)) + for ratio, expected in ( + (0.8, "ok"), + (0.79, "improvement"), + (1.2, "ok"), + (1.21, "regression"), + ): + with self.subTest(ratio=ratio): + self.assertEqual( + reporting.comparisons(self.sample_report(ratio=ratio))[0]["status"], expected + ) + report = self.sample_report(ratio=1.3) + for pair in report["pairs"][:2]: + pair["candidate"]["scenarios"]["numeric_fetchone"]["wall_ms"] = 100 + self.assertEqual(reporting.comparisons(report)[0]["status"], "noisy") + report["pairs"][1]["candidate"]["scenarios"]["numeric_fetchone"]["wall_ms"] = 130 + self.assertEqual(reporting.comparisons(report)[0]["status"], "regression") + for pair in report["pairs"]: + pair["base"]["scenarios"]["numeric_fetchone"]["wall_ms"] = 1 + pair["candidate"]["scenarios"]["numeric_fetchone"]["wall_ms"] = 1.5 + self.assertEqual(reporting.comparisons(report)[0]["status"], "ok") + + def test_schema_rejects_mismatched_or_missing_evidence(self): + mutations = ( + lambda sample: sample.update(mode="diagnostic"), + lambda sample: sample.update(status="running"), + lambda sample: sample.pop("provenance"), + lambda sample: sample["provenance"].update(native_profiling=True), + lambda sample: sample["provenance"].update(source_commit="f" * 40), + lambda sample: sample["provenance"].update(native_sha256="invalid"), + lambda sample: sample["scenarios"]["mixed_fetchval"].update(py={"py::fetch": {}}), + lambda sample: sample["scenarios"]["mixed_fetchval"].update(cpp={"ddbc::fetch": {}}), + lambda sample: sample["scenarios"].pop("numeric_fetchmany"), + lambda sample: sample["scenarios"]["mixed_fetchval"].update(wall_ms=0), + lambda sample: sample["scenarios"]["mixed_fetchval"].update(work="Rows: 1"), + ) + for index, mutate in enumerate(mutations): + with self.subTest(mutation=index): + report = self.sample_report() + mutate(report["pairs"][0]["candidate"]) + with self.assertRaises(ValueError): + reporting.validate(report) + report = self.sample_report() + report["pairs"][1]["candidate"]["provenance"]["native_sha256"] = "f" * 64 + with self.assertRaisesRegex(ValueError, "changed between samples"): + reporting.validate(report) + with self.assertRaisesRegex(ValueError, "different measurement modes"): + reporting.render([self.sample_report(), self.sample_report("route")], "c" * 40, 42) + + def test_route_requires_actual_counts_not_only_timers(self): + for label, calls in (("ddbc::FetchOne_wrap", 1000), ("ddbc::FetchRow::construct_row", 0)): + with self.subTest(label=label): + report = self.sample_report("route") + report["pairs"][0]["candidate"]["scenarios"]["numeric_fetchone"]["cpp"][label][ + "calls" + ] = calls + with self.assertRaises(ValueError): + reporting.validate(report) + report = self.sample_report("route") + report["pairs"][0]["base"]["scenarios"]["mixed_fetchval"]["cpp"] = {} + with self.assertRaises(ValueError): + reporting.validate(report) + + def test_incomplete_measurement_is_unavailable_not_zero(self): + for mode in ("latency", "route"): + report = self.sample_report(mode) + report.update(status="incomplete", pairs=[]) + reporting.validate(report) + body = reporting.render([report], "c" * 40, 42) + self.assertIn("Performance unavailable", body) + self.assertNotIn("0.000 ms", body) + + def test_context_never_enables_python_and_rejects_contamination(self): + with self.assertRaises(ValueError): + controller._FetchContext("latency", MagicMock(), MagicMock()) + for mode in ("latency", "route"): + with self.subTest(mode=mode): + native = MagicMock() if mode == "route" else None + python = MagicMock() + python.is_enabled.return_value = False + python.get_stats.return_value = {} + if native is not None: + native.get_stats.return_value = {"timer": 1} + ctx = controller._FetchContext(mode, native, python) + ctx.enable() + python.enable.assert_not_called() + python.enable_timeline.assert_not_called() + python.reset.assert_called_once() + cpp, py = ctx.collect() + self.assertEqual(py, {}) + self.assertEqual(cpp, {"timer": 1} if native is not None else {}) + python.is_enabled.return_value = True + with self.assertRaisesRegex(RuntimeError, "became enabled"): + ctx.collect() + python.is_enabled.return_value = False + python.get_stats.return_value = {"py::unexpected": {}} + with self.assertRaisesRegex(RuntimeError, "contaminate"): + ctx.collect() + ctx.disable() + if native is not None: + native.enable.assert_called_once() + native.disable_timeline.assert_called() + + def test_six_workloads_time_only_fetch_and_validate_full_results(self): + from unittest.mock import patch + + for name, workload in benchmark_workloads.single_row_registry().items(): + with self.subTest(name=name): + shape, method = name.split("_") + cursor = MagicMock() + cursor.messages = [] + rows = [(n, n + 10 if shape == "numeric" else "text") for n in range(1000)] + values = ( + list(range(1000)) + if method == "fetchval" + else [[row] for row in rows] if method == "fetchmany" else rows + ) + fetch = getattr(cursor, method) + fetch.side_effect = values + ([[]] if method == "fetchmany" else [None]) + conn = MagicMock() + conn.cursor.return_value.__enter__.return_value = cursor + ctx = MagicMock() + ctx.collect.return_value = ({}, {}) + + def clock(): + cursor.execute.assert_called_once() + ctx.enable.assert_called_once() + ctx.collect.assert_not_called() + self.assertIn(fetch.call_count, (0, 1001)) + return 2.0 if fetch.call_count else 1.0 + + package = SimpleNamespace(logger=SimpleNamespace(is_debug_enabled=False)) + with ( + patch.dict(sys.modules, {"mssql_python.logging": package}), + patch.object(benchmark_workloads.time, "perf_counter", clock), + ): + result = workload(conn, ctx) + self.assertEqual(result["wall_ms"], 1000) + self.assertEqual(fetch.call_count, 1001) + if method == "fetchmany": + self.assertTrue( + all( + call.args == (1,) and type(call.args[0]) is int + for call in fetch.call_args_list + ) + ) + for other in {"fetchone", "fetchmany", "fetchval"} - {method}: + getattr(cursor, other).assert_not_called() + ctx.disable.assert_called_once() + conn.cursor.return_value.__exit__.assert_called_once() + + def test_workloads_fail_closed_and_disable_context(self): + from unittest.mock import patch + + for failure in ("type", "value", "eof", "warning", "fetch", "enable", "debug"): + with self.subTest(failure=failure): + cursor = MagicMock() + cursor.messages = [("01000", "warning")] if failure == "warning" else [] + rows = [(n, "text") for n in range(1000)] + if failure == "type": + rows[0] = (0.0, "text") + if failure == "value": + rows[0] = (0, "wrong") + cursor.fetchone.side_effect = ( + RuntimeError("fetch") + if failure == "fetch" + else rows + ([rows[0]] if failure == "eof" else [None]) + ) + conn = MagicMock() + conn.cursor.return_value.__enter__.return_value = cursor + ctx = MagicMock() + ctx.collect.return_value = ({}, {}) + if failure == "enable": + ctx.enable.side_effect = RuntimeError("enable") + package = SimpleNamespace( + logger=SimpleNamespace(is_debug_enabled=failure == "debug") + ) + with patch.dict(sys.modules, {"mssql_python.logging": package}): + with self.assertRaises((AssertionError, RuntimeError, ValueError)): + benchmark_workloads.single_row_fetch(conn, ctx, "fetchone", "mixed") + if failure == "debug": + conn.cursor.assert_not_called() + else: + ctx.disable.assert_called_once() + conn.cursor.return_value.__exit__.assert_called_once() + + def test_off_build_and_worker_mode_are_explicit(self): + from unittest.mock import patch + + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) + process = MagicMock() + process.wait.return_value = 0 + with patch.object(controller.subprocess, "Popen", return_value=process) as start: + controller.build(path, path / "build.log", profiling=False) + self.assertEqual(start.call_args.kwargs["env"]["ENABLE_PROFILING"], "0") + with patch.object(controller.subprocess, "run") as run: + output = path / "sample.json" + run.side_effect = lambda *a, **k: output.write_text('{"status":"complete"}') + controller.measure(path, output, None, mode="latency", revision="a" * 40) + command = run.call_args.args[0] + self.assertEqual(command[command.index("--mode") + 1], "latency") + self.assertEqual(command[command.index("--revision") + 1], "a" * 40) + with patch.object(controller, "resolve_revisions") as resolve: + with self.assertRaisesRegex(ValueError, "cannot reuse"): + controller.run(SimpleNamespace(mode="latency", reuse_candidate=True)) + resolve.assert_not_called() + + def test_worker_provenance_route_counts_and_failure_checkpoint(self): + from unittest.mock import patch + + for mode, failure in ( + ("latency", None), + ("route", None), + ("route", "counts"), + ("latency", "close"), + ): + with ( + self.subTest(mode=mode, failure=failure), + tempfile.TemporaryDirectory() as directory, + ): + root = Path(directory) + native_file = root / "native.so" + native_file.write_bytes(b"fake binary identity; never loaded") + args = SimpleNamespace( + source_root=root, revision="b" * 40, scenarios=None, output=root / "sample.json" + ) + native = SimpleNamespace( + module=SimpleNamespace(__file__=str(native_file)), DDBCSQLFetchRow=object() + ) + if mode == "route": + native.profiling = MagicMock() + python = MagicMock() + conn = MagicMock() + conn.__enter__.return_value = conn + if failure == "close": + conn.__exit__.side_effect = RuntimeError("connection close failed") + conn.cursor.return_value.__enter__.return_value.fetchone.return_value = ("16.0",) + package = SimpleNamespace( + ddbc_bindings=native, perf_timer=python, connect=MagicMock(return_value=conn) + ) + sample = self.sample_report(mode)["pairs"][0]["candidate"] + + def workload(connection, ctx): + ctx.enable() + # Measurement results here are fake; native count validation is real. + name = json.loads(args.output.read_text())["active_scenario"] + result = copy.deepcopy(sample["scenarios"][name]) + result["detail"] = result.pop("work") + if failure == "counts": + result["cpp"]["ddbc::FetchRow::construct_row"]["calls"] = 999 + ctx.disable() + return result + + cases = {name: workload for name in reporting.FETCH_CASES} + with ( + patch.dict(sys.modules, {"mssql_python": package}), + patch.dict(os.environ, {"DB_CONNECTION_STRING": "test-only"}), + patch.object(controller, "check_build") as check, + patch.object(benchmark_workloads, "single_row_registry", return_value=cases), + ): + if failure == "counts": + with self.assertRaisesRegex(RuntimeError, "route was not established"): + controller.fetch_worker(args, mode) + elif failure == "close": + with self.assertRaisesRegex(RuntimeError, "connection close failed"): + controller.fetch_worker(args, mode) + else: + controller.fetch_worker(args, mode) + check.assert_called_once_with(root, profiling=mode == "route") + result = json.loads(args.output.read_text()) + self.assertEqual(result["status"], "running" if failure else "complete") + self.assertEqual(result["provenance"]["native_profiling"], mode == "route") + self.assertEqual(len(result["provenance"]["native_sha256"]), 64) + self.assertEqual(result["provenance"]["source_commit"], "b" * 40) + self.assertEqual(len(result["scenarios"]), 0 if failure == "counts" else 6) + python.enable.assert_not_called() + if mode == "route": + native.profiling.disable.assert_called() + + def test_controller_pairs_use_selected_mode_and_fail_incomplete(self): + from unittest.mock import patch + + for mode, fail in (("latency", False), ("route", False), ("latency", True)): + with self.subTest(mode=mode, fail=fail), tempfile.TemporaryDirectory() as directory: + root = Path(directory) + args = SimpleNamespace( + base="a" * 40, + candidate="b" * 40, + output=root, + mode=mode, + reuse_candidate=False, + leg="Linux-SQL2022", + samples=3, + warmups=1, + scenarios=None, + ) + sample = self.sample_report(mode)["pairs"][0] + calls = [] + + def measure(path, output, scenarios, timeout, **options): + calls.append((path.name, options)) + if fail: + raise RuntimeError("missing measurement") + return copy.deepcopy(sample[path.name]) + + with ( + patch.object( + controller, "resolve_revisions", return_value=("a" * 40, "b" * 40) + ), + patch.object(controller, "checkout"), + patch.object(controller, "build") as build, + patch.object(controller, "measure", side_effect=measure), + ): + if fail: + with self.assertRaisesRegex(RuntimeError, "missing measurement"): + controller.run(args) + else: + controller.run(args) + self.assertEqual(build.call_count, 2) + self.assertTrue( + all( + call.kwargs == {"profiling": mode != "latency"} + for call in build.call_args_list + ) + ) + for side, options in calls: + self.assertEqual( + options, dict(mode=mode, revision=("a" if side == "base" else "b") * 40) + ) + result = reporting.validate(json.loads((root / "report.json").read_text())) + self.assertEqual(result["status"], "incomplete" if fail else "complete") + if not fail: + self.assertEqual( + [side for side, _ in calls], ["base", "candidate", "candidate", "base"] * 2 + ) + self.assertEqual(len(result["pairs"]), 3) + + def test_legacy_diagnostic_schema_still_requires_native_samples(self): + report = self.sample_report() + report.pop("mode") + report["schema_version"] = 1 + for pair in report["pairs"]: + for sample in pair.values(): + sample.pop("mode") + sample.pop("provenance") + sample["scenarios"] = { + name: dict( + wall_ms=100, + work="Rows: 1", + cpp={"ddbc::query": dict(calls=1, total_us=10, min_us=10, max_us=10)}, + py={}, + ) + for name in reporting.CASES + } + reporting.validate(report) + self.assertEqual(len(reporting.comparisons(report)), 22) + self.assertIn("profiling-enabled builds", reporting.render([report], "c" * 40, 42)) + report["pairs"][0]["base"]["scenarios"]["fetchone"]["cpp"] = {} + with self.assertRaisesRegex(ValueError, "profiling data"): + reporting.validate(report) + + def sample_bundle(self): + latency = self.sample_report("latency") + route = self.sample_report("route") + diagnostic = copy.deepcopy(route) + diagnostic.pop("mode") + diagnostic["schema_version"] = 1 + for pair in diagnostic["pairs"]: + for sample in pair.values(): + sample["mode"] = "diagnostic" + cell = sample["scenarios"]["numeric_fetchone"] + sample["scenarios"] = {name: copy.deepcopy(cell) for name in reporting.CASES} + diagnostic.update( + measurement_bundle_version=1, fetch_measurements=dict(latency=latency, route=route) + ) + return diagnostic + + def test_ci_bundle_all_mode_outcomes_have_latency_only_verdicts(self): + for bits in range(8): + with self.subTest(completeness=bits): + bundle = self.sample_bundle() + for index, mode in enumerate(("latency", "route", "diagnostic")): + report = bundle if mode == "diagnostic" else bundle["fetch_measurements"][mode] + if not bits & (1 << index): + report.update( + status="incomplete", + pairs=[], + unavailable_reason="Not started: budget exhausted", + ) + valid, reasons = reporting.ci_mode_reports(bundle) + self.assertEqual(len(valid), 3) + self.assertEqual(len(reasons), 3 - bits.bit_count()) + body = reporting.render_ci_reports([bundle], "c" * 40, 42) + self.assertIn("Primary verdict: native OFF / Python phases OFF latency only", body) + self.assertIn("Diagnostics ON/ON", body) + self.assertEqual("Performance unavailable" in body, not bool(bits & 1)) + self.assertNotIn("Performance improved", body) + self.assertLessEqual(len(body), reporting.MAX_COMMENT_CHARS) + if bits & 2: + self.assertIn("1001 fetch calls per side/sample", body) + section = body.split("Native route attribution", 1)[1].split( + "", 1 + )[0] + self.assertGreater(section.index("Native proof for"), section.rindex("| 1,000")) + if bits != 7: + self.assertIn("Unavailable: Not started: budget exhausted", body) + + def test_ci_bundle_rejects_bad_siblings_and_shared_on_drift(self): + for defect in ("missing", "mode", "commit", "counter", "identity", "environment"): + with self.subTest(defect=defect): + bundle = self.sample_bundle() + route = bundle["fetch_measurements"]["route"] + if defect == "missing": + bundle["fetch_measurements"].pop("route") + elif defect == "mode": + route["mode"] = "latency" + elif defect == "commit": + route["source_commit"] = "f" * 40 + elif defect == "counter": + route["pairs"][0]["candidate"]["scenarios"]["mixed_fetchmany"]["cpp"] = {} + elif defect == "identity": + for pair in route["pairs"]: + pair["candidate"]["provenance"]["native_sha256"] = "f" * 64 + else: + for pair in route["pairs"]: + for sample in pair.values(): + sample["environment"]["python"] = "3.12.0" + valid, reasons = reporting.ci_mode_reports(bundle) + self.assertIn("latency", valid) + self.assertNotIn("route", valid) + self.assertIn("route", reasons) + if defect in ("identity", "environment"): + self.assertNotIn("diagnostic", valid) + body = reporting.render_ci_reports([bundle], "c" * 40, 42) + self.assertIn("Unavailable:", body) + self.assertIn("1,000 mixed rows / fetchmany(1)", body) + bundle = self.sample_bundle() + for pair in bundle["pairs"]: + for side in ("base", "candidate"): + for cell in pair[side]["scenarios"].values(): + cell["cpp"] = { + "ddbc::" + + str(n) + + "_" + * 150: dict( + calls=1 if side == "base" else 2, + total_us=100 if side == "base" else 200, + min_us=1, + max_us=10, + ) + for n in range(3) + } + body = reporting.render_ci_reports([bundle], "c" * 40, 42) + self.assertLessEqual(len(body), reporting.MAX_COMMENT_CHARS) + self.assertIn("raw artifact", body) + for version in (None, True, 2): + bundle = self.sample_bundle() + bundle["measurement_bundle_version"] = version + with self.assertRaises(ValueError): + reporting.validate_ci_header(bundle) + bundle = self.sample_bundle() + bundle.pop("measurement_bundle_version") + with self.assertRaises(ValueError): + reporting.validate_ci_header(bundle) + + def test_ci_bundle_existing_artifact_and_assessment_preserve_trust(self): + bundle = self.sample_bundle() + reporting.validate(bundle) # The legacy schema-1 diagnostic root still works. + + def archive(value, duplicate=False): + buf = io.BytesIO() + with zipfile.ZipFile(buf, "w") as output: + output.writestr("profiler-Linux-SQL2022/report.json", json.dumps(value)) + output.writestr("profiler-Linux-SQL2022/route-base-0.json", "{}") + if duplicate: + output.writestr("route/report.json", "{}") + return buf.getvalue() + + self.assertEqual(reporting.artifact_report(archive(bundle)), bundle) + with self.assertRaisesRegex(ValueError, "exactly one"): + reporting.artifact_report(archive(bundle, True)) + evidence = reporting.AssessmentEvidence( + build=dict(id=42, sourceVersion="b" * 40), + head="c" * 40, + base="a" * 40, + merge_commit=dict(sha="b" * 40, parents=[dict(sha="a" * 40), dict(sha="c" * 40)]), + base_commit=dict(sha="a" * 40), + ) + body = reporting.assess( + evidence, {"Linux-SQL2022": "fake-artifact"}, lambda url: archive(bundle) + ) + self.assertIn("Primary verdict: native OFF", body) + self.assertIn("Row constructions base/candidate 0/1000", body) + bundle["fetch_measurements"]["latency"].update( + status="incomplete", pairs=[], unavailable_reason="OFF build failed" + ) + body = reporting.assess( + evidence, {"Linux-SQL2022": "fake-artifact"}, lambda url: archive(bundle) + ) + self.assertIn("Performance unavailable", body) + self.assertIn("OFF build failed", body) + bundle["head_commit"] = "f" * 40 + body = reporting.assess( + evidence, {"Linux-SQL2022": "fake-artifact"}, lambda url: archive(bundle) + ) + self.assertIn("Performance unavailable", body) + self.assertNotIn("Row constructions base/candidate 0/1000", body) + + def test_ci_orchestration_counts_order_reuse_and_budget_exhaustion(self): + from unittest.mock import patch + + for slow in (False, True): + with self.subTest(slow=slow), tempfile.TemporaryDirectory() as directory: + args = SimpleNamespace( + output=Path(directory), + mode="diagnostic", + base="a" * 40, + candidate="b" * 40, + reuse_candidate=True, + scenarios=None, + samples=5, + warmups=1, + leg="Linux-SQL2022", + ) + clock, builds, workers, archives = [0], [], [], [] + bundle = self.sample_bundle() + samples = { + "diagnostic": bundle["pairs"][0], + **{ + mode: report["pairs"][0] + for mode, report in bundle["fetch_measurements"].items() + }, + } + + def build(path, log, timeout, profiling=True): + builds.append((path, profiling)) + clock[0] += 900 if slow else 1 + + def measure(path, output, scenarios, timeout, **options): + mode = options["mode"] + side = "base" if options["revision"] == "a" * 40 else "candidate" + workers.append((mode, side, path, options["isolated"])) + duration = (360 if mode == "diagnostic" else 30) if slow else 1 + clock[0] += min(timeout, duration) + if duration > timeout: + raise subprocess.TimeoutExpired("fake-worker", timeout) + return copy.deepcopy(samples[mode][side]) + + with ( + patch.object(controller.time, "monotonic", side_effect=lambda: clock[0]), + patch.object( + controller, "resolve_revisions", return_value=("a" * 40, "b" * 40) + ), + patch.object(controller, "git", return_value="b" * 40), + patch.object( + controller, "run_process", side_effect=lambda *a, **k: archives.append(a[0]) + ), + patch.object(controller, "build", side_effect=build), + patch.object(controller, "measure", side_effect=measure), + patch.dict( + os.environ, + {"BUILD_BUILDID": "42", "SYSTEM_PULLREQUEST_SOURCECOMMITID": "c" * 40}, + ), + ): + if slow: + with self.assertRaisesRegex(RuntimeError, "incomplete"): + controller.run_ci_report(args) + else: + controller.run_ci_report(args) + result = json.loads((args.output / "report.json").read_text()) + self.assertEqual([flag for _, flag in builds], [False, False, True]) + self.assertEqual(len(archives), 3) + self.assertTrue(all("--archive-source" in command for command in archives)) + self.assertTrue(all(isolated for _, _, _, isolated in workers)) + self.assertEqual( + [mode for mode, *_ in workers[:24]], ["latency"] * 12 + ["route"] * 12 + ) + for mode in ("latency", "route"): + selected = [(side, path) for kind, side, path, _ in workers if kind == mode] + self.assertEqual( + [side for side, _ in selected], + ["base", "candidate", "candidate", "base"] * 3, + ) + on_paths = { + (mode, side): path for mode, side, path, _ in workers if mode != "latency" + } + for side in ("base", "candidate"): + self.assertEqual(on_paths["route", side], on_paths["diagnostic", side]) + valid, reasons = reporting.ci_mode_reports(result) + self.assertEqual(valid["latency"]["status"], "complete") + self.assertEqual(valid["route"]["status"], "complete") + self.assertEqual(result["status"], "incomplete" if slow else "complete") + self.assertLessEqual( + clock[0], controller.BENCHMARK_TIMEOUT - controller.CI_FINISH_RESERVE + ) + if slow: + self.assertIn("Timeout", reasons["diagnostic"]) + self.assertLess(len(workers), 36) + else: + self.assertEqual(len(workers), 36) + self.assertEqual( + sum(22 if mode == "diagnostic" else 6 for mode, *_ in workers), 408 + ) + self.assertTrue(all(len(report["pairs"]) == 5 for report in valid.values())) + self.assertFalse((args.output / "report.tmp").exists()) + + def test_ci_failures_do_not_retry_or_fabricate_siblings(self): + from unittest.mock import patch + + for failure in ( + "off-build", + "on-build", + "latency", + "route", + "diagnostic", + "identity", + "exhausted", + ): + with self.subTest(failure=failure), tempfile.TemporaryDirectory() as directory: + args = SimpleNamespace( + output=Path(directory), + mode="diagnostic", + base="a" * 40, + candidate="b" * 40, + reuse_candidate=True, + scenarios=None, + samples=5, + warmups=1, + leg="Linux-SQL2022", + ) + clock, workers, builds = [0], [], [] + bundle = self.sample_bundle() + samples = { + "diagnostic": bundle["pairs"][0], + **{ + mode: report["pairs"][0] + for mode, report in bundle["fetch_measurements"].items() + }, + } + + def build(path, log, timeout, profiling=True): + builds.append(path.name) + if failure == ("on-build" if profiling else "off-build"): + raise subprocess.CalledProcessError(1, "fake-build") + if failure == "exhausted": + clock[0] = controller.BENCHMARK_TIMEOUT + + def measure(path, output, scenarios, timeout, **options): + mode = options["mode"] + workers.append(mode) + if mode == failure: + raise subprocess.CalledProcessError(1, "fake-worker") + side = "base" if options["revision"] == "a" * 40 else "candidate" + result = copy.deepcopy(samples[mode][side]) + if failure == "identity" and mode == "diagnostic": + result["provenance"]["native_sha256"] = "f" * 64 + return result + + with ( + patch.object(controller.time, "monotonic", side_effect=lambda: clock[0]), + patch.object( + controller, "resolve_revisions", return_value=("a" * 40, "b" * 40) + ), + patch.object(controller, "git", return_value="b" * 40), + patch.object(controller, "run_process"), + patch.object(controller, "build", side_effect=build), + patch.object(controller, "measure", side_effect=measure), + patch.dict( + os.environ, + {"BUILD_BUILDID": "42", "SYSTEM_PULLREQUEST_SOURCECOMMITID": "c" * 40}, + ), + ): + with self.assertRaisesRegex(RuntimeError, "incomplete"): + controller.run_ci_report(args) + result = json.loads((args.output / "report.json").read_text()) + valid, reasons = reporting.ci_mode_reports(result) + self.assertTrue(reasons) + self.assertEqual(len(builds), len(set(builds))) + self.assertLessEqual(len(workers), 36) + if failure in ("latency", "route", "diagnostic"): + self.assertEqual(workers.count(failure), 1) + if failure in ("off-build", "latency"): + self.assertEqual(valid["route"]["status"], "complete") + self.assertEqual(valid["diagnostic"]["status"], "complete") + elif failure in ("on-build", "route", "diagnostic", "identity"): + self.assertEqual(valid["latency"]["status"], "complete") + if failure == "on-build": + self.assertNotIn("diagnostic", workers) + self.assertIn("Shared ON build unavailable", reasons["diagnostic"]) + if failure == "exhausted": + self.assertEqual(workers, []) + for mode, reason in reasons.items(): + self.assertTrue(reason) + self.assertEqual(valid[mode]["status"], "incomplete") + + def test_ci_atomic_write_preserves_last_report_and_cleanup_aborts(self): + from unittest.mock import patch + + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + output = root / "report.json" + controller.write_report(output, {"status": "incomplete"}) + original = output.read_bytes() + with self.assertRaises(ValueError): + controller.write_report(output, {"wall_ms": float("nan")}) + self.assertEqual(output.read_bytes(), original) + process = MagicMock() + process.wait.side_effect = subprocess.TimeoutExpired("fake-process", 1) + with ( + patch.object(controller.subprocess, "Popen", return_value=process), + patch.object( + controller, "terminate_process_tree", side_effect=OSError("cannot reap") + ) as reap, + ): + with self.assertRaises(controller.ProcessCleanupError): + controller.run_process(["fake-worker"], root / "worker.log", 1) + reap.assert_called_once_with(process) + args = SimpleNamespace( + output=root, + mode="diagnostic", + base="a" * 40, + candidate="b" * 40, + reuse_candidate=True, + scenarios=None, + samples=5, + warmups=1, + leg="Linux-SQL2022", + ) + retained = root / "retained-build-root" + retained.mkdir() + with ( + patch.object(controller, "resolve_revisions", return_value=("a" * 40, "b" * 40)), + patch.object(controller, "git", return_value="b" * 40), + patch.object(controller.tempfile, "mkdtemp", return_value=str(retained)), + patch.object( + controller, + "run_process", + side_effect=controller.ProcessCleanupError("cannot reap"), + ) as launch, + patch.object(controller.shutil, "rmtree") as remove, + patch.dict(os.environ, {"SYSTEM_PULLREQUEST_SOURCECOMMITID": "c" * 40}), + ): + with self.assertRaises(controller.ProcessCleanupError): + controller.run_ci_report(args) + remove.assert_not_called() + self.assertEqual(launch.call_count, 1) + result = json.loads(output.read_text()) + self.assertEqual(result["cleanup_required"], str(retained)) + self.assertEqual(result["fetch_measurements"]["latency"]["status"], "incomplete") + self.assertIn("cleanup", result["fetch_measurements"]["route"]["unavailable_reason"]) + + def test_ci_source_wiring_keeps_matrix_deadlines_and_trust(self): + pipeline = (ROOT / "eng/pipelines/pr-validation-pipeline.yml").read_text(encoding="utf-8") + self.assertEqual(pipeline.count("--ci-report"), 1) + linux = pipeline.split("- job: PytestOnLinux\n", 1)[1].split("\n- job:", 1)[0] + self.assertIn('--reuse-candidate --ci-report --leg "$(profilerLeg)"', linux) + self.assertIn("timeoutInMinutes: 100", linux) + self.assertIn("timeoutInMinutes: 160", linux) + self.assertIn("artifact: profiler-$(profilerLeg)", linux) + self.assertIn("eq(variables['distroName'], 'Ubuntu-SQL2025')", linux) + windows = pipeline.split("# Hosted Windows timings", 1)[1].split("- job:", 1)[0] + self.assertIn("condition: false", windows) + self.assertNotIn("--ci-report", windows) + self.assertEqual( + ( + controller.BENCHMARK_TIMEOUT, + controller.LOCAL_BENCHMARK_TIMEOUT, + controller.WORKER_TIMEOUT, + controller.FETCH_WORKER_TIMEOUT, + ), + (5400, 6300, 360, 30), + ) + workflow = (ROOT / ".github/workflows/pr-profiler-report.yml").read_text(encoding="utf-8") + self.assertIn("timeout-minutes: 230", workflow) + self.assertIn("github.event.pull_request.base.sha", workflow) + + def test_ci_diagnostic_worker_attests_actual_binary_after_cleanup(self): + from unittest.mock import patch + + for fails in (False, True): + with self.subTest(cleanup_failure=fails), tempfile.TemporaryDirectory() as directory: + root = Path(directory) + binary = root / "native.so" + binary.write_bytes(b"fake-not-imported") + native = SimpleNamespace( + module=SimpleNamespace(__file__=str(binary)), DDBCSQLFetchRow=object() + ) + manager = MagicMock() + profiler = manager.__enter__.return_value + profiler.run.return_value = [ + dict( + wall_ms=1, + cpp={"ddbc::query": dict(calls=1, total_us=1, min_us=1, max_us=1)}, + py={}, + detail="Rows: 1", + ) + ] + profiler._conn.cursor.return_value.__enter__.return_value.fetchone.return_value = ( + "16.0", + ) + if fails: + manager.__exit__.side_effect = RuntimeError("profiler cleanup failed") + core = SimpleNamespace(Profiler=MagicMock(return_value=manager)) + suite = SimpleNamespace(registry=lambda: {"connect": (object(), False)}) + args = SimpleNamespace( + source_root=root, + mode="diagnostic", + revision="b" * 40, + scenarios=None, + output=root / "diagnostic.json", + ) + with ( + patch.dict( + sys.modules, {"mssql_python": SimpleNamespace(ddbc_bindings=native)} + ), + patch.object(controller, "check_build") as check, + patch.object(controller, "load_suite", return_value=(core, suite)), + ): + if fails: + with self.assertRaisesRegex(RuntimeError, "profiler cleanup failed"): + controller.worker(args) + else: + controller.worker(args) + check.assert_called_once_with(root, profiling=True) + sample = json.loads(args.output.read_text()) + self.assertEqual(sample["status"], "running" if fails else "complete") + if not fails: + self.assertEqual(sample["mode"], "diagnostic") + self.assertEqual(sample["provenance"]["native_file"], str(binary.resolve())) + self.assertEqual(sample["provenance"]["source_commit"], "b" * 40) + self.assertTrue(sample["provenance"]["native_profiling"]) + self.assertEqual(len(sample["provenance"]["native_sha256"]), 64) + manager.__exit__.assert_called_once() + + def test_ci_rejects_scope_changes_before_work(self): + from unittest.mock import patch + + for change in ( + {"samples": 3}, + {"warmups": 2}, + {"scenarios": ["numeric_fetchone"]}, + {"mode": "route"}, + {"reuse_candidate": False}, + ): + args = SimpleNamespace( + samples=5, warmups=1, scenarios=None, mode="diagnostic", reuse_candidate=True + ) + args.__dict__.update(change) + with ( + self.subTest(change=change), + patch.object(controller, "resolve_revisions") as resolve, + ): + with self.assertRaises(ValueError): + controller.run_ci_report(args) + resolve.assert_not_called() + + command = [ + "controller", + "--reuse-candidate", + "--ci-report", + "--leg", + "Linux-SQL2022", + "--output", + "unused-fake-output", + ] + with ( + patch.object(sys, "argv", command), + patch.object(controller, "run_ci_report") as run, + patch.object(controller, "run") as legacy, + ): + controller.main() + self.assertTrue(run.call_args.args[0].ci_report) + legacy.assert_not_called() + with ( + patch.object(sys, "argv", command + ["--worker"]), + patch.object(controller, "worker") as worker, + ): + with self.assertRaises(SystemExit): + controller.main() + worker.assert_not_called() + + def test_ci_reused_source_and_filesystem_cleanup_fail_closed(self): + from unittest.mock import patch + + with patch.object(controller, "git", return_value="b" * 40) as git: + controller.verify_reused_source(controller.ROOT, "b" * 40) + self.assertEqual(git.call_count, 2) + self.assertIn("--exit-code", git.call_args.args) + with patch.object(controller, "git", return_value="a" * 40): + with self.assertRaisesRegex(ValueError, "revision changed"): + controller.verify_reused_source(controller.ROOT, "b" * 40) + with patch.object( + controller, "git", side_effect=["b" * 40, subprocess.CalledProcessError(1, "git diff")] + ): + with self.assertRaises(subprocess.CalledProcessError): + controller.verify_reused_source(controller.ROOT, "b" * 40) + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + retained = root / "build-root" + retained.mkdir() + args = SimpleNamespace( + output=root, + mode="diagnostic", + base="a" * 40, + candidate="b" * 40, + reuse_candidate=True, + scenarios=None, + samples=5, + warmups=1, + leg="Linux-SQL2022", + ) + with ( + patch.object(controller, "resolve_revisions", return_value=("a" * 40, "b" * 40)), + patch.object(controller, "git", return_value="b" * 40), + patch.object(controller.tempfile, "mkdtemp", return_value=str(retained)), + patch.object( + controller, + "run_process", + side_effect=subprocess.CalledProcessError(1, "fake-archive"), + ), + patch.object( + controller.shutil, "rmtree", side_effect=OSError("cannot remove root") + ), + patch.dict(os.environ, {"SYSTEM_PULLREQUEST_SOURCECOMMITID": "c" * 40}), + ): + with self.assertRaisesRegex( + controller.ProcessCleanupError, "build-directory cleanup" + ): + controller.run_ci_report(args) + result = json.loads((root / "report.json").read_text()) + self.assertEqual(result["status"], "incomplete") + self.assertEqual(result["cleanup_required"], str(retained)) + self.assertIn("cleanup failed", result["unavailable_reason"]) + + def test_ci_existing_publisher_consumes_bundle_without_latency_fallback(self): + from unittest.mock import patch + + for latency_available in (True, False): + with self.subTest(latency_available=latency_available): + bundle = self.sample_bundle() + if not latency_available: + bundle["fetch_measurements"]["latency"].update( + status="incomplete", pairs=[], unavailable_reason="OFF worker failed" + ) + data = {} + for leg in reporting.LEGS: + value = set_leg(bundle, leg) + value["fetch_measurements"] = { + mode: set_leg(child, leg) + for mode, child in value["fetch_measurements"].items() + } + data[leg] = zip_data([("report.json", json.dumps(value))]) + artifacts = [ + { + "name": "profiler-" + leg, + "resource": {"downloadUrl": "https://dev.azure.com/" + leg}, + } + for leg in data + ] + posted, clock = [], [0] + with ( + patch.object( + publisher, + "publish", + side_effect=lambda number, head, body, base=None: posted.append(body), + ), + patch.object(publisher, "github", side_effect=pr_topology()), + patch.object( + publisher, + "api", + side_effect=lambda url: { + "value": artifacts if "/artifacts?" in url else [ado_build()] + }, + ), + patch.object( + publisher, + "fetch", + side_effect=lambda url, **kwargs: data[url.rsplit("/", 1)[-1]], + ), + patch.object(publisher.time, "monotonic", side_effect=lambda: clock[0]), + patch.object( + publisher.time, + "sleep", + side_effect=lambda seconds: clock.__setitem__(0, clock[0] + seconds), + ), + ): + publisher.run(123, "c" * 40, 4) + self.assertEqual(len(posted), 2) + body = posted[-1] + self.assertEqual(body.count(reporting.MARKER), 1) + self.assertEqual(body.count("## PR Performance Report"), 1) + self.assertIn("Primary verdict: native OFF", body) + self.assertIn("Row constructions base/candidate 0/1000", body) + self.assertIn("Legacy diagnostics", body) + self.assertEqual("Performance unavailable" in body, not latency_available) + if not latency_available: + self.assertIn("OFF worker failed", body) + self.assertNotIn("Performance improved", body) + + def test_ci_finalization_deadline_is_checked_after_validation_and_write(self): + from unittest.mock import patch + + for delayed in (None, "validation", "write"): + with self.subTest(delayed=delayed), tempfile.TemporaryDirectory() as directory: + args = SimpleNamespace( + output=Path(directory), + mode="diagnostic", + base="a" * 40, + candidate="b" * 40, + reuse_candidate=True, + scenarios=None, + samples=5, + warmups=1, + leg="Linux-SQL2022", + ) + bundle = self.sample_bundle() + samples = { + "diagnostic": bundle["pairs"][0], + **{ + mode: report["pairs"][0] + for mode, report in bundle["fetch_measurements"].items() + }, + } + clock, cleaned, final_writes = [0], [False], [0] + original_remove = controller.shutil.rmtree + original_validate = controller.ci_mode_reports + original_write = controller.write_report + + def remove(path): + original_remove(path) + cleaned[0] = True + + def validate(value): + result = original_validate(value) + if cleaned[0] and delayed == "validation": + clock[0] = controller.BENCHMARK_TIMEOUT + 1 + return result + + def write(path, value): + original_write(path, value) + if cleaned[0]: + final_writes[0] += 1 + if delayed == "write": + clock[0] = controller.BENCHMARK_TIMEOUT + final_writes[0] + elif delayed is None: + clock[0] = controller.BENCHMARK_TIMEOUT + + def measure(path, output, scenarios, timeout, **options): + side = "base" if options["revision"] == "a" * 40 else "candidate" + return copy.deepcopy(samples[options["mode"]][side]) + + error = None + with ( + patch.object(controller.time, "monotonic", side_effect=lambda: clock[0]), + patch.object( + controller, "resolve_revisions", return_value=("a" * 40, "b" * 40) + ), + patch.object(controller, "git", return_value="b" * 40), + patch.object(controller, "run_process"), + patch.object(controller, "build"), + patch.object(controller, "measure", side_effect=measure), + patch.object(controller.shutil, "rmtree", side_effect=remove), + patch.object(controller, "ci_mode_reports", side_effect=validate), + patch.object(controller, "write_report", side_effect=write), + patch.dict( + os.environ, + {"BUILD_BUILDID": "42", "SYSTEM_PULLREQUEST_SOURCECOMMITID": "c" * 40}, + ), + ): + try: + controller.run_ci_report(args) + except RuntimeError as failure: + error = str(failure) + saved = json.loads((args.output / "report.json").read_text()) + self.assertTrue(cleaned[0]) + self.assertEqual(final_writes[0], 2 if delayed else 1) + self.assertEqual(saved["status"], "incomplete" if delayed else "complete") + self.assertEqual(len(saved["pairs"]), 5) + for mode in ("latency", "route"): + self.assertEqual(saved["fetch_measurements"][mode]["status"], "complete") + self.assertEqual(len(saved["fetch_measurements"][mode]["pairs"]), 5) + if delayed: + self.assertIsNotNone(error) + self.assertIn("incomplete", error) + self.assertIn("finish deadline", saved["unavailable_reason"]) + else: + self.assertIsNone(error) From 3e7fcffad8bb78e7f3755ac8f485d91e6c3877cc Mon Sep 17 00:00:00 2001 From: Jahnvi Thakkar Date: Tue, 6 Oct 2026 22:01:42 +0530 Subject: [PATCH 09/17] PERF: simplify fetchone routing and attest per-method report routes Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- eng/profiler_benchmarks/controller.py | 181 ++++++++++- eng/profiler_benchmarks/report.py | 131 ++++++-- mssql_python/cursor.py | 17 +- profiler/README.md | 53 +++- tests/test_036_profiler_ci.py | 422 +++++++++++++++++++++++++- tests/test_fetch_single_row.py | 33 +- 6 files changed, 768 insertions(+), 69 deletions(-) diff --git a/eng/profiler_benchmarks/controller.py b/eng/profiler_benchmarks/controller.py index 5c79e0122..77a7c1a22 100644 --- a/eng/profiler_benchmarks/controller.py +++ b/eng/profiler_benchmarks/controller.py @@ -1,6 +1,7 @@ """Build and measure base/candidate in isolated directories on the same CI agent.""" import argparse +import ast import contextlib import faulthandler import hashlib @@ -18,8 +19,20 @@ import tarfile import tempfile import time - -from .report import LEGS, MODES, validate, validate_ci_mode, validate_ci_header, ci_mode_reports +import textwrap + +from .report import ( + LEGS, + MODES, + validate, + validate_ci_mode, + validate_ci_header, + ci_mode_reports, + validate_row_route, + expected_constructors, + validate_python_sources, + validate_samples, +) from . import workloads ROOT = Path(__file__).resolve().parents[2] @@ -216,18 +229,141 @@ def verify_reused_source(source_root, revision): ) +# Closed, reviewed dispatch bodies: main666, f539, and the many-only successor. +# Method/comment/docstring edits require review and an explicit fingerprint update. +_LEGACY_PYTHON_ROUTE = "52681c15f92e85c01e0b43a9bd763872051fdc956701046534a5beb2467e6c3c" +_LEGACY_FUSED_ROUTE = "d4c6671b89ace027c22585e761c4f4c3243e297be2162213d9bbb38d5bed03c6" +_MANY_ONLY_ROUTE = "544d5b0ccbf9df1f0b6ec72c79ecbfa49a3517cab0437a5520940b8941966759" + + +def python_source_identity(source_root, revision): + """Read a selected root's descriptive policy, never infer it from measurements.""" + if not SHA.fullmatch(revision): + raise ValueError("Invalid Python source revision") + source = (source_root / "mssql_python" / "cursor.py").read_text(encoding="utf-8") + try: + tree = ast.parse(source) + except SyntaxError as error: + raise ValueError("Invalid Python route source syntax") from error + name = "_DEFAULT_NATIVE_ROW_ROUTE" + mentions = [node for node in ast.walk(tree) if isinstance(node, ast.Name) and node.id == name] + declarations = [ + node + for node in tree.body + if isinstance(node, ast.Assign) + and len(node.targets) == 1 + and isinstance(node.targets[0], ast.Name) + and node.targets[0].id == name + ] + if mentions and (len(mentions) != 1 or len(declarations) != 1): + raise ValueError("Malformed or duplicate Python route declaration") + classes = [ + node for node in tree.body if isinstance(node, ast.ClassDef) and node.name == "Cursor" + ] + if len(classes) != 1 or classes[0].decorator_list: + raise ValueError("Missing, duplicate or decorated Cursor class") + methods = [] + for method in ("fetchone", "fetchmany", "fetchval"): + nodes = [ + node + for node in classes[0].body + if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)) and node.name == method + ] + if len(nodes) != 1: + raise ValueError("Missing or duplicate fetch method") + if nodes[0].decorator_list or isinstance(nodes[0], ast.AsyncFunctionDef): + raise ValueError("Decorated or asynchronous fetch method is not a reviewed route") + methods.append(textwrap.dedent(ast.get_source_segment(source, nodes[0]))) + # Check class-namespace bindings, not locals in methods or nested scopes. + route_names = {"fetchone", "fetchmany", "fetchval"} + pending = list(classes[0].body) + while pending: + node = pending.pop() + if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)): + if node.name in route_names and ( + isinstance(node, ast.ClassDef) or node not in classes[0].body + ): + raise ValueError("Rebound Python fetch method") + pending.extend(node.decorator_list) + if isinstance(node, ast.ClassDef): + pending.extend(node.bases + node.keywords) + else: + pending.append(node.args) + if node.returns is not None: + pending.append(node.returns) + continue + if isinstance(node, ast.Lambda): + pending.append(node.args) + continue + if isinstance(node, (ast.ListComp, ast.SetComp, ast.DictComp, ast.GeneratorExp)): + continue + if ( + isinstance(node, ast.Name) + and isinstance(node.ctx, (ast.Store, ast.Del)) + and node.id in route_names + or isinstance(node, ast.alias) + and (node.asname or node.name.split(".")[0]) in route_names + or isinstance(node, (ast.ExceptHandler, ast.MatchAs, ast.MatchStar)) + and node.name in route_names + or isinstance(node, ast.MatchMapping) + and node.rest in route_names + ): + raise ValueError("Rebound Python fetch method") + pending.extend(ast.iter_child_nodes(node)) + fingerprint = hashlib.sha256( + json.dumps(methods, ensure_ascii=True, separators=(",", ":")).encode() + ).hexdigest() + if fingerprint == _LEGACY_PYTHON_ROUTE: + values = (False, False, False) + elif fingerprint == _LEGACY_FUSED_ROUTE: + values = (True, True, True) + elif fingerprint == _MANY_ONLY_ROUTE: + values = (False, True, False) + else: + raise ValueError("Unknown Python fetch dispatch fingerprint") + expected = dict(version=1, methods=dict(zip(("fetchone", "fetchmany", "fetchval"), values))) + if declarations: + expression = declarations[0].value + # literal_eval alone silently accepts duplicate dictionary keys. + for node in ast.walk(expression): + if isinstance(node, ast.Dict): + keys = [ast.literal_eval(key) for key in node.keys] + if any(type(key) is not str for key in keys) or len(set(keys)) != len(keys): + raise ValueError("Duplicate or invalid route declaration key") + route = ast.literal_eval(expression) + validate_row_route(route) + if route != expected: + raise ValueError("Python source-policy drift") + elif fingerprint == _MANY_ONLY_ROUTE: + raise ValueError("Missing Python route declaration") + return dict( + source_commit=revision, + python_cursor_sha256=hashlib.sha256(source.encode()).hexdigest(), + row_route=expected, + ) + + def native_identity(source_root, revision, profiling): from mssql_python import ddbc_bindings - + import mssql_python.cursor as cursor_module + + if ( + Path(cursor_module.__file__).resolve() + != (source_root / "mssql_python" / "cursor.py").resolve() + ): + raise ValueError("Python cursor is outside the selected checkout") + python_identity = python_source_identity(source_root, revision) + guarded = hasattr(ddbc_bindings, "DDBCSQLFetchRow") + validate_row_route(python_identity["row_route"], guarded) native_file = Path(ddbc_bindings.module.__file__).resolve() if not native_file.is_relative_to(source_root.resolve()): raise RuntimeError("Native binary is outside the selected checkout") return dict( - source_commit=revision, + **python_identity, native_file=str(native_file), native_sha256=hashlib.sha256(native_file.read_bytes()).hexdigest(), native_profiling=profiling, - guarded_row=hasattr(ddbc_bindings, "DDBCSQLFetchRow"), + guarded_row=guarded, ) @@ -280,9 +416,7 @@ def checkpoint(active=None, environment=None): constructors = ( result["cpp"].get("ddbc::FetchRow::construct_row", {}).get("calls", 0) ) - if constructors != ( - workloads.SINGLE_ROW_COUNT if provenance["guarded_row"] else 0 - ): + if constructors != expected_constructors(provenance, name.split("_")[1]): raise RuntimeError("Native Row construction route was not established") output[name] = {key: result[key] for key in ("wall_ms", "cpp", "py")} output[name]["work"] = result["detail"] @@ -513,6 +647,15 @@ def run_ci_report(args): "base": roots["base-off" if mode == "latency" else "base-on"], "candidate": roots["candidate-off"] if mode == "latency" else ROOT, } + sources = { + side: python_source_identity(paths[side], base if side == "base" else candidate) + for side in ("base", "candidate") + } + for previous in modes.values(): + if "python_sources" in previous and previous["python_sources"] != sources: + raise ValueError("Python source identity changed across modes") + report["python_sources"] = sources + validate_python_sources(report) identity = None for sample in range(6): pair = {} @@ -666,6 +809,14 @@ def run(args): remaining(deadline, 900), **build_options, ) + if mode != "diagnostic": + report["python_sources"] = { + side: python_source_identity(paths[side], base if side == "base" else candidate) + for side in ("base", "candidate") + } + validate_python_sources(report) + report["unavailable_reason"] = "Collecting selected workload pairs" + identity = None for sample in range(args.warmups + args.samples): pair = {} order = ("base", "candidate") if sample % 2 == 0 else ("candidate", "base") @@ -684,6 +835,20 @@ def run(args): ) if pair["base"]["environment"] != pair["candidate"]["environment"]: raise RuntimeError("Base and candidate environments differ") + if mode != "diagnostic": + probe = dict( + report, pairs=[*report["pairs"], pair], unavailable_reason="Collecting pairs" + ) + if args.scenarios is None: + validate_ci_mode(probe, mode) + else: + validate_samples(probe, selected_cases=args.scenarios) + observed = { + side: (pair[side]["provenance"], pair[side]["environment"]) for side in pair + } + if identity is not None and identity != observed: + raise ValueError("Worker identity changed after warmup") + identity = observed if sample >= args.warmups: report["pairs"].append(pair) report_path.write_text(json.dumps(report, allow_nan=False), encoding="utf-8") diff --git a/eng/profiler_benchmarks/report.py b/eng/profiler_benchmarks/report.py index 4f714fb63..672a6e3c2 100644 --- a/eng/profiler_benchmarks/report.py +++ b/eng/profiler_benchmarks/report.py @@ -168,16 +168,22 @@ def _validate(report, build_id=None, head=None, source=None, base=None): pairs = report.get("pairs") if not isinstance(pairs, list) or len(pairs) > samples: raise ValueError("Invalid sample pairs") + validate_python_sources(report) if report["status"] == "incomplete": - return report + if "python_sources" in report: + text(report.get("unavailable_reason"), limit=240) + return validate_samples(report) if len(pairs) != samples: raise ValueError("Incomplete sample pairs") return validate_samples(report) -def validate_samples(report): +def validate_samples(report, selected_cases=None): mode = measurement_mode(report) pairs = report["pairs"] + cases = cases_for(report) if selected_cases is None else selected_cases + if not cases or set(cases) - set(cases_for(report)): + raise ValueError("Invalid selected scenarios") environment = None work = {} provenance = {} @@ -188,7 +194,7 @@ def validate_samples(report): sample = pair[side] if not isinstance(sample, dict): raise ValueError("Invalid sample") - if mode != "diagnostic": + if mode != "diagnostic" or "python_sources" in report or "provenance" in sample: if sample.get("mode") != mode or sample.get("status") != "complete": raise ValueError("Incomplete or mismatched measurement mode") native_identity = validate_native_identity(sample, report, side, mode) @@ -216,7 +222,7 @@ def validate_samples(report): scenarios = sample["scenarios"] if not isinstance(scenarios, dict): raise ValueError("Invalid scenarios object") - if set(scenarios) != set(cases_for(report)): + if set(scenarios) != set(cases): raise ValueError("Scenario set incomplete or changed") for name, scenario in scenarios.items(): if not isinstance(scenario, dict): @@ -247,7 +253,7 @@ def validate_samples(report): raise ValueError("Native fetch route count mismatch") constructor = stats.get("ddbc::FetchRow::construct_row", {}) if not isinstance(constructor, dict) or constructor.get("calls", 0) != ( - 1000 if native_identity["guarded_row"] else 0 + expected_constructors(native_identity, method) ): raise ValueError("Native constructor route count mismatch") for layer in ("cpp", "py"): @@ -274,16 +280,84 @@ def validate_samples(report): return report +def validate_row_route(route, guarded=None): + if ( + type(route) is not dict + or set(route) != {"version", "methods"} + or type(route["version"]) is not int + or route["version"] != 1 + ): + raise ValueError("Unsupported Python row route version or shape") + methods = route["methods"] + if ( + type(methods) is not dict + or set(methods) != {"fetchone", "fetchmany", "fetchval"} + or any(type(value) is not bool for value in methods.values()) + ): + raise ValueError("Invalid Python row route methods") + values = tuple(methods[name] for name in ("fetchone", "fetchmany", "fetchval")) + if values not in ((False, False, False), (True, True, True), (False, True, False)): + raise ValueError("Unsupported Python row route policy") + if guarded is not None and (type(guarded) is not bool or any(values) != guarded): + raise ValueError("Native binding contradicts Python row route") + return methods + + +def expected_constructors(identity, method): + if "row_route" not in identity: + return 1000 if identity["guarded_row"] else 0 + return 1000 if validate_row_route(identity["row_route"], identity["guarded_row"])[method] else 0 + + +def route_description(report, method): + descriptions = [] + for side in ("base", "candidate"): + identity = report["pairs"][0][side]["provenance"] + descriptions.append( + f"{side}: binding={identity['guarded_row']}, default {method} native Row=" + f"{bool(expected_constructors(identity, method))}" + ) + return "; ".join(descriptions) + "." + + +def validate_python_sources(report): + if "python_sources" not in report: + return + sources = report["python_sources"] + if type(sources) is not dict or set(sources) != {"base", "candidate"}: + raise ValueError("Missing Python source anchors") + for side, source in sources.items(): + if type(source) is not dict or set(source) != { + "source_commit", + "python_cursor_sha256", + "row_route", + }: + raise ValueError("Invalid Python source anchor") + if source["source_commit"] != report["base_commit" if side == "base" else "source_commit"]: + raise ValueError("Python source revision mismatch") + if not isinstance(source["python_cursor_sha256"], str) or not re.fullmatch( + r"[0-9a-f]{64}", source["python_cursor_sha256"] + ): + raise ValueError("Invalid Python cursor digest") + validate_row_route(source["row_route"]) + + def validate_native_identity(sample, report, side, mode): identity = sample.get("provenance") - if not isinstance(identity, dict) or set(identity) != { - "source_commit", - "native_file", - "native_sha256", - "native_profiling", - "guarded_row", - }: + legacy = {"source_commit", "native_file", "native_sha256", "native_profiling", "guarded_row"} + modern = legacy | {"python_cursor_sha256", "row_route"} + if not isinstance(identity, dict) or set(identity) not in (legacy, modern): raise ValueError("Missing native measurement identity") + if (set(identity) == modern) != ("python_sources" in report): + raise ValueError("Missing Python source anchors or worker route fields") + if set(identity) == modern: + validate_python_sources(report) + validate_row_route(identity["row_route"], identity["guarded_row"]) + anchor = { + key: identity[key] for key in ("source_commit", "python_cursor_sha256", "row_route") + } + if anchor != report["python_sources"][side]: + raise ValueError("Python source-policy drift") if identity["source_commit"] != report["base_commit" if side == "base" else "source_commit"]: raise ValueError("Worker revision mismatch") text(identity["native_file"], limit=4096) @@ -332,7 +406,9 @@ def validate_ci_header(report, build_id=None, head=None, source=None, base=None) raise ValueError("Unsupported CI measurement bundle version") if measurement_mode(report) != "diagnostic": raise ValueError("CI bundle root must be the diagnostic report") - validate(dict(report, status="incomplete", pairs=[]), build_id, head, source, base) + header = dict(report, status="incomplete", pairs=[]) + header.pop("python_sources", None) + validate(header, build_id, head, source, base) if report["samples"] != 5 or report["warmups"] != 1: raise ValueError("CI bundle requires five pairs and one warmup") children = report.get("fetch_measurements") @@ -367,7 +443,7 @@ def ci_mode_reports(report): except (KeyError, TypeError, ValueError) as error: errors[mode] = "Invalid mode data: " + str(error)[:180] on = [valid.get(mode) for mode in ("route", "diagnostic")] - if all(item is not None and item["status"] == "complete" for item in on): + if all(item is not None and item["pairs"] for item in on): if any( on[0]["pairs"][0][side][key] != on[1]["pairs"][0][side][key] for side in ("base", "candidate") @@ -376,13 +452,22 @@ def ci_mode_reports(report): for mode in ("route", "diagnostic"): valid.pop(mode) errors[mode] = "Shared ON binary/environment identity mismatch" + anchored = [(mode, item) for mode, item in valid.items() if "python_sources" in item] + if anchored: + reference = next((item for mode, item in anchored if mode == "latency"), anchored[0][1]) + for mode, item in list(valid.items()): + if item.get("python_sources") != reference["python_sources"] and ( + item["pairs"] or "python_sources" in item + ): + valid.pop(mode) + errors[mode] = "Python source identity differs across modes" latency = valid.get("latency") - if latency is not None and latency["status"] == "complete": + if latency is not None and latency["pairs"]: for mode in ("route", "diagnostic"): item = valid.get(mode) if ( item is not None - and item["status"] == "complete" + and item["pairs"] and item["pairs"][0]["base"]["environment"] != latency["pairs"][0]["base"]["environment"] ): @@ -486,7 +571,8 @@ def render_ci_reports(reports, head, build_id, issues=()): for side in ("base", "candidate") ] notes.append( - f"Native proof for {row['name']}: 1001 fetch calls per side/sample; Row constructions base/candidate {counts[0]}/{counts[1]}." + f"Native proof for {row['name']}: 1001 fetch calls per side/sample; Row constructions base/candidate {counts[0]}/{counts[1]}. " + + route_description(report, row["name"].split("_")[1]) ) lines += [""] + notes lines.append( @@ -949,7 +1035,16 @@ def render(reports, head, build_id, issues=(), default_mode="diagnostic"): identity = report["pairs"][0][side]["provenance"] lines.append( f"- {environment_name(leg)} {side} native SHA256: `{identity['native_sha256']}`; " - f"guarded Row entry available: {identity['guarded_row']}." + f"guarded Row entry available: {identity['guarded_row']}. " + + ( + "Default native Row routes: " + + ", ".join( + f"{method}={enabled}" + for method, enabled in identity["row_route"]["methods"].items() + ) + if "row_route" in identity + else "Historical all-or-none route contract." + ) ) lines += [ "", diff --git a/mssql_python/cursor.py b/mssql_python/cursor.py index 996961bbf..d3fb01a08 100644 --- a/mssql_python/cursor.py +++ b/mssql_python/cursor.py @@ -47,6 +47,11 @@ _DEFAULT_FAST_ROW_CREATE = Row._fast_create _DEFAULT_FAST_ROW_CODE = _DEFAULT_FAST_ROW_CREATE.__code__ _DEFAULT_FAST_ROW_DESCRIPTOR = vars(Row)["_fast_create"] +# Describes the default routes; reporting reads this outside the fetch hot path. +_DEFAULT_NATIVE_ROW_ROUTE = { + "version": 1, + "methods": {"fetchone": False, "fetchmany": True, "fetchval": False}, +} _DEFAULT_NATIVE_FETCH_ONE = ddbc_bindings.DDBCSQLFetchOne _DEFAULT_NATIVE_FETCH_MANY = ddbc_bindings.DDBCSQLFetchMany _NATIVE_ROW_PLAN = object() @@ -2884,18 +2889,6 @@ def fetchone(self) -> Union[None, Row]: started = perf_start() try: fetch = ddbc_bindings.DDBCSQLFetchOne - if not started and fetch is _DEFAULT_NATIVE_FETCH_ONE and _native_row_eligible(Row): - return ddbc_bindings.DDBCSQLFetchRow( - self.hstmt, - row_data, - char_enc, - wchar_enc, - self._cached_char_ctype, - self.messages, - False, - self._finish_fetchone, - _NATIVE_ROW_PLAN, - ) ret = fetch( self.hstmt, row_data, diff --git a/profiler/README.md b/profiler/README.md index 119e78082..e4fd81375 100644 --- a/profiler/README.md +++ b/profiler/README.md @@ -294,9 +294,11 @@ uses its public API, not a first-column-only native shortcut. profiling-enabled CI extension. Setup/execute and validation are outside the timed window; the API loop, result retention and EOF call are inside it. - `--mode route` uses native instrumentation ON with Python phases OFF. Each case - must record 1,001 native fetch calls and, when the guarded Row entry exists, - exactly 1,000 native Row constructions. An older base without that entry must - record no such constructions. This is route attribution, not production latency. + must record 1,001 native fetch calls. The version-1 default route contract requires + 1,000 native Row constructions for `fetchmany(1)` and none for `fetchone` or + `fetchval` in the successor. Here `1` is a built-in integer for both numeric and + mixed shapes; this is not the numeric-column-only native fetching shortcut. + Legacy f539 requires 1,000 constructions for all three APIs; main666 requires none. This is route attribution, not production latency. Existing native regression tests separately cover converter fallback, all-column callbacks, repeated EOF, and customization failures. @@ -350,7 +352,9 @@ The controller retains a **90-minute aggregate budget**, including a three-minut finish reserve; the pipeline step remains 100 minutes and its job 160 minutes. Archives/preflight, builds and workers receive bounded timeouts clipped to the remaining work budget. Fetch workers receive at most 30 seconds; diagnostic workers -retain their six-minute cap. The fetch allowance is not yet runtime-qualified. +retain their six-minute cap. All workers completed within the allowance in ADO180876 +on the two observed agents. That observation is not a future-host guarantee or +qualification of this successor. The proposal already required 132 minutes if its build/worker caps and planning overhead were all consumed. Metadata operations also consume the shared deadline; they never extend it. Completing every mode is **not guaranteed**. Unused early time flows to later modes. Exhaustion or @@ -375,11 +379,46 @@ The updated collector validates shared PR/build/base/merge identity, then each m independently. The primary performance verdict comes **only from OFF/OFF latency**. Native route counts/timings and both-ON diagnostics appear in separately labeled sections; neither replaces missing latency. Missing, malformed and unstarted modes -are explicitly unavailable. Legacy readers can still read the diagnostic root; -standalone schema-1/schema-2 reports and interactive Profiler defaults are unchanged. +are explicitly unavailable. The updated reader accepts historical schema-1/schema-2 +reports as well as the additive route contract. Old strict five-key provenance readers +reject new fetch workers; deploy producer and trusted consumer changes together rather +than stripping fields. Interactive Profiler defaults remain unchanged. Fork reporting continues to use trusted base code. Existing artifact, download, comment-size and publisher time limits are not expanded. -This wiring is source-only until approved exact-head CI runs it. Source/fake-boundary +This successor is source-only until approved exact-head CI runs it. Source/fake-boundary controls and prior-head CI do not establish native correctness, completion within these allowances, or a speedup. + +### Default route identity + +`guarded_row` records native binding capability, not which Python API uses it. +New workers carry exactly seven provenance fields: `source_commit`, `native_file`, +`native_sha256`, `native_profiling`, `guarded_row`, `python_cursor_sha256`, and +`row_route`. The latter is `{version: 1, methods: {fetchone: false, fetchmany: true, +fetchval: false}}` for the successor. The private literal in `cursor.py` is descriptive; +it is never consulted by fetching. `fetchval` still calls the dynamically resolved +`self.fetchone()`. The private `_finish_fetchone` completion now observes `native=False` +for default fetchone/fetchval instead of the previous eligible `native=True` route. + +The controller independently reads each selected source root before measurement and +records `python_sources.base` and `.candidate` anchors (revision, cursor digest, route). +Workers verify their imported cursor origin and match these anchors. A closed recognizer +accepts only reviewed main666, f539 and successor method bodies. Its constants hash the +LF-normalized, dedented source segments of fetchone/fetchmany/fetchval, in that order, +as a compact ASCII-escaped JSON list. Decorated methods, duplicate methods or literal +keys, and non-definition class-scope bindings of the three route names are rejected. +The binding check does not inspect method locals or nested scopes; it is a narrow +recognizer guard, not analysis of arbitrary dynamic Python namespace mutations. Method-body, comment or docstring edits require explicit fingerprint +review/update; this maintenance coupling is deliberate, not a semantic or security proof. +The whole cursor digest still binds all other source bytes. Neither side labels nor +observed counts determine policy. Future bases with either reviewed route work identically. + +Warmups and retained/partial pairs must preserve source, policy, native and environment +identity. OFF and ON modes share Python anchors, but have distinct native binaries; +route and diagnostic workers share the exact ON identity even for available partial data. +Unknown versions, missing new fields/anchors, source-policy drift or contradictory native +capability make evidence unavailable. Historical schema-2 workers without either new field +retain the strict all-or-none `guarded_row` expectation; historical artifacts are not rewritten. +An absent constructor timer is acceptable only for an expected-zero method with positive +1,001-call native fetch proof. It never supplies a missing timing or successful measurement. diff --git a/tests/test_036_profiler_ci.py b/tests/test_036_profiler_ci.py index 37cb3243d..c3fbd1aad 100644 --- a/tests/test_036_profiler_ci.py +++ b/tests/test_036_profiler_ci.py @@ -1919,6 +1919,33 @@ def sample(side): pairs=[dict(base=sample("base"), candidate=sample("candidate")) for _ in range(5)], ) + def source_anchor(self, source_root, revision): + return dict( + source_commit=revision, + python_cursor_sha256=("1" if revision == "a" * 40 else "2") * 64, + row_route=dict( + version=1, + methods=dict(fetchone=False, fetchmany=revision != "a" * 40, fetchval=False), + ), + ) + + def new_sample_report(self, mode="latency", ratio=1.1): + report = self.sample_report(mode, ratio) + report["python_sources"] = { + side: self.source_anchor( + None, report["base_commit" if side == "base" else "source_commit"] + ) + for side in ("base", "candidate") + } + for pair in report["pairs"]: + for side, sample in pair.items(): + sample["provenance"].update(copy.deepcopy(report["python_sources"][side])) + if mode == "route": + for name, case in sample["scenarios"].items(): + if not name.endswith("fetchmany"): + case["cpp"].pop("ddbc::FetchRow::construct_row", None) + return report + def test_modes_validate_and_keep_subthreshold_changes_visible(self): self.assertEqual(tuple(benchmark_workloads.single_row_registry()), reporting.FETCH_CASES) self.assertEqual(len(reporting.CASES), 22) @@ -2162,6 +2189,10 @@ def test_worker_provenance_route_counts_and_failure_checkpoint(self): tempfile.TemporaryDirectory() as directory, ): root = Path(directory) + source = root / "mssql_python" / "cursor.py" + source.parent.mkdir() + source.write_bytes((ROOT / "mssql_python" / "cursor.py").read_bytes()) + cursor_module = SimpleNamespace(__file__=str(source)) native_file = root / "native.so" native_file.write_bytes(b"fake binary identity; never loaded") args = SimpleNamespace( @@ -2179,9 +2210,12 @@ def test_worker_provenance_route_counts_and_failure_checkpoint(self): conn.__exit__.side_effect = RuntimeError("connection close failed") conn.cursor.return_value.__enter__.return_value.fetchone.return_value = ("16.0",) package = SimpleNamespace( - ddbc_bindings=native, perf_timer=python, connect=MagicMock(return_value=conn) + ddbc_bindings=native, + perf_timer=python, + cursor=cursor_module, + connect=MagicMock(return_value=conn), ) - sample = self.sample_report(mode)["pairs"][0]["candidate"] + sample = self.new_sample_report(mode)["pairs"][0]["candidate"] def workload(connection, ctx): ctx.enable() @@ -2190,15 +2224,20 @@ def workload(connection, ctx): result = copy.deepcopy(sample["scenarios"][name]) result["detail"] = result.pop("work") if failure == "counts": - result["cpp"]["ddbc::FetchRow::construct_row"]["calls"] = 999 + result["cpp"]["ddbc::FetchRow::construct_row"] = dict(calls=999) ctx.disable() return result cases = {name: workload for name in reporting.FETCH_CASES} with ( - patch.dict(sys.modules, {"mssql_python": package}), + patch.dict( + sys.modules, {"mssql_python": package, "mssql_python.cursor": cursor_module} + ), patch.dict(os.environ, {"DB_CONNECTION_STRING": "test-only"}), patch.object(controller, "check_build") as check, + patch.object(controller.platform, "system", return_value="Linux"), + patch.object(controller.platform, "machine", return_value="x86_64"), + patch.object(controller.platform, "python_version", return_value="3.13.7"), patch.object(benchmark_workloads, "single_row_registry", return_value=cases), ): if failure == "counts": @@ -2237,7 +2276,7 @@ def test_controller_pairs_use_selected_mode_and_fail_incomplete(self): warmups=1, scenarios=None, ) - sample = self.sample_report(mode)["pairs"][0] + sample = self.new_sample_report(mode)["pairs"][0] calls = [] def measure(path, output, scenarios, timeout, **options): @@ -2251,6 +2290,9 @@ def measure(path, output, scenarios, timeout, **options): controller, "resolve_revisions", return_value=("a" * 40, "b" * 40) ), patch.object(controller, "checkout"), + patch.object( + controller, "python_source_identity", side_effect=self.source_anchor + ), patch.object(controller, "build") as build, patch.object(controller, "measure", side_effect=measure), ): @@ -2302,9 +2344,10 @@ def test_legacy_diagnostic_schema_still_requires_native_samples(self): with self.assertRaisesRegex(ValueError, "profiling data"): reporting.validate(report) - def sample_bundle(self): - latency = self.sample_report("latency") - route = self.sample_report("route") + def sample_bundle(self, modern=False): + factory = self.new_sample_report if modern else self.sample_report + latency = factory("latency") + route = factory("route") diagnostic = copy.deepcopy(route) diagnostic.pop("mode") diagnostic["schema_version"] = 1 @@ -2466,7 +2509,7 @@ def test_ci_orchestration_counts_order_reuse_and_budget_exhaustion(self): leg="Linux-SQL2022", ) clock, builds, workers, archives = [0], [], [], [] - bundle = self.sample_bundle() + bundle = self.sample_bundle(modern=True) samples = { "diagnostic": bundle["pairs"][0], **{ @@ -2498,6 +2541,9 @@ def measure(path, output, scenarios, timeout, **options): patch.object( controller, "run_process", side_effect=lambda *a, **k: archives.append(a[0]) ), + patch.object( + controller, "python_source_identity", side_effect=self.source_anchor + ), patch.object(controller, "build", side_effect=build), patch.object(controller, "measure", side_effect=measure), patch.dict( @@ -2572,7 +2618,7 @@ def test_ci_failures_do_not_retry_or_fabricate_siblings(self): leg="Linux-SQL2022", ) clock, workers, builds = [0], [], [] - bundle = self.sample_bundle() + bundle = self.sample_bundle(modern=True) samples = { "diagnostic": bundle["pairs"][0], **{ @@ -2606,6 +2652,9 @@ def measure(path, output, scenarios, timeout, **options): ), patch.object(controller, "git", return_value="b" * 40), patch.object(controller, "run_process"), + patch.object( + controller, "python_source_identity", side_effect=self.source_anchor + ), patch.object(controller, "build", side_effect=build), patch.object(controller, "measure", side_effect=measure), patch.dict( @@ -2723,6 +2772,10 @@ def test_ci_diagnostic_worker_attests_actual_binary_after_cleanup(self): for fails in (False, True): with self.subTest(cleanup_failure=fails), tempfile.TemporaryDirectory() as directory: root = Path(directory) + source = root / "mssql_python" / "cursor.py" + source.parent.mkdir() + source.write_bytes((ROOT / "mssql_python" / "cursor.py").read_bytes()) + cursor_module = SimpleNamespace(__file__=str(source)) binary = root / "native.so" binary.write_bytes(b"fake-not-imported") native = SimpleNamespace( @@ -2754,9 +2807,18 @@ def test_ci_diagnostic_worker_attests_actual_binary_after_cleanup(self): ) with ( patch.dict( - sys.modules, {"mssql_python": SimpleNamespace(ddbc_bindings=native)} + sys.modules, + { + "mssql_python": SimpleNamespace( + ddbc_bindings=native, cursor=cursor_module + ), + "mssql_python.cursor": cursor_module, + }, ), patch.object(controller, "check_build") as check, + patch.object(controller.platform, "system", return_value="Linux"), + patch.object(controller.platform, "machine", return_value="x86_64"), + patch.object(controller.platform, "python_version", return_value="3.13.7"), patch.object(controller, "load_suite", return_value=(core, suite)), ): if fails: @@ -2878,9 +2940,14 @@ def test_ci_reused_source_and_filesystem_cleanup_fail_closed(self): def test_ci_existing_publisher_consumes_bundle_without_latency_fallback(self): from unittest.mock import patch - for latency_available in (True, False): - with self.subTest(latency_available=latency_available): - bundle = self.sample_bundle() + for modern, latency_available in ( + (False, True), + (False, False), + (True, True), + (True, False), + ): + with self.subTest(modern=modern, latency_available=latency_available): + bundle = self.sample_bundle(modern=modern) if not latency_available: bundle["fetch_measurements"]["latency"].update( status="incomplete", pairs=[], unavailable_reason="OFF worker failed" @@ -2956,7 +3023,7 @@ def test_ci_finalization_deadline_is_checked_after_validation_and_write(self): warmups=1, leg="Linux-SQL2022", ) - bundle = self.sample_bundle() + bundle = self.sample_bundle(modern=True) samples = { "diagnostic": bundle["pairs"][0], **{ @@ -3000,6 +3067,9 @@ def measure(path, output, scenarios, timeout, **options): ), patch.object(controller, "git", return_value="b" * 40), patch.object(controller, "run_process"), + patch.object( + controller, "python_source_identity", side_effect=self.source_anchor + ), patch.object(controller, "build"), patch.object(controller, "measure", side_effect=measure), patch.object(controller.shutil, "rmtree", side_effect=remove), @@ -3028,3 +3098,325 @@ def measure(path, output, scenarios, timeout, **options): self.assertIn("finish deadline", saved["unavailable_reason"]) else: self.assertIsNone(error) + + def test_route_source_recognizer_exact_bytes_and_rejects_drift(self): + import ast + + source = (ROOT / "mssql_python" / "cursor.py").read_text(encoding="utf-8") + tree = ast.parse(source) + declaration = next( + node + for node in tree.body + if isinstance(node, ast.Assign) + and any( + isinstance(t, ast.Name) and t.id == "_DEFAULT_NATIVE_ROW_ROUTE" + for t in node.targets + ) + ) + literal = ast.get_source_segment(source, declaration) + route = {"version": 1, "methods": {"fetchone": False, "fetchmany": True, "fetchval": False}} + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + target = root / "mssql_python" / "cursor.py" + target.parent.mkdir() + hashes = [] + for text in (source, source.replace("\n", "\r\n"), "\n\n" + source): + target.write_bytes(text.encode()) + observed = controller.python_source_identity(root, "b" * 40) + self.assertEqual(observed["row_route"], route) + hashes.append(observed["python_cursor_sha256"]) + self.assertEqual(hashes[0], hashes[1]) + self.assertNotEqual(hashes[0], hashes[2]) + mutations = [ + source.replace(literal, ""), + source + "\n" + literal, + source.replace(literal, "_DEFAULT_NATIVE_ROW_ROUTE: dict = " + repr(route)), + source.replace(literal, "ALIAS = _DEFAULT_NATIVE_ROW_ROUTE = " + repr(route)), + source.replace(literal, "_DEFAULT_NATIVE_ROW_ROUTE = dict(version=1)"), + source.replace( + " def fetchone(", " @unreviewed_dispatch_wrapper\n def fetchone(" + ), + source.replace(" def fetchmany(", " @staticmethod\n def fetchmany("), + source.replace(" def fetchval(", " @property\n def fetchval("), + source.replace(" def fetchone(", " async def fetchone("), + source.replace( + " def fetchone(", " def fetchone(self): pass\n\n def fetchone(" + ), + source + "\nclass Cursor: pass\n", + source.replace("class Cursor:", "@unreviewed_dispatch_wrapper\nclass Cursor:"), + source.replace( + " Fetch the next row of a query result set.", + " Unknown changed method.", + ), + source + "\nprobe = _DEFAULT_NATIVE_ROW_ROUTE\n", + "invalid Python syntax !", + ] + for bad in ( + {"version": True, "methods": route["methods"]}, + {"version": 2, "methods": route["methods"]}, + {"methods": route["methods"]}, + {"version": 1, "methods": {"fetchone": 0, "fetchmany": True, "fetchval": False}}, + {"version": 1, "methods": {"fetchone": True, "fetchmany": True, "fetchval": True}}, + {"version": 1, "methods": {"fetchone": False, "fetchmany": True}}, + ): + mutations.append( + source.replace(literal, "_DEFAULT_NATIVE_ROW_ROUTE = " + repr(bad)) + ) + mutations.append( + source.replace( + literal, + "_DEFAULT_NATIVE_ROW_ROUTE = {'version':1,'version':1,'methods':" + + repr(route["methods"]) + + "}", + ) + ) + mutations.append( + source.replace( + literal, + "_DEFAULT_NATIVE_ROW_ROUTE = {'version':1,'methods':{'fetchone':False,'fetchone':False,'fetchmany':True,'fetchval':False}}", + ) + ) + cursor_class = next( + node + for node in tree.body + if isinstance(node, ast.ClassDef) and node.name == "Cursor" + ) + lines = source.splitlines(keepends=True) + for method in ("fetchone", "fetchmany", "fetchval"): + for binding in ( + f"{method} = replacement", + f"{method}: object = replacement", + f"{method}: object", + f"alias = {method} = replacement", + f"{method}, alias = replacements", + f"{method} += replacement", + f"del {method}", + f"import replacement as {method}", + f"from replacement import value as {method}", + f"from replacement import {method}", + f"for {method} in replacements: pass", + f"with replacement as {method}: pass", + f"if flag: {method} = replacement", + f"({method} := replacement)", + f"def helper(self, value=({method} := replacement)): pass", + f"def helper(self) -> ({method} := replacement): pass", + f"async def helper(self) -> ({method} := replacement): pass", + f"def helper(self, value: ({method} := replacement)): pass", + f"async def helper(self, value: ({method} := replacement)): pass", + f"helper = lambda value=({method} := replacement): value", + f"class Helper(({method} := replacement)): pass", + f"try: pass\nexcept Exception as {method}: pass", + f"match replacement:\n case {{'value': {method}}}: pass", + f"class {method}: pass", + f"if flag:\n def {method}(self): pass", + ): + insertion = ( + "\n" + "\n".join(" " + line for line in binding.splitlines()) + "\n" + ) + mutations.append( + "".join(lines[: cursor_class.end_lineno]) + + insertion + + "".join(lines[cursor_class.end_lineno :]) + ) + for index, text in enumerate(mutations): + with self.subTest(mutation=index): + self.assertNotEqual(text, source) + target.write_bytes(text.encode()) + with self.assertRaises(ValueError): + controller.python_source_identity(root, "b" * 40) + + def test_versioned_route_standalone_and_fully_populated_bundles(self): + for modern in (False, True): + bundle = self.sample_bundle(modern=modern) + for mode in ("latency", "route"): + report = bundle["fetch_measurements"][mode] + reporting.validate(report) + self.assertIn("Python phases OFF", reporting.render([report], "c" * 40, 42)) + reporting.validate(bundle) + valid, errors = reporting.ci_mode_reports(bundle) + self.assertEqual(set(valid), set(reporting.MODES)) + self.assertEqual(errors, {}) + body = reporting.render_ci_reports([bundle], "c" * 40, 42) + self.assertIn("Row constructions base/candidate 0/1000", body) + if modern: + self.assertIn("Row constructions base/candidate 0/0", body) + self.assertIn("binding=True, default fetchone native Row=False", body) + for policy in ((True, True, True), (False, True, False)) if modern else (): + # A future base chooses its own policy; candidate role does not select it. + future = copy.deepcopy(bundle) + for mode in reporting.MODES: + report = future if mode == "diagnostic" else future["fetch_measurements"][mode] + anchor = report["python_sources"]["base"] + anchor["row_route"]["methods"] = dict( + zip(("fetchone", "fetchmany", "fetchval"), policy) + ) + for pair in report["pairs"]: + sample = pair["base"] + sample["provenance"].update(copy.deepcopy(anchor), guarded_row=True) + if mode == "route": + for name, case in sample["scenarios"].items(): + if anchor["row_route"]["methods"][name.split("_")[1]]: + case["cpp"]["ddbc::FetchRow::construct_row"] = dict( + calls=1000, total_us=10000, min_us=1, max_us=100 + ) + self.assertEqual(reporting.ci_mode_reports(future)[1], {}) + self.assertIn( + "Primary verdict: native OFF", + reporting.render_ci_reports([future], "c" * 40, 42), + ) + + def test_versioned_route_rejects_bad_identity_policy_counts_and_partial_data(self): + for mutation in ( + lambda r: r.pop("python_sources"), + lambda r: r["python_sources"].pop("base"), + lambda r: r["python_sources"]["candidate"].update(source_commit="f" * 40), + lambda r: r["python_sources"]["candidate"].update(python_cursor_sha256="bad"), + lambda r: r["pairs"][0]["candidate"]["provenance"].pop("row_route"), + lambda r: r["pairs"][0]["candidate"]["provenance"].pop("python_cursor_sha256"), + lambda r: r["pairs"][0]["candidate"]["provenance"].update( + python_cursor_sha256="f" * 64 + ), + lambda r: r["pairs"][0]["candidate"]["provenance"].update(guarded_row=False), + lambda r: r["pairs"][0]["candidate"]["provenance"]["row_route"].update(version=True), + lambda r: r["pairs"][0]["candidate"]["provenance"]["row_route"].update(version=2), + lambda r: r["pairs"][0]["candidate"]["provenance"]["row_route"].pop("version"), + lambda r: r["pairs"][0]["candidate"]["provenance"]["row_route"]["methods"].pop( + "fetchval" + ), + lambda r: r["pairs"][0]["candidate"]["provenance"]["row_route"]["methods"].update( + fetchone=0 + ), + lambda r: r["pairs"][0]["candidate"]["provenance"]["row_route"]["methods"].update( + fetchval=True + ), + lambda r: r["pairs"][0]["candidate"]["scenarios"]["numeric_fetchone"]["cpp"].update( + {"ddbc::FetchRow::construct_row": dict(calls=1000)} + ), + lambda r: r["pairs"][0]["candidate"]["scenarios"]["mixed_fetchmany"]["cpp"].pop( + "ddbc::FetchRow::construct_row" + ), + lambda r: r["pairs"][0]["candidate"]["scenarios"]["mixed_fetchone"].update(cpp={}), + lambda r: r["pairs"][0]["candidate"].update(status="running"), + lambda r: r["pairs"][1]["candidate"]["provenance"].update(native_sha256="f" * 64), + ): + for partial in (False, True): + with self.subTest(mutation=mutation, partial=partial): + report = self.new_sample_report("route") + if partial: + report.update( + status="incomplete", + pairs=report["pairs"][:2], + unavailable_reason="Deadline before next pair", + ) + mutation(report) + with self.assertRaises((ValueError, KeyError)): + reporting.validate(report) + report = self.new_sample_report("route") + report.update( + status="incomplete", + pairs=report["pairs"][:2], + unavailable_reason="Deadline before next pair", + ) + reporting.validate(report) + self.assertIn("Performance unavailable", reporting.render([report], "c" * 40, 42)) + + def test_versioned_route_partial_siblings_and_off_on_identity(self): + for defect in ( + "native", + "python", + "missing-anchor", + "diagnostic-anchor", + "off-environment", + ): + bundle = self.sample_bundle(modern=True) + route = bundle["fetch_measurements"]["route"] + route.update( + status="incomplete", pairs=route["pairs"][:2], unavailable_reason="Deadline" + ) + if defect == "native": + for pair in route["pairs"]: + pair["candidate"]["provenance"]["native_sha256"] = "f" * 64 + elif defect == "missing-anchor": + route.pop("python_sources") + elif defect == "diagnostic-anchor": + bundle["python_sources"]["candidate"]["python_cursor_sha256"] = "f" * 64 + elif defect == "off-environment": + for item in (route, bundle): + for pair in item["pairs"]: + for sample in pair.values(): + sample["environment"]["python"] = "3.12.0" + else: + route["python_sources"]["candidate"]["python_cursor_sha256"] = "f" * 64 + for pair in route["pairs"]: + pair["candidate"]["provenance"]["python_cursor_sha256"] = "f" * 64 + valid, errors = reporting.ci_mode_reports(bundle) + self.assertIn("latency", valid) + self.assertIn("diagnostic" if defect == "diagnostic-anchor" else "route", errors) + self.assertIn( + "Primary verdict: native OFF", reporting.render_ci_reports([bundle], "c" * 40, 42) + ) + + def test_versioned_controller_rejects_warmup_drift_and_keeps_subset_incomplete(self): + from unittest.mock import patch + + for defect in (None, "warmup", "native-drift", "source-drift", "subset"): + with self.subTest(defect=defect), tempfile.TemporaryDirectory() as directory: + root = Path(directory) + args = SimpleNamespace( + base="a" * 40, + candidate="b" * 40, + output=root, + mode="route", + reuse_candidate=False, + leg="Linux-SQL2022", + samples=3, + warmups=1, + scenarios=["mixed_fetchmany"] if defect == "subset" else None, + ) + calls = [] + samples = self.new_sample_report("route")["pairs"][0] + + def measure(path, output, scenarios, timeout, **options): + result = copy.deepcopy(samples[path.name]) + calls.append(path.name) + if defect == "warmup" and len(calls) == 1: + result["scenarios"]["mixed_fetchmany"]["cpp"] = {} + if len(calls) > 2 and path.name == "base": + if defect == "native-drift": + result["provenance"]["native_sha256"] = "f" * 64 + if defect == "source-drift": + result["provenance"]["python_cursor_sha256"] = "f" * 64 + if scenarios: + result["scenarios"] = { + name: result["scenarios"][name] for name in scenarios + } + return result + + with ( + patch.object( + controller, "resolve_revisions", return_value=("a" * 40, "b" * 40) + ), + patch.object(controller, "checkout"), + patch.object(controller, "build"), + patch.object( + controller, "python_source_identity", side_effect=self.source_anchor + ), + patch.object(controller, "measure", side_effect=measure), + ): + if defect in (None, "subset"): + controller.run(args) + else: + with self.assertRaises(ValueError): + controller.run(args) + saved = json.loads((root / "report.json").read_text()) + self.assertEqual(saved["status"], "complete" if defect is None else "incomplete") + if defect not in (None, "subset"): + self.assertEqual(saved["pairs"], []) + self.assertLessEqual(len(calls), 4) + if defect == "subset": + self.assertEqual(len(saved["pairs"]), 3) + self.assertEqual( + set(saved["pairs"][0]["base"]["scenarios"]), {"mixed_fetchmany"} + ) + with self.assertRaises(ValueError): + reporting.validate(saved) diff --git a/tests/test_fetch_single_row.py b/tests/test_fetch_single_row.py index 501ba8d50..db4790886 100644 --- a/tests/test_fetch_single_row.py +++ b/tests/test_fetch_single_row.py @@ -781,7 +781,7 @@ def names(self): ): with pytest.raises(RuntimeError, match="factory changed after native advancement"): cursor.fetchmany(1) if method == "fetchmany" else getattr(cursor, method)() - fused.assert_called_once() + assert fused.call_count == (1 if method == "fetchmany" else 0) assert Row._fast_create is factory assert cursor.rowcount == 1 and cursor.rownumber == 0 finally: @@ -797,6 +797,13 @@ def test_single_row_fusion_uses_native_constructor(cursor, method): cursor.execute("SELECT 42 AS number, CAST(N'text' AS NVARCHAR(10)) AS label") factory_code = Row._fast_create.__code__ calls = [] + completion = cursor._finish_fetchmany if method == "fetchmany" else cursor._finish_fetchone + native_flags = [] + + def finish(ret, data, native=False): + native_flags.append(native) + return completion(ret, data, native) + previous_profile = sys.getprofile() phases_enabled = perf_timer.is_enabled() @@ -806,22 +813,30 @@ def profile(frame, event, arg): perf_timer.disable() try: - with patch.object( - ddbc_bindings, "DDBCSQLFetchRow", wraps=ddbc_bindings.DDBCSQLFetchRow - ) as fused: + with ( + patch.object( + ddbc_bindings, "DDBCSQLFetchRow", wraps=ddbc_bindings.DDBCSQLFetchRow + ) as fused, + patch.object( + cursor, + "_finish_fetchmany" if method == "fetchmany" else "_finish_fetchone", + side_effect=finish, + ), + ): sys.setprofile(profile) try: result = cursor.fetchmany(1) if method == "fetchmany" else getattr(cursor, method)() finally: sys.setprofile(previous_profile) - fused.assert_called_once() + assert fused.call_count == (1 if method == "fetchmany" else 0) if method == "fetchval": assert result == 42 else: row = result[0] if method == "fetchmany" else result assert type(row) is Row assert tuple(row) == (42, "text") - assert calls == [] + assert calls == ([] if method == "fetchmany" else ["call"]) + assert native_flags == [method == "fetchmany"] assert cursor.rowcount == 1 and cursor.rownumber == 0 finally: if phases_enabled: @@ -868,7 +883,7 @@ def names(self): ) as fused, ): result = cursor.fetchmany(1) if method == "fetchmany" else getattr(cursor, method)() - fused.assert_called_once() + assert fused.call_count == (1 if method == "fetchmany" else 0) if method == "fetchval": assert result == 1 else: @@ -924,12 +939,12 @@ def fetch(): p.reset() p.enable() assert fetch() == (1 if method == "fetchval" else (1, 11, "text")) - assert calls("ddbc::FetchRow::construct_row") == 1 + assert calls("ddbc::FetchRow::construct_row") == (1 if method == "fetchmany" else 0) assert calls(fetch_timer) == 1 assert fetch() == (2 if method == "fetchval" else (2, 12, "text")) assert fetch() is None assert fetch() is None - assert calls("ddbc::FetchRow::construct_row") == 2 + assert calls("ddbc::FetchRow::construct_row") == (2 if method == "fetchmany" else 0) assert calls(fetch_timer) == 4 assert perf_timer.get_stats() == {} assert cursor.rowcount == 2 and cursor.rownumber == 1 From 8a4472b98148472ebdda9996076a2bc5ed445a6d Mon Sep 17 00:00:00 2001 From: Jahnvi Thakkar Date: Wed, 7 Oct 2026 08:09:06 +0530 Subject: [PATCH 10/17] FIX: Align fetch recovery spies and annotate source fingerprints Select the native bridge used by each public fetch route while retaining all failure and recovery assertions. Mark three reviewed SHA256 source fingerprints with narrow DevSkim rule-specific suppressions. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- eng/profiler_benchmarks/controller.py | 13 ++++++++++--- tests/test_fetch_single_row.py | 4 ++-- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/eng/profiler_benchmarks/controller.py b/eng/profiler_benchmarks/controller.py index 77a7c1a22..3c07096c7 100644 --- a/eng/profiler_benchmarks/controller.py +++ b/eng/profiler_benchmarks/controller.py @@ -231,9 +231,16 @@ def verify_reused_source(source_root, revision): # Closed, reviewed dispatch bodies: main666, f539, and the many-only successor. # Method/comment/docstring edits require review and an explicit fingerprint update. -_LEGACY_PYTHON_ROUTE = "52681c15f92e85c01e0b43a9bd763872051fdc956701046534a5beb2467e6c3c" -_LEGACY_FUSED_ROUTE = "d4c6671b89ace027c22585e761c4f4c3243e297be2162213d9bbb38d5bed03c6" -_MANY_ONLY_ROUTE = "544d5b0ccbf9df1f0b6ec72c79ecbfa49a3517cab0437a5520940b8941966759" +# These reviewed method-source SHA256 digests are not credentials. +_LEGACY_PYTHON_ROUTE = ( + "52681c15f92e85c01e0b43a9bd763872051fdc956701046534a5beb2467e6c3c" # DevSkim: ignore DS173237 +) +_LEGACY_FUSED_ROUTE = ( + "d4c6671b89ace027c22585e761c4f4c3243e297be2162213d9bbb38d5bed03c6" # DevSkim: ignore DS173237 +) +_MANY_ONLY_ROUTE = ( + "544d5b0ccbf9df1f0b6ec72c79ecbfa49a3517cab0437a5520940b8941966759" # DevSkim: ignore DS173237 +) def python_source_identity(source_root, revision): diff --git a/tests/test_fetch_single_row.py b/tests/test_fetch_single_row.py index db4790886..50d45503c 100644 --- a/tests/test_fetch_single_row.py +++ b/tests/test_fetch_single_row.py @@ -672,8 +672,8 @@ def test_single_row_construction_failure_keeps_fetch_position(cursor, method, fa cursor.execute("SELECT n AS number FROM (VALUES (1), (2)) AS v(n) ORDER BY n") from mssql_python.row import Row - bridge_name = "DDBCSQLFetchRow" - bridge = ddbc_bindings.DDBCSQLFetchRow + bridge_name = "DDBCSQLFetchRow" if method == "fetchmany" else "DDBCSQLFetchOne" + bridge = getattr(ddbc_bindings, bridge_name) def fetch(): value = cursor.fetchmany(1) if method == "fetchmany" else getattr(cursor, method)() From 06eba447cec66030cfc4f17be3d1dfa55465bc74 Mon Sep 17 00:00:00 2001 From: Jahnvi Thakkar Date: Wed, 7 Oct 2026 09:13:48 +0530 Subject: [PATCH 11/17] FIX: Restore original PR performance report presentation Restore the original 22-task diagnostic headline and report layout. Keep separate OFF/OFF latency and ON/OFF route artifacts, validation, thresholds and collection unchanged, with an explicit scope caveat. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- eng/profiler_benchmarks/report.py | 115 +++++---------------------- profiler/README.md | 11 ++- tests/test_036_profiler_ci.py | 126 ++++++++++++++++++++++-------- 3 files changed, 117 insertions(+), 135 deletions(-) diff --git a/eng/profiler_benchmarks/report.py b/eng/profiler_benchmarks/report.py index 672a6e3c2..9960a5e11 100644 --- a/eng/profiler_benchmarks/report.py +++ b/eng/profiler_benchmarks/report.py @@ -477,8 +477,8 @@ def ci_mode_reports(report): def render_ci_reports(reports, head, build_id, issues=()): - modes = {mode: [] for mode in MODES} - reasons = {} + diagnostics = [] + diagnostic_issues = list(issues) seen = set() for report in reports: leg = report["leg"] @@ -490,104 +490,25 @@ def render_ci_reports(reports, head, build_id, issues=()): else: validate(report) valid = {measurement_mode(report): report} - errors = { - mode: "Mode not supplied by this legacy artifact" - for mode in MODES - if mode not in valid - } - for mode, item in valid.items(): - if item["status"] == "complete": - modes[mode].append(item) - for mode, reason in errors.items(): - reasons[leg, mode] = reason - latency_issues = list(issues) + [ - leg + " (" + reasons.get((leg, "latency"), "missing latency measurement") + ")" - for leg in LEGS - if not any(r["leg"] == leg for r in modes["latency"]) - ] - primary = render(modes["latency"], head, build_id, latency_issues, default_mode="latency") - primary = primary.replace( - "## PR Performance Report", - "## PR Performance Report\n\n**Primary verdict: native OFF / Python phases OFF latency only.**", + errors = {} + diagnostic = valid.get("diagnostic") + if diagnostic is not None and diagnostic["status"] == "complete": + diagnostics.append(diagnostic) + else: + reason = errors.get("diagnostic", "missing or incomplete diagnostic measurement") + diagnostic_issues.append(leg + " (" + reason + ")") + body = render(diagnostics, head, build_id, diagnostic_issues) + artifact_note = "Raw samples and logs are attached to the ADO run as `profiler-*` artifacts." + body = body.replace( + artifact_note, + "This headline uses the original 22-task profiling-enabled diagnostics; separate " + "OFF/OFF latency and ON/OFF route measurements, when available, are retained in the raw artifacts " + "and are not headline inputs.\n\n" + artifact_note, 1, ) - lines = [ - "", - "### Measurement availability", - "", - "| Environment | Latency OFF/OFF | Route ON/OFF | Diagnostics ON/ON |", - "|---|---|---|---|", - ] - for leg in LEGS: - cells = [] - for mode in ("latency", "route", "diagnostic"): - complete = any(item["leg"] == leg for item in modes[mode]) - cells.append( - "Complete" - if complete - else "Unavailable: " + escape(reasons.get((leg, mode), "artifact missing")) - ) - lines.append("| " + environment_name(leg) + " | " + " | ".join(cells) + " |") - for mode, label in ( - ( - "route", - "Native route attribution (instrumented, Python phases OFF; not production latency)", - ), - ("diagnostic", "Legacy diagnostics (native and Python phases ON; not production latency)"), - ): - lines += ["", "
", "" + label + "", ""] - if not modes[mode]: - lines.append("Unavailable. No latency inference is made from this section.") - for report in modes[mode]: - lines += [ - "", - "#### " + environment_name(report["leg"]), - "", - "| Task | Before ms | After ms | Paired change | Ratio median [min, max] |", - "|---|---:|---:|---:|---:|", - ] - notes = [] - for row in comparisons(report): - lines.append( - f"| {TASK_NAMES[row['name']]} | {row['base_ms']:.3f} | {row['candidate_ms']:.3f} | {row['change_pct']:+.1f}% | {row['ratio']:.3f} [{row['ratio_min']:.3f}, {row['ratio_max']:.3f}] |" - ) - if mode == "diagnostic" and (row["phases"] or row["counts"]): - phases = "; ".join( - f"{escape(label)} {delta:+.3f} ms" for delta, label in row["phases"] - ) - counts_text = "; ".join(escape(count) for count in row["counts"]) - notes.append( - "Inclusive phase changes: " - + (phases or "none recorded") - + ". Call changes: " - + (counts_text or "none recorded") - + "." - ) - if mode == "route": - counts = [ - report["pairs"][0][side]["scenarios"][row["name"]]["cpp"] - .get("ddbc::FetchRow::construct_row", {}) - .get("calls", 0) - for side in ("base", "candidate") - ] - notes.append( - f"Native proof for {row['name']}: 1001 fetch calls per side/sample; Row constructions base/candidate {counts[0]}/{counts[1]}. " - + route_description(report, row["name"].split("_")[1]) - ) - lines += [""] + notes - lines.append( - "Raw paired counters and worker/native identities are retained in the artifact; inclusive phases must not be summed." - ) - lines += ["", "
"] - appendix = "\n".join(lines) - if len(primary) + len(appendix) + 1 > MAX_COMMENT_CHARS: - appendix = "\n".join( - line for line in lines if not line.startswith("Inclusive phase changes:") - ) - appendix += "\nExpanded phase details exceed the comment budget; full counters remain in the raw artifact." - if len(primary) + len(appendix) + 1 > MAX_COMMENT_CHARS: + if len(body) > MAX_COMMENT_CHARS: raise ValueError("CI performance comment exceeds its bounded size") - return primary + "\n" + appendix + return body def assess(evidence, artifact_urls, load_artifact, issues=()): diff --git a/profiler/README.md b/profiler/README.md index e4fd81375..e81a7fc99 100644 --- a/profiler/README.md +++ b/profiler/README.md @@ -376,10 +376,13 @@ loop. Raw build logs and mode-prefixed worker JSON/log files stay in the same ar without nested files named `report.json`. The updated collector validates shared PR/build/base/merge identity, then each mode -independently. The primary performance verdict comes **only from OFF/OFF latency**. -Native route counts/timings and both-ON diagnostics appear in separately labeled -sections; neither replaces missing latency. Missing, malformed and unstarted modes -are explicitly unavailable. The updated reader accepts historical schema-1/schema-2 +independently. The PR comment restores the original 22-task profiling-enabled +**diagnostic** headline, section ordering and timing tables; it is not a shipped-latency +verdict. Separate OFF/OFF latency and ON/OFF route measurements, when available, remain +in the raw artifacts and are not headline inputs. Missing, malformed or incomplete +diagnostics are explicitly unavailable even when the latency or route measurements +complete. The standalone latency and route reports remain available with their existing +labels. The updated reader accepts historical schema-1/schema-2 reports as well as the additive route contract. Old strict five-key provenance readers reject new fetch workers; deploy producer and trusted consumer changes together rather than stripping fields. Interactive Profiler defaults remain unchanged. diff --git a/tests/test_036_profiler_ci.py b/tests/test_036_profiler_ci.py index c3fbd1aad..2f423141b 100644 --- a/tests/test_036_profiler_ci.py +++ b/tests/test_036_profiler_ci.py @@ -2361,7 +2361,7 @@ def sample_bundle(self, modern=False): ) return diagnostic - def test_ci_bundle_all_mode_outcomes_have_latency_only_verdicts(self): + def test_ci_bundle_all_mode_outcomes_restore_diagnostic_verdicts(self): for bits in range(8): with self.subTest(completeness=bits): bundle = self.sample_bundle() @@ -2377,19 +2377,55 @@ def test_ci_bundle_all_mode_outcomes_have_latency_only_verdicts(self): self.assertEqual(len(valid), 3) self.assertEqual(len(reasons), 3 - bits.bit_count()) body = reporting.render_ci_reports([bundle], "c" * 40, 42) - self.assertIn("Primary verdict: native OFF / Python phases OFF latency only", body) - self.assertIn("Diagnostics ON/ON", body) - self.assertEqual("Performance unavailable" in body, not bool(bits & 1)) + self.assertIn("original 22-task profiling-enabled diagnostics", body) + self.assertIn("not headline inputs", body) + self.assertIn("when available", body) + self.assertIn("do not represent production-wheel latency", body) + self.assertNotIn("Primary verdict:", body) + self.assertNotIn("### Measurement availability", body) + self.assertNotIn("Native route attribution", body) + self.assertEqual("Performance unavailable" in body, not bool(bits & 4)) self.assertNotIn("Performance improved", body) self.assertLessEqual(len(body), reporting.MAX_COMMENT_CHARS) - if bits & 2: - self.assertIn("1001 fetch calls per side/sample", body) - section = body.split("Native route attribution", 1)[1].split( - "", 1 - )[0] - self.assertGreater(section.index("Native proof for"), section.rindex("| 1,000")) - if bits != 7: - self.assertIn("Unavailable: Not started: budget exhausted", body) + headings = [ + "Performance diagnostics", + "All database tasks and timings", + "Build and measurement details", + ] + positions = [body.index(heading) for heading in headings] + self.assertEqual(positions, sorted(positions)) + if bits & 4: + self.assertIn( + "| Database task | Before | After | Paired change | Result |", body + ) + table = body.split(headings[1], 1)[1].split("", 1)[0] + for name in reporting.CASES: + self.assertIn("| " + reporting.TASK_NAMES[name] + " |", table) + self.assertNotIn("Ratio median", table) + else: + self.assertIn("Not started: budget exhausted", body) + + def test_ci_restored_headline_does_not_substitute_other_modes(self): + for diagnostic_ratio, latency_ratio in ((0.6, 1.5), (1.5, 0.6)): + with self.subTest(diagnostic_ratio=diagnostic_ratio): + bundle = self.sample_bundle() + for report, ratio in ( + (bundle, diagnostic_ratio), + (bundle["fetch_measurements"]["latency"], latency_ratio), + ): + for pair in report["pairs"]: + for name, cell in pair["candidate"]["scenarios"].items(): + cell["wall_ms"] = pair["base"]["scenarios"][name]["wall_ms"] * ratio + body = reporting.render_ci_reports([bundle], "c" * 40, 42) + self.assertEqual("Performance improved" in body, diagnostic_ratio < 1) + self.assertEqual("Performance regression detected" in body, diagnostic_ratio > 1) + self.assertIn("not headline inputs", body) + self.assertIn("when available", body) + bundle["pairs"][0]["candidate"]["scenarios"].pop("fetchone") + body = reporting.render_ci_reports([bundle], "c" * 40, 42) + self.assertIn("Performance unavailable", body) + self.assertIn("Invalid mode data", body) + self.assertNotIn("Performance improved", body) def test_ci_bundle_rejects_bad_siblings_and_shared_on_drift(self): for defect in ("missing", "mode", "commit", "counter", "identity", "environment"): @@ -2419,7 +2455,11 @@ def test_ci_bundle_rejects_bad_siblings_and_shared_on_drift(self): self.assertNotIn("diagnostic", valid) body = reporting.render_ci_reports([bundle], "c" * 40, 42) self.assertIn("Unavailable:", body) - self.assertIn("1,000 mixed rows / fetchmany(1)", body) + self.assertEqual("Performance unavailable" in body, "diagnostic" not in valid) + if "diagnostic" in valid: + self.assertIn("| Row-by-row fetching |", body) + else: + self.assertIn(reasons["diagnostic"], body) bundle = self.sample_bundle() for pair in bundle["pairs"]: for side in ("base", "candidate"): @@ -2475,22 +2515,28 @@ def archive(value, duplicate=False): body = reporting.assess( evidence, {"Linux-SQL2022": "fake-artifact"}, lambda url: archive(bundle) ) - self.assertIn("Primary verdict: native OFF", body) - self.assertIn("Row constructions base/candidate 0/1000", body) + self.assertIn("original 22-task profiling-enabled diagnostics", body) + self.assertIn("| Row-by-row fetching |", body) bundle["fetch_measurements"]["latency"].update( status="incomplete", pairs=[], unavailable_reason="OFF build failed" ) body = reporting.assess( evidence, {"Linux-SQL2022": "fake-artifact"}, lambda url: archive(bundle) ) + self.assertNotIn("Performance unavailable", body) + self.assertIn("| Row-by-row fetching |", body) + bundle.update(status="incomplete", pairs=[], unavailable_reason="Diagnostic worker failed") + body = reporting.assess( + evidence, {"Linux-SQL2022": "fake-artifact"}, lambda url: archive(bundle) + ) self.assertIn("Performance unavailable", body) - self.assertIn("OFF build failed", body) + self.assertIn("Diagnostic worker failed", body) bundle["head_commit"] = "f" * 40 body = reporting.assess( evidence, {"Linux-SQL2022": "fake-artifact"}, lambda url: archive(bundle) ) self.assertIn("Performance unavailable", body) - self.assertNotIn("Row constructions base/candidate 0/1000", body) + self.assertNotIn("| Row-by-row fetching |", body) def test_ci_orchestration_counts_order_reuse_and_budget_exhaustion(self): from unittest.mock import patch @@ -2937,20 +2983,20 @@ def test_ci_reused_source_and_filesystem_cleanup_fail_closed(self): self.assertEqual(result["cleanup_required"], str(retained)) self.assertIn("cleanup failed", result["unavailable_reason"]) - def test_ci_existing_publisher_consumes_bundle_without_latency_fallback(self): + def test_ci_existing_publisher_restores_diagnostics_without_sibling_fallback(self): from unittest.mock import patch - for modern, latency_available in ( + for modern, diagnostic_available in ( (False, True), (False, False), (True, True), (True, False), ): - with self.subTest(modern=modern, latency_available=latency_available): + with self.subTest(modern=modern, diagnostic_available=diagnostic_available): bundle = self.sample_bundle(modern=modern) - if not latency_available: - bundle["fetch_measurements"]["latency"].update( - status="incomplete", pairs=[], unavailable_reason="OFF worker failed" + if not diagnostic_available: + bundle.update( + status="incomplete", pairs=[], unavailable_reason="Diagnostic worker failed" ) data = {} for leg in reporting.LEGS: @@ -2999,12 +3045,13 @@ def test_ci_existing_publisher_consumes_bundle_without_latency_fallback(self): body = posted[-1] self.assertEqual(body.count(reporting.MARKER), 1) self.assertEqual(body.count("## PR Performance Report"), 1) - self.assertIn("Primary verdict: native OFF", body) - self.assertIn("Row constructions base/candidate 0/1000", body) - self.assertIn("Legacy diagnostics", body) - self.assertEqual("Performance unavailable" in body, not latency_available) - if not latency_available: - self.assertIn("OFF worker failed", body) + self.assertIn("original 22-task profiling-enabled diagnostics", body) + self.assertIn("not headline inputs", body) + self.assertIn("when available", body) + self.assertNotIn("Primary verdict:", body) + self.assertEqual("Performance unavailable" in body, not diagnostic_available) + if not diagnostic_available: + self.assertIn("Diagnostic worker failed", body) self.assertNotIn("Performance improved", body) def test_ci_finalization_deadline_is_checked_after_validation_and_write(self): @@ -3237,10 +3284,20 @@ def test_versioned_route_standalone_and_fully_populated_bundles(self): self.assertEqual(set(valid), set(reporting.MODES)) self.assertEqual(errors, {}) body = reporting.render_ci_reports([bundle], "c" * 40, 42) - self.assertIn("Row constructions base/candidate 0/1000", body) + self.assertIn("original 22-task profiling-enabled diagnostics", body) + route = valid["route"] + for method in ("fetchone", "fetchmany", "fetchval"): + for side in ("base", "candidate"): + identity = route["pairs"][0][side]["provenance"] + expected = ( + 1000 if side == "candidate" and (not modern or method == "fetchmany") else 0 + ) + self.assertEqual(reporting.expected_constructors(identity, method), expected) if modern: - self.assertIn("Row constructions base/candidate 0/0", body) - self.assertIn("binding=True, default fetchone native Row=False", body) + self.assertIn( + "binding=True, default fetchone native Row=False", + reporting.route_description(route, "fetchone"), + ) for policy in ((True, True, True), (False, True, False)) if modern else (): # A future base chooses its own policy; candidate role does not select it. future = copy.deepcopy(bundle) @@ -3261,7 +3318,7 @@ def test_versioned_route_standalone_and_fully_populated_bundles(self): ) self.assertEqual(reporting.ci_mode_reports(future)[1], {}) self.assertIn( - "Primary verdict: native OFF", + "original 22-task profiling-enabled diagnostics", reporting.render_ci_reports([future], "c" * 40, 42), ) @@ -3353,7 +3410,8 @@ def test_versioned_route_partial_siblings_and_off_on_identity(self): self.assertIn("latency", valid) self.assertIn("diagnostic" if defect == "diagnostic-anchor" else "route", errors) self.assertIn( - "Primary verdict: native OFF", reporting.render_ci_reports([bundle], "c" * 40, 42) + "original 22-task profiling-enabled diagnostics", + reporting.render_ci_reports([bundle], "c" * 40, 42), ) def test_versioned_controller_rejects_warmup_drift_and_keeps_subset_incomplete(self): From 4dbfb55aebe792c0dd2f3d85abdcf4d6d7cc1c78 Mon Sep 17 00:00:00 2001 From: Jahnvi Thakkar Date: Wed, 7 Oct 2026 11:39:25 +0530 Subject: [PATCH 12/17] PERF: Use inline scratch storage for small NVARCHAR fetches Keep logical buffer capacity, initialization and conversion unchanged, with heap fallback for larger bounded values. Preserve existing tests and append boundary, strict-error and isolated buffer-contract cases. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- mssql_python/pybind/ddbc_bindings.cpp | 18 ++++- tests/test_017_fetch_bounded_text.py | 101 ++++++++++++++++++++++++++ 2 files changed, 115 insertions(+), 4 deletions(-) diff --git a/mssql_python/pybind/ddbc_bindings.cpp b/mssql_python/pybind/ddbc_bindings.cpp index 4e7554ea1..882f3d0f7 100644 --- a/mssql_python/pybind/ddbc_bindings.cpp +++ b/mssql_python/pybind/ddbc_bindings.cpp @@ -3806,9 +3806,19 @@ SQLRETURN SQLGetData_wrap(SqlHandlePtr StatementHandle, SQLUSMALLINT colCount, p } else { uint64_t fetchBufferSize = (columnSize + 1) * sizeof(SQLWCHAR); // +1 for null terminator - std::vector dataBuffer(columnSize + 1); + const size_t bufferChars = columnSize + 1; + SQLWCHAR inlineBuffer[64]; + std::vector heapBuffer; + SQLWCHAR* dataBuffer; + if (bufferChars <= std::size(inlineBuffer)) { + std::fill_n(inlineBuffer, bufferChars, SQLWCHAR{}); + dataBuffer = inlineBuffer; + } else { + heapBuffer.resize(bufferChars); + dataBuffer = heapBuffer.data(); + } SQLLEN dataLen; - ret = SQLGetData_ptr(hStmt, i, SQL_C_WCHAR, dataBuffer.data(), fetchBufferSize, + ret = SQLGetData_ptr(hStmt, i, SQL_C_WCHAR, dataBuffer, fetchBufferSize, &dataLen); CaptureFetchDiagnostics( hStmt, ret, messages, @@ -3818,14 +3828,14 @@ SQLRETURN SQLGetData_wrap(SqlHandlePtr StatementHandle, SQLUSMALLINT colCount, p if (SQL_SUCCEEDED(ret)) { if (dataLen > 0) { uint64_t numCharsInData = dataLen / sizeof(SQLWCHAR); - if (numCharsInData < dataBuffer.size()) { + if (numCharsInData < bufferChars) { // Construct with explicit length: SQLGetData reports the // exact number of characters via dataLen, so do not rely on // null termination. This preserves embedded NULs and avoids // any risk of reading past the valid range if the driver // omits the terminator. row.append(FetchText::from_utf16_native( - reinterpret_cast(dataBuffer.data()), + reinterpret_cast(dataBuffer), static_cast(numCharsInData * sizeof(SQLWCHAR)))); LOG("SQLGetData: Appended NVARCHAR string " "length=%lu for column %d", diff --git a/tests/test_017_fetch_bounded_text.py b/tests/test_017_fetch_bounded_text.py index e300485d7..7627b8091 100644 --- a/tests/test_017_fetch_bounded_text.py +++ b/tests/test_017_fetch_bounded_text.py @@ -193,3 +193,104 @@ def test_bounded_nvarchar_batch_malformed_fallback(db_connection, raw, method): assert _fetch_rows(cursor, method) == [(expected,)] cursor.execute("SELECT CAST(N'recovered' AS nvarchar(64))") assert cursor.fetchone()[0] == "recovered" + + +@pytest.mark.parametrize( + "width", [10, 63, 64, 4000], ids=["small", "inline-edge", "heap-edge", "large"] +) +@pytest.mark.parametrize("method", ["fetchone", "fetchval"]) +def test_bounded_nvarchar_single_row_buffer_boundary(db_connection, width, method): + expected = [None, "", "A\0B", "\ufeff\ufffe", "x" * (width - 2) + "\U0001f642"] + values = ", ".join( + f"({index}, CAST(" + + ("NULL" if value is None else f"0x{value.encode('utf-16le').hex()}") + + f" AS nvarchar({width})))" + for index, value in enumerate(expected) + ) + with db_connection.cursor() as cursor: + cursor.execute(f"SELECT payload FROM (VALUES {values}) AS v(n, payload) ORDER BY n") + for value in expected: + row = getattr(cursor, method)() + actual = row[0] if method == "fetchone" else row + assert actual == value + assert actual is None or type(actual) is str + assert getattr(cursor, method)() is None + assert getattr(cursor, method)() is None + cursor.execute("SELECT CAST(N'reused' AS nvarchar(10))") + assert cursor.fetchone()[0] == "reused" + + +@pytest.mark.parametrize("width", [63, 64], ids=["inline-edge", "heap-edge"]) +@pytest.mark.parametrize("method", ["fetchone", "fetchval"]) +@pytest.mark.parametrize("raw", ["00D8", "00DC"], ids=["unpaired-high", "unpaired-low"]) +def test_bounded_nvarchar_buffer_boundary_strict_error(db_connection, width, method, raw): + with db_connection.cursor() as cursor: + cursor.execute(f"SELECT CAST(0x{raw} AS nvarchar({width}))") + with pytest.raises(UnicodeDecodeError): + getattr(cursor, method)() + cursor.execute("SELECT CAST(N'recovered' AS nvarchar(10))") + row = getattr(cursor, method)() + assert (row[0] if method == "fetchone" else row) == "recovered" + + +@pytest.mark.skipif( + sys.platform == "win32", + reason="Windows does not export the native driver function-pointer globals", +) +@pytest.mark.parametrize("width", [63, 64], ids=["inline-edge", "heap-edge"]) +def test_bounded_nvarchar_getdata_buffer_contract(conn_str, width): + import os + import subprocess + import textwrap + + script = textwrap.dedent(""" + import ctypes as c + import os + import sys + import mssql_python + from mssql_python import ddbc_bindings as ddbc + + width = int(sys.argv[1]) + pointer, short, length = c.c_void_p, c.c_short, c.c_ssize_t + get_type = c.CFUNCTYPE(short, pointer, c.c_ushort, short, pointer, length, pointer) + library = c.CDLL(ddbc.module.__file__) + slot = pointer.in_dll(library, "SQLGetData_ptr") + calls, errors = [], [] + + @get_type + def getdata(handle, column, ctype, buffer, capacity, indicator): + try: + assert column == 1 and ctype == -8 # SQL_C_WCHAR + assert capacity == (width + 1) * 2 + assert c.string_at(buffer, capacity) == bytes(capacity) + calls.append(capacity) + return original(handle, column, ctype, buffer, capacity, indicator) + except BaseException as error: + errors.append(repr(error)) + return -1 + + with mssql_python.connect(os.environ["DB_CONNECTION_STRING"]) as connection: + with connection.cursor() as cursor: + cursor.execute(f"SELECT CAST(REPLICATE(N'x', {width}) AS nvarchar({width}))") + saved = slot.value + assert saved + original = get_type(saved) + slot.value = c.cast(getdata, pointer).value + try: + assert cursor.fetchone()[0] == "x" * width + assert cursor.fetchone() is None + finally: + slot.value = saved + assert calls == [(width + 1) * 2] and not errors, (calls, errors) + cursor.execute("SELECT CAST(N'recovered' AS nvarchar(10))") + assert cursor.fetchval() == "recovered" + """) + result = subprocess.run( + [sys.executable, "-c", script, str(width)], + env={**os.environ, "DB_CONNECTION_STRING": conn_str}, + capture_output=True, + text=True, + timeout=45, + check=False, + ) + assert result.returncode == 0, result.stdout + result.stderr From 269590030639acf2b59fe09f83340141e3559823 Mon Sep 17 00:00:00 2001 From: Jahnvi Thakkar Date: Thu, 8 Oct 2026 11:23:50 +0530 Subject: [PATCH 13/17] CHORE: Keep fetch tests focused on driver coverage Consolidate essential fetch regressions into existing suites and remove PR-added benchmark-only and unified-coverage-skipped tests. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- tests/test_017_fetch_bounded_text.py | 124 +- tests/test_036_profiler_ci.py | 1619 ------------------- tests/test_fetch_settings_cache.py | 724 ++++++++- tests/test_fetch_single_row.py | 979 ----------- tests/test_fetch_single_row_construction.py | 444 ----- 5 files changed, 846 insertions(+), 3044 deletions(-) delete mode 100644 tests/test_fetch_single_row.py delete mode 100644 tests/test_fetch_single_row_construction.py diff --git a/tests/test_017_fetch_bounded_text.py b/tests/test_017_fetch_bounded_text.py index 7627b8091..14065700b 100644 --- a/tests/test_017_fetch_bounded_text.py +++ b/tests/test_017_fetch_bounded_text.py @@ -7,11 +7,14 @@ belongs to the separate LOB decoder and is not covered by this regression. """ +import os +import subprocess import sys +import textwrap import pytest -from mssql_python import SQL_CHAR, SQL_WCHAR +from mssql_python import SQL_CHAR, SQL_WCHAR, ddbc_bindings as ddbc @pytest.fixture(scope="module") @@ -294,3 +297,122 @@ def getdata(handle, column, ctype, buffer, capacity, indicator): check=False, ) assert result.returncode == 0, result.stdout + result.stderr + + +def _run_fetch_script(conn_str, script, *arguments, timeout=45): + result = subprocess.run( + [sys.executable, "-c", script, *arguments], + env={**os.environ, "DB_CONNECTION_STRING": conn_str}, + capture_output=True, + text=True, + timeout=timeout, + check=False, + ) + assert result.returncode == 0, result.stdout + result.stderr + + +@pytest.mark.parametrize("size", (1, 2)) +def test_getdata_decode_error_keeps_preexisting_and_completed_cells(cursor, size): + cursor.execute("SELECT 7, CAST(0x00D8 AS NVARCHAR(1))") + assert ddbc.DDBCSQLFetch(cursor.hstmt) == 0 + marker = object() + row = [marker] + if size == 1: + assert ddbc.DDBCSQLGetData(cursor.hstmt, size, row, "utf-16le", "utf-16le", -8) == 0 + else: + with pytest.raises(UnicodeDecodeError): + ddbc.DDBCSQLGetData(cursor.hstmt, size, row, "utf-16le", "utf-16le", -8) + assert row == [marker, 7] + cursor.execute("SELECT 42") + assert cursor.fetchval() == 42 + + +@pytest.mark.skipif( + sys.platform == "win32", + reason="Windows does not export the native driver function-pointer globals", +) +@pytest.mark.parametrize("payload", ("utf8", "empty", "null", "invalid", "warning")) +def test_narrow_getdata_decoding_in_subprocess(conn_str, payload): + script = textwrap.dedent(""" + import ctypes as c + import os + import sys + import mssql_python + from mssql_python import ddbc_bindings as ddbc + + mode = sys.argv[1] + data = b"\\xff" if mode == "invalid" else ( + b"" if mode == "empty" else "\\ufeffA\\x00\\U0001f600".encode("utf-8") + ) + pointer, short, length = c.c_void_p, c.c_short, c.c_ssize_t + get_type = c.CFUNCTYPE(short, pointer, c.c_ushort, short, pointer, length, pointer) + diag_type = c.CFUNCTYPE( + short, short, pointer, short, pointer, pointer, pointer, short, pointer + ) + library = c.CDLL(ddbc.module.__file__) + slot = pointer.in_dll(library, "SQLGetData_ptr") + diag_slot = pointer.in_dll(library, "SQLGetDiagRec_ptr") + calls, errors = [], [] + + @diag_type + def diagnostic(handle_type, handle, record, state, native, message, capacity, size): + try: + if record > 1: + return 100 + text = "getdata warning".encode("utf-16le") + assert capacity > len(text) // 2 + c.memmove(state, "01000\\0".encode("utf-16le"), 12) + c.memmove(message, text + b"\\0\\0", len(text) + 2) + c.cast(native, c.POINTER(c.c_int))[0] = 0 + c.cast(size, c.POINTER(short))[0] = len(text) // 2 + return 0 + except BaseException as error: + errors.append(type(error).__name__) + return -1 + + @get_type + def getdata(handle, column, ctype, buffer, capacity, indicator): + try: + assert ctype == 1 # SQL_C_CHAR + calls.append(column) + result = original(handle, column, ctype, buffer, capacity, indicator) + assert result == 0 and capacity > len(data) + c.memmove(buffer, data + b"\\0", len(data) + 1) + c.cast(indicator, c.POINTER(length))[0] = -1 if mode == "null" else len(data) + return 1 if mode == "warning" else result + except BaseException as error: + errors.append(type(error).__name__) + return -1 + + with mssql_python.connect(os.environ["DB_CONNECTION_STRING"]) as connection: + with connection.cursor() as cursor: + cursor.execute("SELECT CAST('text' AS VARCHAR(30))") + saved, saved_diag = slot.value, diag_slot.value + assert saved and saved_diag + original = get_type(saved) + assert ddbc.DDBCSQLFetch(cursor.hstmt) == 0 + marker = object() + row = [marker] + slot.value = c.cast(getdata, pointer).value + if mode == "warning": + diag_slot.value = c.cast(diagnostic, pointer).value + try: + ret = ddbc.DDBCSQLGetData( + cursor.hstmt, 1, row, "utf-8", "utf-16le", 1, cursor.messages + ) + assert ret == (1 if mode == "warning" else 0) + finally: + slot.value, diag_slot.value = saved, saved_diag + expected = None if mode == "null" else ( + data if mode == "invalid" else data.decode("utf-8") + ) + assert row == [marker, expected] + assert type(row[1]) is type(expected) + assert calls == [1] and not errors, (calls, errors) + assert cursor.messages == ( + [("[01000] (0)", "getdata warning")] if mode == "warning" else [] + ) + cursor.execute("SELECT 42") + assert cursor.fetchval() == 42 + """) + _run_fetch_script(conn_str, script, payload) diff --git a/tests/test_036_profiler_ci.py b/tests/test_036_profiler_ci.py index 2f423141b..8f5e24eed 100644 --- a/tests/test_036_profiler_ci.py +++ b/tests/test_036_profiler_ci.py @@ -13,8 +13,6 @@ import sys import tarfile import time -import tempfile -import unittest from types import SimpleNamespace from unittest.mock import MagicMock from urllib.error import URLError @@ -1861,1620 +1859,3 @@ def test_comment_workflow_separates_same_repo_and_fork_trust(): "head.ref" not in workflow and "head.sha }}" not in workflow.split("ref:", 1)[1].split("persist", 1)[0] ) - - -class TestSingleRowMeasurementModes(unittest.TestCase): - """Pure fake-boundary contracts; these do not qualify native timing.""" - - def sample_report(self, mode="latency", ratio=0.9): - def counter(calls): - return dict(calls=calls, total_us=10000, min_us=1, max_us=100) - - def sample(side): - guarded = side == "candidate" - cases = {} - for name in reporting.FETCH_CASES: - shape, method = name.split("_") - cpp = {} - if mode == "route": - timer = ( - "ddbc::FetchMany_wrap" if method == "fetchmany" else "ddbc::FetchOne_wrap" - ) - cpp[timer] = counter(1001) - if guarded: - cpp["ddbc::FetchRow::construct_row"] = counter(1000) - cases[name] = dict( - wall_ms=100 * (ratio if guarded else 1), - cpp=cpp, - py={}, - work=f"Rows: 1000; shape: {shape}; API: {method}; EOF: 1", - ) - return dict( - status="complete", - mode=mode, - provenance=dict( - source_commit=("b" if guarded else "a") * 40, - native_file="/source/native.so", - native_sha256=("d" if guarded else "e") * 64, - native_profiling=mode == "route", - guarded_row=guarded, - ), - environment=dict( - os="Linux", architecture="x86_64", python="3.13.7", sql_version="16.0" - ), - scenarios=cases, - ) - - return dict( - schema_version=2, - mode=mode, - status="complete", - leg="Linux-SQL2022", - base_commit="a" * 40, - source_commit="b" * 40, - head_commit="c" * 40, - build_id=42, - samples=5, - warmups=1, - pairs=[dict(base=sample("base"), candidate=sample("candidate")) for _ in range(5)], - ) - - def source_anchor(self, source_root, revision): - return dict( - source_commit=revision, - python_cursor_sha256=("1" if revision == "a" * 40 else "2") * 64, - row_route=dict( - version=1, - methods=dict(fetchone=False, fetchmany=revision != "a" * 40, fetchval=False), - ), - ) - - def new_sample_report(self, mode="latency", ratio=1.1): - report = self.sample_report(mode, ratio) - report["python_sources"] = { - side: self.source_anchor( - None, report["base_commit" if side == "base" else "source_commit"] - ) - for side in ("base", "candidate") - } - for pair in report["pairs"]: - for side, sample in pair.items(): - sample["provenance"].update(copy.deepcopy(report["python_sources"][side])) - if mode == "route": - for name, case in sample["scenarios"].items(): - if not name.endswith("fetchmany"): - case["cpp"].pop("ddbc::FetchRow::construct_row", None) - return report - - def test_modes_validate_and_keep_subthreshold_changes_visible(self): - self.assertEqual(tuple(benchmark_workloads.single_row_registry()), reporting.FETCH_CASES) - self.assertEqual(len(reporting.CASES), 22) - for mode in ("latency", "route"): - for ratio in (0.9, 1.1): - with self.subTest(mode=mode, ratio=ratio): - report = self.sample_report(mode, ratio) - reporting.validate(report) - self.assertTrue( - all(row["status"] == "ok" for row in reporting.comparisons(report)) - ) - body = reporting.render([report], "c" * 40, 42) - self.assertIn(f"{(ratio - 1) * 100:+.1f}%", body) - self.assertIn("Python phases OFF", body) - self.assertIn("All database tasks and timings", body) - self.assertIn(f"{ratio:.3f} [{ratio:.3f}, {ratio:.3f}]", body) - self.assertIn("1,000 mixed rows / fetchmany(1)", body) - if mode == "route": - self.assertIn("not production latency", body) - - def test_threshold_policy_is_unchanged(self): - self.assertEqual((reporting.THRESHOLD, reporting.MIN_DELTA_MS), (0.20, 1.0)) - for ratio, expected in ( - (0.8, "ok"), - (0.79, "improvement"), - (1.2, "ok"), - (1.21, "regression"), - ): - with self.subTest(ratio=ratio): - self.assertEqual( - reporting.comparisons(self.sample_report(ratio=ratio))[0]["status"], expected - ) - report = self.sample_report(ratio=1.3) - for pair in report["pairs"][:2]: - pair["candidate"]["scenarios"]["numeric_fetchone"]["wall_ms"] = 100 - self.assertEqual(reporting.comparisons(report)[0]["status"], "noisy") - report["pairs"][1]["candidate"]["scenarios"]["numeric_fetchone"]["wall_ms"] = 130 - self.assertEqual(reporting.comparisons(report)[0]["status"], "regression") - for pair in report["pairs"]: - pair["base"]["scenarios"]["numeric_fetchone"]["wall_ms"] = 1 - pair["candidate"]["scenarios"]["numeric_fetchone"]["wall_ms"] = 1.5 - self.assertEqual(reporting.comparisons(report)[0]["status"], "ok") - - def test_schema_rejects_mismatched_or_missing_evidence(self): - mutations = ( - lambda sample: sample.update(mode="diagnostic"), - lambda sample: sample.update(status="running"), - lambda sample: sample.pop("provenance"), - lambda sample: sample["provenance"].update(native_profiling=True), - lambda sample: sample["provenance"].update(source_commit="f" * 40), - lambda sample: sample["provenance"].update(native_sha256="invalid"), - lambda sample: sample["scenarios"]["mixed_fetchval"].update(py={"py::fetch": {}}), - lambda sample: sample["scenarios"]["mixed_fetchval"].update(cpp={"ddbc::fetch": {}}), - lambda sample: sample["scenarios"].pop("numeric_fetchmany"), - lambda sample: sample["scenarios"]["mixed_fetchval"].update(wall_ms=0), - lambda sample: sample["scenarios"]["mixed_fetchval"].update(work="Rows: 1"), - ) - for index, mutate in enumerate(mutations): - with self.subTest(mutation=index): - report = self.sample_report() - mutate(report["pairs"][0]["candidate"]) - with self.assertRaises(ValueError): - reporting.validate(report) - report = self.sample_report() - report["pairs"][1]["candidate"]["provenance"]["native_sha256"] = "f" * 64 - with self.assertRaisesRegex(ValueError, "changed between samples"): - reporting.validate(report) - with self.assertRaisesRegex(ValueError, "different measurement modes"): - reporting.render([self.sample_report(), self.sample_report("route")], "c" * 40, 42) - - def test_route_requires_actual_counts_not_only_timers(self): - for label, calls in (("ddbc::FetchOne_wrap", 1000), ("ddbc::FetchRow::construct_row", 0)): - with self.subTest(label=label): - report = self.sample_report("route") - report["pairs"][0]["candidate"]["scenarios"]["numeric_fetchone"]["cpp"][label][ - "calls" - ] = calls - with self.assertRaises(ValueError): - reporting.validate(report) - report = self.sample_report("route") - report["pairs"][0]["base"]["scenarios"]["mixed_fetchval"]["cpp"] = {} - with self.assertRaises(ValueError): - reporting.validate(report) - - def test_incomplete_measurement_is_unavailable_not_zero(self): - for mode in ("latency", "route"): - report = self.sample_report(mode) - report.update(status="incomplete", pairs=[]) - reporting.validate(report) - body = reporting.render([report], "c" * 40, 42) - self.assertIn("Performance unavailable", body) - self.assertNotIn("0.000 ms", body) - - def test_context_never_enables_python_and_rejects_contamination(self): - with self.assertRaises(ValueError): - controller._FetchContext("latency", MagicMock(), MagicMock()) - for mode in ("latency", "route"): - with self.subTest(mode=mode): - native = MagicMock() if mode == "route" else None - python = MagicMock() - python.is_enabled.return_value = False - python.get_stats.return_value = {} - if native is not None: - native.get_stats.return_value = {"timer": 1} - ctx = controller._FetchContext(mode, native, python) - ctx.enable() - python.enable.assert_not_called() - python.enable_timeline.assert_not_called() - python.reset.assert_called_once() - cpp, py = ctx.collect() - self.assertEqual(py, {}) - self.assertEqual(cpp, {"timer": 1} if native is not None else {}) - python.is_enabled.return_value = True - with self.assertRaisesRegex(RuntimeError, "became enabled"): - ctx.collect() - python.is_enabled.return_value = False - python.get_stats.return_value = {"py::unexpected": {}} - with self.assertRaisesRegex(RuntimeError, "contaminate"): - ctx.collect() - ctx.disable() - if native is not None: - native.enable.assert_called_once() - native.disable_timeline.assert_called() - - def test_six_workloads_time_only_fetch_and_validate_full_results(self): - from unittest.mock import patch - - for name, workload in benchmark_workloads.single_row_registry().items(): - with self.subTest(name=name): - shape, method = name.split("_") - cursor = MagicMock() - cursor.messages = [] - rows = [(n, n + 10 if shape == "numeric" else "text") for n in range(1000)] - values = ( - list(range(1000)) - if method == "fetchval" - else [[row] for row in rows] if method == "fetchmany" else rows - ) - fetch = getattr(cursor, method) - fetch.side_effect = values + ([[]] if method == "fetchmany" else [None]) - conn = MagicMock() - conn.cursor.return_value.__enter__.return_value = cursor - ctx = MagicMock() - ctx.collect.return_value = ({}, {}) - - def clock(): - cursor.execute.assert_called_once() - ctx.enable.assert_called_once() - ctx.collect.assert_not_called() - self.assertIn(fetch.call_count, (0, 1001)) - return 2.0 if fetch.call_count else 1.0 - - package = SimpleNamespace(logger=SimpleNamespace(is_debug_enabled=False)) - with ( - patch.dict(sys.modules, {"mssql_python.logging": package}), - patch.object(benchmark_workloads.time, "perf_counter", clock), - ): - result = workload(conn, ctx) - self.assertEqual(result["wall_ms"], 1000) - self.assertEqual(fetch.call_count, 1001) - if method == "fetchmany": - self.assertTrue( - all( - call.args == (1,) and type(call.args[0]) is int - for call in fetch.call_args_list - ) - ) - for other in {"fetchone", "fetchmany", "fetchval"} - {method}: - getattr(cursor, other).assert_not_called() - ctx.disable.assert_called_once() - conn.cursor.return_value.__exit__.assert_called_once() - - def test_workloads_fail_closed_and_disable_context(self): - from unittest.mock import patch - - for failure in ("type", "value", "eof", "warning", "fetch", "enable", "debug"): - with self.subTest(failure=failure): - cursor = MagicMock() - cursor.messages = [("01000", "warning")] if failure == "warning" else [] - rows = [(n, "text") for n in range(1000)] - if failure == "type": - rows[0] = (0.0, "text") - if failure == "value": - rows[0] = (0, "wrong") - cursor.fetchone.side_effect = ( - RuntimeError("fetch") - if failure == "fetch" - else rows + ([rows[0]] if failure == "eof" else [None]) - ) - conn = MagicMock() - conn.cursor.return_value.__enter__.return_value = cursor - ctx = MagicMock() - ctx.collect.return_value = ({}, {}) - if failure == "enable": - ctx.enable.side_effect = RuntimeError("enable") - package = SimpleNamespace( - logger=SimpleNamespace(is_debug_enabled=failure == "debug") - ) - with patch.dict(sys.modules, {"mssql_python.logging": package}): - with self.assertRaises((AssertionError, RuntimeError, ValueError)): - benchmark_workloads.single_row_fetch(conn, ctx, "fetchone", "mixed") - if failure == "debug": - conn.cursor.assert_not_called() - else: - ctx.disable.assert_called_once() - conn.cursor.return_value.__exit__.assert_called_once() - - def test_off_build_and_worker_mode_are_explicit(self): - from unittest.mock import patch - - with tempfile.TemporaryDirectory() as directory: - path = Path(directory) - process = MagicMock() - process.wait.return_value = 0 - with patch.object(controller.subprocess, "Popen", return_value=process) as start: - controller.build(path, path / "build.log", profiling=False) - self.assertEqual(start.call_args.kwargs["env"]["ENABLE_PROFILING"], "0") - with patch.object(controller.subprocess, "run") as run: - output = path / "sample.json" - run.side_effect = lambda *a, **k: output.write_text('{"status":"complete"}') - controller.measure(path, output, None, mode="latency", revision="a" * 40) - command = run.call_args.args[0] - self.assertEqual(command[command.index("--mode") + 1], "latency") - self.assertEqual(command[command.index("--revision") + 1], "a" * 40) - with patch.object(controller, "resolve_revisions") as resolve: - with self.assertRaisesRegex(ValueError, "cannot reuse"): - controller.run(SimpleNamespace(mode="latency", reuse_candidate=True)) - resolve.assert_not_called() - - def test_worker_provenance_route_counts_and_failure_checkpoint(self): - from unittest.mock import patch - - for mode, failure in ( - ("latency", None), - ("route", None), - ("route", "counts"), - ("latency", "close"), - ): - with ( - self.subTest(mode=mode, failure=failure), - tempfile.TemporaryDirectory() as directory, - ): - root = Path(directory) - source = root / "mssql_python" / "cursor.py" - source.parent.mkdir() - source.write_bytes((ROOT / "mssql_python" / "cursor.py").read_bytes()) - cursor_module = SimpleNamespace(__file__=str(source)) - native_file = root / "native.so" - native_file.write_bytes(b"fake binary identity; never loaded") - args = SimpleNamespace( - source_root=root, revision="b" * 40, scenarios=None, output=root / "sample.json" - ) - native = SimpleNamespace( - module=SimpleNamespace(__file__=str(native_file)), DDBCSQLFetchRow=object() - ) - if mode == "route": - native.profiling = MagicMock() - python = MagicMock() - conn = MagicMock() - conn.__enter__.return_value = conn - if failure == "close": - conn.__exit__.side_effect = RuntimeError("connection close failed") - conn.cursor.return_value.__enter__.return_value.fetchone.return_value = ("16.0",) - package = SimpleNamespace( - ddbc_bindings=native, - perf_timer=python, - cursor=cursor_module, - connect=MagicMock(return_value=conn), - ) - sample = self.new_sample_report(mode)["pairs"][0]["candidate"] - - def workload(connection, ctx): - ctx.enable() - # Measurement results here are fake; native count validation is real. - name = json.loads(args.output.read_text())["active_scenario"] - result = copy.deepcopy(sample["scenarios"][name]) - result["detail"] = result.pop("work") - if failure == "counts": - result["cpp"]["ddbc::FetchRow::construct_row"] = dict(calls=999) - ctx.disable() - return result - - cases = {name: workload for name in reporting.FETCH_CASES} - with ( - patch.dict( - sys.modules, {"mssql_python": package, "mssql_python.cursor": cursor_module} - ), - patch.dict(os.environ, {"DB_CONNECTION_STRING": "test-only"}), - patch.object(controller, "check_build") as check, - patch.object(controller.platform, "system", return_value="Linux"), - patch.object(controller.platform, "machine", return_value="x86_64"), - patch.object(controller.platform, "python_version", return_value="3.13.7"), - patch.object(benchmark_workloads, "single_row_registry", return_value=cases), - ): - if failure == "counts": - with self.assertRaisesRegex(RuntimeError, "route was not established"): - controller.fetch_worker(args, mode) - elif failure == "close": - with self.assertRaisesRegex(RuntimeError, "connection close failed"): - controller.fetch_worker(args, mode) - else: - controller.fetch_worker(args, mode) - check.assert_called_once_with(root, profiling=mode == "route") - result = json.loads(args.output.read_text()) - self.assertEqual(result["status"], "running" if failure else "complete") - self.assertEqual(result["provenance"]["native_profiling"], mode == "route") - self.assertEqual(len(result["provenance"]["native_sha256"]), 64) - self.assertEqual(result["provenance"]["source_commit"], "b" * 40) - self.assertEqual(len(result["scenarios"]), 0 if failure == "counts" else 6) - python.enable.assert_not_called() - if mode == "route": - native.profiling.disable.assert_called() - - def test_controller_pairs_use_selected_mode_and_fail_incomplete(self): - from unittest.mock import patch - - for mode, fail in (("latency", False), ("route", False), ("latency", True)): - with self.subTest(mode=mode, fail=fail), tempfile.TemporaryDirectory() as directory: - root = Path(directory) - args = SimpleNamespace( - base="a" * 40, - candidate="b" * 40, - output=root, - mode=mode, - reuse_candidate=False, - leg="Linux-SQL2022", - samples=3, - warmups=1, - scenarios=None, - ) - sample = self.new_sample_report(mode)["pairs"][0] - calls = [] - - def measure(path, output, scenarios, timeout, **options): - calls.append((path.name, options)) - if fail: - raise RuntimeError("missing measurement") - return copy.deepcopy(sample[path.name]) - - with ( - patch.object( - controller, "resolve_revisions", return_value=("a" * 40, "b" * 40) - ), - patch.object(controller, "checkout"), - patch.object( - controller, "python_source_identity", side_effect=self.source_anchor - ), - patch.object(controller, "build") as build, - patch.object(controller, "measure", side_effect=measure), - ): - if fail: - with self.assertRaisesRegex(RuntimeError, "missing measurement"): - controller.run(args) - else: - controller.run(args) - self.assertEqual(build.call_count, 2) - self.assertTrue( - all( - call.kwargs == {"profiling": mode != "latency"} - for call in build.call_args_list - ) - ) - for side, options in calls: - self.assertEqual( - options, dict(mode=mode, revision=("a" if side == "base" else "b") * 40) - ) - result = reporting.validate(json.loads((root / "report.json").read_text())) - self.assertEqual(result["status"], "incomplete" if fail else "complete") - if not fail: - self.assertEqual( - [side for side, _ in calls], ["base", "candidate", "candidate", "base"] * 2 - ) - self.assertEqual(len(result["pairs"]), 3) - - def test_legacy_diagnostic_schema_still_requires_native_samples(self): - report = self.sample_report() - report.pop("mode") - report["schema_version"] = 1 - for pair in report["pairs"]: - for sample in pair.values(): - sample.pop("mode") - sample.pop("provenance") - sample["scenarios"] = { - name: dict( - wall_ms=100, - work="Rows: 1", - cpp={"ddbc::query": dict(calls=1, total_us=10, min_us=10, max_us=10)}, - py={}, - ) - for name in reporting.CASES - } - reporting.validate(report) - self.assertEqual(len(reporting.comparisons(report)), 22) - self.assertIn("profiling-enabled builds", reporting.render([report], "c" * 40, 42)) - report["pairs"][0]["base"]["scenarios"]["fetchone"]["cpp"] = {} - with self.assertRaisesRegex(ValueError, "profiling data"): - reporting.validate(report) - - def sample_bundle(self, modern=False): - factory = self.new_sample_report if modern else self.sample_report - latency = factory("latency") - route = factory("route") - diagnostic = copy.deepcopy(route) - diagnostic.pop("mode") - diagnostic["schema_version"] = 1 - for pair in diagnostic["pairs"]: - for sample in pair.values(): - sample["mode"] = "diagnostic" - cell = sample["scenarios"]["numeric_fetchone"] - sample["scenarios"] = {name: copy.deepcopy(cell) for name in reporting.CASES} - diagnostic.update( - measurement_bundle_version=1, fetch_measurements=dict(latency=latency, route=route) - ) - return diagnostic - - def test_ci_bundle_all_mode_outcomes_restore_diagnostic_verdicts(self): - for bits in range(8): - with self.subTest(completeness=bits): - bundle = self.sample_bundle() - for index, mode in enumerate(("latency", "route", "diagnostic")): - report = bundle if mode == "diagnostic" else bundle["fetch_measurements"][mode] - if not bits & (1 << index): - report.update( - status="incomplete", - pairs=[], - unavailable_reason="Not started: budget exhausted", - ) - valid, reasons = reporting.ci_mode_reports(bundle) - self.assertEqual(len(valid), 3) - self.assertEqual(len(reasons), 3 - bits.bit_count()) - body = reporting.render_ci_reports([bundle], "c" * 40, 42) - self.assertIn("original 22-task profiling-enabled diagnostics", body) - self.assertIn("not headline inputs", body) - self.assertIn("when available", body) - self.assertIn("do not represent production-wheel latency", body) - self.assertNotIn("Primary verdict:", body) - self.assertNotIn("### Measurement availability", body) - self.assertNotIn("Native route attribution", body) - self.assertEqual("Performance unavailable" in body, not bool(bits & 4)) - self.assertNotIn("Performance improved", body) - self.assertLessEqual(len(body), reporting.MAX_COMMENT_CHARS) - headings = [ - "Performance diagnostics", - "All database tasks and timings", - "Build and measurement details", - ] - positions = [body.index(heading) for heading in headings] - self.assertEqual(positions, sorted(positions)) - if bits & 4: - self.assertIn( - "| Database task | Before | After | Paired change | Result |", body - ) - table = body.split(headings[1], 1)[1].split("", 1)[0] - for name in reporting.CASES: - self.assertIn("| " + reporting.TASK_NAMES[name] + " |", table) - self.assertNotIn("Ratio median", table) - else: - self.assertIn("Not started: budget exhausted", body) - - def test_ci_restored_headline_does_not_substitute_other_modes(self): - for diagnostic_ratio, latency_ratio in ((0.6, 1.5), (1.5, 0.6)): - with self.subTest(diagnostic_ratio=diagnostic_ratio): - bundle = self.sample_bundle() - for report, ratio in ( - (bundle, diagnostic_ratio), - (bundle["fetch_measurements"]["latency"], latency_ratio), - ): - for pair in report["pairs"]: - for name, cell in pair["candidate"]["scenarios"].items(): - cell["wall_ms"] = pair["base"]["scenarios"][name]["wall_ms"] * ratio - body = reporting.render_ci_reports([bundle], "c" * 40, 42) - self.assertEqual("Performance improved" in body, diagnostic_ratio < 1) - self.assertEqual("Performance regression detected" in body, diagnostic_ratio > 1) - self.assertIn("not headline inputs", body) - self.assertIn("when available", body) - bundle["pairs"][0]["candidate"]["scenarios"].pop("fetchone") - body = reporting.render_ci_reports([bundle], "c" * 40, 42) - self.assertIn("Performance unavailable", body) - self.assertIn("Invalid mode data", body) - self.assertNotIn("Performance improved", body) - - def test_ci_bundle_rejects_bad_siblings_and_shared_on_drift(self): - for defect in ("missing", "mode", "commit", "counter", "identity", "environment"): - with self.subTest(defect=defect): - bundle = self.sample_bundle() - route = bundle["fetch_measurements"]["route"] - if defect == "missing": - bundle["fetch_measurements"].pop("route") - elif defect == "mode": - route["mode"] = "latency" - elif defect == "commit": - route["source_commit"] = "f" * 40 - elif defect == "counter": - route["pairs"][0]["candidate"]["scenarios"]["mixed_fetchmany"]["cpp"] = {} - elif defect == "identity": - for pair in route["pairs"]: - pair["candidate"]["provenance"]["native_sha256"] = "f" * 64 - else: - for pair in route["pairs"]: - for sample in pair.values(): - sample["environment"]["python"] = "3.12.0" - valid, reasons = reporting.ci_mode_reports(bundle) - self.assertIn("latency", valid) - self.assertNotIn("route", valid) - self.assertIn("route", reasons) - if defect in ("identity", "environment"): - self.assertNotIn("diagnostic", valid) - body = reporting.render_ci_reports([bundle], "c" * 40, 42) - self.assertIn("Unavailable:", body) - self.assertEqual("Performance unavailable" in body, "diagnostic" not in valid) - if "diagnostic" in valid: - self.assertIn("| Row-by-row fetching |", body) - else: - self.assertIn(reasons["diagnostic"], body) - bundle = self.sample_bundle() - for pair in bundle["pairs"]: - for side in ("base", "candidate"): - for cell in pair[side]["scenarios"].values(): - cell["cpp"] = { - "ddbc::" - + str(n) - + "_" - * 150: dict( - calls=1 if side == "base" else 2, - total_us=100 if side == "base" else 200, - min_us=1, - max_us=10, - ) - for n in range(3) - } - body = reporting.render_ci_reports([bundle], "c" * 40, 42) - self.assertLessEqual(len(body), reporting.MAX_COMMENT_CHARS) - self.assertIn("raw artifact", body) - for version in (None, True, 2): - bundle = self.sample_bundle() - bundle["measurement_bundle_version"] = version - with self.assertRaises(ValueError): - reporting.validate_ci_header(bundle) - bundle = self.sample_bundle() - bundle.pop("measurement_bundle_version") - with self.assertRaises(ValueError): - reporting.validate_ci_header(bundle) - - def test_ci_bundle_existing_artifact_and_assessment_preserve_trust(self): - bundle = self.sample_bundle() - reporting.validate(bundle) # The legacy schema-1 diagnostic root still works. - - def archive(value, duplicate=False): - buf = io.BytesIO() - with zipfile.ZipFile(buf, "w") as output: - output.writestr("profiler-Linux-SQL2022/report.json", json.dumps(value)) - output.writestr("profiler-Linux-SQL2022/route-base-0.json", "{}") - if duplicate: - output.writestr("route/report.json", "{}") - return buf.getvalue() - - self.assertEqual(reporting.artifact_report(archive(bundle)), bundle) - with self.assertRaisesRegex(ValueError, "exactly one"): - reporting.artifact_report(archive(bundle, True)) - evidence = reporting.AssessmentEvidence( - build=dict(id=42, sourceVersion="b" * 40), - head="c" * 40, - base="a" * 40, - merge_commit=dict(sha="b" * 40, parents=[dict(sha="a" * 40), dict(sha="c" * 40)]), - base_commit=dict(sha="a" * 40), - ) - body = reporting.assess( - evidence, {"Linux-SQL2022": "fake-artifact"}, lambda url: archive(bundle) - ) - self.assertIn("original 22-task profiling-enabled diagnostics", body) - self.assertIn("| Row-by-row fetching |", body) - bundle["fetch_measurements"]["latency"].update( - status="incomplete", pairs=[], unavailable_reason="OFF build failed" - ) - body = reporting.assess( - evidence, {"Linux-SQL2022": "fake-artifact"}, lambda url: archive(bundle) - ) - self.assertNotIn("Performance unavailable", body) - self.assertIn("| Row-by-row fetching |", body) - bundle.update(status="incomplete", pairs=[], unavailable_reason="Diagnostic worker failed") - body = reporting.assess( - evidence, {"Linux-SQL2022": "fake-artifact"}, lambda url: archive(bundle) - ) - self.assertIn("Performance unavailable", body) - self.assertIn("Diagnostic worker failed", body) - bundle["head_commit"] = "f" * 40 - body = reporting.assess( - evidence, {"Linux-SQL2022": "fake-artifact"}, lambda url: archive(bundle) - ) - self.assertIn("Performance unavailable", body) - self.assertNotIn("| Row-by-row fetching |", body) - - def test_ci_orchestration_counts_order_reuse_and_budget_exhaustion(self): - from unittest.mock import patch - - for slow in (False, True): - with self.subTest(slow=slow), tempfile.TemporaryDirectory() as directory: - args = SimpleNamespace( - output=Path(directory), - mode="diagnostic", - base="a" * 40, - candidate="b" * 40, - reuse_candidate=True, - scenarios=None, - samples=5, - warmups=1, - leg="Linux-SQL2022", - ) - clock, builds, workers, archives = [0], [], [], [] - bundle = self.sample_bundle(modern=True) - samples = { - "diagnostic": bundle["pairs"][0], - **{ - mode: report["pairs"][0] - for mode, report in bundle["fetch_measurements"].items() - }, - } - - def build(path, log, timeout, profiling=True): - builds.append((path, profiling)) - clock[0] += 900 if slow else 1 - - def measure(path, output, scenarios, timeout, **options): - mode = options["mode"] - side = "base" if options["revision"] == "a" * 40 else "candidate" - workers.append((mode, side, path, options["isolated"])) - duration = (360 if mode == "diagnostic" else 30) if slow else 1 - clock[0] += min(timeout, duration) - if duration > timeout: - raise subprocess.TimeoutExpired("fake-worker", timeout) - return copy.deepcopy(samples[mode][side]) - - with ( - patch.object(controller.time, "monotonic", side_effect=lambda: clock[0]), - patch.object( - controller, "resolve_revisions", return_value=("a" * 40, "b" * 40) - ), - patch.object(controller, "git", return_value="b" * 40), - patch.object( - controller, "run_process", side_effect=lambda *a, **k: archives.append(a[0]) - ), - patch.object( - controller, "python_source_identity", side_effect=self.source_anchor - ), - patch.object(controller, "build", side_effect=build), - patch.object(controller, "measure", side_effect=measure), - patch.dict( - os.environ, - {"BUILD_BUILDID": "42", "SYSTEM_PULLREQUEST_SOURCECOMMITID": "c" * 40}, - ), - ): - if slow: - with self.assertRaisesRegex(RuntimeError, "incomplete"): - controller.run_ci_report(args) - else: - controller.run_ci_report(args) - result = json.loads((args.output / "report.json").read_text()) - self.assertEqual([flag for _, flag in builds], [False, False, True]) - self.assertEqual(len(archives), 3) - self.assertTrue(all("--archive-source" in command for command in archives)) - self.assertTrue(all(isolated for _, _, _, isolated in workers)) - self.assertEqual( - [mode for mode, *_ in workers[:24]], ["latency"] * 12 + ["route"] * 12 - ) - for mode in ("latency", "route"): - selected = [(side, path) for kind, side, path, _ in workers if kind == mode] - self.assertEqual( - [side for side, _ in selected], - ["base", "candidate", "candidate", "base"] * 3, - ) - on_paths = { - (mode, side): path for mode, side, path, _ in workers if mode != "latency" - } - for side in ("base", "candidate"): - self.assertEqual(on_paths["route", side], on_paths["diagnostic", side]) - valid, reasons = reporting.ci_mode_reports(result) - self.assertEqual(valid["latency"]["status"], "complete") - self.assertEqual(valid["route"]["status"], "complete") - self.assertEqual(result["status"], "incomplete" if slow else "complete") - self.assertLessEqual( - clock[0], controller.BENCHMARK_TIMEOUT - controller.CI_FINISH_RESERVE - ) - if slow: - self.assertIn("Timeout", reasons["diagnostic"]) - self.assertLess(len(workers), 36) - else: - self.assertEqual(len(workers), 36) - self.assertEqual( - sum(22 if mode == "diagnostic" else 6 for mode, *_ in workers), 408 - ) - self.assertTrue(all(len(report["pairs"]) == 5 for report in valid.values())) - self.assertFalse((args.output / "report.tmp").exists()) - - def test_ci_failures_do_not_retry_or_fabricate_siblings(self): - from unittest.mock import patch - - for failure in ( - "off-build", - "on-build", - "latency", - "route", - "diagnostic", - "identity", - "exhausted", - ): - with self.subTest(failure=failure), tempfile.TemporaryDirectory() as directory: - args = SimpleNamespace( - output=Path(directory), - mode="diagnostic", - base="a" * 40, - candidate="b" * 40, - reuse_candidate=True, - scenarios=None, - samples=5, - warmups=1, - leg="Linux-SQL2022", - ) - clock, workers, builds = [0], [], [] - bundle = self.sample_bundle(modern=True) - samples = { - "diagnostic": bundle["pairs"][0], - **{ - mode: report["pairs"][0] - for mode, report in bundle["fetch_measurements"].items() - }, - } - - def build(path, log, timeout, profiling=True): - builds.append(path.name) - if failure == ("on-build" if profiling else "off-build"): - raise subprocess.CalledProcessError(1, "fake-build") - if failure == "exhausted": - clock[0] = controller.BENCHMARK_TIMEOUT - - def measure(path, output, scenarios, timeout, **options): - mode = options["mode"] - workers.append(mode) - if mode == failure: - raise subprocess.CalledProcessError(1, "fake-worker") - side = "base" if options["revision"] == "a" * 40 else "candidate" - result = copy.deepcopy(samples[mode][side]) - if failure == "identity" and mode == "diagnostic": - result["provenance"]["native_sha256"] = "f" * 64 - return result - - with ( - patch.object(controller.time, "monotonic", side_effect=lambda: clock[0]), - patch.object( - controller, "resolve_revisions", return_value=("a" * 40, "b" * 40) - ), - patch.object(controller, "git", return_value="b" * 40), - patch.object(controller, "run_process"), - patch.object( - controller, "python_source_identity", side_effect=self.source_anchor - ), - patch.object(controller, "build", side_effect=build), - patch.object(controller, "measure", side_effect=measure), - patch.dict( - os.environ, - {"BUILD_BUILDID": "42", "SYSTEM_PULLREQUEST_SOURCECOMMITID": "c" * 40}, - ), - ): - with self.assertRaisesRegex(RuntimeError, "incomplete"): - controller.run_ci_report(args) - result = json.loads((args.output / "report.json").read_text()) - valid, reasons = reporting.ci_mode_reports(result) - self.assertTrue(reasons) - self.assertEqual(len(builds), len(set(builds))) - self.assertLessEqual(len(workers), 36) - if failure in ("latency", "route", "diagnostic"): - self.assertEqual(workers.count(failure), 1) - if failure in ("off-build", "latency"): - self.assertEqual(valid["route"]["status"], "complete") - self.assertEqual(valid["diagnostic"]["status"], "complete") - elif failure in ("on-build", "route", "diagnostic", "identity"): - self.assertEqual(valid["latency"]["status"], "complete") - if failure == "on-build": - self.assertNotIn("diagnostic", workers) - self.assertIn("Shared ON build unavailable", reasons["diagnostic"]) - if failure == "exhausted": - self.assertEqual(workers, []) - for mode, reason in reasons.items(): - self.assertTrue(reason) - self.assertEqual(valid[mode]["status"], "incomplete") - - def test_ci_atomic_write_preserves_last_report_and_cleanup_aborts(self): - from unittest.mock import patch - - with tempfile.TemporaryDirectory() as directory: - root = Path(directory) - output = root / "report.json" - controller.write_report(output, {"status": "incomplete"}) - original = output.read_bytes() - with self.assertRaises(ValueError): - controller.write_report(output, {"wall_ms": float("nan")}) - self.assertEqual(output.read_bytes(), original) - process = MagicMock() - process.wait.side_effect = subprocess.TimeoutExpired("fake-process", 1) - with ( - patch.object(controller.subprocess, "Popen", return_value=process), - patch.object( - controller, "terminate_process_tree", side_effect=OSError("cannot reap") - ) as reap, - ): - with self.assertRaises(controller.ProcessCleanupError): - controller.run_process(["fake-worker"], root / "worker.log", 1) - reap.assert_called_once_with(process) - args = SimpleNamespace( - output=root, - mode="diagnostic", - base="a" * 40, - candidate="b" * 40, - reuse_candidate=True, - scenarios=None, - samples=5, - warmups=1, - leg="Linux-SQL2022", - ) - retained = root / "retained-build-root" - retained.mkdir() - with ( - patch.object(controller, "resolve_revisions", return_value=("a" * 40, "b" * 40)), - patch.object(controller, "git", return_value="b" * 40), - patch.object(controller.tempfile, "mkdtemp", return_value=str(retained)), - patch.object( - controller, - "run_process", - side_effect=controller.ProcessCleanupError("cannot reap"), - ) as launch, - patch.object(controller.shutil, "rmtree") as remove, - patch.dict(os.environ, {"SYSTEM_PULLREQUEST_SOURCECOMMITID": "c" * 40}), - ): - with self.assertRaises(controller.ProcessCleanupError): - controller.run_ci_report(args) - remove.assert_not_called() - self.assertEqual(launch.call_count, 1) - result = json.loads(output.read_text()) - self.assertEqual(result["cleanup_required"], str(retained)) - self.assertEqual(result["fetch_measurements"]["latency"]["status"], "incomplete") - self.assertIn("cleanup", result["fetch_measurements"]["route"]["unavailable_reason"]) - - def test_ci_source_wiring_keeps_matrix_deadlines_and_trust(self): - pipeline = (ROOT / "eng/pipelines/pr-validation-pipeline.yml").read_text(encoding="utf-8") - self.assertEqual(pipeline.count("--ci-report"), 1) - linux = pipeline.split("- job: PytestOnLinux\n", 1)[1].split("\n- job:", 1)[0] - self.assertIn('--reuse-candidate --ci-report --leg "$(profilerLeg)"', linux) - self.assertIn("timeoutInMinutes: 100", linux) - self.assertIn("timeoutInMinutes: 160", linux) - self.assertIn("artifact: profiler-$(profilerLeg)", linux) - self.assertIn("eq(variables['distroName'], 'Ubuntu-SQL2025')", linux) - windows = pipeline.split("# Hosted Windows timings", 1)[1].split("- job:", 1)[0] - self.assertIn("condition: false", windows) - self.assertNotIn("--ci-report", windows) - self.assertEqual( - ( - controller.BENCHMARK_TIMEOUT, - controller.LOCAL_BENCHMARK_TIMEOUT, - controller.WORKER_TIMEOUT, - controller.FETCH_WORKER_TIMEOUT, - ), - (5400, 6300, 360, 30), - ) - workflow = (ROOT / ".github/workflows/pr-profiler-report.yml").read_text(encoding="utf-8") - self.assertIn("timeout-minutes: 230", workflow) - self.assertIn("github.event.pull_request.base.sha", workflow) - - def test_ci_diagnostic_worker_attests_actual_binary_after_cleanup(self): - from unittest.mock import patch - - for fails in (False, True): - with self.subTest(cleanup_failure=fails), tempfile.TemporaryDirectory() as directory: - root = Path(directory) - source = root / "mssql_python" / "cursor.py" - source.parent.mkdir() - source.write_bytes((ROOT / "mssql_python" / "cursor.py").read_bytes()) - cursor_module = SimpleNamespace(__file__=str(source)) - binary = root / "native.so" - binary.write_bytes(b"fake-not-imported") - native = SimpleNamespace( - module=SimpleNamespace(__file__=str(binary)), DDBCSQLFetchRow=object() - ) - manager = MagicMock() - profiler = manager.__enter__.return_value - profiler.run.return_value = [ - dict( - wall_ms=1, - cpp={"ddbc::query": dict(calls=1, total_us=1, min_us=1, max_us=1)}, - py={}, - detail="Rows: 1", - ) - ] - profiler._conn.cursor.return_value.__enter__.return_value.fetchone.return_value = ( - "16.0", - ) - if fails: - manager.__exit__.side_effect = RuntimeError("profiler cleanup failed") - core = SimpleNamespace(Profiler=MagicMock(return_value=manager)) - suite = SimpleNamespace(registry=lambda: {"connect": (object(), False)}) - args = SimpleNamespace( - source_root=root, - mode="diagnostic", - revision="b" * 40, - scenarios=None, - output=root / "diagnostic.json", - ) - with ( - patch.dict( - sys.modules, - { - "mssql_python": SimpleNamespace( - ddbc_bindings=native, cursor=cursor_module - ), - "mssql_python.cursor": cursor_module, - }, - ), - patch.object(controller, "check_build") as check, - patch.object(controller.platform, "system", return_value="Linux"), - patch.object(controller.platform, "machine", return_value="x86_64"), - patch.object(controller.platform, "python_version", return_value="3.13.7"), - patch.object(controller, "load_suite", return_value=(core, suite)), - ): - if fails: - with self.assertRaisesRegex(RuntimeError, "profiler cleanup failed"): - controller.worker(args) - else: - controller.worker(args) - check.assert_called_once_with(root, profiling=True) - sample = json.loads(args.output.read_text()) - self.assertEqual(sample["status"], "running" if fails else "complete") - if not fails: - self.assertEqual(sample["mode"], "diagnostic") - self.assertEqual(sample["provenance"]["native_file"], str(binary.resolve())) - self.assertEqual(sample["provenance"]["source_commit"], "b" * 40) - self.assertTrue(sample["provenance"]["native_profiling"]) - self.assertEqual(len(sample["provenance"]["native_sha256"]), 64) - manager.__exit__.assert_called_once() - - def test_ci_rejects_scope_changes_before_work(self): - from unittest.mock import patch - - for change in ( - {"samples": 3}, - {"warmups": 2}, - {"scenarios": ["numeric_fetchone"]}, - {"mode": "route"}, - {"reuse_candidate": False}, - ): - args = SimpleNamespace( - samples=5, warmups=1, scenarios=None, mode="diagnostic", reuse_candidate=True - ) - args.__dict__.update(change) - with ( - self.subTest(change=change), - patch.object(controller, "resolve_revisions") as resolve, - ): - with self.assertRaises(ValueError): - controller.run_ci_report(args) - resolve.assert_not_called() - - command = [ - "controller", - "--reuse-candidate", - "--ci-report", - "--leg", - "Linux-SQL2022", - "--output", - "unused-fake-output", - ] - with ( - patch.object(sys, "argv", command), - patch.object(controller, "run_ci_report") as run, - patch.object(controller, "run") as legacy, - ): - controller.main() - self.assertTrue(run.call_args.args[0].ci_report) - legacy.assert_not_called() - with ( - patch.object(sys, "argv", command + ["--worker"]), - patch.object(controller, "worker") as worker, - ): - with self.assertRaises(SystemExit): - controller.main() - worker.assert_not_called() - - def test_ci_reused_source_and_filesystem_cleanup_fail_closed(self): - from unittest.mock import patch - - with patch.object(controller, "git", return_value="b" * 40) as git: - controller.verify_reused_source(controller.ROOT, "b" * 40) - self.assertEqual(git.call_count, 2) - self.assertIn("--exit-code", git.call_args.args) - with patch.object(controller, "git", return_value="a" * 40): - with self.assertRaisesRegex(ValueError, "revision changed"): - controller.verify_reused_source(controller.ROOT, "b" * 40) - with patch.object( - controller, "git", side_effect=["b" * 40, subprocess.CalledProcessError(1, "git diff")] - ): - with self.assertRaises(subprocess.CalledProcessError): - controller.verify_reused_source(controller.ROOT, "b" * 40) - with tempfile.TemporaryDirectory() as directory: - root = Path(directory) - retained = root / "build-root" - retained.mkdir() - args = SimpleNamespace( - output=root, - mode="diagnostic", - base="a" * 40, - candidate="b" * 40, - reuse_candidate=True, - scenarios=None, - samples=5, - warmups=1, - leg="Linux-SQL2022", - ) - with ( - patch.object(controller, "resolve_revisions", return_value=("a" * 40, "b" * 40)), - patch.object(controller, "git", return_value="b" * 40), - patch.object(controller.tempfile, "mkdtemp", return_value=str(retained)), - patch.object( - controller, - "run_process", - side_effect=subprocess.CalledProcessError(1, "fake-archive"), - ), - patch.object( - controller.shutil, "rmtree", side_effect=OSError("cannot remove root") - ), - patch.dict(os.environ, {"SYSTEM_PULLREQUEST_SOURCECOMMITID": "c" * 40}), - ): - with self.assertRaisesRegex( - controller.ProcessCleanupError, "build-directory cleanup" - ): - controller.run_ci_report(args) - result = json.loads((root / "report.json").read_text()) - self.assertEqual(result["status"], "incomplete") - self.assertEqual(result["cleanup_required"], str(retained)) - self.assertIn("cleanup failed", result["unavailable_reason"]) - - def test_ci_existing_publisher_restores_diagnostics_without_sibling_fallback(self): - from unittest.mock import patch - - for modern, diagnostic_available in ( - (False, True), - (False, False), - (True, True), - (True, False), - ): - with self.subTest(modern=modern, diagnostic_available=diagnostic_available): - bundle = self.sample_bundle(modern=modern) - if not diagnostic_available: - bundle.update( - status="incomplete", pairs=[], unavailable_reason="Diagnostic worker failed" - ) - data = {} - for leg in reporting.LEGS: - value = set_leg(bundle, leg) - value["fetch_measurements"] = { - mode: set_leg(child, leg) - for mode, child in value["fetch_measurements"].items() - } - data[leg] = zip_data([("report.json", json.dumps(value))]) - artifacts = [ - { - "name": "profiler-" + leg, - "resource": {"downloadUrl": "https://dev.azure.com/" + leg}, - } - for leg in data - ] - posted, clock = [], [0] - with ( - patch.object( - publisher, - "publish", - side_effect=lambda number, head, body, base=None: posted.append(body), - ), - patch.object(publisher, "github", side_effect=pr_topology()), - patch.object( - publisher, - "api", - side_effect=lambda url: { - "value": artifacts if "/artifacts?" in url else [ado_build()] - }, - ), - patch.object( - publisher, - "fetch", - side_effect=lambda url, **kwargs: data[url.rsplit("/", 1)[-1]], - ), - patch.object(publisher.time, "monotonic", side_effect=lambda: clock[0]), - patch.object( - publisher.time, - "sleep", - side_effect=lambda seconds: clock.__setitem__(0, clock[0] + seconds), - ), - ): - publisher.run(123, "c" * 40, 4) - self.assertEqual(len(posted), 2) - body = posted[-1] - self.assertEqual(body.count(reporting.MARKER), 1) - self.assertEqual(body.count("## PR Performance Report"), 1) - self.assertIn("original 22-task profiling-enabled diagnostics", body) - self.assertIn("not headline inputs", body) - self.assertIn("when available", body) - self.assertNotIn("Primary verdict:", body) - self.assertEqual("Performance unavailable" in body, not diagnostic_available) - if not diagnostic_available: - self.assertIn("Diagnostic worker failed", body) - self.assertNotIn("Performance improved", body) - - def test_ci_finalization_deadline_is_checked_after_validation_and_write(self): - from unittest.mock import patch - - for delayed in (None, "validation", "write"): - with self.subTest(delayed=delayed), tempfile.TemporaryDirectory() as directory: - args = SimpleNamespace( - output=Path(directory), - mode="diagnostic", - base="a" * 40, - candidate="b" * 40, - reuse_candidate=True, - scenarios=None, - samples=5, - warmups=1, - leg="Linux-SQL2022", - ) - bundle = self.sample_bundle(modern=True) - samples = { - "diagnostic": bundle["pairs"][0], - **{ - mode: report["pairs"][0] - for mode, report in bundle["fetch_measurements"].items() - }, - } - clock, cleaned, final_writes = [0], [False], [0] - original_remove = controller.shutil.rmtree - original_validate = controller.ci_mode_reports - original_write = controller.write_report - - def remove(path): - original_remove(path) - cleaned[0] = True - - def validate(value): - result = original_validate(value) - if cleaned[0] and delayed == "validation": - clock[0] = controller.BENCHMARK_TIMEOUT + 1 - return result - - def write(path, value): - original_write(path, value) - if cleaned[0]: - final_writes[0] += 1 - if delayed == "write": - clock[0] = controller.BENCHMARK_TIMEOUT + final_writes[0] - elif delayed is None: - clock[0] = controller.BENCHMARK_TIMEOUT - - def measure(path, output, scenarios, timeout, **options): - side = "base" if options["revision"] == "a" * 40 else "candidate" - return copy.deepcopy(samples[options["mode"]][side]) - - error = None - with ( - patch.object(controller.time, "monotonic", side_effect=lambda: clock[0]), - patch.object( - controller, "resolve_revisions", return_value=("a" * 40, "b" * 40) - ), - patch.object(controller, "git", return_value="b" * 40), - patch.object(controller, "run_process"), - patch.object( - controller, "python_source_identity", side_effect=self.source_anchor - ), - patch.object(controller, "build"), - patch.object(controller, "measure", side_effect=measure), - patch.object(controller.shutil, "rmtree", side_effect=remove), - patch.object(controller, "ci_mode_reports", side_effect=validate), - patch.object(controller, "write_report", side_effect=write), - patch.dict( - os.environ, - {"BUILD_BUILDID": "42", "SYSTEM_PULLREQUEST_SOURCECOMMITID": "c" * 40}, - ), - ): - try: - controller.run_ci_report(args) - except RuntimeError as failure: - error = str(failure) - saved = json.loads((args.output / "report.json").read_text()) - self.assertTrue(cleaned[0]) - self.assertEqual(final_writes[0], 2 if delayed else 1) - self.assertEqual(saved["status"], "incomplete" if delayed else "complete") - self.assertEqual(len(saved["pairs"]), 5) - for mode in ("latency", "route"): - self.assertEqual(saved["fetch_measurements"][mode]["status"], "complete") - self.assertEqual(len(saved["fetch_measurements"][mode]["pairs"]), 5) - if delayed: - self.assertIsNotNone(error) - self.assertIn("incomplete", error) - self.assertIn("finish deadline", saved["unavailable_reason"]) - else: - self.assertIsNone(error) - - def test_route_source_recognizer_exact_bytes_and_rejects_drift(self): - import ast - - source = (ROOT / "mssql_python" / "cursor.py").read_text(encoding="utf-8") - tree = ast.parse(source) - declaration = next( - node - for node in tree.body - if isinstance(node, ast.Assign) - and any( - isinstance(t, ast.Name) and t.id == "_DEFAULT_NATIVE_ROW_ROUTE" - for t in node.targets - ) - ) - literal = ast.get_source_segment(source, declaration) - route = {"version": 1, "methods": {"fetchone": False, "fetchmany": True, "fetchval": False}} - with tempfile.TemporaryDirectory() as directory: - root = Path(directory) - target = root / "mssql_python" / "cursor.py" - target.parent.mkdir() - hashes = [] - for text in (source, source.replace("\n", "\r\n"), "\n\n" + source): - target.write_bytes(text.encode()) - observed = controller.python_source_identity(root, "b" * 40) - self.assertEqual(observed["row_route"], route) - hashes.append(observed["python_cursor_sha256"]) - self.assertEqual(hashes[0], hashes[1]) - self.assertNotEqual(hashes[0], hashes[2]) - mutations = [ - source.replace(literal, ""), - source + "\n" + literal, - source.replace(literal, "_DEFAULT_NATIVE_ROW_ROUTE: dict = " + repr(route)), - source.replace(literal, "ALIAS = _DEFAULT_NATIVE_ROW_ROUTE = " + repr(route)), - source.replace(literal, "_DEFAULT_NATIVE_ROW_ROUTE = dict(version=1)"), - source.replace( - " def fetchone(", " @unreviewed_dispatch_wrapper\n def fetchone(" - ), - source.replace(" def fetchmany(", " @staticmethod\n def fetchmany("), - source.replace(" def fetchval(", " @property\n def fetchval("), - source.replace(" def fetchone(", " async def fetchone("), - source.replace( - " def fetchone(", " def fetchone(self): pass\n\n def fetchone(" - ), - source + "\nclass Cursor: pass\n", - source.replace("class Cursor:", "@unreviewed_dispatch_wrapper\nclass Cursor:"), - source.replace( - " Fetch the next row of a query result set.", - " Unknown changed method.", - ), - source + "\nprobe = _DEFAULT_NATIVE_ROW_ROUTE\n", - "invalid Python syntax !", - ] - for bad in ( - {"version": True, "methods": route["methods"]}, - {"version": 2, "methods": route["methods"]}, - {"methods": route["methods"]}, - {"version": 1, "methods": {"fetchone": 0, "fetchmany": True, "fetchval": False}}, - {"version": 1, "methods": {"fetchone": True, "fetchmany": True, "fetchval": True}}, - {"version": 1, "methods": {"fetchone": False, "fetchmany": True}}, - ): - mutations.append( - source.replace(literal, "_DEFAULT_NATIVE_ROW_ROUTE = " + repr(bad)) - ) - mutations.append( - source.replace( - literal, - "_DEFAULT_NATIVE_ROW_ROUTE = {'version':1,'version':1,'methods':" - + repr(route["methods"]) - + "}", - ) - ) - mutations.append( - source.replace( - literal, - "_DEFAULT_NATIVE_ROW_ROUTE = {'version':1,'methods':{'fetchone':False,'fetchone':False,'fetchmany':True,'fetchval':False}}", - ) - ) - cursor_class = next( - node - for node in tree.body - if isinstance(node, ast.ClassDef) and node.name == "Cursor" - ) - lines = source.splitlines(keepends=True) - for method in ("fetchone", "fetchmany", "fetchval"): - for binding in ( - f"{method} = replacement", - f"{method}: object = replacement", - f"{method}: object", - f"alias = {method} = replacement", - f"{method}, alias = replacements", - f"{method} += replacement", - f"del {method}", - f"import replacement as {method}", - f"from replacement import value as {method}", - f"from replacement import {method}", - f"for {method} in replacements: pass", - f"with replacement as {method}: pass", - f"if flag: {method} = replacement", - f"({method} := replacement)", - f"def helper(self, value=({method} := replacement)): pass", - f"def helper(self) -> ({method} := replacement): pass", - f"async def helper(self) -> ({method} := replacement): pass", - f"def helper(self, value: ({method} := replacement)): pass", - f"async def helper(self, value: ({method} := replacement)): pass", - f"helper = lambda value=({method} := replacement): value", - f"class Helper(({method} := replacement)): pass", - f"try: pass\nexcept Exception as {method}: pass", - f"match replacement:\n case {{'value': {method}}}: pass", - f"class {method}: pass", - f"if flag:\n def {method}(self): pass", - ): - insertion = ( - "\n" + "\n".join(" " + line for line in binding.splitlines()) + "\n" - ) - mutations.append( - "".join(lines[: cursor_class.end_lineno]) - + insertion - + "".join(lines[cursor_class.end_lineno :]) - ) - for index, text in enumerate(mutations): - with self.subTest(mutation=index): - self.assertNotEqual(text, source) - target.write_bytes(text.encode()) - with self.assertRaises(ValueError): - controller.python_source_identity(root, "b" * 40) - - def test_versioned_route_standalone_and_fully_populated_bundles(self): - for modern in (False, True): - bundle = self.sample_bundle(modern=modern) - for mode in ("latency", "route"): - report = bundle["fetch_measurements"][mode] - reporting.validate(report) - self.assertIn("Python phases OFF", reporting.render([report], "c" * 40, 42)) - reporting.validate(bundle) - valid, errors = reporting.ci_mode_reports(bundle) - self.assertEqual(set(valid), set(reporting.MODES)) - self.assertEqual(errors, {}) - body = reporting.render_ci_reports([bundle], "c" * 40, 42) - self.assertIn("original 22-task profiling-enabled diagnostics", body) - route = valid["route"] - for method in ("fetchone", "fetchmany", "fetchval"): - for side in ("base", "candidate"): - identity = route["pairs"][0][side]["provenance"] - expected = ( - 1000 if side == "candidate" and (not modern or method == "fetchmany") else 0 - ) - self.assertEqual(reporting.expected_constructors(identity, method), expected) - if modern: - self.assertIn( - "binding=True, default fetchone native Row=False", - reporting.route_description(route, "fetchone"), - ) - for policy in ((True, True, True), (False, True, False)) if modern else (): - # A future base chooses its own policy; candidate role does not select it. - future = copy.deepcopy(bundle) - for mode in reporting.MODES: - report = future if mode == "diagnostic" else future["fetch_measurements"][mode] - anchor = report["python_sources"]["base"] - anchor["row_route"]["methods"] = dict( - zip(("fetchone", "fetchmany", "fetchval"), policy) - ) - for pair in report["pairs"]: - sample = pair["base"] - sample["provenance"].update(copy.deepcopy(anchor), guarded_row=True) - if mode == "route": - for name, case in sample["scenarios"].items(): - if anchor["row_route"]["methods"][name.split("_")[1]]: - case["cpp"]["ddbc::FetchRow::construct_row"] = dict( - calls=1000, total_us=10000, min_us=1, max_us=100 - ) - self.assertEqual(reporting.ci_mode_reports(future)[1], {}) - self.assertIn( - "original 22-task profiling-enabled diagnostics", - reporting.render_ci_reports([future], "c" * 40, 42), - ) - - def test_versioned_route_rejects_bad_identity_policy_counts_and_partial_data(self): - for mutation in ( - lambda r: r.pop("python_sources"), - lambda r: r["python_sources"].pop("base"), - lambda r: r["python_sources"]["candidate"].update(source_commit="f" * 40), - lambda r: r["python_sources"]["candidate"].update(python_cursor_sha256="bad"), - lambda r: r["pairs"][0]["candidate"]["provenance"].pop("row_route"), - lambda r: r["pairs"][0]["candidate"]["provenance"].pop("python_cursor_sha256"), - lambda r: r["pairs"][0]["candidate"]["provenance"].update( - python_cursor_sha256="f" * 64 - ), - lambda r: r["pairs"][0]["candidate"]["provenance"].update(guarded_row=False), - lambda r: r["pairs"][0]["candidate"]["provenance"]["row_route"].update(version=True), - lambda r: r["pairs"][0]["candidate"]["provenance"]["row_route"].update(version=2), - lambda r: r["pairs"][0]["candidate"]["provenance"]["row_route"].pop("version"), - lambda r: r["pairs"][0]["candidate"]["provenance"]["row_route"]["methods"].pop( - "fetchval" - ), - lambda r: r["pairs"][0]["candidate"]["provenance"]["row_route"]["methods"].update( - fetchone=0 - ), - lambda r: r["pairs"][0]["candidate"]["provenance"]["row_route"]["methods"].update( - fetchval=True - ), - lambda r: r["pairs"][0]["candidate"]["scenarios"]["numeric_fetchone"]["cpp"].update( - {"ddbc::FetchRow::construct_row": dict(calls=1000)} - ), - lambda r: r["pairs"][0]["candidate"]["scenarios"]["mixed_fetchmany"]["cpp"].pop( - "ddbc::FetchRow::construct_row" - ), - lambda r: r["pairs"][0]["candidate"]["scenarios"]["mixed_fetchone"].update(cpp={}), - lambda r: r["pairs"][0]["candidate"].update(status="running"), - lambda r: r["pairs"][1]["candidate"]["provenance"].update(native_sha256="f" * 64), - ): - for partial in (False, True): - with self.subTest(mutation=mutation, partial=partial): - report = self.new_sample_report("route") - if partial: - report.update( - status="incomplete", - pairs=report["pairs"][:2], - unavailable_reason="Deadline before next pair", - ) - mutation(report) - with self.assertRaises((ValueError, KeyError)): - reporting.validate(report) - report = self.new_sample_report("route") - report.update( - status="incomplete", - pairs=report["pairs"][:2], - unavailable_reason="Deadline before next pair", - ) - reporting.validate(report) - self.assertIn("Performance unavailable", reporting.render([report], "c" * 40, 42)) - - def test_versioned_route_partial_siblings_and_off_on_identity(self): - for defect in ( - "native", - "python", - "missing-anchor", - "diagnostic-anchor", - "off-environment", - ): - bundle = self.sample_bundle(modern=True) - route = bundle["fetch_measurements"]["route"] - route.update( - status="incomplete", pairs=route["pairs"][:2], unavailable_reason="Deadline" - ) - if defect == "native": - for pair in route["pairs"]: - pair["candidate"]["provenance"]["native_sha256"] = "f" * 64 - elif defect == "missing-anchor": - route.pop("python_sources") - elif defect == "diagnostic-anchor": - bundle["python_sources"]["candidate"]["python_cursor_sha256"] = "f" * 64 - elif defect == "off-environment": - for item in (route, bundle): - for pair in item["pairs"]: - for sample in pair.values(): - sample["environment"]["python"] = "3.12.0" - else: - route["python_sources"]["candidate"]["python_cursor_sha256"] = "f" * 64 - for pair in route["pairs"]: - pair["candidate"]["provenance"]["python_cursor_sha256"] = "f" * 64 - valid, errors = reporting.ci_mode_reports(bundle) - self.assertIn("latency", valid) - self.assertIn("diagnostic" if defect == "diagnostic-anchor" else "route", errors) - self.assertIn( - "original 22-task profiling-enabled diagnostics", - reporting.render_ci_reports([bundle], "c" * 40, 42), - ) - - def test_versioned_controller_rejects_warmup_drift_and_keeps_subset_incomplete(self): - from unittest.mock import patch - - for defect in (None, "warmup", "native-drift", "source-drift", "subset"): - with self.subTest(defect=defect), tempfile.TemporaryDirectory() as directory: - root = Path(directory) - args = SimpleNamespace( - base="a" * 40, - candidate="b" * 40, - output=root, - mode="route", - reuse_candidate=False, - leg="Linux-SQL2022", - samples=3, - warmups=1, - scenarios=["mixed_fetchmany"] if defect == "subset" else None, - ) - calls = [] - samples = self.new_sample_report("route")["pairs"][0] - - def measure(path, output, scenarios, timeout, **options): - result = copy.deepcopy(samples[path.name]) - calls.append(path.name) - if defect == "warmup" and len(calls) == 1: - result["scenarios"]["mixed_fetchmany"]["cpp"] = {} - if len(calls) > 2 and path.name == "base": - if defect == "native-drift": - result["provenance"]["native_sha256"] = "f" * 64 - if defect == "source-drift": - result["provenance"]["python_cursor_sha256"] = "f" * 64 - if scenarios: - result["scenarios"] = { - name: result["scenarios"][name] for name in scenarios - } - return result - - with ( - patch.object( - controller, "resolve_revisions", return_value=("a" * 40, "b" * 40) - ), - patch.object(controller, "checkout"), - patch.object(controller, "build"), - patch.object( - controller, "python_source_identity", side_effect=self.source_anchor - ), - patch.object(controller, "measure", side_effect=measure), - ): - if defect in (None, "subset"): - controller.run(args) - else: - with self.assertRaises(ValueError): - controller.run(args) - saved = json.loads((root / "report.json").read_text()) - self.assertEqual(saved["status"], "complete" if defect is None else "incomplete") - if defect not in (None, "subset"): - self.assertEqual(saved["pairs"], []) - self.assertLessEqual(len(calls), 4) - if defect == "subset": - self.assertEqual(len(saved["pairs"]), 3) - self.assertEqual( - set(saved["pairs"][0]["base"]["scenarios"]), {"mixed_fetchmany"} - ) - with self.assertRaises(ValueError): - reporting.validate(saved) diff --git a/tests/test_fetch_settings_cache.py b/tests/test_fetch_settings_cache.py index 828283b0c..4421d19fd 100644 --- a/tests/test_fetch_settings_cache.py +++ b/tests/test_fetch_settings_cache.py @@ -3,20 +3,26 @@ Licensed under the MIT license. Regression and operation-count tests for fetch settings and diagnostic preservation. -All integration queries are read-only and each test owns its connection. +Read-only fetch regressions; native fault injection runs in isolated child processes. """ import datetime +import decimal +import gc +import os from pathlib import Path import subprocess import sys +import textwrap import uuid import weakref from unittest.mock import Mock, patch import pytest import mssql_python +from mssql_python import ddbc_bindings as ddbc from mssql_python.constants import ConstantsDDBC +from mssql_python.cursor import Cursor from mssql_python.row import Row FETCH_METHODS = ("fetchone", "fetchmany", "fetchall") @@ -1387,3 +1393,719 @@ def fetch_row(handle): count_pointer.value, fetch_pointer.value = saved_count, saved_fetch cursor.execute("SELECT 42") assert cursor.fetchone()[0] == 42 + + +def _run_fetch_script(conn_str, script, *arguments, timeout=45): + result = subprocess.run( + [sys.executable, "-c", script, *arguments], + env={**os.environ, "DB_CONNECTION_STRING": conn_str}, + capture_output=True, + text=True, + timeout=timeout, + check=False, + ) + assert result.returncode == 0, result.stdout + result.stderr + + +@pytest.mark.parametrize( + ("sql_type", "literal", "expected"), + ( + ("INT", "-2147483648", -2147483648), + ("INT", "2147483647", 2147483647), + ("SMALLINT", "-32768", -32768), + ("SMALLINT", "32767", 32767), + ("BIGINT", "-9223372036854775808", -9223372036854775808), + ("BIGINT", "9223372036854775807", 9223372036854775807), + ("TINYINT", "255", 255), + ("BIT", "0", False), + ("REAL", "-1.25", -1.25), + ("FLOAT", "1.7976931348623157E308", 1.7976931348623157e308), + ("FLOAT", "-2.2250738585072014E-308", -2.2250738585072014e-308), + ("FLOAT", "-0.0", 0.0), + ), + ids=( + "int_min", + "int_max", + "smallint_min", + "smallint_max", + "bigint_min", + "bigint_max", + "tinyint_max", + "bit_zero", + "real_negative", + "float_max", + "float_tiny", + "float_zero", + ), +) +def test_single_numeric_row_bound_path_parity(cursor, sql_type, literal, expected): + query = f"SELECT CAST({literal} AS {sql_type}) AS a, CAST(NULL AS {sql_type}) AS b" + cursor.execute(query) + bound = cursor.fetchmany(2)[0] + cursor.execute(query) + single = cursor.fetchmany(1)[0] + assert tuple(single) == tuple(bound) == (expected, None) + assert type(single[0]) is type(bound[0]) is type(expected) + + +def test_single_row_converters_run_in_column_order(cursor): + events = [] + + def convert(value): + events.append(value) + if value == 2: + raise ValueError("keep original second column") + return value + 10 + + cursor.connection.add_output_converter(mssql_python.SQL_INTEGER, convert) + try: + cursor.execute("SELECT 1 AS a, 2 AS b, CAST(NULL AS INT) AS c") + assert cursor.fetchval() == 11 + assert events == [1, 2] + events.clear() + cursor.execute("SELECT 1 AS a, 2 AS b, CAST(NULL AS INT) AS c") + assert tuple(cursor.fetchmany(1)[0]) == (11, 2, None) + assert events == [1, 2] + finally: + cursor.connection.remove_output_converter(mssql_python.SQL_INTEGER) + + +def test_single_row_wrapper_does_not_enter_batch_factory(cursor): + from mssql_python import ddbc_bindings + + cursor.execute( + "SELECT n AS a, CAST(N'text' AS NVARCHAR(10)) AS b " + "FROM (VALUES (1),(2),(3),(4)) AS v(n) ORDER BY n" + ) + with patch.object(ddbc_bindings, "construct_rows", wraps=ddbc_bindings.construct_rows) as batch: + retained = cursor.fetchmany(1)[0] + assert tuple(retained) == (1, "text") + batch.assert_not_called() + assert [row[0] for row in cursor.fetchmany(2)] == [2, 3] + batch.assert_called_once() + assert cursor.fetchmany(2)[0][0] == 4 + assert batch.call_count == 2 + assert tuple(retained) == (1, "text") + + +@pytest.mark.parametrize("override_fast_create", (False, True)) +def test_fetchmany_preserves_substituted_row_class(cursor, override_fast_create): + import importlib + from mssql_python.row import Row + + class DerivedRow(Row): + pass + + def forbidden(*args): + raise AssertionError("batch wrapping must not call a substituted Row's factory") + + if override_fast_create: + DerivedRow._fast_create = staticmethod(forbidden) + cursor_module = importlib.import_module("mssql_python.cursor") + cursor.execute("SELECT 1 AS a") + with patch.object(cursor_module, "Row", DerivedRow): + row = cursor.fetchmany(1)[0] + assert type(row) is DerivedRow + assert row.a == 1 + + +def test_fetchmany_preserves_replaced_fast_factory(cursor): + from mssql_python.row import Row + + cursor.execute("SELECT 1 AS a") + with patch.object(Row, "_fast_create", side_effect=AssertionError("must use batch factory")): + assert cursor.fetchmany(1)[0].a == 1 + + +@pytest.mark.parametrize("via_arraysize", (False, True)) +@pytest.mark.parametrize("raises", (False, True)) +def test_fetchmany_size_subclass_equality_is_not_called(cursor, via_arraysize, raises): + from mssql_python import ddbc_bindings + + calls = [] + + class Size(int): + def __eq__(self, other): + calls.append(other) + if raises: + raise AssertionError("size equality must not run after native fetch") + return super().__eq__(other) + + cursor.execute("SELECT 1 AS a UNION ALL SELECT 2") + with patch.object(ddbc_bindings, "construct_rows", wraps=ddbc_bindings.construct_rows) as batch: + if via_arraysize: + cursor.arraysize = Size(1) + result = cursor.fetchmany() + else: + result = cursor.fetchmany(Size(1)) + assert result[0][0] == 1 + batch.assert_called_once() + assert calls == [] + assert cursor.fetchone()[0] == 2 + + +def test_real_subclass_and_instance_fetchone_overrides(cursor): + calls = [] + + class DerivedCursor(Cursor): + def fetchone(self): + calls.append("derived") + return (71, 72) + + with DerivedCursor(cursor.connection) as derived: + derived.execute("SELECT 1 AS a UNION ALL SELECT 2") + assert derived.fetchval() == 71 + assert next(derived) == (71, 72) + assert calls == ["derived", "derived"] + # fetchmany must not acquire fetchone's Python override semantics. + assert derived.fetchmany(1)[0][0] == 1 + assert calls == ["derived", "derived"] + with patch.object(derived, "fetchone", return_value=(81, 82)) as override: + assert derived.fetchval() == 81 + assert next(derived) == (81, 82) + assert derived.fetchmany(1)[0][0] == 2 + assert override.call_count == 2 + + +@pytest.mark.skipif( + sys.platform == "win32", + reason="Windows does not export the native driver function-pointer globals", +) +@pytest.mark.parametrize("first_fetch", ("fetchone", "fetchmany")) +def test_count_generation_change_with_unbound_marker(conn_str, first_fetch): + """A count obtained across invalidation cannot authorize either cache.""" + script = textwrap.dedent(""" + import ctypes as c + import os + import sys + import mssql_python + from mssql_python import ddbc_bindings as ddbc + + library = c.CDLL(ddbc.module.__file__) + pointer = c.c_void_p + count_type = c.CFUNCTYPE(c.c_short, pointer, c.POINTER(c.c_short)) + unbind_type = c.CFUNCTYPE(c.c_short, pointer, c.c_ushort) + count_slot = pointer.in_dll(library, "SQLNumResultCols_ptr") + unbind_slot = pointer.in_dll(library, "SQLFreeStmt_ptr") + counts, unbinds, invalidations, callback_errors = [], [], [], [] + + @count_type + def counted(handle, value): + try: + counts.append(handle) + ret = original_count(handle, value) + if not invalidations: + invalidations.append(True) + # Change only the cache generation, not the result shape. + assert ddbc.DDBCSQLSetStmtAttr(cursor.hstmt, 0, 0) == 0 + return ret + except BaseException as error: + callback_errors.append(type(error).__name__) + return -1 + + @unbind_type + def unbound(handle, option): + try: + if option == 2: + unbinds.append(handle) + return original_unbind(handle, option) + except BaseException as error: + callback_errors.append(type(error).__name__) + return -1 + + with mssql_python.connect(os.environ["DB_CONNECTION_STRING"]) as connection: + with connection.cursor() as cursor: + cursor.execute( + "SELECT n AS a, n + 10 AS b FROM (VALUES (1),(2),(3),(4)) v(n) ORDER BY n" + ) + saved_count, saved_unbind = count_slot.value, unbind_slot.value + assert saved_count and saved_unbind + original_count = count_type(saved_count) + original_unbind = unbind_type(saved_unbind) + count_slot.value = c.cast(counted, pointer).value + unbind_slot.value = c.cast(unbound, pointer).value + try: + first = ( + cursor.fetchmany(1)[0] if sys.argv[1] == "fetchmany" + else cursor.fetchone() + ) + assert tuple(first) == (1, 11) + assert invalidations == [True] + before_count, before_unbind = len(counts), len(unbinds) + assert tuple(cursor.fetchone()) == (2, 12) + assert len(counts) == before_count + 1 + assert len(unbinds) == before_unbind + 1 + before_count, before_unbind = len(counts), len(unbinds) + assert cursor.fetchval() == 3 + assert tuple(next(cursor)) == (4, 14) + assert cursor.fetchone() is None + assert len(counts) == before_count + assert len(unbinds) == before_unbind + # Direct count calls remain uncached even after a warm fetch. + assert ddbc.DDBCSQLNumResultCols(cursor.hstmt) == 2 + assert ddbc.DDBCSQLNumResultCols(cursor.hstmt) == 2 + assert len(counts) == before_count + 2 + assert tuple(first) == (1, 11) + assert not callback_errors, callback_errors + assert not cursor.messages + finally: + count_slot.value, unbind_slot.value = saved_count, saved_unbind + cursor.execute("SELECT 42") + assert cursor.fetchval() == 42 + """) + _run_fetch_script(conn_str, script, first_fetch) + + +@pytest.mark.parametrize("mutation", ("new", "setattr", "code", "abstract", "descriptor")) +def test_fast_row_inplace_customization(monkeypatch, mutation): + import weakref + from mssql_python import ddbc_bindings + from mssql_python.row import Row + + factory = Row._fast_create + events = [] + + def custom_new(cls): + events.append("new") + return object.__new__(cls) + + def custom_setattr(self, name, value): + events.append(name) + object.__setattr__(self, name, value) + + def custom_factory(values, column_map, cursor, column_map_lower=None, column_names=None): + raise RuntimeError("customized factory code") + + def fail_descriptor(self, value): + events.append(weakref.ref(self)) + raise RuntimeError("customized descriptor") + + if mutation == "new": + monkeypatch.setattr(Row, "__new__", staticmethod(custom_new)) + elif mutation == "setattr": + monkeypatch.setattr(Row, "__setattr__", custom_setattr) + elif mutation == "code": + monkeypatch.setattr(factory, "__code__", custom_factory.__code__) + elif mutation == "abstract": + monkeypatch.setattr(Row, "__abstractmethods__", frozenset({"required"}), raising=False) + else: + monkeypatch.setattr(Row, "_column_names", property(fset=fail_descriptor)) + + assert Row._fast_create is factory + with patch.object(ddbc_bindings, "construct_row", wraps=ddbc_bindings.construct_row) as native: + if mutation in ("code", "descriptor"): + with pytest.raises(RuntimeError, match="customized"): + factory([42], {"number": 0}, None) + elif mutation == "abstract": + with pytest.raises(TypeError, match="abstract"): + factory([42], {"number": 0}, None) + else: + assert factory([42], {"number": 0}, None).number == 42 + native.assert_not_called() + if mutation == "new": + assert events == ["new"] + elif mutation == "setattr": + assert events == ["_values", "_column_map", "_cursor", "_column_map_lower", "_column_names"] + elif mutation == "descriptor": + assert len(events) == 1 and events[0]() is None + + +@pytest.mark.parametrize("method", ("fetchone", "fetchmany", "fetchval")) +@pytest.mark.parametrize("failure", ("maps", "factory")) +def test_single_row_construction_failure_keeps_fetch_position(cursor, method, failure): + from mssql_python import ddbc_bindings + + cursor.execute("SELECT n AS number FROM (VALUES (1), (2)) AS v(n) ORDER BY n") + from mssql_python.row import Row + + bridge_name = "DDBCSQLFetchRow" if method == "fetchmany" else "DDBCSQLFetchOne" + bridge = getattr(ddbc_bindings, bridge_name) + + def fetch(): + value = cursor.fetchmany(1) if method == "fetchmany" else getattr(cursor, method)() + return value[0][0] if method == "fetchmany" else value if method == "fetchval" else value[0] + + def fail(*args): + assert cursor.rowcount == 1 + assert cursor.rownumber == 0 + assert cursor._next_row_index == 1 + raise RuntimeError("injected post-fetch failure") + + failed_stage = Mock(side_effect=fail) + failure_patch = ( + patch.object(cursor, "_get_column_and_converter_maps", failed_stage) + if failure == "maps" + else patch.object(Row, "_column_names", property(fset=failed_stage)) + ) + with patch.object(ddbc_bindings, bridge_name, wraps=bridge) as native_fetch, failure_patch: + with pytest.raises(RuntimeError, match="injected post-fetch failure"): + fetch() + native_fetch.assert_called_once() + failed_stage.assert_called_once() + assert fetch() == 2 + assert cursor.rowcount == 2 + assert cursor.rownumber == 1 + + +@pytest.mark.parametrize("target", ("__new__", "__setattr__")) +def test_native_row_guard_does_not_invoke_descriptors(monkeypatch, target): + from mssql_python.cursor import _native_row_eligible + from mssql_python.row import Row + + events = [] + + class Descriptor: + def __get__(self, instance, owner): + events.append("lookup") + if target == "__new__": + if len(events) > 1: + raise RuntimeError("duplicate allocator lookup") + + def allocate(cls): + events.append("allocate") + return object.__new__(cls) + + return allocate + return lambda name, value: object.__setattr__(instance, name, value) + + monkeypatch.setattr(Row, target, Descriptor()) + assert not _native_row_eligible(Row) + assert events == [] + row = Row._fast_create([42], {"number": 0}, None) + assert row.number == 42 + assert events == (["lookup", "allocate"] if target == "__new__" else ["lookup"] * 5) + + +def test_native_row_guard_does_not_invoke_metaclass_hooks(): + from mssql_python.cursor import _native_row_eligible + from mssql_python.row import Row + + class Meta(type): + def __getattribute__(cls, name): + raise AssertionError("guard must not inspect substituted class through metaclass") + + class CustomRow(Row, metaclass=Meta): + pass + + assert not _native_row_eligible(CustomRow) + + +@pytest.mark.parametrize("method", ("fetchone", "fetchmany", "fetchval")) +@pytest.mark.parametrize("when", ("maps", "final_argument")) +def test_single_row_fusion_handles_post_fetch_factory_change(cursor, method, when): + from mssql_python import ddbc_bindings + from mssql_python.row import Row + + original_maps = cursor._get_column_and_converter_maps + factory = Row._fast_create + + def replacement(values, column_map, cursor, column_map_lower=None, column_names=None): + raise RuntimeError("factory changed after native advancement") + + def maps(): + factory.__code__ = replacement.__code__ + return original_maps() + + def names(self): + factory.__code__ = replacement.__code__ + return self.__dict__["_cached_result_columns"] + + cursor.execute("SELECT n AS number FROM (VALUES (1), (2)) AS v(n) ORDER BY n") + cursor._get_column_and_converter_maps() + change = ( + patch.object(cursor, "_get_column_and_converter_maps", side_effect=maps) + if when == "maps" + else patch.object(type(cursor), "_cached_result_columns", property(names), create=True) + ) + old_code = factory.__code__ + try: + with ( + change, + patch.object( + ddbc_bindings, "DDBCSQLFetchRow", wraps=ddbc_bindings.DDBCSQLFetchRow + ) as fused, + ): + with pytest.raises(RuntimeError, match="factory changed after native advancement"): + cursor.fetchmany(1) if method == "fetchmany" else getattr(cursor, method)() + assert fused.call_count == (1 if method == "fetchmany" else 0) + assert Row._fast_create is factory + assert cursor.rowcount == 1 and cursor.rownumber == 0 + finally: + factory.__code__ = old_code + assert cursor.fetchone().number == 2 + + +@pytest.mark.parametrize("method", ("fetchone", "fetchmany", "fetchval")) +def test_single_row_late_allocator_preserves_factory_global_lookup(cursor, method): + from mssql_python import ddbc_bindings + from mssql_python.row import Row + + original_row = Row + + class ChangedRow(original_row): + pass + + events = [] + factory = original_row._fast_create + assert "__new__" not in vars(original_row) + + class Allocator: + def __get__(self, instance, owner): + events.append("new_lookup") + factory.__globals__["Row"] = ChangedRow + + def allocate(cls): + events.append("allocate:" + cls.__name__) + return object.__new__(cls) + + return allocate + + def names(self): + events.append("names_lookup") + original_row.__new__ = Allocator() + return self.__dict__["_cached_result_columns"] + + cursor.execute("SELECT n AS number FROM (VALUES (1), (2)) AS v(n) ORDER BY n") + cursor._get_column_and_converter_maps() + try: + with ( + patch.object(type(cursor), "_cached_result_columns", property(names), create=True), + patch.object( + ddbc_bindings, "DDBCSQLFetchRow", wraps=ddbc_bindings.DDBCSQLFetchRow + ) as fused, + ): + result = cursor.fetchmany(1) if method == "fetchmany" else getattr(cursor, method)() + assert fused.call_count == (1 if method == "fetchmany" else 0) + if method == "fetchval": + assert result == 1 + else: + row = result[0] if method == "fetchmany" else result + assert type(row) is ChangedRow + assert row.number == 1 + assert events == ["names_lookup", "new_lookup", "allocate:ChangedRow"] + assert cursor.rowcount == 1 and cursor.rownumber == 0 + finally: + factory.__globals__["Row"] = original_row + if "__new__" in vars(original_row): + delattr(original_row, "__new__") + assert cursor.fetchone().number == 2 + + +def test_getdata_appends_to_existing_list_without_python_append(cursor): + class Destination(list): + def append(self, value): + raise AssertionError("native append must not dispatch to list overrides") + + class Marker: + pass + + marker = Marker() + reference = weakref.ref(marker) + destination = Destination([marker]) + alias = destination + cursor.execute( + "SELECT CAST(-2147483648 AS INT), CAST(-32768 AS SMALLINT), " + "CAST(-9223372036854775808 AS BIGINT), CAST(255 AS TINYINT), " + "CAST(1 AS BIT), CAST(-1.25 AS REAL), CAST(1.5 AS FLOAT), " + "CAST(NULL AS INT), CAST(NULL AS SMALLINT), CAST(NULL AS BIGINT), " + "CAST(NULL AS TINYINT), CAST(NULL AS BIT), CAST(NULL AS REAL), " + "CAST(NULL AS FLOAT), CAST(12.50 AS DECIMAL(5,2)), " + "CAST('2024-02-29' AS DATE), CAST(0x0001FF AS VARBINARY(3))" + ) + assert ddbc.DDBCSQLFetch(cursor.hstmt) == 0 + assert ddbc.DDBCSQLGetData(cursor.hstmt, 17, destination, "utf-16le", "utf-16le", -8) == 0 + expected = [ + -2147483648, + -32768, + -9223372036854775808, + 255, + True, + -1.25, + 1.5, + *([None] * 7), + decimal.Decimal("12.50"), + datetime.date(2024, 2, 29), + b"\x00\x01\xff", + ] + assert destination is alias and destination[0] is marker + assert destination[1:] == expected + assert list(map(type, destination[1:])) == list(map(type, expected)) + del marker, destination, alias + gc.collect() + assert reference() is None + cursor.execute("SELECT 42") + assert cursor.fetchval() == 42 + + +@pytest.mark.skipif( + sys.platform == "win32", + reason="Windows does not export the native driver function-pointer globals", +) +@pytest.mark.parametrize( + "mode", + ( + "numeric", + "mixed", + "lob", + "prefix", + "generation", + "count_error", + "nextset", + "warning", + "decode_error", + ), +) +def test_fetchmany_one_full_count_calls_in_subprocess(conn_str, mode): + script = textwrap.dedent(""" + import ctypes as c + import os + import sys + import mssql_python + from mssql_python import ddbc_bindings as ddbc + + mode = sys.argv[1] + pointer, short, ushort = c.c_void_p, c.c_short, c.c_ushort + count_type = c.CFUNCTYPE(short, pointer, c.POINTER(short)) + diag_type = c.CFUNCTYPE( + short, short, pointer, short, pointer, pointer, pointer, short, pointer + ) + library = c.CDLL(ddbc.module.__file__) + count_slot = pointer.in_dll(library, "SQLNumResultCols_ptr") + diag_slot = pointer.in_dll(library, "SQLGetDiagRec_ptr") + calls, errors, injected = [], [], [] + warning_pending = False + + @count_type + def counted(handle, count): + global warning_pending + try: + calls.append(handle) + if mode == "count_error" and not injected: + injected.append(True) + return -1 + result = original_count(handle, count) + if mode == "generation" and not injected: + injected.append(True) + assert ddbc.DDBCSQLSetStmtAttr(cursor.hstmt, 0, 0) == 0 + if mode == "warning" and not injected: + injected.append(True) + warning_pending = True + return 1 + return result + except BaseException as error: + errors.append(type(error).__name__) + return -1 + + @diag_type + def diagnostic(handle_type, handle, record, state, native, message, capacity, size): + global warning_pending + try: + if not warning_pending: + return original_diag( + handle_type, handle, record, state, native, message, capacity, size + ) + if record > 1: + warning_pending = False + return 100 + text = "count warning".encode("utf-16le") + assert capacity > len(text) // 2 + c.memmove(state, "01000\\0".encode("utf-16le"), 12) + c.memmove(message, text + b"\\0\\0", len(text) + 2) + c.cast(native, c.POINTER(c.c_int))[0] = 0 + c.cast(size, c.POINTER(short))[0] = len(text) // 2 + return 0 + except BaseException as error: + errors.append(type(error).__name__) + return -1 + + with mssql_python.connect(os.environ["DB_CONNECTION_STRING"]) as connection: + with connection.cursor() as cursor: + value_sql = ( + "CASE WHEN n = 2 THEN CAST(0x00D8 AS NVARCHAR(10)) " + "ELSE CAST(N'text' AS NVARCHAR(10)) END" if mode == "decode_error" else + "CAST(N'text' AS NVARCHAR(MAX))" if mode == "lob" else + "CAST(N'text' AS NVARCHAR(10))" if mode in ("mixed", "prefix") else + "n + 10" + ) + query = ( + f"SELECT n AS a, {value_sql} AS b FROM " + "(VALUES (1),(2),(3),(4),(5),(6)) v(n) ORDER BY n" + ) + cursor.execute(query) + saved_count, saved_diag = count_slot.value, diag_slot.value + assert saved_count and saved_diag + original_count, original_diag = count_type(saved_count), diag_type(saved_diag) + count_slot.value = c.cast(counted, pointer).value + diag_slot.value = c.cast(diagnostic, pointer).value + try: + expected = 1 + if mode == "prefix": + assert ddbc.DDBCSQLFetch(cursor.hstmt) == 0 + prefix = [] + assert ddbc.DDBCSQLGetData( + cursor.hstmt, 1, prefix, "utf-16le", "utf-16le", -8 + ) == 0 + assert prefix == [1] and calls == [] + expected = 2 + if mode == "count_error": + try: + cursor.fetchmany(1) + except mssql_python.DatabaseError: + pass + else: + raise AssertionError("count failure was not propagated") + assert len(calls) == 1 + calls.clear() + first = cursor.fetchmany(1)[0] + assert first[0] == expected and len(first) == 2 + assert first[1] == ( + "text" if mode in ("mixed", "prefix", "lob", "decode_error") + else expected + 10 + ) + # Cold eager count and DescribeColumns' independent count. + assert len(calls) == 2, (mode, calls) + if mode == "decode_error": + try: + cursor.fetchone() + except UnicodeDecodeError: + pass + else: + raise AssertionError("invalid UTF-16 must fail GetData decoding") + assert len(calls) == 2 + assert tuple(cursor.fetchmany(1)[0]) == (3, "text") + assert len(calls) == 4 # failure invalidated count and metadata + expected = 3 + cold = len(calls) + assert cursor.fetchmany(1)[0][0] == expected + 1 + assert len(calls) == cold + (1 if mode == "generation" else 0) + warm = len(calls) + assert cursor.fetchval() == expected + 2 + assert cursor.fetchmany(1)[0][0] == expected + 3 + assert len(calls) == warm + assert ddbc.DDBCSQLNumResultCols(cursor.hstmt) == 2 + assert ddbc.DDBCSQLNumResultCols(cursor.hstmt) == 2 + assert len(calls) == warm + 2 # public API is still uncached + assert not errors, errors + if mode == "warning": + assert cursor.messages == [("[01000] (0)", "count warning")] + else: + assert not cursor.messages + while cursor.fetchmany(1): + pass + assert len(calls) == warm + 2 # EOF does not reacquire the count + if mode == "nextset": + cursor.execute("SELECT 7 AS a; SELECT 8 AS a, 9 AS b") + calls.clear() + assert tuple(cursor.fetchmany(1)[0]) == (7,) + assert len(calls) == 2 + assert cursor.nextset() + calls.clear() + assert tuple(cursor.fetchmany(1)[0]) == (8, 9) + assert len(calls) == 2 + assert cursor.fetchmany(1) == [] + assert len(calls) == 2 + finally: + count_slot.value, diag_slot.value = saved_count, saved_diag + assert not errors, errors + """) + _run_fetch_script(conn_str, script, mode, timeout=60) diff --git a/tests/test_fetch_single_row.py b/tests/test_fetch_single_row.py deleted file mode 100644 index 50d45503c..000000000 --- a/tests/test_fetch_single_row.py +++ /dev/null @@ -1,979 +0,0 @@ -"""Shared single-row fetch contracts; native lifetime checks run in a child process.""" - -import os -import subprocess -import sys -import textwrap -from unittest.mock import Mock, patch - -import pytest - -import mssql_python -from mssql_python.cursor import Cursor - -NUMERIC_TYPES = ("INT", "SMALLINT", "BIGINT", "TINYINT", "BIT", "REAL", "FLOAT") - - -@pytest.mark.parametrize( - ("sql_type", "literal", "expected"), - ( - ("INT", "-2147483648", -2147483648), - ("INT", "2147483647", 2147483647), - ("SMALLINT", "-32768", -32768), - ("SMALLINT", "32767", 32767), - ("BIGINT", "-9223372036854775808", -9223372036854775808), - ("BIGINT", "9223372036854775807", 9223372036854775807), - ("TINYINT", "255", 255), - ("BIT", "0", False), - ("REAL", "-1.25", -1.25), - ("FLOAT", "1.7976931348623157E308", 1.7976931348623157e308), - ("FLOAT", "-2.2250738585072014E-308", -2.2250738585072014e-308), - ("FLOAT", "-0.0", 0.0), - ), - ids=( - "int_min", - "int_max", - "smallint_min", - "smallint_max", - "bigint_min", - "bigint_max", - "tinyint_max", - "bit_zero", - "real_negative", - "float_max", - "float_tiny", - "float_zero", - ), -) -def test_single_numeric_row_bound_path_parity(cursor, sql_type, literal, expected): - query = f"SELECT CAST({literal} AS {sql_type}) AS a, CAST(NULL AS {sql_type}) AS b" - cursor.execute(query) - bound = cursor.fetchmany(2)[0] - cursor.execute(query) - single = cursor.fetchmany(1)[0] - assert tuple(single) == tuple(bound) == (expected, None) - assert type(single[0]) is type(bound[0]) is type(expected) - - -@pytest.mark.parametrize("sql_type", NUMERIC_TYPES) -@pytest.mark.parametrize("method", ("fetchone", "iterator", "fetchmany", "fetchval")) -def test_numeric_single_row_values_and_eof(cursor, sql_type, method): - cursor.execute( - f"SELECT CAST(CASE WHEN n = 2 THEN NULL ELSE 1 END AS {sql_type}) AS value " - "FROM (VALUES (1), (2), (3)) AS v(n) ORDER BY n" - ) - - def fetch(): - if method == "fetchmany": - rows = cursor.fetchmany(1) - return rows[0][0] if rows else "EOF" - if method == "fetchval": - return cursor.fetchval() - row = next(cursor, None) if method == "iterator" else cursor.fetchone() - return row[0] if row is not None else "EOF" - - values = [fetch(), fetch(), fetch()] - expected = True if sql_type == "BIT" else 1.0 if sql_type in ("REAL", "FLOAT") else 1 - assert values == [expected, None, expected] - assert type(values[0]) is type(expected) - assert cursor.rowcount == 3 - assert cursor.rownumber == 2 - assert fetch() == (None if method == "fetchval" else "EOF") - assert cursor.rowcount == 3 - assert not cursor.messages - - -def test_single_row_converters_run_in_column_order(cursor): - events = [] - - def convert(value): - events.append(value) - if value == 2: - raise ValueError("keep original second column") - return value + 10 - - cursor.connection.add_output_converter(mssql_python.SQL_INTEGER, convert) - try: - cursor.execute("SELECT 1 AS a, 2 AS b, CAST(NULL AS INT) AS c") - assert cursor.fetchval() == 11 - assert events == [1, 2] - events.clear() - cursor.execute("SELECT 1 AS a, 2 AS b, CAST(NULL AS INT) AS c") - assert tuple(cursor.fetchmany(1)[0]) == (11, 2, None) - assert events == [1, 2] - finally: - cursor.connection.remove_output_converter(mssql_python.SQL_INTEGER) - - -def test_single_row_wrapper_does_not_enter_batch_factory(cursor): - from mssql_python import ddbc_bindings - - cursor.execute( - "SELECT n AS a, CAST(N'text' AS NVARCHAR(10)) AS b " - "FROM (VALUES (1),(2),(3),(4)) AS v(n) ORDER BY n" - ) - with patch.object(ddbc_bindings, "construct_rows", wraps=ddbc_bindings.construct_rows) as batch: - retained = cursor.fetchmany(1)[0] - assert tuple(retained) == (1, "text") - batch.assert_not_called() - assert [row[0] for row in cursor.fetchmany(2)] == [2, 3] - batch.assert_called_once() - assert cursor.fetchmany(2)[0][0] == 4 - assert batch.call_count == 2 - assert tuple(retained) == (1, "text") - - -@pytest.mark.parametrize("override_fast_create", (False, True)) -def test_fetchmany_preserves_substituted_row_class(cursor, override_fast_create): - import importlib - from mssql_python.row import Row - - class DerivedRow(Row): - pass - - def forbidden(*args): - raise AssertionError("batch wrapping must not call a substituted Row's factory") - - if override_fast_create: - DerivedRow._fast_create = staticmethod(forbidden) - cursor_module = importlib.import_module("mssql_python.cursor") - cursor.execute("SELECT 1 AS a") - with patch.object(cursor_module, "Row", DerivedRow): - row = cursor.fetchmany(1)[0] - assert type(row) is DerivedRow - assert row.a == 1 - - -def test_fetchmany_preserves_replaced_fast_factory(cursor): - from mssql_python.row import Row - - cursor.execute("SELECT 1 AS a") - with patch.object(Row, "_fast_create", side_effect=AssertionError("must use batch factory")): - assert cursor.fetchmany(1)[0].a == 1 - - -@pytest.mark.parametrize("via_arraysize", (False, True)) -@pytest.mark.parametrize("raises", (False, True)) -def test_fetchmany_size_subclass_equality_is_not_called(cursor, via_arraysize, raises): - from mssql_python import ddbc_bindings - - calls = [] - - class Size(int): - def __eq__(self, other): - calls.append(other) - if raises: - raise AssertionError("size equality must not run after native fetch") - return super().__eq__(other) - - cursor.execute("SELECT 1 AS a UNION ALL SELECT 2") - with patch.object(ddbc_bindings, "construct_rows", wraps=ddbc_bindings.construct_rows) as batch: - if via_arraysize: - cursor.arraysize = Size(1) - result = cursor.fetchmany() - else: - result = cursor.fetchmany(Size(1)) - assert result[0][0] == 1 - batch.assert_called_once() - assert calls == [] - assert cursor.fetchone()[0] == 2 - - -def test_real_subclass_and_instance_fetchone_overrides(cursor): - calls = [] - - class DerivedCursor(Cursor): - def fetchone(self): - calls.append("derived") - return (71, 72) - - with DerivedCursor(cursor.connection) as derived: - derived.execute("SELECT 1 AS a UNION ALL SELECT 2") - assert derived.fetchval() == 71 - assert next(derived) == (71, 72) - assert calls == ["derived", "derived"] - # fetchmany must not acquire fetchone's Python override semantics. - assert derived.fetchmany(1)[0][0] == 1 - assert calls == ["derived", "derived"] - with patch.object(derived, "fetchone", return_value=(81, 82)) as override: - assert derived.fetchval() == 81 - assert next(derived) == (81, 82) - assert derived.fetchmany(1)[0][0] == 2 - assert override.call_count == 2 - - -@pytest.mark.parametrize("enabled", (False, True)) -@pytest.mark.parametrize("method", ("fetchone", "fetchmany", "fetchval", "iterator")) -def test_single_row_python_phase_counts(cursor, enabled, method): - from mssql_python import perf_timer - - perf_timer.disable() - perf_timer.reset() - if enabled: - perf_timer.enable() - try: - cursor.execute("SELECT 1 AS a") - perf_timer.reset() - if method == "fetchmany": - assert cursor.fetchmany(1)[0][0] == 1 - elif method == "fetchval": - assert cursor.fetchval() == 1 - elif method == "iterator": - assert next(cursor)[0] == 1 - else: - assert cursor.fetchone()[0] == 1 - stats = perf_timer.get_stats() - prefix = "py::fetchmany" if method == "fetchmany" else "py::fetchone" - assert {name: item["calls"] for name, item in stats.items()} == ( - {prefix + "::cpp_call": 1, prefix + "::row_wrap": 1} if enabled else {} - ) - finally: - perf_timer.disable() - perf_timer.reset() - - -def test_single_row_native_transitions_in_subprocess(conn_str): - """Counters describe actual native work, not mocked Python bridge calls.""" - from mssql_python import ddbc_bindings - - if not hasattr(ddbc_bindings, "profiling"): - pytest.skip("requires a profiling-enabled native build") - script = textwrap.dedent(""" - import os - import mssql_python - from mssql_python import ddbc_bindings as ddbc - - p = ddbc.profiling - assert hasattr(p, "enable") - with mssql_python.connect(os.environ["DB_CONNECTION_STRING"]) as connection: - with connection.cursor() as cursor: - query = ( - "SELECT n AS a, n + 10 AS b FROM " - "(VALUES (1),(2),(3),(4),(5),(6),(7),(8)) AS v(n) ORDER BY n" - ) - cursor.execute(query) - p.reset() - p.enable() - retained = cursor.fetchone() - assert retained[0] == 1 - assert next(cursor)[0] == 2 - assert cursor.fetchval() == 3 - assert p.get_stats()["ddbc::FetchSingleRow::SQL_UNBIND"]["calls"] == 1 - assert cursor.fetchmany(1)[0][0] == 4 - assert "ddbc::SQLBindColums" not in p.get_stats() - assert p.get_stats()["ddbc::FetchMany::single_numeric_row"]["calls"] == 1 - # Cleanup deliberately forgets the unbound marker. - assert cursor.fetchone()[0] == 5 - assert p.get_stats()["ddbc::FetchSingleRow::SQL_UNBIND"]["calls"] == 2 - assert [r[0] for r in cursor.fetchmany(2)] == [6, 7] - assert p.get_stats()["ddbc::SQLBindColums"]["calls"] == 1 - assert cursor.fetchone()[0] == 8 - assert cursor.fetchone() is None - assert cursor.fetchone() is None - assert p.get_stats()["ddbc::FetchSingleRow::SQL_UNBIND"]["calls"] == 3 - assert tuple(retained) == (1, 11) - assert retained.a == 1 - assert not cursor.messages - - cursor.execute(query) - assert cursor.fetchone()[0] == 1 - assert p.get_stats()["ddbc::FetchSingleRow::SQL_UNBIND"]["calls"] == 4 - # Public statement-attribute changes must invalidate the marker. - assert ddbc.DDBCSQLSetStmtAttr(cursor.hstmt, 0, 0) == 0 # SQL_ATTR_QUERY_TIMEOUT - assert cursor.fetchone()[0] == 2 - assert p.get_stats()["ddbc::FetchSingleRow::SQL_UNBIND"]["calls"] == 5 - cursor.execute("SELECT 21 AS a; SELECT 22 AS a") - assert cursor.fetchone()[0] == 21 - assert cursor.nextset() - assert cursor.fetchone()[0] == 22 - assert p.get_stats()["ddbc::FetchSingleRow::SQL_UNBIND"]["calls"] == 7 - - cursor.execute("SELECT CAST(N'text' AS NVARCHAR(10)) AS a") - assert cursor.fetchmany(1)[0][0] == "text" - assert p.get_stats()["ddbc::SQLBindColums"]["calls"] == 2 - cursor.execute(query) - assert cursor.fetchone()[0] == 1 - cursor.skip(1) - before = p.get_stats()["ddbc::FetchSingleRow::SQL_UNBIND"]["calls"] - assert cursor.fetchone()[0] == 3 - assert p.get_stats()["ddbc::FetchSingleRow::SQL_UNBIND"]["calls"] == before + 1 - batch = cursor.arrow_batch(2) - assert batch.to_pydict() == {"a": [4, 5], "b": [14, 15]} - before = p.get_stats()["ddbc::FetchSingleRow::SQL_UNBIND"]["calls"] - assert cursor.fetchone()[0] == 6 - assert p.get_stats()["ddbc::FetchSingleRow::SQL_UNBIND"]["calls"] == before + 1 - assert batch.to_pydict() == {"a": [4, 5], "b": [14, 15]} - # Cancel invalidates the result generation; execute recovers the handle. - cursor.hstmt._cancel() - cursor.execute(query) - before = p.get_stats()["ddbc::FetchSingleRow::SQL_UNBIND"]["calls"] - assert cursor.fetchone()[0] == 1 - assert p.get_stats()["ddbc::FetchSingleRow::SQL_UNBIND"]["calls"] == before + 1 - cursor.close() - with connection.cursor() as replacement: - replacement.execute(query) - before = p.get_stats()["ddbc::FetchSingleRow::SQL_UNBIND"]["calls"] - assert replacement.fetchone()[0] == 1 - assert p.get_stats()["ddbc::FetchSingleRow::SQL_UNBIND"]["calls"] == before + 1 - p.disable() - """) - environment = os.environ.copy() - environment["DB_CONNECTION_STRING"] = conn_str - result = subprocess.run( - [sys.executable, "-c", script], - env=environment, - capture_output=True, - text=True, - timeout=45, - check=False, - ) - assert result.returncode == 0, result.stdout + result.stderr - - -@pytest.mark.skipif( - sys.platform == "win32", - reason="Windows does not export the native driver function-pointer globals", -) -@pytest.mark.parametrize("first_fetch", ("fetchone", "fetchmany")) -def test_count_generation_change_with_unbound_marker(conn_str, first_fetch): - """A count obtained across invalidation cannot authorize either cache.""" - script = textwrap.dedent(""" - import ctypes as c - import os - import sys - import mssql_python - from mssql_python import ddbc_bindings as ddbc - - library = c.CDLL(ddbc.module.__file__) - pointer = c.c_void_p - count_type = c.CFUNCTYPE(c.c_short, pointer, c.POINTER(c.c_short)) - unbind_type = c.CFUNCTYPE(c.c_short, pointer, c.c_ushort) - count_slot = pointer.in_dll(library, "SQLNumResultCols_ptr") - unbind_slot = pointer.in_dll(library, "SQLFreeStmt_ptr") - counts, unbinds, invalidations, callback_errors = [], [], [], [] - - @count_type - def counted(handle, value): - try: - counts.append(handle) - ret = original_count(handle, value) - if not invalidations: - invalidations.append(True) - # Change only the cache generation, not the result shape. - assert ddbc.DDBCSQLSetStmtAttr(cursor.hstmt, 0, 0) == 0 - return ret - except BaseException as error: - callback_errors.append(type(error).__name__) - return -1 - - @unbind_type - def unbound(handle, option): - try: - if option == 2: - unbinds.append(handle) - return original_unbind(handle, option) - except BaseException as error: - callback_errors.append(type(error).__name__) - return -1 - - with mssql_python.connect(os.environ["DB_CONNECTION_STRING"]) as connection: - with connection.cursor() as cursor: - cursor.execute( - "SELECT n AS a, n + 10 AS b FROM (VALUES (1),(2),(3),(4)) v(n) ORDER BY n" - ) - saved_count, saved_unbind = count_slot.value, unbind_slot.value - assert saved_count and saved_unbind - original_count = count_type(saved_count) - original_unbind = unbind_type(saved_unbind) - count_slot.value = c.cast(counted, pointer).value - unbind_slot.value = c.cast(unbound, pointer).value - try: - first = ( - cursor.fetchmany(1)[0] if sys.argv[1] == "fetchmany" - else cursor.fetchone() - ) - assert tuple(first) == (1, 11) - assert invalidations == [True] - before_count, before_unbind = len(counts), len(unbinds) - assert tuple(cursor.fetchone()) == (2, 12) - assert len(counts) == before_count + 1 - assert len(unbinds) == before_unbind + 1 - before_count, before_unbind = len(counts), len(unbinds) - assert cursor.fetchval() == 3 - assert tuple(next(cursor)) == (4, 14) - assert cursor.fetchone() is None - assert len(counts) == before_count - assert len(unbinds) == before_unbind - # Direct count calls remain uncached even after a warm fetch. - assert ddbc.DDBCSQLNumResultCols(cursor.hstmt) == 2 - assert ddbc.DDBCSQLNumResultCols(cursor.hstmt) == 2 - assert len(counts) == before_count + 2 - assert tuple(first) == (1, 11) - assert not callback_errors, callback_errors - assert not cursor.messages - finally: - count_slot.value, unbind_slot.value = saved_count, saved_unbind - cursor.execute("SELECT 42") - assert cursor.fetchval() == 42 - """) - environment = os.environ.copy() - environment["DB_CONNECTION_STRING"] = conn_str - result = subprocess.run( - [sys.executable, "-c", script, first_fetch], - env=environment, - capture_output=True, - text=True, - timeout=45, - check=False, - ) - assert result.returncode == 0, result.stdout + result.stderr - - -@pytest.mark.parametrize( - "failure", - ( - "unbind", - "unbind_warning", - "partial_bind", - "configure", - "cleanup", - "rows_fetched_cleanup", - "column_count", - "column_name", - "fetch", - "getdata", - ), -) -def test_single_row_native_failure_recovery_in_subprocess(conn_str, failure): - """Inject one native failure; restore the entry point before recovery.""" - from mssql_python import ddbc_bindings - - if sys.platform == "win32": - pytest.skip("native function-pointer globals are not exported on Windows") - if not hasattr(ddbc_bindings, "profiling"): - pytest.skip("requires a profiling-enabled native build") - script = textwrap.dedent(""" - import ctypes as c - import os - import sys - import mssql_python - from mssql_python import ddbc_bindings as ddbc - - failure = sys.argv[1] - library = c.CDLL(ddbc.module.__file__) - short, ushort, pointer, length = c.c_short, c.c_ushort, c.c_void_p, c.c_ssize_t - signatures = { - "unbind": ("SQLFreeStmt_ptr", (pointer, ushort)), - "unbind_warning": ("SQLFreeStmt_ptr", (pointer, ushort)), - "partial_bind": ("SQLBindCol_ptr", (pointer, ushort, short, pointer, length, pointer)), - "configure": ("SQLSetStmtAttr_ptr", (pointer, c.c_int, pointer, c.c_int)), - "cleanup": ("SQLSetStmtAttr_ptr", (pointer, c.c_int, pointer, c.c_int)), - "rows_fetched_cleanup": ("SQLSetStmtAttr_ptr", (pointer, c.c_int, pointer, c.c_int)), - "column_count": ("SQLNumResultCols_ptr", (pointer, pointer)), - "fetch": ("SQLFetchScroll_ptr", (pointer, short, length)), - "getdata": ("SQLGetData_ptr", (pointer, ushort, short, pointer, length, pointer)), - "column_name": ("SQLDescribeCol_ptr", - (pointer, ushort, pointer, short, pointer, pointer, - pointer, pointer, pointer)), - } - symbol, arguments = signatures[failure] - prototype = c.CFUNCTYPE(short, *arguments) - slot = pointer.in_dll(library, symbol) - calls = [] - fired = [] - callback_errors = [] - - def inject(*args): - calls.append(args) - hit = not fired - if failure == "partial_bind": - hit = hit and args[1] == 2 - elif failure in ("unbind", "unbind_warning"): - hit = hit and args[1] == 2 # SQL_UNBIND - elif failure == "rows_fetched_cleanup": - matching = [item for item in calls if item[1] == 26] - hit = hit and args[1] == 26 and len(matching) == 2 - elif failure in ("configure", "cleanup"): - matching = [item for item in calls if item[1] == 27] - hit = hit and args[1] == 27 and len(matching) == ( - 1 if failure == "configure" else 2 - ) - if hit: - fired.append(True) - if failure == "unbind_warning": - result = original(*args) - assert result == 0 - return 1 # SQL_SUCCESS_WITH_INFO after a real successful unbind - if failure == "column_name": - result = original(*args) - # Invalid UTF-16 must fail eager name validation before fetch. - c.cast(args[2], c.POINTER(ushort))[0] = 0xD800 - c.cast(args[4], c.POINTER(short))[0] = 1 - return result - return -1 # SQL_ERROR - return original(*args) - - @prototype - def injected(*args): - try: - return inject(*args) - except BaseException as error: - callback_errors.append(type(error).__name__) - return -1 - - diag_slot = pointer.in_dll(library, "SQLGetDiagRec_ptr") - diag_type = c.CFUNCTYPE( - short, short, pointer, short, pointer, pointer, pointer, short, pointer - ) - - @diag_type - def warning(handle_type, handle, record, state, native, message, capacity, size): - try: - if record > 1: - return 100 - text = "unbind warning".encode("utf-16le") - assert capacity > len(text) // 2 - c.memmove(state, "01000\\0".encode("utf-16le"), 12) - c.memmove(message, text + b"\\0\\0", len(text) + 2) - c.cast(native, c.POINTER(c.c_int))[0] = 0 - c.cast(size, c.POINTER(short))[0] = len(text) // 2 - return 0 - except BaseException as error: - callback_errors.append(type(error).__name__) - return -1 - - with mssql_python.connect(os.environ["DB_CONNECTION_STRING"]) as connection: - with connection.cursor() as cursor: - cursor.execute( - "SELECT n AS a, n + 10 AS b FROM (VALUES (1),(2),(3),(4)) v(n) ORDER BY n" - ) - # The bundled driver resolves function pointers lazily on connect. - saved, saved_diag = slot.value, diag_slot.value - assert saved and saved_diag - original = prototype(saved) - ddbc.profiling.reset() - ddbc.profiling.enable() - slot.value = c.cast(injected, pointer).value - if failure == "unbind_warning": - diag_slot.value = c.cast(warning, pointer).value - try: - if failure == "unbind_warning": - assert cursor.fetchone()[0] == 1 - assert cursor.messages == [("[01000] (0)", "unbind warning")] - assert cursor.fetchone()[0] == 2 - assert len(calls) == 1 - assert cursor.messages == [("[01000] (0)", "unbind warning")] - else: - try: - if failure == "unbind": - cursor.fetchone() - else: - cursor.fetchmany(2 if failure == "partial_bind" else 1) - except (mssql_python.Error, UnicodeError, RuntimeError) as error: - if failure == "partial_bind": - assert type(error) is RuntimeError - assert str(error) == ( - "Failed to bind column - b, Type - 4, column ID - 2" - ) - elif failure == "column_name": - assert isinstance(error, UnicodeError) - else: - assert isinstance(error, mssql_python.Error) - else: - raise AssertionError("injected failure was not propagated") - finally: - slot.value = saved - diag_slot.value = saved_diag - assert fired == [True], (failure, fired) - assert not callback_errors, callback_errors - # GetData and cleanup failures happen after consuming the row. - expected = ( - 3 if failure == "unbind_warning" else - 2 if failure in ("cleanup", "rows_fetched_cleanup", "getdata") else 1 - ) - assert cursor.fetchone()[0] == expected, failure - before = ddbc.profiling.get_stats()["ddbc::FetchSingleRow::SQL_UNBIND"]["calls"] - assert cursor.fetchone()[0] == expected + 1 - assert ddbc.profiling.get_stats()["ddbc::FetchSingleRow::SQL_UNBIND"]["calls"] == before - ddbc.profiling.disable() - """) - environment = os.environ.copy() - environment["DB_CONNECTION_STRING"] = conn_str - result = subprocess.run( - [sys.executable, "-c", script, failure], - env=environment, - capture_output=True, - text=True, - timeout=45, - check=False, - ) - assert result.returncode == 0, result.stdout + result.stderr - - -@pytest.mark.parametrize("mutation", ("new", "setattr", "code", "abstract", "descriptor")) -def test_fast_row_inplace_customization(monkeypatch, mutation): - import weakref - from mssql_python import ddbc_bindings - from mssql_python.row import Row - - factory = Row._fast_create - events = [] - - def custom_new(cls): - events.append("new") - return object.__new__(cls) - - def custom_setattr(self, name, value): - events.append(name) - object.__setattr__(self, name, value) - - def custom_factory(values, column_map, cursor, column_map_lower=None, column_names=None): - raise RuntimeError("customized factory code") - - def fail_descriptor(self, value): - events.append(weakref.ref(self)) - raise RuntimeError("customized descriptor") - - if mutation == "new": - monkeypatch.setattr(Row, "__new__", staticmethod(custom_new)) - elif mutation == "setattr": - monkeypatch.setattr(Row, "__setattr__", custom_setattr) - elif mutation == "code": - monkeypatch.setattr(factory, "__code__", custom_factory.__code__) - elif mutation == "abstract": - monkeypatch.setattr(Row, "__abstractmethods__", frozenset({"required"}), raising=False) - else: - monkeypatch.setattr(Row, "_column_names", property(fset=fail_descriptor)) - - assert Row._fast_create is factory - with patch.object(ddbc_bindings, "construct_row", wraps=ddbc_bindings.construct_row) as native: - if mutation in ("code", "descriptor"): - with pytest.raises(RuntimeError, match="customized"): - factory([42], {"number": 0}, None) - elif mutation == "abstract": - with pytest.raises(TypeError, match="abstract"): - factory([42], {"number": 0}, None) - else: - assert factory([42], {"number": 0}, None).number == 42 - native.assert_not_called() - if mutation == "new": - assert events == ["new"] - elif mutation == "setattr": - assert events == ["_values", "_column_map", "_cursor", "_column_map_lower", "_column_names"] - elif mutation == "descriptor": - assert len(events) == 1 and events[0]() is None - - -@pytest.mark.parametrize("method", ("fetchone", "fetchmany", "fetchval")) -@pytest.mark.parametrize("failure", ("maps", "factory")) -def test_single_row_construction_failure_keeps_fetch_position(cursor, method, failure): - from mssql_python import ddbc_bindings - - cursor.execute("SELECT n AS number FROM (VALUES (1), (2)) AS v(n) ORDER BY n") - from mssql_python.row import Row - - bridge_name = "DDBCSQLFetchRow" if method == "fetchmany" else "DDBCSQLFetchOne" - bridge = getattr(ddbc_bindings, bridge_name) - - def fetch(): - value = cursor.fetchmany(1) if method == "fetchmany" else getattr(cursor, method)() - return value[0][0] if method == "fetchmany" else value if method == "fetchval" else value[0] - - def fail(*args): - assert cursor.rowcount == 1 - assert cursor.rownumber == 0 - assert cursor._next_row_index == 1 - raise RuntimeError("injected post-fetch failure") - - failed_stage = Mock(side_effect=fail) - failure_patch = ( - patch.object(cursor, "_get_column_and_converter_maps", failed_stage) - if failure == "maps" - else patch.object(Row, "_column_names", property(fset=failed_stage)) - ) - with patch.object(ddbc_bindings, bridge_name, wraps=bridge) as native_fetch, failure_patch: - with pytest.raises(RuntimeError, match="injected post-fetch failure"): - fetch() - native_fetch.assert_called_once() - failed_stage.assert_called_once() - assert fetch() == 2 - assert cursor.rowcount == 2 - assert cursor.rownumber == 1 - - -@pytest.mark.parametrize("target", ("__new__", "__setattr__")) -def test_native_row_guard_does_not_invoke_descriptors(monkeypatch, target): - from mssql_python.cursor import _native_row_eligible - from mssql_python.row import Row - - events = [] - - class Descriptor: - def __get__(self, instance, owner): - events.append("lookup") - if target == "__new__": - if len(events) > 1: - raise RuntimeError("duplicate allocator lookup") - - def allocate(cls): - events.append("allocate") - return object.__new__(cls) - - return allocate - return lambda name, value: object.__setattr__(instance, name, value) - - monkeypatch.setattr(Row, target, Descriptor()) - assert not _native_row_eligible(Row) - assert events == [] - row = Row._fast_create([42], {"number": 0}, None) - assert row.number == 42 - assert events == (["lookup", "allocate"] if target == "__new__" else ["lookup"] * 5) - - -def test_native_row_guard_does_not_invoke_metaclass_hooks(): - from mssql_python.cursor import _native_row_eligible - from mssql_python.row import Row - - class Meta(type): - def __getattribute__(cls, name): - raise AssertionError("guard must not inspect substituted class through metaclass") - - class CustomRow(Row, metaclass=Meta): - pass - - assert not _native_row_eligible(CustomRow) - - -@pytest.mark.parametrize("method", ("fetchone", "fetchmany", "fetchval")) -@pytest.mark.parametrize("when", ("maps", "final_argument")) -def test_single_row_fusion_handles_post_fetch_factory_change(cursor, method, when): - from mssql_python import ddbc_bindings - from mssql_python.row import Row - - original_maps = cursor._get_column_and_converter_maps - factory = Row._fast_create - - def replacement(values, column_map, cursor, column_map_lower=None, column_names=None): - raise RuntimeError("factory changed after native advancement") - - def maps(): - factory.__code__ = replacement.__code__ - return original_maps() - - def names(self): - factory.__code__ = replacement.__code__ - return self.__dict__["_cached_result_columns"] - - cursor.execute("SELECT n AS number FROM (VALUES (1), (2)) AS v(n) ORDER BY n") - cursor._get_column_and_converter_maps() - change = ( - patch.object(cursor, "_get_column_and_converter_maps", side_effect=maps) - if when == "maps" - else patch.object(type(cursor), "_cached_result_columns", property(names), create=True) - ) - old_code = factory.__code__ - try: - with ( - change, - patch.object( - ddbc_bindings, "DDBCSQLFetchRow", wraps=ddbc_bindings.DDBCSQLFetchRow - ) as fused, - ): - with pytest.raises(RuntimeError, match="factory changed after native advancement"): - cursor.fetchmany(1) if method == "fetchmany" else getattr(cursor, method)() - assert fused.call_count == (1 if method == "fetchmany" else 0) - assert Row._fast_create is factory - assert cursor.rowcount == 1 and cursor.rownumber == 0 - finally: - factory.__code__ = old_code - assert cursor.fetchone().number == 2 - - -@pytest.mark.parametrize("method", ("fetchone", "fetchmany", "fetchval")) -def test_single_row_fusion_uses_native_constructor(cursor, method): - from mssql_python import ddbc_bindings, perf_timer - from mssql_python.row import Row - - cursor.execute("SELECT 42 AS number, CAST(N'text' AS NVARCHAR(10)) AS label") - factory_code = Row._fast_create.__code__ - calls = [] - completion = cursor._finish_fetchmany if method == "fetchmany" else cursor._finish_fetchone - native_flags = [] - - def finish(ret, data, native=False): - native_flags.append(native) - return completion(ret, data, native) - - previous_profile = sys.getprofile() - phases_enabled = perf_timer.is_enabled() - - def profile(frame, event, arg): - if event == "call" and frame.f_code is factory_code: - calls.append(event) - - perf_timer.disable() - try: - with ( - patch.object( - ddbc_bindings, "DDBCSQLFetchRow", wraps=ddbc_bindings.DDBCSQLFetchRow - ) as fused, - patch.object( - cursor, - "_finish_fetchmany" if method == "fetchmany" else "_finish_fetchone", - side_effect=finish, - ), - ): - sys.setprofile(profile) - try: - result = cursor.fetchmany(1) if method == "fetchmany" else getattr(cursor, method)() - finally: - sys.setprofile(previous_profile) - assert fused.call_count == (1 if method == "fetchmany" else 0) - if method == "fetchval": - assert result == 42 - else: - row = result[0] if method == "fetchmany" else result - assert type(row) is Row - assert tuple(row) == (42, "text") - assert calls == ([] if method == "fetchmany" else ["call"]) - assert native_flags == [method == "fetchmany"] - assert cursor.rowcount == 1 and cursor.rownumber == 0 - finally: - if phases_enabled: - perf_timer.enable() - - -@pytest.mark.parametrize("method", ("fetchone", "fetchmany", "fetchval")) -def test_single_row_late_allocator_preserves_factory_global_lookup(cursor, method): - from mssql_python import ddbc_bindings - from mssql_python.row import Row - - original_row = Row - - class ChangedRow(original_row): - pass - - events = [] - factory = original_row._fast_create - assert "__new__" not in vars(original_row) - - class Allocator: - def __get__(self, instance, owner): - events.append("new_lookup") - factory.__globals__["Row"] = ChangedRow - - def allocate(cls): - events.append("allocate:" + cls.__name__) - return object.__new__(cls) - - return allocate - - def names(self): - events.append("names_lookup") - original_row.__new__ = Allocator() - return self.__dict__["_cached_result_columns"] - - cursor.execute("SELECT n AS number FROM (VALUES (1), (2)) AS v(n) ORDER BY n") - cursor._get_column_and_converter_maps() - try: - with ( - patch.object(type(cursor), "_cached_result_columns", property(names), create=True), - patch.object( - ddbc_bindings, "DDBCSQLFetchRow", wraps=ddbc_bindings.DDBCSQLFetchRow - ) as fused, - ): - result = cursor.fetchmany(1) if method == "fetchmany" else getattr(cursor, method)() - assert fused.call_count == (1 if method == "fetchmany" else 0) - if method == "fetchval": - assert result == 1 - else: - row = result[0] if method == "fetchmany" else result - assert type(row) is ChangedRow - assert row.number == 1 - assert events == ["names_lookup", "new_lookup", "allocate:ChangedRow"] - assert cursor.rowcount == 1 and cursor.rownumber == 0 - finally: - factory.__globals__["Row"] = original_row - if "__new__" in vars(original_row): - delattr(original_row, "__new__") - assert cursor.fetchone().number == 2 - - -@pytest.mark.parametrize("method", ("fetchone", "fetchmany", "fetchval")) -def test_single_row_fusion_native_counters_in_subprocess(conn_str, method): - from mssql_python import ddbc_bindings - - if not hasattr(ddbc_bindings, "profiling"): - pytest.skip("requires a profiling-enabled native build; Python phases remain disabled") - script = textwrap.dedent(""" - import os - import sys - import mssql_python - from mssql_python import ddbc_bindings as ddbc, perf_timer - - method = sys.argv[1] - p = ddbc.profiling - perf_timer.disable() - perf_timer.reset() - p.disable() - query = ( - "SELECT n AS a, n + 10 AS b, CAST(N'text' AS NVARCHAR(10)) AS label " - "FROM (VALUES (1), (2)) AS v(n) ORDER BY n" - ) - fetch_timer = "ddbc::FetchMany_wrap" if method == "fetchmany" else "ddbc::FetchOne_wrap" - - def calls(name): - return p.get_stats().get(name, {}).get("calls", 0) - - with mssql_python.connect(os.environ["DB_CONNECTION_STRING"]) as connection: - with connection.cursor() as cursor: - def fetch(): - if method == "fetchval": - return cursor.fetchval() - result = cursor.fetchmany(1) if method == "fetchmany" else cursor.fetchone() - row = result[0] if result and method == "fetchmany" else result - return tuple(row) if row else None - - try: - cursor.execute(query) - p.reset() - p.enable() - assert fetch() == (1 if method == "fetchval" else (1, 11, "text")) - assert calls("ddbc::FetchRow::construct_row") == (1 if method == "fetchmany" else 0) - assert calls(fetch_timer) == 1 - assert fetch() == (2 if method == "fetchval" else (2, 12, "text")) - assert fetch() is None - assert fetch() is None - assert calls("ddbc::FetchRow::construct_row") == (2 if method == "fetchmany" else 0) - assert calls(fetch_timer) == 4 - assert perf_timer.get_stats() == {} - assert cursor.rowcount == 2 and cursor.rownumber == 1 - - p.disable() - converted = [] - def convert(value): - converted.append(value) - return value + 100 - connection.add_output_converter(mssql_python.SQL_INTEGER, convert) - cursor.execute(query) - p.reset() - p.enable() - assert fetch() == (101 if method == "fetchval" else (101, 111, "text")) - assert converted == [1, 11] - assert calls("ddbc::FetchRow::construct_row") == 0 - assert calls(fetch_timer) == 1 - assert perf_timer.get_stats() == {} - finally: - p.disable() - """) - environment = os.environ.copy() - environment["DB_CONNECTION_STRING"] = conn_str - result = subprocess.run( - [sys.executable, "-c", script, method], - env=environment, - capture_output=True, - text=True, - timeout=45, - check=False, - ) - assert result.returncode == 0, result.stdout + result.stderr diff --git a/tests/test_fetch_single_row_construction.py b/tests/test_fetch_single_row_construction.py deleted file mode 100644 index 7386c9992..000000000 --- a/tests/test_fetch_single_row_construction.py +++ /dev/null @@ -1,444 +0,0 @@ -"""Count reuse and checked row-wise construction; driver faults stay in subprocesses.""" - -import datetime -import decimal -import gc -import os -import subprocess -import sys -import textwrap -import weakref - -import pytest - -from mssql_python import ddbc_bindings as ddbc - - -def test_getdata_appends_to_existing_list_without_python_append(cursor): - class Destination(list): - def append(self, value): - raise AssertionError("native append must not dispatch to list overrides") - - class Marker: - pass - - marker = Marker() - reference = weakref.ref(marker) - destination = Destination([marker]) - alias = destination - cursor.execute( - "SELECT CAST(-2147483648 AS INT), CAST(-32768 AS SMALLINT), " - "CAST(-9223372036854775808 AS BIGINT), CAST(255 AS TINYINT), " - "CAST(1 AS BIT), CAST(-1.25 AS REAL), CAST(1.5 AS FLOAT), " - "CAST(NULL AS INT), CAST(NULL AS SMALLINT), CAST(NULL AS BIGINT), " - "CAST(NULL AS TINYINT), CAST(NULL AS BIT), CAST(NULL AS REAL), " - "CAST(NULL AS FLOAT), CAST(12.50 AS DECIMAL(5,2)), " - "CAST('2024-02-29' AS DATE), CAST(0x0001FF AS VARBINARY(3))" - ) - assert ddbc.DDBCSQLFetch(cursor.hstmt) == 0 - assert ddbc.DDBCSQLGetData(cursor.hstmt, 17, destination, "utf-16le", "utf-16le", -8) == 0 - expected = [ - -2147483648, - -32768, - -9223372036854775808, - 255, - True, - -1.25, - 1.5, - *([None] * 7), - decimal.Decimal("12.50"), - datetime.date(2024, 2, 29), - b"\x00\x01\xff", - ] - assert destination is alias and destination[0] is marker - assert destination[1:] == expected - assert list(map(type, destination[1:])) == list(map(type, expected)) - del marker, destination, alias - gc.collect() - assert reference() is None - cursor.execute("SELECT 42") - assert cursor.fetchval() == 42 - - -@pytest.mark.parametrize("size", (1, 2)) -def test_getdata_decode_error_keeps_preexisting_and_completed_cells(cursor, size): - cursor.execute("SELECT 7, CAST(0x00D8 AS NVARCHAR(1))") - assert ddbc.DDBCSQLFetch(cursor.hstmt) == 0 - marker = object() - row = [marker] - if size == 1: - assert ddbc.DDBCSQLGetData(cursor.hstmt, size, row, "utf-16le", "utf-16le", -8) == 0 - else: - with pytest.raises(UnicodeDecodeError): - ddbc.DDBCSQLGetData(cursor.hstmt, size, row, "utf-16le", "utf-16le", -8) - assert row == [marker, 7] - cursor.execute("SELECT 42") - assert cursor.fetchval() == 42 - - -@pytest.mark.skipif( - sys.platform == "win32", - reason="Windows does not export the native driver function-pointer globals", -) -@pytest.mark.parametrize( - "mode", - ( - "numeric", - "mixed", - "lob", - "prefix", - "generation", - "count_error", - "nextset", - "warning", - "decode_error", - ), -) -def test_fetchmany_one_full_count_calls_in_subprocess(conn_str, mode): - script = textwrap.dedent(""" - import ctypes as c - import os - import sys - import mssql_python - from mssql_python import ddbc_bindings as ddbc - - mode = sys.argv[1] - pointer, short, ushort = c.c_void_p, c.c_short, c.c_ushort - count_type = c.CFUNCTYPE(short, pointer, c.POINTER(short)) - diag_type = c.CFUNCTYPE( - short, short, pointer, short, pointer, pointer, pointer, short, pointer - ) - library = c.CDLL(ddbc.module.__file__) - count_slot = pointer.in_dll(library, "SQLNumResultCols_ptr") - diag_slot = pointer.in_dll(library, "SQLGetDiagRec_ptr") - calls, errors, injected = [], [], [] - warning_pending = False - - @count_type - def counted(handle, count): - global warning_pending - try: - calls.append(handle) - if mode == "count_error" and not injected: - injected.append(True) - return -1 - result = original_count(handle, count) - if mode == "generation" and not injected: - injected.append(True) - assert ddbc.DDBCSQLSetStmtAttr(cursor.hstmt, 0, 0) == 0 - if mode == "warning" and not injected: - injected.append(True) - warning_pending = True - return 1 - return result - except BaseException as error: - errors.append(type(error).__name__) - return -1 - - @diag_type - def diagnostic(handle_type, handle, record, state, native, message, capacity, size): - global warning_pending - try: - if not warning_pending: - return original_diag( - handle_type, handle, record, state, native, message, capacity, size - ) - if record > 1: - warning_pending = False - return 100 - text = "count warning".encode("utf-16le") - assert capacity > len(text) // 2 - c.memmove(state, "01000\\0".encode("utf-16le"), 12) - c.memmove(message, text + b"\\0\\0", len(text) + 2) - c.cast(native, c.POINTER(c.c_int))[0] = 0 - c.cast(size, c.POINTER(short))[0] = len(text) // 2 - return 0 - except BaseException as error: - errors.append(type(error).__name__) - return -1 - - with mssql_python.connect(os.environ["DB_CONNECTION_STRING"]) as connection: - with connection.cursor() as cursor: - value_sql = ( - "CASE WHEN n = 2 THEN CAST(0x00D8 AS NVARCHAR(10)) " - "ELSE CAST(N'text' AS NVARCHAR(10)) END" if mode == "decode_error" else - "CAST(N'text' AS NVARCHAR(MAX))" if mode == "lob" else - "CAST(N'text' AS NVARCHAR(10))" if mode in ("mixed", "prefix") else - "n + 10" - ) - query = ( - f"SELECT n AS a, {value_sql} AS b FROM " - "(VALUES (1),(2),(3),(4),(5),(6)) v(n) ORDER BY n" - ) - cursor.execute(query) - saved_count, saved_diag = count_slot.value, diag_slot.value - assert saved_count and saved_diag - original_count, original_diag = count_type(saved_count), diag_type(saved_diag) - count_slot.value = c.cast(counted, pointer).value - diag_slot.value = c.cast(diagnostic, pointer).value - try: - expected = 1 - if mode == "prefix": - assert ddbc.DDBCSQLFetch(cursor.hstmt) == 0 - prefix = [] - assert ddbc.DDBCSQLGetData( - cursor.hstmt, 1, prefix, "utf-16le", "utf-16le", -8 - ) == 0 - assert prefix == [1] and calls == [] - expected = 2 - if mode == "count_error": - try: - cursor.fetchmany(1) - except mssql_python.DatabaseError: - pass - else: - raise AssertionError("count failure was not propagated") - assert len(calls) == 1 - calls.clear() - first = cursor.fetchmany(1)[0] - assert first[0] == expected and len(first) == 2 - assert first[1] == ( - "text" if mode in ("mixed", "prefix", "lob", "decode_error") - else expected + 10 - ) - # Cold eager count and DescribeColumns' independent count. - assert len(calls) == 2, (mode, calls) - if mode == "decode_error": - try: - cursor.fetchone() - except UnicodeDecodeError: - pass - else: - raise AssertionError("invalid UTF-16 must fail GetData decoding") - assert len(calls) == 2 - assert tuple(cursor.fetchmany(1)[0]) == (3, "text") - assert len(calls) == 4 # failure invalidated count and metadata - expected = 3 - cold = len(calls) - assert cursor.fetchmany(1)[0][0] == expected + 1 - assert len(calls) == cold + (1 if mode == "generation" else 0) - warm = len(calls) - assert cursor.fetchval() == expected + 2 - assert cursor.fetchmany(1)[0][0] == expected + 3 - assert len(calls) == warm - assert ddbc.DDBCSQLNumResultCols(cursor.hstmt) == 2 - assert ddbc.DDBCSQLNumResultCols(cursor.hstmt) == 2 - assert len(calls) == warm + 2 # public API is still uncached - assert not errors, errors - if mode == "warning": - assert cursor.messages == [("[01000] (0)", "count warning")] - else: - assert not cursor.messages - while cursor.fetchmany(1): - pass - assert len(calls) == warm + 2 # EOF does not reacquire the count - if mode == "nextset": - cursor.execute("SELECT 7 AS a; SELECT 8 AS a, 9 AS b") - calls.clear() - assert tuple(cursor.fetchmany(1)[0]) == (7,) - assert len(calls) == 2 - assert cursor.nextset() - calls.clear() - assert tuple(cursor.fetchmany(1)[0]) == (8, 9) - assert len(calls) == 2 - assert cursor.fetchmany(1) == [] - assert len(calls) == 2 - finally: - count_slot.value, diag_slot.value = saved_count, saved_diag - assert not errors, errors - """) - result = subprocess.run( - [sys.executable, "-c", script, mode], - env={**os.environ, "DB_CONNECTION_STRING": conn_str}, - capture_output=True, - text=True, - timeout=60, - check=False, - ) - assert result.returncode == 0, result.stdout + result.stderr - - -@pytest.mark.skipif( - sys.platform == "win32", - reason="Windows does not export the native driver function-pointer globals", -) -@pytest.mark.parametrize("payload", ("utf8", "empty", "null", "invalid", "warning")) -def test_narrow_getdata_decoding_in_subprocess(conn_str, payload): - script = textwrap.dedent(""" - import ctypes as c - import os - import sys - import mssql_python - from mssql_python import ddbc_bindings as ddbc - - mode = sys.argv[1] - data = b"\\xff" if mode == "invalid" else ( - b"" if mode == "empty" else "\\ufeffA\\x00\\U0001f600".encode("utf-8") - ) - pointer, short, length = c.c_void_p, c.c_short, c.c_ssize_t - get_type = c.CFUNCTYPE(short, pointer, c.c_ushort, short, pointer, length, pointer) - diag_type = c.CFUNCTYPE( - short, short, pointer, short, pointer, pointer, pointer, short, pointer - ) - library = c.CDLL(ddbc.module.__file__) - slot = pointer.in_dll(library, "SQLGetData_ptr") - diag_slot = pointer.in_dll(library, "SQLGetDiagRec_ptr") - calls, errors = [], [] - - @diag_type - def diagnostic(handle_type, handle, record, state, native, message, capacity, size): - try: - if record > 1: - return 100 - text = "getdata warning".encode("utf-16le") - assert capacity > len(text) // 2 - c.memmove(state, "01000\\0".encode("utf-16le"), 12) - c.memmove(message, text + b"\\0\\0", len(text) + 2) - c.cast(native, c.POINTER(c.c_int))[0] = 0 - c.cast(size, c.POINTER(short))[0] = len(text) // 2 - return 0 - except BaseException as error: - errors.append(type(error).__name__) - return -1 - - @get_type - def getdata(handle, column, ctype, buffer, capacity, indicator): - try: - assert ctype == 1 # SQL_C_CHAR - calls.append(column) - result = original(handle, column, ctype, buffer, capacity, indicator) - assert result == 0 and capacity > len(data) - c.memmove(buffer, data + b"\\0", len(data) + 1) - c.cast(indicator, c.POINTER(length))[0] = -1 if mode == "null" else len(data) - return 1 if mode == "warning" else result - except BaseException as error: - errors.append(type(error).__name__) - return -1 - - with mssql_python.connect(os.environ["DB_CONNECTION_STRING"]) as connection: - with connection.cursor() as cursor: - cursor.execute("SELECT CAST('text' AS VARCHAR(30))") - saved, saved_diag = slot.value, diag_slot.value - assert saved and saved_diag - original = get_type(saved) - assert ddbc.DDBCSQLFetch(cursor.hstmt) == 0 - marker = object() - row = [marker] - slot.value = c.cast(getdata, pointer).value - if mode == "warning": - diag_slot.value = c.cast(diagnostic, pointer).value - try: - ret = ddbc.DDBCSQLGetData( - cursor.hstmt, 1, row, "utf-8", "utf-16le", 1, cursor.messages - ) - assert ret == (1 if mode == "warning" else 0) - finally: - slot.value, diag_slot.value = saved, saved_diag - expected = None if mode == "null" else ( - data if mode == "invalid" else data.decode("utf-8") - ) - assert row == [marker, expected] - assert type(row[1]) is type(expected) - assert calls == [1] and not errors, (calls, errors) - assert cursor.messages == ( - [("[01000] (0)", "getdata warning")] if mode == "warning" else [] - ) - cursor.execute("SELECT 42") - assert cursor.fetchval() == 42 - """) - result = subprocess.run( - [sys.executable, "-c", script, payload], - env={**os.environ, "DB_CONNECTION_STRING": conn_str}, - capture_output=True, - text=True, - timeout=45, - check=False, - ) - assert result.returncode == 0, result.stdout + result.stderr - - -@pytest.mark.skipif( - sys.platform != "win32", - reason="Unix narrow fetches intentionally decode the driver's UTF-8, not a custom codec", -) -@pytest.mark.parametrize("failure", ("memory", "codec", "embedded_nul", "len_error", "len_quiet")) -def test_narrow_custom_codec_failure_in_subprocess(conn_str, failure): - script = textwrap.dedent(""" - import codecs - import os - import sys - import mssql_python - from mssql_python import ddbc_bindings as ddbc - from mssql_python.logging import logger - - calls = [] - length_calls = [] - saved_level = logger._cached_level - - class Decoded(str): - def __len__(self): - length_calls.append(True) - raise ValueError("injected decoded length failure") - - def decode(data, errors="strict"): - calls.append(bytes(data)) - if sys.argv[1] == "memory": - raise MemoryError("injected codec allocation failure") - if sys.argv[1] in ("len_error", "len_quiet"): - return Decoded("text"), len(data) - raise ValueError("injected codec failure") - - def search(name): - if name == "mssql_fetch_test_codec": - return codecs.CodecInfo(name=name, encode=None, decode=decode) - return None - - codecs.register(search) - try: - with mssql_python.connect(os.environ["DB_CONNECTION_STRING"]) as connection: - with connection.cursor() as cursor: - cursor.execute("SELECT 7, CAST('text' AS VARCHAR(10))") - assert ddbc.DDBCSQLFetch(cursor.hstmt) == 0 - marker = object() - row = [marker] - encoding = ( - "utf-8\\0suffix" if sys.argv[1] == "embedded_nul" - else "mssql_fetch_test_codec" - ) - if sys.argv[1] in ("len_error", "len_quiet"): - # Native LOG arguments are evaluated only at DEBUG. - ddbc.update_log_level(10 if sys.argv[1] == "len_error" else 50) - try: - result = ddbc.DDBCSQLGetData( - cursor.hstmt, 2, row, encoding, "utf-16le", 1 - ) - except MemoryError: - assert sys.argv[1] == "memory" - assert row == [marker, 7] - else: - assert result == 0 - if sys.argv[1] in ("len_error", "len_quiet"): - assert type(row[2]) is Decoded - assert row[:3] == [marker, 7, "text"] - assert row[3:] == ([b"text"] if sys.argv[1] == "len_error" else []) - assert length_calls == ([True] if sys.argv[1] == "len_error" else []) - else: - assert sys.argv[1] in ("codec", "embedded_nul") - assert row == [marker, 7, b"text"] - assert type(row[2]) is bytes - assert calls == ([] if sys.argv[1] == "embedded_nul" else [b"text"]) - cursor.execute("SELECT 42") - assert cursor.fetchval() == 42 - finally: - ddbc.update_log_level(saved_level) - codecs.unregister(search) - """) - result = subprocess.run( - [sys.executable, "-c", script, failure], - env={**os.environ, "DB_CONNECTION_STRING": conn_str}, - capture_output=True, - text=True, - timeout=45, - check=False, - ) - assert result.returncode == 0, result.stdout + result.stderr From 1468a231315c98cec4289fd7bba52e450a087f80 Mon Sep 17 00:00:00 2001 From: Jahnvi Thakkar Date: Thu, 8 Oct 2026 13:41:23 +0530 Subject: [PATCH 14/17] PERF: Remove fetchmany single-row construction detour Use existing Python Row completion while retaining native fetch behavior. Keep route provenance accurate and update existing regression expectations. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- eng/profiler_benchmarks/controller.py | 9 ++++++--- eng/profiler_benchmarks/report.py | 2 +- mssql_python/cursor.py | 20 +------------------- tests/test_fetch_settings_cache.py | 14 ++++++++++---- 4 files changed, 18 insertions(+), 27 deletions(-) diff --git a/eng/profiler_benchmarks/controller.py b/eng/profiler_benchmarks/controller.py index 3c07096c7..0438ed841 100644 --- a/eng/profiler_benchmarks/controller.py +++ b/eng/profiler_benchmarks/controller.py @@ -229,7 +229,7 @@ def verify_reused_source(source_root, revision): ) -# Closed, reviewed dispatch bodies: main666, f539, and the many-only successor. +# Closed, reviewed dispatch bodies: main666, f539, many-only, and plain completion. # Method/comment/docstring edits require review and an explicit fingerprint update. # These reviewed method-source SHA256 digests are not credentials. _LEGACY_PYTHON_ROUTE = ( @@ -241,6 +241,9 @@ def verify_reused_source(source_root, revision): _MANY_ONLY_ROUTE = ( "544d5b0ccbf9df1f0b6ec72c79ecbfa49a3517cab0437a5520940b8941966759" # DevSkim: ignore DS173237 ) +_PLAIN_COMPLETION_ROUTE = ( + "776ebc4532b69fe2492317ef6af9d4b2bb08e8a0f112613a7ce2600500bc37d8" # DevSkim: ignore DS173237 +) def python_source_identity(source_root, revision): @@ -320,7 +323,7 @@ def python_source_identity(source_root, revision): fingerprint = hashlib.sha256( json.dumps(methods, ensure_ascii=True, separators=(",", ":")).encode() ).hexdigest() - if fingerprint == _LEGACY_PYTHON_ROUTE: + if fingerprint in (_LEGACY_PYTHON_ROUTE, _PLAIN_COMPLETION_ROUTE): values = (False, False, False) elif fingerprint == _LEGACY_FUSED_ROUTE: values = (True, True, True) @@ -341,7 +344,7 @@ def python_source_identity(source_root, revision): validate_row_route(route) if route != expected: raise ValueError("Python source-policy drift") - elif fingerprint == _MANY_ONLY_ROUTE: + elif fingerprint in (_MANY_ONLY_ROUTE, _PLAIN_COMPLETION_ROUTE): raise ValueError("Missing Python route declaration") return dict( source_commit=revision, diff --git a/eng/profiler_benchmarks/report.py b/eng/profiler_benchmarks/report.py index 9960a5e11..749756dd5 100644 --- a/eng/profiler_benchmarks/report.py +++ b/eng/profiler_benchmarks/report.py @@ -298,7 +298,7 @@ def validate_row_route(route, guarded=None): values = tuple(methods[name] for name in ("fetchone", "fetchmany", "fetchval")) if values not in ((False, False, False), (True, True, True), (False, True, False)): raise ValueError("Unsupported Python row route policy") - if guarded is not None and (type(guarded) is not bool or any(values) != guarded): + if guarded is not None and (type(guarded) is not bool or (any(values) and not guarded)): raise ValueError("Native binding contradicts Python row route") return methods diff --git a/mssql_python/cursor.py b/mssql_python/cursor.py index d3fb01a08..2bbb90229 100644 --- a/mssql_python/cursor.py +++ b/mssql_python/cursor.py @@ -50,7 +50,7 @@ # Describes the default routes; reporting reads this outside the fetch hot path. _DEFAULT_NATIVE_ROW_ROUTE = { "version": 1, - "methods": {"fetchone": False, "fetchmany": True, "fetchval": False}, + "methods": {"fetchone": False, "fetchmany": False, "fetchval": False}, } _DEFAULT_NATIVE_FETCH_ONE = ddbc_bindings.DDBCSQLFetchOne _DEFAULT_NATIVE_FETCH_MANY = ddbc_bindings.DDBCSQLFetchMany @@ -2990,24 +2990,6 @@ def fetchmany(self, size: Optional[int] = None) -> List[Row]: started = perf_start() try: fetch = ddbc_bindings.DDBCSQLFetchMany - if ( - not started - and type(size) is int - and size == 1 - and fetch is _DEFAULT_NATIVE_FETCH_MANY - and _native_row_eligible(Row) - ): - return ddbc_bindings.DDBCSQLFetchRow( - self.hstmt, - rows_data, - char_enc, - wchar_enc, - self._cached_char_ctype, - self.messages, - True, - self._finish_fetchmany, - _NATIVE_ROW_PLAN, - ) ret = fetch( self.hstmt, rows_data, diff --git a/tests/test_fetch_settings_cache.py b/tests/test_fetch_settings_cache.py index 4421d19fd..2883d393d 100644 --- a/tests/test_fetch_settings_cache.py +++ b/tests/test_fetch_settings_cache.py @@ -1477,7 +1477,12 @@ def test_single_row_wrapper_does_not_enter_batch_factory(cursor): "SELECT n AS a, CAST(N'text' AS NVARCHAR(10)) AS b " "FROM (VALUES (1),(2),(3),(4)) AS v(n) ORDER BY n" ) - with patch.object(ddbc_bindings, "construct_rows", wraps=ddbc_bindings.construct_rows) as batch: + with ( + patch.object(ddbc_bindings, "construct_rows", wraps=ddbc_bindings.construct_rows) as batch, + patch.object( + ddbc_bindings, "DDBCSQLFetchRow", wraps=ddbc_bindings.DDBCSQLFetchRow + ) as fused, + ): retained = cursor.fetchmany(1)[0] assert tuple(retained) == (1, "text") batch.assert_not_called() @@ -1486,6 +1491,7 @@ def test_single_row_wrapper_does_not_enter_batch_factory(cursor): assert cursor.fetchmany(2)[0][0] == 4 assert batch.call_count == 2 assert tuple(retained) == (1, "text") + fused.assert_not_called() @pytest.mark.parametrize("override_fast_create", (False, True)) @@ -1718,7 +1724,7 @@ def test_single_row_construction_failure_keeps_fetch_position(cursor, method, fa cursor.execute("SELECT n AS number FROM (VALUES (1), (2)) AS v(n) ORDER BY n") from mssql_python.row import Row - bridge_name = "DDBCSQLFetchRow" if method == "fetchmany" else "DDBCSQLFetchOne" + bridge_name = "DDBCSQLFetchMany" if method == "fetchmany" else "DDBCSQLFetchOne" bridge = getattr(ddbc_bindings, bridge_name) def fetch(): @@ -1827,7 +1833,7 @@ def names(self): ): with pytest.raises(RuntimeError, match="factory changed after native advancement"): cursor.fetchmany(1) if method == "fetchmany" else getattr(cursor, method)() - assert fused.call_count == (1 if method == "fetchmany" else 0) + fused.assert_not_called() assert Row._fast_create is factory assert cursor.rowcount == 1 and cursor.rownumber == 0 finally: @@ -1875,7 +1881,7 @@ def names(self): ) as fused, ): result = cursor.fetchmany(1) if method == "fetchmany" else getattr(cursor, method)() - assert fused.call_count == (1 if method == "fetchmany" else 0) + fused.assert_not_called() if method == "fetchval": assert result == 1 else: From c928b63a946d87bcea85e37e63dc497ece3f78b7 Mon Sep 17 00:00:00 2001 From: Jahnvi Thakkar Date: Thu, 8 Oct 2026 17:14:56 +0530 Subject: [PATCH 15/17] PERF: Reduce fetch options, LOB copies, and converter dispatch Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- eng/profiler_benchmarks/controller.py | 8 +- mssql_python/cursor.py | 61 +++-- mssql_python/pybind/ddbc_bindings.cpp | 43 +++- mssql_python/pybind/ddbc_bindings.h | 9 + mssql_python/pybind/row_factory.hpp | 82 +++++++ mssql_python/row.py | 9 + tests/test_017_fetch_bounded_text.py | 160 ++++++++++++- tests/test_fetch_settings_cache.py | 313 +++++++++++++++++++++++++- 8 files changed, 654 insertions(+), 31 deletions(-) diff --git a/eng/profiler_benchmarks/controller.py b/eng/profiler_benchmarks/controller.py index 0438ed841..661e39ca9 100644 --- a/eng/profiler_benchmarks/controller.py +++ b/eng/profiler_benchmarks/controller.py @@ -230,6 +230,7 @@ def verify_reused_source(source_root, revision): # Closed, reviewed dispatch bodies: main666, f539, many-only, and plain completion. +# The options snapshot changes marshalling, not the plain Row-completion policy. # Method/comment/docstring edits require review and an explicit fingerprint update. # These reviewed method-source SHA256 digests are not credentials. _LEGACY_PYTHON_ROUTE = ( @@ -244,6 +245,9 @@ def verify_reused_source(source_root, revision): _PLAIN_COMPLETION_ROUTE = ( "776ebc4532b69fe2492317ef6af9d4b2bb08e8a0f112613a7ce2600500bc37d8" # DevSkim: ignore DS173237 ) +_FETCH_OPTIONS_ROUTE = ( + "6c8545f38956380e22b8cc9376e215e032d45ec68d1a29c669201f09477ec507" # DevSkim: ignore DS173237 +) def python_source_identity(source_root, revision): @@ -323,7 +327,7 @@ def python_source_identity(source_root, revision): fingerprint = hashlib.sha256( json.dumps(methods, ensure_ascii=True, separators=(",", ":")).encode() ).hexdigest() - if fingerprint in (_LEGACY_PYTHON_ROUTE, _PLAIN_COMPLETION_ROUTE): + if fingerprint in (_LEGACY_PYTHON_ROUTE, _PLAIN_COMPLETION_ROUTE, _FETCH_OPTIONS_ROUTE): values = (False, False, False) elif fingerprint == _LEGACY_FUSED_ROUTE: values = (True, True, True) @@ -344,7 +348,7 @@ def python_source_identity(source_root, revision): validate_row_route(route) if route != expected: raise ValueError("Python source-policy drift") - elif fingerprint in (_MANY_ONLY_ROUTE, _PLAIN_COMPLETION_ROUTE): + elif fingerprint in (_MANY_ONLY_ROUTE, _PLAIN_COMPLETION_ROUTE, _FETCH_OPTIONS_ROUTE): raise ValueError("Missing Python route declaration") return dict( source_commit=revision, diff --git a/mssql_python/cursor.py b/mssql_python/cursor.py index 2bbb90229..d87953590 100644 --- a/mssql_python/cursor.py +++ b/mssql_python/cursor.py @@ -677,8 +677,19 @@ def _refresh_decoding_cache(self): self._cached_char_encoding = char_decoding.get("encoding", "utf-16le") self._cached_char_ctype = char_decoding.get("ctype", ddbc_sql_const.SQL_WCHAR.value) self._cached_wchar_encoding = wchar_encoding + self._cached_fetch_options = None self._cached_decoding_generation = generation + def _create_fetch_options(self): + generation = self._cached_decoding_generation + options = ddbc_bindings._FetchOptions( + self._cached_char_encoding, self._cached_wchar_encoding, self._cached_char_ctype + ) + # Allocation callbacks can refresh decoding; never install an older snapshot. + if self._cached_decoding_generation == generation: + self._cached_fetch_options = options + return options + def _get_decoding_settings(self, sql_type): """ Get decoding settings for a specific SQL type. @@ -2889,14 +2900,22 @@ def fetchone(self) -> Union[None, Row]: started = perf_start() try: fetch = ddbc_bindings.DDBCSQLFetchOne - ret = fetch( - self.hstmt, - row_data, - char_enc, - wchar_enc, - self._cached_char_ctype, - self.messages, - ) + if fetch is _DEFAULT_NATIVE_FETCH_ONE: + options = self._cached_fetch_options + if options is None: + options = self._create_fetch_options() + ret = ddbc_bindings._fetchone_with_options( + self.hstmt, row_data, options, self.messages + ) + else: + ret = fetch( + self.hstmt, + row_data, + char_enc, + wchar_enc, + self._cached_char_ctype, + self.messages, + ) finally: if started: perf_stop("py::fetchone::cpp_call", started) @@ -2990,15 +3009,23 @@ def fetchmany(self, size: Optional[int] = None) -> List[Row]: started = perf_start() try: fetch = ddbc_bindings.DDBCSQLFetchMany - ret = fetch( - self.hstmt, - rows_data, - size, - char_enc, - wchar_enc, - self._cached_char_ctype, - self.messages, - ) + if fetch is _DEFAULT_NATIVE_FETCH_MANY: + options = self._cached_fetch_options + if options is None: + options = self._create_fetch_options() + ret = ddbc_bindings._fetchmany_with_options( + self.hstmt, rows_data, size, options, self.messages + ) + else: + ret = fetch( + self.hstmt, + rows_data, + size, + char_enc, + wchar_enc, + self._cached_char_ctype, + self.messages, + ) finally: if started: perf_stop("py::fetchmany::cpp_call", started) diff --git a/mssql_python/pybind/ddbc_bindings.cpp b/mssql_python/pybind/ddbc_bindings.cpp index 882f3d0f7..2d0caf3f4 100644 --- a/mssql_python/pybind/ddbc_bindings.cpp +++ b/mssql_python/pybind/ddbc_bindings.cpp @@ -23,6 +23,7 @@ #include // For std::memcpy #include #include +#include #include // std::forward #include // CPython datetime API (PyDateTime_IMPORT, PyDateTime_GET_*, etc.) @@ -3280,12 +3281,18 @@ py::object FetchLobColumnData(SQLHSTMT hStmt, SQLUSMALLINT colIndex, SQLSMALLINT while (true) { ++loopCount; - std::vector chunk(DAE_CHUNK_SIZE, 0); + const size_t offset = buffer.size(); + if (buffer.max_size() - offset < DAE_CHUNK_SIZE) { + throw std::length_error("LOB data exceeds maximum buffer size"); + } + // Keep the existing read windows and zero-filled terminator room. + buffer.resize(offset + DAE_CHUNK_SIZE, 0); + char* chunk = buffer.data() + offset; SQLLEN actualRead = 0; { // Release the GIL during blocking SQLGetData LOB streaming py::gil_scoped_release release; - ret = SQLGetData_ptr(hStmt, colIndex, cType, chunk.data(), DAE_CHUNK_SIZE, &actualRead); + ret = SQLGetData_ptr(hStmt, colIndex, cType, chunk, DAE_CHUNK_SIZE, &actualRead); } CaptureFetchDiagnostics(hStmt, ret, messages, true); @@ -3328,11 +3335,13 @@ py::object FetchLobColumnData(SQLHSTMT hStmt, SQLUSMALLINT colIndex, SQLSMALLINT // Wide characters size_t wcharSize = sizeof(SQLWCHAR); if (bytesRead >= wcharSize && (bytesRead % wcharSize == 0)) { - size_t wcharCount = bytesRead / wcharSize; - std::vector alignedBuf(wcharCount); - std::memcpy(alignedBuf.data(), chunk.data(), bytesRead); - while (wcharCount > 0 && alignedBuf[wcharCount - 1] == 0) { - --wcharCount; + while (bytesRead >= wcharSize) { + SQLWCHAR lastChar; + // The byte destination need not be aligned for SQLWCHAR. + std::memcpy(&lastChar, chunk + bytesRead - wcharSize, wcharSize); + if (lastChar != 0) { + break; + } bytesRead -= wcharSize; } if (bytesRead < DAE_CHUNK_SIZE) { @@ -3343,8 +3352,8 @@ py::object FetchLobColumnData(SQLHSTMT hStmt, SQLUSMALLINT colIndex, SQLSMALLINT } } } + buffer.resize(offset + bytesRead); if (bytesRead > 0) { - buffer.insert(buffer.end(), chunk.begin(), chunk.begin() + bytesRead); LOG("FetchLobColumnData: Appended %zu bytes at loop %d", bytesRead, loopCount); } if (ret == SQL_SUCCESS) { @@ -6667,6 +6676,20 @@ PYBIND11_MODULE(ddbc_bindings, m) { py::arg("StatementHandle"), py::arg("colCount"), py::arg("row"), py::arg("charEncoding"), py::arg("wcharEncoding"), py::arg("charCtype"), py::arg("messages") = py::none()); m.def("DDBCSQLMoreResults", &SQLMoreResults_wrap, "Check for more results in the result set"); + py::class_(m, "_FetchOptions") + .def(py::init()); + m.def("_fetchone_with_options", + [](SqlHandlePtr statement, py::list& row, const FetchOptions& options, + py::handle messages) { + return FetchOne_wrap(std::move(statement), row, options.charEncoding, + options.wcharEncoding, options.charCtype, messages); + }); + m.def("_fetchmany_with_options", + [](SqlHandlePtr statement, py::list& rows, int size, const FetchOptions& options, + py::handle messages) { + return FetchMany_wrap(std::move(statement), rows, size, options.charEncoding, + options.wcharEncoding, options.charCtype, messages); + }); m.def("DDBCSQLFetchOne", &FetchOne_wrap, "Fetch one row from the result set", py::arg("StatementHandle"), py::arg("row"), py::arg("charEncoding") = "utf-16le", py::arg("wcharEncoding") = "utf-16le", py::arg("charCtype") = SQL_C_WCHAR, @@ -6778,6 +6801,10 @@ PYBIND11_MODULE(ddbc_bindings, m) { // Add a version attribute m.attr("__version__") = "1.0.0"; + m.def("_apply_output_converters", &RowFactory::apply_output_converters, + "Apply a cached converter map after native value materialization", + py::arg("values"), py::arg("converters")); + // Fast Row construction in C++ — replaces Python list comprehension m.def("construct_rows", &RowFactory::construct_rows, "Build Row objects in C++ for fetchall/fetchmany fast path", diff --git a/mssql_python/pybind/ddbc_bindings.h b/mssql_python/pybind/ddbc_bindings.h index 3494fc833..bd0c19c6a 100644 --- a/mssql_python/pybind/ddbc_bindings.h +++ b/mssql_python/pybind/ddbc_bindings.h @@ -426,6 +426,15 @@ struct DateTimeOffset { SQLSMALLINT timezone_minute; // Offset minutes from UTC }; +struct FetchOptions { + const std::string charEncoding; + const std::string wcharEncoding; + const int charCtype; + + FetchOptions(const std::string& charEncoding, const std::string& wcharEncoding, int charCtype) + : charEncoding(charEncoding), wcharEncoding(wcharEncoding), charCtype(charCtype) {} +}; + // Struct to hold data buffers and indicators for each column struct ColumnBuffers { std::vector> charBuffers; diff --git a/mssql_python/pybind/row_factory.hpp b/mssql_python/pybind/row_factory.hpp index 6ef1a7298..61c33b66c 100644 --- a/mssql_python/pybind/row_factory.hpp +++ b/mssql_python/pybind/row_factory.hpp @@ -5,8 +5,90 @@ #include "py_ref.hpp" +#include + namespace RowFactory { +// Only the post-fetch cached-map loop: no ODBC access, Row allocation, or UUID work. +inline py::list apply_output_converters(const py::object& values, const py::object& converters) { + if (!PyList_CheckExact(values.ptr()) || !PyList_CheckExact(converters.ptr())) { + throw py::type_error("converter values and map must be exact lists"); + } + py::list result = + steal(PyList_GetSlice(values.ptr(), 0, PyList_GET_SIZE(values.ptr()))); + if (!result) + throw py::error_already_set(); + + // Keep the Python iterators: their retained tuples affect finalizer timing + // when a callback replaces itself or mutates the source lists. + py::object pairs = steal(PyObject_CallFunctionObjArgs(reinterpret_cast(&PyZip_Type), + values.ptr(), converters.ptr(), nullptr)); + if (!pairs) + throw py::error_already_set(); + py::object items = + steal(PyObject_CallOneArg(reinterpret_cast(&PyEnum_Type), pairs.ptr())); + if (!items) + throw py::error_already_set(); + pairs = py::object(); + + // Retain the current inputs and last encoded value like the Python locals. + py::object value, converter, value_bytes; + for (Py_ssize_t i = 0;; ++i) { + py::object item = steal(PyIter_Next(items.ptr())); + if (!item) { + if (PyErr_Occurred()) + throw py::error_already_set(); + break; + } + PyObject* pair = PyTuple_GET_ITEM(item.ptr(), 1); + py::object next_value = borrow(PyTuple_GET_ITEM(pair, 0)); + py::object next_converter = borrow(PyTuple_GET_ITEM(pair, 1)); + value = std::move(next_value); + converter = std::move(next_converter); + item = py::object(); + const int enabled = PyObject_IsTrue(converter.ptr()); + if (enabled < 0) + throw py::error_already_set(); + if (!enabled || value.is_none()) + continue; + + try { + const int is_string = + PyObject_IsInstance(value.ptr(), reinterpret_cast(&PyUnicode_Type)); + if (is_string < 0) + throw py::error_already_set(); + PyObject* argument = value.ptr(); + if (is_string) { + // Match str.encode's codec lookup, including the spelling. + // Subclasses and __class__ proxies still dispatch encode dynamically. + py::object encoded = + steal(PyUnicode_CheckExact(value.ptr()) + ? PyUnicode_AsEncodedString(value.ptr(), "utf-16-le", nullptr) + : PyObject_CallMethod(value.ptr(), "encode", "s", "utf-16-le")); + if (!encoded) + throw py::error_already_set(); + value_bytes = std::move(encoded); + argument = value_bytes.ptr(); + } + py::object converted = steal(PyObject_CallOneArg(converter.ptr(), argument)); + if (!converted) + throw py::error_already_set(); + // Checked assignment also preserves Python's caught IndexError if a + // callback grows the input lists beyond the initial result copy. + if (PyList_SetItem(result.ptr(), i, converted.release().ptr()) < 0) + throw py::error_already_set(); + } catch (py::error_already_set& error) { + if (!error.matches(PyExc_Exception)) + throw; + // Preserve the existing cached path's keep-original-on-Exception contract. + error.restore(); + PyErr_Clear(); + } + } + items = py::object(); + return result; +} + inline void initialize_row( const py::object& row, PyObject* row_data, const py::object& column_map, const py::object& cursor_obj, const py::object& column_map_lower, diff --git a/mssql_python/row.py b/mssql_python/row.py index ccec53787..7eb766b0d 100644 --- a/mssql_python/row.py +++ b/mssql_python/row.py @@ -9,6 +9,7 @@ import uuid as _uuid from collections.abc import Mapping from typing import Any +from mssql_python import ddbc_bindings from mssql_python.logging import logger @@ -198,6 +199,9 @@ def _apply_output_converters_optimized(self, values, converter_map): """ Apply output converters using pre-computed converter map for optimal performance. + Native materialization has already completed. Exact lists use a native + dispatch loop; both paths copy values and preserve dynamic string encoding. + Args: values: Raw values from the database converter_map: Pre-computed list of converters (one per column, None if no converter) @@ -205,6 +209,11 @@ def _apply_output_converters_optimized(self, values, converter_map): Returns: List of converted values """ + # Native fetches and the cursor cache supply exact lists. Keep arbitrary + # iterables on the Python path, including their iteration side effects. + if type(values) is list and type(converter_map) is list: + return ddbc_bindings._apply_output_converters(values, converter_map) + converted_values = list(values) for i, (value, converter) in enumerate(zip(values, converter_map)): diff --git a/tests/test_017_fetch_bounded_text.py b/tests/test_017_fetch_bounded_text.py index 14065700b..7f0be4a54 100644 --- a/tests/test_017_fetch_bounded_text.py +++ b/tests/test_017_fetch_bounded_text.py @@ -1,10 +1,10 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT license. -"""Bounded text payload fidelity, including row-wise routing beside a MAX column. +"""Bounded text payload fidelity and LOB storage regression coverage. -The MAX value here is only a routing control. Actual MAX text BOM/NUL fidelity -belongs to the separate LOB decoder and is not covered by this regression. +The bounded cases use MAX only as a routing control. The LOB cases preserve the +existing streaming/decoder behavior; they do not redefine MAX text BOM/NUL fidelity. """ import os @@ -416,3 +416,157 @@ def getdata(handle, column, ctype, buffer, capacity, indicator): assert cursor.fetchval() == 42 """) _run_fetch_script(conn_str, script, payload) + + +@pytest.mark.parametrize("method", ["fetchone", "fetchmany", "fetchall"]) +@pytest.mark.parametrize("size", [None, 0, 1, 8191, 8192, 8193, 16384, 262144]) +def test_lob_binary_read_boundaries(db_connection, method, size): + expected = None if size is None else (bytes(range(256)) * ((size + 255) // 256))[:size] + literal = "NULL" if expected is None else "0x" + expected.hex() + with db_connection.cursor() as cursor: + cursor.execute(f"SELECT CAST({literal} AS varbinary(max))") + rows = _fetch_rows(cursor, method) + assert rows == [(expected,)] + assert type(rows[0][0]) is type(expected) + assert cursor.messages == [] + cursor.execute("SELECT 42") + assert cursor.fetchval() == 42 + + +@pytest.mark.parametrize("method", ["fetchone", "fetchmany", "fetchall"]) +@pytest.mark.parametrize("kind", ["nvarchar", "varchar-wide", "varchar-narrow"]) +def test_lob_text_read_boundaries(db_connection, method, kind): + suffix = "\U0001f642\0caf\u00e9-tail" if kind == "nvarchar" else "caf\u00e9\0tail" + expected = [None, ""] + [ + "x" * boundary + suffix for boundary in (4093, 4094, 4095, 8187, 8190, 8191, 8192, 262144) + ] + original = db_connection.getdecoding(SQL_CHAR) + try: + narrow = kind == "varchar-narrow" + db_connection.setdecoding( + SQL_CHAR, + encoding="latin-1" if narrow else "utf-16le", + ctype=SQL_CHAR if narrow else SQL_WCHAR, + ) + expressions = [] + for index, value in enumerate(expected): + literal = "NULL" if value is None else "0x" + value.encode("utf-16le").hex() + expression = f"CAST({literal} AS nvarchar(max))" + if kind != "nvarchar": + expression = f"CAST({expression} COLLATE Latin1_General_100_BIN2 AS varchar(max))" + expressions.append(f"({index}, {expression})") + with db_connection.cursor() as cursor: + cursor.execute( + "SELECT payload FROM (VALUES " + + ", ".join(expressions) + + ") AS v(n, payload) ORDER BY n" + ) + rows = _fetch_rows(cursor, method) + assert rows == [(value,) for value in expected] + assert all(row[0] is None or type(row[0]) is str for row in rows) + assert cursor.messages == [] + cursor.execute("SELECT 42") + assert cursor.fetchval() == 42 + finally: + db_connection.setdecoding(SQL_CHAR, encoding=original["encoding"], ctype=original["ctype"]) + + +@pytest.mark.skipif( + sys.platform == "win32", + reason="Windows does not export the native driver function-pointer globals", +) +@pytest.mark.parametrize("kind", ["binary", "narrow", "wide"]) +@pytest.mark.parametrize( + "mode", ["known", "unknown", "negative", "split", "zero", "null", "error", "no-data"] +) +def test_lob_getdata_storage_contract(conn_str, kind, mode): + script = textwrap.dedent(""" + import ctypes as c + import os + import sys + import mssql_python + from mssql_python import ddbc_bindings as ddbc + + kind, mode = sys.argv[1:] + binary, wide = kind == "binary", kind == "wide" + ctype = -2 if binary else (-8 if wide else 1) + sqltype = "varbinary(max)" if binary else ("nvarchar(max)" if wide else "varchar(max)") + codec = "utf-16le" if wide else "utf-8" + unit = 2 if wide else 1 + terminator = b"" if binary else bytes(unit) + head_text = "A\\0B" + "x" * ((8192 - len(terminator)) // unit - 3) + tail_text = "Z\\0Y\\0\\0" + head, tail = head_text.encode(codec), tail_text.encode(codec) + expected = head + tail if binary else head_text + tail_text.rstrip("\\0") + if mode == "split" and not binary: + text = head_text[:-1] + "\\U0001f642" + tail_text + encoded = text.encode(codec) + head, tail = encoded[:len(head)], encoded[len(head):] + expected = text.rstrip("\\0") + elif mode == "zero": + expected = head if binary else head_text + elif mode == "null": + expected = None + + pointer, short, length = c.c_void_p, c.c_short, c.c_ssize_t + get_type = c.CFUNCTYPE(short, pointer, c.c_ushort, short, pointer, length, pointer) + library = c.CDLL(ddbc.module.__file__) + slot = pointer.in_dll(library, "SQLGetData_ptr") + calls, errors = [], [] + + @get_type + def getdata(handle, column, actual_ctype, buffer, capacity, indicator): + try: + assert column == 1 and actual_ctype == ctype + assert capacity == 8192 + assert c.string_at(buffer, capacity) == bytes(capacity) + calls.append(capacity) + assert len(calls) <= 2 + first = len(calls) == 1 + data = head if first else tail + c.memmove(buffer, data + terminator, len(data) + len(terminator)) + reported = len(head) + len(tail) if first else len(tail) + if first and mode in ("unknown", "negative"): + reported = -4 if mode == "unknown" else -9 + if not first: + if mode == "zero": + reported = 0 + elif mode in ("null", "error", "no-data"): + reported = -1 + c.cast(indicator, c.POINTER(length))[0] = reported + if not first and mode in ("error", "no-data"): + return -1 if mode == "error" else 100 + return 1 if first else 0 + except BaseException as error: + errors.append(repr(error)) + return -1 + + with mssql_python.connect(os.environ["DB_CONNECTION_STRING"]) as connection: + with connection.cursor() as cursor: + cursor.execute(f"SELECT CAST(NULL AS {sqltype})") + assert ddbc.DDBCSQLFetch(cursor.hstmt) == 0 + saved = slot.value + assert saved + marker = object() + row = [marker] + slot.value = c.cast(getdata, pointer).value + try: + try: + result = ddbc.DDBCSQLGetData( + cursor.hstmt, 1, row, "utf-8", "utf-16le", ctype, cursor.messages + ) + except RuntimeError as error: + assert mode in ("error", "no-data"), str(error) + assert "Error fetching LOB" in str(error) + assert row == [marker] + else: + assert mode not in ("error", "no-data") + assert result == 0 and row == [marker, expected] + assert type(row[1]) is type(expected) + finally: + slot.value = saved + assert calls == [8192, 8192] and not errors, (calls, errors) + cursor.execute("SELECT 42") + assert cursor.fetchval() == 42 + """) + _run_fetch_script(conn_str, script, kind, mode) diff --git a/tests/test_fetch_settings_cache.py b/tests/test_fetch_settings_cache.py index 2883d393d..7a3b4dfe9 100644 --- a/tests/test_fetch_settings_cache.py +++ b/tests/test_fetch_settings_cache.py @@ -194,18 +194,33 @@ def test_wchar_decoding_forwarded_to_live_fetch_bridge(connection, method, bridg def test_decoding_cache_reuse_and_multiple_cursors(connection): - with patch.object(connection, "getdecoding", wraps=connection.getdecoding) as reads: + with ( + patch.object(connection, "getdecoding", wraps=connection.getdecoding) as reads, + patch.object(ddbc, "_FetchOptions", wraps=ddbc._FetchOptions) as options, + patch.object(ddbc, "_fetchone_with_options", wraps=ddbc._fetchone_with_options) as one, + patch.object(ddbc, "_fetchmany_with_options", wraps=ddbc._fetchmany_with_options) as many, + ): with connection.cursor() as first, connection.cursor() as second: assert reads.call_count == 4 + assert options.call_count == 0 for cursor in (first, second): + cursor.execute("SELECT 42") + assert cursor.fetchall()[0][0] == 42 + assert cursor._cached_fetch_options is None cursor.execute("SELECT n FROM (VALUES (1), (2), (3)) AS v(n) ORDER BY n") assert cursor.fetchone()[0] == 1 + snapshot = cursor._cached_fetch_options assert cursor.fetchmany(1)[0][0] == 2 assert cursor.fetchall()[0][0] == 3 + assert one.call_args.args[-2] is many.call_args.args[-2] is snapshot + assert one.call_args.args[-1] is many.call_args.args[-1] is cursor.messages + assert cursor._cached_fetch_options is snapshot assert reads.call_count == 4 + assert options.call_count == one.call_count == many.call_count == 2 connection.setdecoding(mssql_python.SQL_CHAR, encoding="latin-1") for index, cursor in enumerate((first, second), 1): + snapshot = cursor._cached_fetch_options cursor.execute( "SELECT CONVERT(VARCHAR(1), 0xE9) AS txt FROM (VALUES (1), (2), (3)) AS v(n)" ) @@ -213,6 +228,35 @@ def test_decoding_cache_reuse_and_multiple_cursors(connection): assert cursor.fetchmany(1)[0].txt == "\u00e9" assert cursor.fetchall()[0].txt == "\u00e9" assert reads.call_count == 4 + 2 * index + assert options.call_count == 2 + index + assert cursor._cached_fetch_options is not snapshot + + +@pytest.mark.parametrize("method", ("fetchone", "fetchmany", "fetchval")) +def test_native_fetch_options_refresh_failure_is_retried(connection, method): + with connection.cursor() as cursor: + cursor.execute("SELECT 42") + assert cursor.fetchone()[0] == 42 + assert cursor._cached_fetch_options is not None + cursor.execute("SELECT CONVERT(VARCHAR(1), 0xE9) AS txt") + connection.setdecoding(mssql_python.SQL_CHAR, encoding="latin-1") + + def fetch(): + return cursor.fetchmany(1) if method == "fetchmany" else getattr(cursor, method)() + + with patch.object(ddbc, "_FetchOptions", side_effect=MemoryError("options allocation")): + with pytest.raises(MemoryError, match="options allocation"): + fetch() + assert cursor._cached_decoding_generation == connection._decoding_generation + assert cursor._cached_fetch_options is None + assert cursor._next_row_index == 0 + value = fetch() + actual = ( + value[0][0] if method == "fetchmany" else value if method == "fetchval" else value[0] + ) + assert actual == "\u00e9" + assert cursor._cached_fetch_options is not None + assert cursor.rowcount == 1 @pytest.mark.parametrize("method", FETCH_METHODS) @@ -518,6 +562,273 @@ def test_direct_row_without_converters_is_zero_copy(): assert row._values is values +@pytest.mark.parametrize("container", (list, tuple)) +def test_converter_leaf_inputs_and_copy(container): + text = "A\0\u00e9\U0001f600" + values = container( + [ + text, + b"\0\xff", + 42, + decimal.Decimal("1.25"), + datetime.date(2026, 1, 2), + uuid.UUID(UUID_TEXT), + None, + ] + ) + seen = [] + + def convert(value): + seen.append(value) + return value + + with patch.object( + ddbc, "_apply_output_converters", wraps=ddbc._apply_output_converters + ) as leaf: + row = Row(values, {}, converter_map=[convert] * len(values), uuid_str_indices=(5,)) + expected = [b"A\0\0\0\xe9\0\x3d\xd8\0\xde", *values[1:-1]] + assert seen == expected + assert [type(value) for value in seen] == [type(value) for value in expected] + assert row._values is not values + assert list(row) == [*expected[:5], UUID_TEXT, None] + assert isinstance(values[5], uuid.UUID) + assert leaf.call_count == int(container is list) + if container is list: + values[0] = "changed" + assert row[0] == expected[0] + + +@pytest.mark.parametrize("container", (list, tuple)) +@pytest.mark.parametrize("map_container", (list, tuple)) +@pytest.mark.parametrize("map_length", (1, 4)) +def test_converter_leaf_map_length(container, map_container, map_length): + calls = [] + converters = map_container([lambda value: calls.append(value)] * map_length) + row = Row(container([1, 2]), {}, converter_map=converters) + assert calls == ([1] if map_length == 1 else [1, 2]) + assert list(row) == ([None, 2] if map_length == 1 else [None, None]) + + +@pytest.mark.parametrize("container", (list, tuple)) +def test_converter_leaf_dynamic_encode(container): + events = [] + token = object() + + class Text(str): + def encode(self, encoding): + events.append((str(self), encoding)) + Text.encode = lambda self, encoding: token + return b"first" + + class StringProxy: + __class__ = property(lambda self: str) + + def encode(self, encoding): + events.append(("proxy", encoding)) + return token + + values = container([Text("one"), Text("two"), StringProxy()]) + row = Row(values, {}, converter_map=[lambda value: value] * 3) + assert events == [("one", "utf-16-le"), ("proxy", "utf-16-le")] + assert row[0] == b"first" + assert row[1] is row[2] is token + + +def test_converter_leaf_releases_owned_references(): + references = [] + + class Value: + pass + + class Text(str): + def encode(self, encoding): + value = Value() + references.append(weakref.ref(value)) + return value + + class Converter: + def __call__(self, value): + assert value is references[-1]() + result = Value() + references.append(weakref.ref(result)) + return result + + values, converters = [Text("text")], [Converter()] + references.extend([weakref.ref(values[0]), weakref.ref(converters[0])]) + row = Row(values, {}, converter_map=converters) + assert references[2]() is None + assert references[3]() is row[0] + del values, converters, row + gc.collect() + assert all(reference() is None for reference in references) + + +@pytest.mark.parametrize("container", (list, tuple)) +@pytest.mark.parametrize("stage", ("bool", "encode", "call")) +@pytest.mark.parametrize("error_type", (ValueError, KeyboardInterrupt)) +def test_converter_leaf_exception_boundaries(container, stage, error_type): + class Text(str): + def encode(self, encoding): + if stage == "encode": + raise error_type("converter failure") + return super().encode(encoding) + + class Converter: + def __bool__(self): + if stage == "bool": + raise error_type("converter failure") + return True + + def __call__(self, value): + raise error_type("converter failure") + + values = container([Text("text")]) + if stage == "bool" or error_type is KeyboardInterrupt: + with pytest.raises(error_type, match="converter failure"): + Row(values, {}, converter_map=[Converter()]) + else: + assert Row(values, {}, converter_map=[Converter()])[0] is values[0] + + +@pytest.mark.parametrize("native", (False, True)) +@pytest.mark.parametrize("mutation", ("replace", "grow", "shrink")) +def test_converter_leaf_live_lists(native, mutation): + events = [] + values = [1, 2, None] + + class Values(list): + pass + + if not native: + values = Values(values) + + class Converter: + def __bool__(self): + events.append("bool") + return True + + def __call__(self, value): + events.append(value) + if value == 1: + if mutation == "replace": + values[1] = 20 + converters[1] = lambda value: value + 100 + elif mutation == "grow": + values.append(4) + converters.append(self) + else: + values.clear() + converters.clear() + return value + 10 + + converters = [Converter()] * 3 + row = Row(values, {}, converter_map=converters) + assert ( + list(row) + == { + "replace": [11, 120, None], + "grow": [11, 12, None], + "shrink": [11, 2, None], + }[mutation] + ) + assert ( + events + == { + "replace": ["bool", 1, "bool"], + "grow": ["bool", 1, "bool", 2, "bool", "bool", 4], + "shrink": ["bool", 1], + }[mutation] + ) + + +def test_converter_leaf_self_replacement_finalizer_order(): + def run(container): + events = [] + + class Converter: + def __call__(self, value): + converters[0] = None + events.append(value) + return value + + def __del__(self): + events.append("released") + + converters = [Converter(), events.append, events.append] + row = Row(container([1, 2, 3]), {}, converter_map=converters) + return events, list(row) + + assert run(list) == run(tuple) + + +def test_converter_leaf_codec_lookup_in_subprocess(): + script = textwrap.dedent(""" + import codecs + import encodings + from mssql_python.row import Row + + events = [] + def encode(value, errors="strict"): + events.append((value, errors)) + return b"custom", len(value) + def search(name): + if name == "utf_16_le": + return codecs.CodecInfo(name=name, encode=encode, decode=None) + codecs.unregister(encodings.search_function) + codecs.register(search) + codecs.register(encodings.search_function) + values = ["A\\0\\u00e9", "\\U0001f600"] + expected = [value.encode("utf-16-le") for value in values] + expected_events = events[:] + events.clear() + row = Row(values, {}, converter_map=[lambda value: value] * 2) + assert list(row) == expected + assert events == expected_events + """) + result = subprocess.run( + [sys.executable, "-c", script], capture_output=True, text=True, timeout=30 + ) + assert result.returncode == 0, result.stdout + result.stderr + + +@pytest.mark.parametrize("method", FETCH_METHODS) +def test_converter_leaf_registry_changes_during_rows(connection, method): + calls = [] + + def replacement(value): + return value + 100 + + def original(value): + calls.append(value) + connection.add_output_converter(mssql_python.SQL_INTEGER, replacement) + return value + 10 + + connection.add_output_converter(mssql_python.SQL_INTEGER, original) + with connection.cursor() as cursor: + cursor.execute("SELECT n AS a, n + 10 AS b FROM (VALUES (1), (2)) AS v(n) ORDER BY n") + rows = fetch_rows(cursor, method) + count = 1 if method == "fetchone" else 2 + assert calls == [value for n in range(1, count + 1) for value in (n, n + 10)] + assert [list(row) for row in rows] == [[n + 10, n + 20] for n in range(1, count + 1)] + if method == "fetchone": + assert list(cursor.fetchone()) == [102, 112] + cursor.execute("SELECT 3 AS renamed") + assert cursor.fetchval() == 103 + assert list(rows[0]) == [11, 21] + assert dict(rows[0]._mapping) == {"a": 11, "b": 21} + + +@pytest.mark.parametrize("method", ("fetchone", "fetchval")) +def test_converter_leaf_native_error_precedes_callbacks(connection, method): + calls = [] + connection.add_output_converter(mssql_python.SQL_INTEGER, lambda value: calls.append(value)) + with connection.cursor() as cursor: + cursor.execute("SELECT 1 AS a, CAST(0x00D8 AS NVARCHAR(10)) AS invalid_utf16") + with pytest.raises(UnicodeDecodeError): + getattr(cursor, method)() + assert calls == [] + + def test_decoding_cache_refresh_failure_is_retried(connection): with connection.cursor() as cursor: cursor.execute("SELECT CONVERT(VARCHAR(1), 0xE9) AS txt") From 84db936ec56b90afaf31756b0a1a411377de645e Mon Sep 17 00:00:00 2001 From: Jahnvi Thakkar Date: Thu, 8 Oct 2026 17:47:21 +0530 Subject: [PATCH 16/17] FIX: Correct LOB fixtures and benchmark review findings Preserve existing LOB semantics, require Release benchmark configurations, and cover aggregate report failures and numeric fetch errors. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- eng/profiler_benchmarks/controller.py | 39 ++++- mssql_python/pybind/build.sh | 6 +- mssql_python/pybind/ddbc_bindings.cpp | 6 +- profiler/README.md | 27 +++- tests/test_017_fetch_bounded_text.py | 7 +- tests/test_036_profiler_ci.py | 219 +++++++++++++++++++++++++- tests/test_fetch_settings_cache.py | 56 +++++++ 7 files changed, 344 insertions(+), 16 deletions(-) diff --git a/eng/profiler_benchmarks/controller.py b/eng/profiler_benchmarks/controller.py index 661e39ca9..db5577cc1 100644 --- a/eng/profiler_benchmarks/controller.py +++ b/eng/profiler_benchmarks/controller.py @@ -121,11 +121,47 @@ def terminate_process_tree(process): def build(path, log, timeout=900, profiling=True): - env = dict(os.environ, ENABLE_PROFILING="1" if profiling else "0") + # CMake >= 3.22 also initializes archived, older build scripts from this variable. + env = dict(os.environ, ENABLE_PROFILING="1" if profiling else "0", CMAKE_BUILD_TYPE="Release") # build scripts find Python via PATH; keep the controller's interpreter. env["PATH"] = str(Path(sys.executable).parent) + os.pathsep + env["PATH"] command = ["cmd", "/c", "build.bat"] if os.name == "nt" else ["bash", "build.sh"] run_process(command, log, timeout, cwd=path / "mssql_python/pybind", env=env) + configuration = release_build_configuration(path) + with log.open("a", encoding="utf-8") as output: + output.write("Verified Release configuration: " + json.dumps(configuration) + "\n") + + +def release_build_configuration(source_root): + cache = source_root / "mssql_python/pybind/build/CMakeCache.txt" + if not cache.is_file(): + tag = f"py{sys.version_info.major}{sys.version_info.minor}" + candidates = list(cache.parent.glob(f"*/{tag}/CMakeCache.txt")) + if len(candidates) > 1: + raise ValueError("Ambiguous native build configuration for this Python version") + if candidates: + cache = candidates[0] + entries = dict( + re.findall(r"^(CMAKE_[A-Z_]+):[^=\n]+=(.*)$", cache.read_text(encoding="utf-8"), re.M) + ) + configurations = entries.get("CMAKE_CONFIGURATION_TYPES", "") + if ( + "Release" not in configurations.split(";") + if configurations + else entries.get("CMAKE_BUILD_TYPE") != "Release" + ): + raise ValueError( + "Benchmarks require a Release native build (CMake >= 3.22 for old scripts)" + ) + return { + key: entries[key] + for key in ( + "CMAKE_GENERATOR", + "CMAKE_CXX_FLAGS", + "CMAKE_CXX_FLAGS_RELEASE", + "CMAKE_CONFIGURATION_TYPES" if configurations else "CMAKE_BUILD_TYPE", + ) + } def run_process(command, log, timeout, **options): @@ -151,6 +187,7 @@ def run_process(command, log, timeout, **options): def check_build(source_root, profiling): + print("Verified Release configuration: " + json.dumps(release_build_configuration(source_root))) sys.path.insert(0, str(source_root)) import mssql_python import mssql_python_odbc diff --git a/mssql_python/pybind/build.sh b/mssql_python/pybind/build.sh index 05bc69568..d12b5dd0c 100755 --- a/mssql_python/pybind/build.sh +++ b/mssql_python/pybind/build.sh @@ -121,10 +121,12 @@ if [[ "$COVERAGE_MODE" == "true" && "$OS" == "Linux" ]]; then else if [[ "$OS" == "macOS" ]]; then echo "[ACTION] Configuring for macOS (default build)" - cmake -DMACOS_STRING_FIX=ON $PROFILING_FLAG "${SOURCE_DIR}" + cmake -DMACOS_STRING_FIX=ON -DCMAKE_BUILD_TYPE="${CMAKE_BUILD_TYPE:-Release}" \ + $PROFILING_FLAG "${SOURCE_DIR}" else echo "[ACTION] Configuring for Linux with architecture: $DETECTED_ARCH" - cmake -DARCHITECTURE="$DETECTED_ARCH" $PROFILING_FLAG "${SOURCE_DIR}" + cmake -DARCHITECTURE="$DETECTED_ARCH" -DCMAKE_BUILD_TYPE="${CMAKE_BUILD_TYPE:-Release}" \ + $PROFILING_FLAG "${SOURCE_DIR}" fi fi diff --git a/mssql_python/pybind/ddbc_bindings.cpp b/mssql_python/pybind/ddbc_bindings.cpp index 2d0caf3f4..0575b435e 100644 --- a/mssql_python/pybind/ddbc_bindings.cpp +++ b/mssql_python/pybind/ddbc_bindings.cpp @@ -3336,10 +3336,10 @@ py::object FetchLobColumnData(SQLHSTMT hStmt, SQLUSMALLINT colIndex, SQLSMALLINT size_t wcharSize = sizeof(SQLWCHAR); if (bytesRead >= wcharSize && (bytesRead % wcharSize == 0)) { while (bytesRead >= wcharSize) { - SQLWCHAR lastChar; // The byte destination need not be aligned for SQLWCHAR. - std::memcpy(&lastChar, chunk + bytesRead - wcharSize, wcharSize); - if (lastChar != 0) { + const char* lastChar = chunk + bytesRead - wcharSize; + if (std::any_of(lastChar, lastChar + wcharSize, + [](char byte) { return byte != '\0'; })) { break; } bytesRead -= wcharSize; diff --git a/profiler/README.md b/profiler/README.md index e81a7fc99..a918c2a03 100644 --- a/profiler/README.md +++ b/profiler/README.md @@ -280,8 +280,17 @@ adding timers costs nothing in released builds. ### Explicit single-row comparison modes The paired controller retains its existing `diagnostic` default: native and Python -recording are enabled, so the guarded Row path deliberately uses the split route. -That report is not a latency measurement of ordinary fused fetching. +recording are enabled. That instrumented report is not an uninstrumented latency +measurement. Default fetching completes Rows in Python, even when the native +Row-construction binding is available. + +Comparisons require Release builds on both sides. The controller sets +`CMAKE_BUILD_TYPE=Release` for archived sources as well as the candidate; old Unix +build scripts require CMake 3.22 or newer to consume that environment setting. +Build checks reject a missing or non-Release configuration and retain the generator +and C++ flags from `CMakeCache.txt` in build/worker logs. Unix non-coverage builds +also explicitly configure Release by default. Historical measurements without +verified Release configuration are not evidence of shipped Release latency. Two opt-in modes use the same six read-only workloads: `numeric_fetchone`, `numeric_fetchmany`, `numeric_fetchval`, and their `mixed_` equivalents. Each fetches @@ -295,10 +304,12 @@ uses its public API, not a first-column-only native shortcut. timed window; the API loop, result retention and EOF call are inside it. - `--mode route` uses native instrumentation ON with Python phases OFF. Each case must record 1,001 native fetch calls. The version-1 default route contract requires - 1,000 native Row constructions for `fetchmany(1)` and none for `fetchone` or - `fetchval` in the successor. Here `1` is a built-in integer for both numeric and + zero native Row constructions for all three APIs in the current implementation. + Here `1` is a built-in integer for both numeric and mixed shapes; this is not the numeric-column-only native fetching shortcut. - Legacy f539 requires 1,000 constructions for all three APIs; main666 requires none. This is route attribution, not production latency. + Legacy f539 requires 1,000 constructions for all three APIs; the intermediate + many-only route requires 1,000 for `fetchmany(1)`; main666 and the current route + require none. This is route attribution, not production latency. Existing native regression tests separately cover converter fallback, all-column callbacks, repeated EOF, and customization failures. @@ -398,8 +409,10 @@ these allowances, or a speedup. `guarded_row` records native binding capability, not which Python API uses it. New workers carry exactly seven provenance fields: `source_commit`, `native_file`, `native_sha256`, `native_profiling`, `guarded_row`, `python_cursor_sha256`, and -`row_route`. The latter is `{version: 1, methods: {fetchone: false, fetchmany: true, -fetchval: false}}` for the successor. The private literal in `cursor.py` is descriptive; +`row_route`. The latter is `{version: 1, methods: {fetchone: false, fetchmany: false, +fetchval: false}}` for the current implementation. `guarded_row` may still be true: +the low-level binding remains available but is not used by these default routes. +The private literal in `cursor.py` is descriptive; it is never consulted by fetching. `fetchval` still calls the dynamically resolved `self.fetchone()`. The private `_finish_fetchone` completion now observes `native=False` for default fetchone/fetchval instead of the previous eligible `native=True` route. diff --git a/tests/test_017_fetch_bounded_text.py b/tests/test_017_fetch_bounded_text.py index 7f0be4a54..76d6bb862 100644 --- a/tests/test_017_fetch_bounded_text.py +++ b/tests/test_017_fetch_bounded_text.py @@ -436,7 +436,8 @@ def test_lob_binary_read_boundaries(db_connection, method, size): @pytest.mark.parametrize("method", ["fetchone", "fetchmany", "fetchall"]) @pytest.mark.parametrize("kind", ["nvarchar", "varchar-wide", "varchar-narrow"]) def test_lob_text_read_boundaries(db_connection, method, kind): - suffix = "\U0001f642\0caf\u00e9-tail" if kind == "nvarchar" else "caf\u00e9\0tail" + # Keep embedded NUL inside a chunk, outside the existing trailing-NUL trimming. + suffix = "\U0001f642c\0af\u00e9-tail" if kind == "nvarchar" else "caf\u00e9\0tail" expected = [None, ""] + [ "x" * boundary + suffix for boundary in (4093, 4094, 4095, 8187, 8190, 8191, 8192, 262144) ] @@ -490,7 +491,9 @@ def test_lob_getdata_storage_contract(conn_str, kind, mode): kind, mode = sys.argv[1:] binary, wide = kind == "binary", kind == "wide" ctype = -2 if binary else (-8 if wide else 1) - sqltype = "varbinary(max)" if binary else ("nvarchar(max)" if wide else "varchar(max)") + # VARCHAR(MAX) enters the LOB helper directly, including SQL_C_WCHAR. + # NVARCHAR(MAX) can first issue a bounded probe when its reported size is zero. + sqltype = "varbinary(max)" if binary else "varchar(max)" codec = "utf-16le" if wide else "utf-8" unit = 2 if wide else 1 terminator = b"" if binary else bytes(unit) diff --git a/tests/test_036_profiler_ci.py b/tests/test_036_profiler_ci.py index 8f5e24eed..e71795159 100644 --- a/tests/test_036_profiler_ci.py +++ b/tests/test_036_profiler_ci.py @@ -1081,7 +1081,77 @@ def test_unix_profiler_step_does_not_put_database_password_on_command_line(): assert "Pwd=$DB_PASSWORD" in benchmark -def test_build_check_rejects_foreign_provider_and_enabled_recording(tmp_path, monkeypatch): +@pytest.fixture +def release_cache(tmp_path): + cache = tmp_path / "mssql_python/pybind/build/CMakeCache.txt" + cache.parent.mkdir(parents=True) + cache.write_text( + "CMAKE_GENERATOR:INTERNAL=Unix Makefiles\n" + "CMAKE_CXX_FLAGS:STRING=\n" + "CMAKE_CXX_FLAGS_RELEASE:STRING=-O3 -DNDEBUG\n" + "CMAKE_BUILD_TYPE:STRING=Release\n", + encoding="utf-8", + ) + return cache + + +@pytest.mark.parametrize("profiling", [False, True]) +def test_benchmark_build_requests_and_records_release( + tmp_path, monkeypatch, release_cache, profiling +): + def run(command, log, timeout, **options): + assert options["env"]["CMAKE_BUILD_TYPE"] == "Release" + assert options["env"]["ENABLE_PROFILING"] == str(int(profiling)) + assert options["cwd"] == tmp_path / "mssql_python/pybind" + assert timeout == 37 + log.write_text("build output\n", encoding="utf-8") + + monkeypatch.setenv("CMAKE_BUILD_TYPE", "Debug") + monkeypatch.setattr(controller, "run_process", run) + log = tmp_path / "build.log" + controller.build(tmp_path, log, 37, profiling=profiling) + assert 'CMAKE_CXX_FLAGS_RELEASE": "-O3 -DNDEBUG"' in log.read_text(encoding="utf-8") + + +@pytest.mark.parametrize("configuration", ["", "Debug", "Release", "Debug;Release"]) +def test_release_build_configuration(tmp_path, release_cache, configuration): + key = "CMAKE_CONFIGURATION_TYPES" if ";" in configuration else "CMAKE_BUILD_TYPE" + content = release_cache.read_text(encoding="utf-8") + release_cache.write_text( + content.replace("CMAKE_BUILD_TYPE:STRING=Release", f"{key}:STRING={configuration}"), + encoding="utf-8", + ) + if ";" in configuration: + tag = f"py{sys.version_info.major}{sys.version_info.minor}" + windows_cache = release_cache.parent / "x64" / tag / release_cache.name + windows_cache.parent.mkdir(parents=True) + release_cache.replace(windows_cache) + release_cache = windows_cache + if "Release" not in configuration.split(";"): + with pytest.raises(ValueError, match="Release native build"): + controller.release_build_configuration(tmp_path) + else: + assert controller.release_build_configuration(tmp_path)[key] == configuration + release_cache.unlink() + with pytest.raises(FileNotFoundError): + controller.release_build_configuration(tmp_path) + + +def test_release_cache_rejects_ambiguous_windows_builds(tmp_path, release_cache): + content = release_cache.read_text(encoding="utf-8") + tag = f"py{sys.version_info.major}{sys.version_info.minor}" + for arch in ("x64", "arm64"): + cache = release_cache.parent / arch / tag / release_cache.name + cache.parent.mkdir(parents=True) + cache.write_text(content, encoding="utf-8") + release_cache.unlink() + with pytest.raises(ValueError, match="Ambiguous"): + controller.release_build_configuration(tmp_path) + + +def test_build_check_rejects_foreign_provider_and_enabled_recording( + tmp_path, monkeypatch, release_cache +): native = SimpleNamespace( __file__=str(tmp_path / "binding.so"), profiling=SimpleNamespace(is_enabled=lambda: False) ) @@ -1103,6 +1173,153 @@ def test_build_check_rejects_foreign_provider_and_enabled_recording(tmp_path, mo controller.check_build(tmp_path, True) +@pytest.mark.parametrize( + "failure", [None, "timeout", "identity", "deadline", "process-cleanup", "directory-cleanup"] +) +def test_ci_bundle_modes_identity_deadline_cleanup_and_headline( + report, tmp_path, monkeypatch, failure +): + clock, measured, built = [0], [], [] + directory = tmp_path / "build-roots" + directory.mkdir() + monkeypatch.setattr(controller.time, "monotonic", lambda: clock[0]) + monkeypatch.setattr(controller, "resolve_revisions", lambda *a, **k: ("a" * 40, "b" * 40)) + monkeypatch.setattr(controller, "git", lambda *a, **k: "b" * 40) + monkeypatch.setattr(controller.tempfile, "mkdtemp", lambda **k: str(directory)) + monkeypatch.setattr(controller, "run_process", lambda *a, **k: None) + monkeypatch.setattr( + controller, "build", lambda path, *a, **k: built.append((path.name, k["profiling"])) + ) + monkeypatch.setenv("BUILD_BUILDID", "42") + monkeypatch.setenv("SYSTEM_PULLREQUEST_SOURCECOMMITID", "c" * 40) + route = dict(version=1, methods=dict.fromkeys(("fetchone", "fetchmany", "fetchval"), False)) + + def source(path, revision): + return dict(source_commit=revision, python_cursor_sha256="d" * 64, row_route=route) + + monkeypatch.setattr(controller, "python_source_identity", source) + + def measure(path, output, scenarios, timeout, *, mode, revision, isolated): + measured.append(output.name) + assert isolated and scenarios is None + assert timeout == (controller.WORKER_TIMEOUT if mode == "diagnostic" else 30) + if output.name == "latency-base-1.json": + if failure == "timeout": + raise subprocess.TimeoutExpired("worker", timeout) + if failure == "process-cleanup": + raise controller.ProcessCleanupError("injected process cleanup failure") + if failure == "deadline": + clock[0] = controller.BENCHMARK_TIMEOUT + 1 + side = "base" if revision == "a" * 40 else "candidate" + sample = copy.deepcopy(report["pairs"][0][side]) + sample.update( + status="complete", + mode=mode, + provenance=dict( + **source(path, revision), + native_file=str(path / "native.so"), + native_sha256=("1" if mode == "latency" else "2") * 64, + native_profiling=mode != "latency", + guarded_row=True, + ), + ) + if failure == "identity" and output.name == "latency-base-1.json": + sample["provenance"]["native_sha256"] = "3" * 64 + if mode != "diagnostic": + sample["scenarios"] = {} + for name in reporting.FETCH_CASES: + shape, method = name.split("_") + timer = "ddbc::FetchMany_wrap" if method == "fetchmany" else "ddbc::FetchOne_wrap" + sample["scenarios"][name] = dict( + wall_ms=10, + work=f"Rows: 1000; shape: {shape}; API: {method}; EOF: 1", + cpp=( + {timer: dict(calls=1001, total_us=1001, min_us=1, max_us=1)} + if mode == "route" + else {} + ), + py={}, + ) + return sample + + monkeypatch.setattr(controller, "measure", measure) + cleanup = MagicMock(wraps=controller.shutil.rmtree) + if failure == "directory-cleanup": + cleanup.side_effect = OSError("injected directory cleanup failure") + monkeypatch.setattr(controller.shutil, "rmtree", cleanup) + args = SimpleNamespace( + base=None, + candidate="HEAD", + output=tmp_path, + leg=report["leg"], + samples=5, + warmups=1, + reuse_candidate=True, + scenarios=None, + mode="diagnostic", + ) + if failure: + with pytest.raises(RuntimeError): + controller.run_ci_report(args) + else: + controller.run_ci_report(args) + bundle = json.loads((tmp_path / "report.json").read_text(encoding="utf-8")) + modes, errors = reporting.ci_mode_reports(bundle) + if failure in ("process-cleanup", "directory-cleanup"): + assert bundle["cleanup_required"] == str(directory) and directory.exists() + assert cleanup.call_count == (failure == "directory-cleanup") + else: + cleanup.assert_called_once_with(str(directory)) + assert not directory.exists() + if failure in ("process-cleanup", "deadline"): + assert measured == [ + "latency-base-0.json", + "latency-candidate-0.json", + "latency-candidate-1.json", + "latency-base-1.json", + ] + assert all(mode["status"] == "incomplete" for mode in modes.values()) + elif failure in ("timeout", "identity"): + assert modes["latency"]["status"] == "incomplete" and "latency" in errors + assert modes["route"]["status"] == modes["diagnostic"]["status"] == "complete" + elif failure is None: + assert not errors and all(len(mode["pairs"]) == 5 for mode in modes.values()) + assert built == [("base-off", False), ("candidate-off", False), ("base-on", True)] + assert measured == [ + f"{mode}-{side}-{index}.json" + for mode in ("latency", "route", "diagnostic") + for index in range(6) + for side in (("base", "candidate") if index % 2 == 0 else ("candidate", "base")) + ] + headline = reporting.render_ci_reports([bundle], "c" * 40, 42) + del bundle["fetch_measurements"]["latency"] + assert "latency" in reporting.ci_mode_reports(bundle)[1] + assert reporting.render_ci_reports([bundle], "c" * 40, 42) == headline + for field, value in (("source_commit", "e" * 40), ("mode", "latency")): + invalid = copy.deepcopy(bundle) + invalid["fetch_measurements"]["route"][field] = value + assert "route" in reporting.ci_mode_reports(invalid)[1] + assert reporting.render_ci_reports([invalid], "c" * 40, 42) == headline + for pair in bundle["fetch_measurements"]["route"]["pairs"]: + pair["candidate"]["provenance"]["native_sha256"] = "e" * 64 + valid, errors = reporting.ci_mode_reports(bundle) + assert not valid and set(errors) == {"latency", "route", "diagnostic"} + assert "Shared ON" in errors["diagnostic"] + + +@pytest.mark.parametrize("active,guarded", [(False, True), (True, False), (False, 1)]) +def test_default_row_route_distinguishes_capability_from_dispatch(active, guarded): + route = dict(version=1, methods=dict.fromkeys(("fetchone", "fetchmany", "fetchval"), active)) + identity = dict(row_route=route, guarded_row=guarded) + if active or type(guarded) is not bool: + with pytest.raises(ValueError, match="binding contradicts"): + reporting.expected_constructors(identity, "fetchmany") + else: + assert all( + reporting.expected_constructors(identity, method) == 0 for method in route["methods"] + ) + + def test_head_moving_while_listing_comments_prevents_publish(monkeypatch): calls = [] reads = 0 diff --git a/tests/test_fetch_settings_cache.py b/tests/test_fetch_settings_cache.py index 7a3b4dfe9..19d8c1a01 100644 --- a/tests/test_fetch_settings_cache.py +++ b/tests/test_fetch_settings_cache.py @@ -1718,6 +1718,62 @@ def _run_fetch_script(conn_str, script, *arguments, timeout=45): assert result.returncode == 0, result.stdout + result.stderr +@pytest.mark.skipif( + sys.platform == "win32", + reason="Windows does not export the native driver function-pointer globals", +) +@pytest.mark.parametrize("method", ["fetchone", "fetchmany", "fetchval"]) +def test_first_numeric_getdata_error_stops_before_second_column(conn_str, method): + script = textwrap.dedent(""" + import ctypes as c + import os + import sys + from types import SimpleNamespace + from unittest.mock import patch + import mssql_python + from mssql_python import ddbc_bindings as ddbc + + pointer, short, length = c.c_void_p, c.c_short, c.c_ssize_t + get_type = c.CFUNCTYPE(short, pointer, c.c_ushort, short, pointer, length, pointer) + library = c.CDLL(ddbc.module.__file__) + slot = pointer.in_dll(library, "SQLGetData_ptr") + calls = [] + + @get_type + def getdata(handle, column, ctype, buffer, capacity, indicator): + calls.append(column) + if column == 1: + return -1 + return original(handle, column, ctype, buffer, capacity, indicator) + + with mssql_python.connect(os.environ["DB_CONNECTION_STRING"]) as connection: + with connection.cursor() as cursor: + cursor.execute("SELECT CAST(7 AS INT), CAST(8 AS INT)") + position = cursor._next_row_index + saved = slot.value + assert saved + original = get_type(saved) + diagnostic = SimpleNamespace(sqlState="HY000", ddbcErrorMsg="first column failed") + with patch.object(ddbc, "DDBCSQLCheckError", return_value=diagnostic): + slot.value = c.cast(getdata, pointer).value + try: + method = sys.argv[1] + try: + getattr(cursor, method)(*([1] if method == "fetchmany" else [])) + except mssql_python.DatabaseError as error: + assert "first column failed" in str(error) + else: + raise AssertionError("first-column SQL_ERROR was masked") + finally: + slot.value = saved + assert calls == [1], calls + assert cursor._next_row_index == position + cursor.execute("SELECT 42, 43") + assert tuple(cursor.fetchone()) == (42, 43) + """) + _run_fetch_script(conn_str, script, method) + + @pytest.mark.parametrize( ("sql_type", "literal", "expected"), ( From d533112b82f46dc813d53fa225d53d0fcfa28972 Mon Sep 17 00:00:00 2001 From: Jahnvi Thakkar Date: Sat, 10 Oct 2026 12:42:13 +0530 Subject: [PATCH 17/17] DOC: Explain bounded fetchmany batching Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- README.md | 39 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/README.md b/README.md index 40fc409e7..352ebe61c 100644 --- a/README.md +++ b/README.md @@ -207,6 +207,45 @@ for row in rows: connection.close() ``` + +### Processing large results in batches + +Use `cursor.fetchmany(1000)` to retrieve up to 1,000 rows per call and process each +batch before fetching the next. The final batch can be smaller; an empty list +means there are no more rows. Compared with calling `fetchone()` for every row, +batching can reduce per-call overhead without retaining the entire result in +application memory. + +Set `DB_CONNECTION_STRING` for your database before running this example, which +uses the same AdventureWorksLT sample schema as above: + +```python +import os + +import mssql_python + +with mssql_python.connect(os.environ["DB_CONNECTION_STRING"]) as connection: + with connection.cursor() as cursor: + cursor.execute("SELECT CustomerID, FirstName, LastName FROM SalesLT.Customer") + while rows := cursor.fetchmany(1000): + for row in rows: + print(row) +``` + +Replace `print(row)` with processing that does not retain earlier rows or batches. +Choose a batch size for your workload, row widths, and available memory; 1,000 is +an example, not a universally optimal size. Larger batches retain more rows at a +time, and large individual values can still require substantial memory. + +`fetchall()` returns all remaining rows in a list. Similarly, accumulating every +batch in a list retains the full result and loses the memory benefit of consuming +and discarding batches. + +This uses the existing `fetchmany(size)` API and does not change `cursor.arraysize` +or any driver defaults. Without an explicit size, `fetchmany()` uses +`cursor.arraysize`, whose default is 1. Batching requires changing the application's +fetch loop; it does not add read-ahead to, or automatically accelerate, existing +`fetchone()` loops. ## Still have questions?