Skip to content

Commit fa3c64d

Browse files
gargsaumyaCopilot
andcommitted
Merge PR 802 native boundary fixes
Preserve the PR 803 malformed-driver suite while adding geometric LOB growth and execute-boundary regressions from the stacked base. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
2 parents 941e5b7 + 48a2f23 commit fa3c64d

3 files changed

Lines changed: 167 additions & 13 deletions

File tree

‎mssql_python/pybind/ddbc_bindings.cpp‎

Lines changed: 43 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -581,6 +581,29 @@ void EnsureNativeFetchBufferSize(std::vector<ElementType>& buffer, size_t requir
581581
ResizeNativeFetchBuffer(buffer, newSize, reservedBytes);
582582
}
583583

584+
size_t ExtendNativeFetchBuffer(std::vector<char>& buffer, size_t currentSize, size_t appendSize,
585+
size_t& reservedBytes) {
586+
const size_t requiredSize =
587+
CheckedAddSize(currentSize, appendSize, "Native fetch buffer size is too large");
588+
EnsureNativeFetchBufferSize(buffer, requiredSize, reservedBytes);
589+
return requiredSize;
590+
}
591+
592+
py::tuple TestLobBufferGrowth(size_t chunkCount, size_t chunkSize) {
593+
std::vector<char> buffer;
594+
size_t dataSize = 0;
595+
size_t reservedBytes = 0;
596+
size_t growthCount = 0;
597+
for (size_t chunk = 0; chunk < chunkCount; ++chunk) {
598+
const size_t previousAllocation = buffer.size();
599+
dataSize = ExtendNativeFetchBuffer(buffer, dataSize, chunkSize, reservedBytes);
600+
if (buffer.size() != previousAllocation) {
601+
++growthCount;
602+
}
603+
}
604+
return py::make_tuple(growthCount, dataSize, buffer.size(), reservedBytes);
605+
}
606+
584607
template <typename ElementType>
585608
std::unique_ptr<ElementType[]> AllocateArrowArray(size_t count, size_t& reservedBytes,
586609
const char* errorMessage) {
@@ -2484,6 +2507,7 @@ SQLRETURN SQLExecute_wrap(const SqlHandlePtr statementHandle,
24842507
bool use_prepare,
24852508
const py::dict& encoding_settings) {
24862509
PERF_TIMER("SQLExecute_wrap");
2510+
ValidateInputSizes(input_sizes.ptr());
24872511
if (!statementHandle || !statementHandle->get()) {
24882512
return SQL_INVALID_HANDLE;
24892513
}
@@ -3306,9 +3330,14 @@ SQLRETURN BindParameterArray(SqlHandle& handle, SQLHANDLE hStmt, const py::list&
33063330

33073331
SQLRETURN SQLExecuteMany_wrap(const SqlHandlePtr statementHandle, const std::u16string& query,
33083332
const py::list& columnwise_params,
3309-
std::vector<ParamInfo>& paramInfos, size_t paramSetSize,
3333+
std::vector<ParamInfo>& paramInfos, py::handle paramSetSizeArg,
33103334
const py::dict& encodingSettings) {
33113335
PERF_TIMER("SQLExecuteMany_wrap");
3336+
const size_t paramSetSize = static_cast<size_t>(
3337+
ValidateNativeRowCountArgument(paramSetSizeArg, "Parameter set size", false));
3338+
if (!statementHandle || !statementHandle->get()) {
3339+
return SQL_INVALID_HANDLE;
3340+
}
33123341
statementHandle->resultMetadata.clear();
33133342
LOG("SQLExecuteMany: Starting batch execution - param_count=%zu, "
33143343
"param_set_size=%zu",
@@ -3646,6 +3675,7 @@ static py::object FetchLobColumnDataImpl(
36463675
SQLGetDataFunc getData, bool (*hasTruncationDiagnostic)(SQLHSTMT)) {
36473676
PERF_TIMER("FetchLobColumnData");
36483677
std::vector<char> buffer;
3678+
size_t dataSize = 0;
36493679
size_t reservedBytes = 0;
36503680
ReserveNativeFetchBytes(reservedBytes, DAE_CHUNK_SIZE, sizeof(char));
36513681
SQLRETURN ret = SQL_SUCCESS_WITH_INFO;
@@ -3742,10 +3772,8 @@ static py::object FetchLobColumnDataImpl(
37423772
ThrowStdException("LOB fetch truncation made no progress");
37433773
}
37443774
if (bytesRead > 0) {
3745-
const size_t previousSize = buffer.size();
3746-
const size_t requiredSize =
3747-
CheckedAddSize(previousSize, bytesRead, "LOB fetch buffer is too large");
3748-
ResizeNativeFetchBuffer(buffer, requiredSize, reservedBytes);
3775+
const size_t previousSize = dataSize;
3776+
dataSize = ExtendNativeFetchBuffer(buffer, dataSize, bytesRead, reservedBytes);
37493777
std::copy_n(chunk.data(), bytesRead, buffer.data() + previousSize);
37503778
LOG("FetchLobColumnData: Appended %zu bytes at loop %d", bytesRead, loopCount);
37513779
}
@@ -3755,21 +3783,21 @@ static py::object FetchLobColumnDataImpl(
37553783
break;
37563784
}
37573785
}
3758-
LOG("FetchLobColumnData: Total bytes collected=%zu for column %d", buffer.size(), colIndex);
3786+
LOG("FetchLobColumnData: Total bytes collected=%zu for column %d", dataSize, colIndex);
37593787

3760-
if (buffer.empty()) {
3788+
if (dataSize == 0) {
37613789
if (isBinary) {
37623790
return py::bytes("");
37633791
}
37643792
return py::str("");
37653793
}
37663794
if (isWideChar) {
3767-
if (buffer.size() % sizeof(SQLWCHAR) != 0) {
3795+
if (dataSize % sizeof(SQLWCHAR) != 0) {
37683796
ThrowStdException("Wide-character LOB data has an invalid byte length");
37693797
}
37703798
int byteOrder = -1;
37713799
PyObject* decoded =
3772-
PyUnicode_DecodeUTF16(buffer.data(), static_cast<Py_ssize_t>(buffer.size()),
3800+
PyUnicode_DecodeUTF16(buffer.data(), static_cast<Py_ssize_t>(dataSize),
37733801
"strict", &byteOrder);
37743802
if (decoded == nullptr) {
37753803
throw py::error_already_set();
@@ -3779,18 +3807,18 @@ static py::object FetchLobColumnDataImpl(
37793807
if (isBinary) {
37803808
LOG("FetchLobColumnData: Returning binary data - %zu bytes for column "
37813809
"%d",
3782-
buffer.size(), colIndex);
3783-
return py::bytes(buffer.data(), buffer.size());
3810+
dataSize, colIndex);
3811+
return py::bytes(buffer.data(), dataSize);
37843812
}
37853813

37863814
// For SQL_C_CHAR data, decode using the appropriate encoding.
37873815
const std::string effectiveCharEncoding = GetEffectiveCharDecoding(charEncoding);
3788-
py::bytes raw_bytes(buffer.data(), buffer.size());
3816+
py::bytes raw_bytes(buffer.data(), dataSize);
37893817
try {
37903818
py::object decoded = raw_bytes.attr("decode")(effectiveCharEncoding, "strict");
37913819
LOG("FetchLobColumnData: Decoded narrow string with '%s' - %zu bytes -> %zu chars for "
37923820
"column %d",
3793-
effectiveCharEncoding.c_str(), buffer.size(), py::len(decoded), colIndex);
3821+
effectiveCharEncoding.c_str(), dataSize, py::len(decoded), colIndex);
37943822
return decoded;
37953823
} catch (const py::error_already_set& e) {
37963824
LOG_ERROR("FetchLobColumnData: Failed to decode with '%s' for column %d: %s",
@@ -7538,6 +7566,8 @@ PYBIND11_MODULE(ddbc_bindings, m) {
75387566
"Fetch an arrow batch of given length from the result set", py::arg("StatementHandle"),
75397567
py::arg("capsules"), py::arg("arrowBatchSize"), py::arg("charCtype"),
75407568
py::arg("messages") = py::none());
7569+
m.def("_test_lob_buffer_growth", &TestLobBufferGrowth, py::arg("chunk_count"),
7570+
py::arg("chunk_size"));
75417571
m.def("DDBCSQLFreeHandle", &SQLFreeHandle_wrap, "Free a handle");
75427572
m.def("DDBCSQLResetStmt", &SQLResetStmt_wrap,
75437573
"Close cursor and unbind params without freeing HSTMT");

‎mssql_python/pybind/param_detect.hpp‎

Lines changed: 88 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -274,6 +274,89 @@ inline void NormalizeTimeParam(PyObject* params, Py_ssize_t index, SQLULEN& colu
274274
}
275275
}
276276

277+
inline long long ValidatedInputSizeInteger(PyObject* value, const char* fieldName) {
278+
if (!PyLong_Check(value) || PyBool_Check(value)) {
279+
throw py::type_error(std::string(fieldName) + " must be an integer");
280+
}
281+
int overflow = 0;
282+
const long long result = PyLong_AsLongLongAndOverflow(value, &overflow);
283+
if ((result == -1 && PyErr_Occurred()) || overflow != 0) {
284+
PyErr_Clear();
285+
throw py::value_error(std::string(fieldName) + " is out of range");
286+
}
287+
return result;
288+
}
289+
290+
inline void ValidateInputSizes(PyObject* inputSizes) {
291+
if (inputSizes == Py_None) {
292+
return;
293+
}
294+
if (!PyList_Check(inputSizes)) {
295+
throw py::type_error("inputSizes must be None or a list");
296+
}
297+
298+
const Py_ssize_t count = PyList_GET_SIZE(inputSizes);
299+
for (Py_ssize_t index = 0; index < count; ++index) {
300+
PyObject* entry = PyList_GET_ITEM(inputSizes, index);
301+
if (!PyTuple_Check(entry) || PyTuple_GET_SIZE(entry) != 4) {
302+
throw py::type_error("each inputSizes entry must be a four-item tuple");
303+
}
304+
305+
const long long sqlType =
306+
ValidatedInputSizeInteger(PyTuple_GET_ITEM(entry, 0), "SQL type");
307+
const long long cType =
308+
ValidatedInputSizeInteger(PyTuple_GET_ITEM(entry, 1), "C type");
309+
PyObject* columnSize = PyTuple_GET_ITEM(entry, 2);
310+
PyObject* decimalDigits = PyTuple_GET_ITEM(entry, 3);
311+
if (!PyLong_Check(columnSize) || PyBool_Check(columnSize) ||
312+
!PyLong_Check(decimalDigits) || PyBool_Check(decimalDigits)) {
313+
throw py::type_error("column size and decimal digits must be integers");
314+
}
315+
316+
if (sqlType < std::numeric_limits<SQLSMALLINT>::min() ||
317+
sqlType > std::numeric_limits<SQLSMALLINT>::max() ||
318+
cType < std::numeric_limits<SQLSMALLINT>::min() ||
319+
cType > std::numeric_limits<SQLSMALLINT>::max()) {
320+
throw py::value_error("SQL and C types must fit in SQLSMALLINT");
321+
}
322+
const bool isNumeric = sqlType == SQL_DECIMAL || sqlType == SQL_NUMERIC;
323+
py::int_ zero(0);
324+
const int negativeColumnSize =
325+
PyObject_RichCompareBool(columnSize, zero.ptr(), Py_LT);
326+
const int negativeDecimalDigits =
327+
PyObject_RichCompareBool(decimalDigits, zero.ptr(), Py_LT);
328+
if (negativeColumnSize == -1 || negativeDecimalDigits == -1) {
329+
throw py::error_already_set();
330+
}
331+
if (negativeColumnSize == 1) {
332+
throw py::value_error("column size must be non-negative");
333+
}
334+
if (negativeDecimalDigits == 1) {
335+
throw py::value_error("decimal digits must be non-negative");
336+
}
337+
if (!isNumeric) {
338+
const unsigned long long requestedSize = PyLong_AsUnsignedLongLong(columnSize);
339+
if (requestedSize == static_cast<unsigned long long>(-1) && PyErr_Occurred()) {
340+
PyErr_Clear();
341+
throw py::value_error("column size is out of range");
342+
}
343+
if (requestedSize > std::numeric_limits<SQLULEN>::max()) {
344+
throw py::value_error("column size is out of range");
345+
}
346+
const unsigned long long requestedDigits =
347+
PyLong_AsUnsignedLongLong(decimalDigits);
348+
if (requestedDigits == static_cast<unsigned long long>(-1) && PyErr_Occurred()) {
349+
PyErr_Clear();
350+
throw py::value_error("decimal digits are out of range");
351+
}
352+
if (requestedDigits >
353+
static_cast<unsigned long long>(std::numeric_limits<SQLSMALLINT>::max())) {
354+
throw py::value_error("decimal digits are out of range");
355+
}
356+
}
357+
}
358+
}
359+
277360
inline void ApplyInputSizeOverride(PyObject* params, PyObject* inputSize, Py_ssize_t index,
278361
ParamInfo& info, const std::string& charEncoding) {
279362
py::tuple values = borrow<py::tuple>(inputSize);
@@ -391,6 +474,11 @@ inline std::vector<ParamInfo> DetectParamTypes(PyObject* params, PyObject* input
391474
const std::string& charEncoding = "utf-8") {
392475
PyTypeCache::initialize();
393476

477+
if (!PyList_Check(params)) {
478+
throw py::type_error("params must be a list");
479+
}
480+
ValidateInputSizes(inputSizes);
481+
394482
const Py_ssize_t n = PyList_GET_SIZE(params);
395483
const Py_ssize_t inputSizeCount = inputSizes == Py_None ? 0 : PyList_GET_SIZE(inputSizes);
396484
std::vector<ParamInfo> infos(n);

‎tests/test_010_pybind_functions.py‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -139,6 +139,42 @@ def test_lob_unrelated_warning_with_progress_completes():
139139
assert calls == 1
140140

141141

142+
@pytest.mark.skipif(not DDBC_AVAILABLE, reason="ddbc_bindings not available")
143+
@pytest.mark.parametrize("param_set_size", [-1, 0, 1_000_001, True])
144+
def test_executemany_rejects_unsafe_size_before_handle_access(param_set_size):
145+
with pytest.raises(RuntimeError, match="Parameter set size"):
146+
ddbc.SQLExecuteMany(None, "", [], [], param_set_size, {})
147+
148+
149+
@pytest.mark.skipif(not DDBC_AVAILABLE, reason="ddbc_bindings not available")
150+
@pytest.mark.parametrize(
151+
("input_sizes", "message"),
152+
[
153+
((12, -8, 1, 0), "inputSizes must be None or a list"),
154+
([None], "four-item tuple"),
155+
([(12, -8, 1)], "four-item tuple"),
156+
([(12, -8, True, 0)], "must be integers"),
157+
([(12, -8, -1, 0)], "column size must be non-negative"),
158+
([(12, -8, 2**128, 0)], "column size is out of range"),
159+
([(12, -8, 1, 2**128)], "decimal digits are out of range"),
160+
],
161+
)
162+
def test_execute_rejects_malformed_input_sizes_before_handle_access(input_sizes, message):
163+
with pytest.raises((TypeError, ValueError), match=message):
164+
ddbc.DDBCSQLExecute(None, "", [], input_sizes, [False], True, {})
165+
166+
167+
@pytest.mark.skipif(not DDBC_AVAILABLE, reason="ddbc_bindings not available")
168+
def test_lob_buffer_growth_is_geometric():
169+
growth_count, data_size, allocation_size, reserved_bytes = ddbc._test_lob_buffer_growth(
170+
64, 64 * 1024
171+
)
172+
assert data_size == 4 * 1024 * 1024
173+
assert allocation_size == data_size
174+
assert reserved_bytes == allocation_size
175+
assert growth_count == 7
176+
177+
142178
@pytest.mark.skipif(not DDBC_AVAILABLE, reason="ddbc_bindings not available")
143179
class TestPybindModuleInfo:
144180
"""Test module information and architecture detection."""

0 commit comments

Comments
 (0)