From 2df4d3f7eeb91f906acd0db3b5828885dc12a675 Mon Sep 17 00:00:00 2001 From: Spenser Sun Date: Tue, 25 Aug 2026 02:15:27 +0000 Subject: [PATCH] [SPARK-58766][PYTHON][TEST][FOLLOWUP] Render NaT-sentinel time64 scalars in golden files Rendering a `time64[ns]` scalar goes through pandas, since Python's `time` has only microsecond resolution, and pandas reads INT64_MIN as its NaT sentinel and refuses that one value. `_scalar_str` now records it as `NaT collision`, beside the existing `temporal overflow` and raw-bytes fallbacks, and the four affected type-scalar golden cells are regenerated. Those cells previously read `ERR@ValueError`, which claims the conversion raised; the conversion in fact succeeds and only the rendering failed. The broad cell try/except recorded that as if it were a conversion error, and narrowing it in #58091 turned the mislabelling into a test error. --- python/pyspark/testing/goldenutils.py | 16 +++++-- ...row_array_from_pandas_type_scalar_safe.csv | 4 +- ...rrow_array_from_pandas_type_scalar_safe.md | 42 +++++++++---------- ...w_array_from_pandas_type_scalar_unsafe.csv | 4 +- ...ow_array_from_pandas_type_scalar_unsafe.md | 42 +++++++++---------- 5 files changed, 59 insertions(+), 49 deletions(-) diff --git a/python/pyspark/testing/goldenutils.py b/python/pyspark/testing/goldenutils.py index f9ba1f66fd986..6d60c4a8be922 100644 --- a/python/pyspark/testing/goldenutils.py +++ b/python/pyspark/testing/goldenutils.py @@ -316,13 +316,16 @@ def _scalar_str(scalar: Any) -> str: """ Render one PyArrow scalar for a golden cell via PyArrow's own ``str(scalar)``. - Some values are valid in Arrow yet unrenderable via ``str``: + Some values that Arrow stores are unrenderable via ``str``: - a temporal value past Python's ``datetime`` range (e.g. date32 past year 9999) raises ``OverflowError`` -> the marker ``temporal overflow``; - an unsafe ``binary``->``string`` cast relabels bytes without UTF-8 validation, so non-UTF-8 bytes raise ``UnicodeDecodeError`` -> the raw bytes, so the golden - still tracks what Arrow stored for them. - An unexpected error (non-temporal overflow, non-string decode error) propagates. + still tracks what Arrow stored for them; + - a nanosecond ``time64`` holding INT64_MIN collides with pandas' NaT sentinel and + raises ``ValueError`` -> the marker ``NaT collision``. + An unexpected error (non-temporal overflow, non-string decode error, or a + ValueError from any other value) propagates. """ try: return str(scalar).replace("\x00", "\\0") @@ -341,6 +344,13 @@ def _scalar_str(scalar: Any) -> str: if pa.types.is_string(scalar.type) or pa.types.is_large_string(scalar.type): return repr(scalar.as_buffer().to_pybytes()) raise + except ValueError: + # A nanosecond time64 renders through pandas (Python's ``time`` is microsecond + # resolution), and pandas reads INT64_MIN as its NaT sentinel, so it refuses + # that one value. Any other ValueError is unexpected, so re-raise. + if pa.types.is_time(scalar.type) and scalar.value == pd.NaT.value: + return "NaT collision" + raise @classmethod def repr_arrow_value( diff --git a/python/pyspark/tests/upstream/pyarrow/golden_pyarrow_array_from_pandas_type_scalar_safe.csv b/python/pyspark/tests/upstream/pyarrow/golden_pyarrow_array_from_pandas_type_scalar_safe.csv index a054d3a83712b..0ef13693498db 100644 --- a/python/pyspark/tests/upstream/pyarrow/golden_pyarrow_array_from_pandas_type_scalar_safe.csv +++ b/python/pyspark/tests/upstream/pyarrow/golden_pyarrow_array_from_pandas_type_scalar_safe.csv @@ -1,5 +1,5 @@ source \ target int8 int64 float32 timestamp[us] date32[day] duration[us] time64[ns] string binary -int64:standard ERR@ArrowInvalid [0, 1, -1, 9223372036854775807, -9223372036854775808]@int64 ERR@ArrowInvalid [1970-01-01 00:00:00, 1970-01-01 00:00:00.000001, 1969-12-31 23:59:59.999999, temporal overflow, temporal overflow]@timestamp[us] ERR@ArrowNotImplementedError [0:00:00, 0:00:00.000001, -1 day, 23:59:59.999999, 106751991 days, 4:00:54.775807, -106751992 days, 19:59:05.224192]@duration[us] ERR@ValueError ERR@ArrowTypeError [b'', b'\x01', b'\xff\xff\xff\xff\xff\xff\xff\xff', b'\xff\xff\xff\xff\xff\xff\xff\x7f', b'']@binary +int64:standard ERR@ArrowInvalid [0, 1, -1, 9223372036854775807, -9223372036854775808]@int64 ERR@ArrowInvalid [1970-01-01 00:00:00, 1970-01-01 00:00:00.000001, 1969-12-31 23:59:59.999999, temporal overflow, temporal overflow]@timestamp[us] ERR@ArrowNotImplementedError [0:00:00, 0:00:00.000001, -1 day, 23:59:59.999999, 106751991 days, 4:00:54.775807, -106751992 days, 19:59:05.224192]@duration[us] [00:00:00, 00:00:00, 23:59:59.999999, 23:47:16.854775, NaT collision]@time64[ns] ERR@ArrowTypeError [b'', b'\x01', b'\xff\xff\xff\xff\xff\xff\xff\xff', b'\xff\xff\xff\xff\xff\xff\xff\x7f', b'']@binary float64:standard ERR@ArrowInvalid ERR@ArrowInvalid [0.0, 1.5, -1.5]@float32 ERR@ArrowNotImplementedError ERR@ArrowNotImplementedError ERR@ArrowNotImplementedError ERR@ArrowNotImplementedError ERR@ArrowTypeError [b'', b'', b'']@binary bool:standard [1, 0, 1]@int8 [1, 0, 1]@int64 [1.0, 0.0, 1.0]@float32 ERR@ArrowNotImplementedError ERR@ArrowNotImplementedError ERR@ArrowNotImplementedError ERR@ArrowNotImplementedError ERR@ArrowTypeError [b'\x01', b'', b'\x01']@binary object:string ERR@ArrowInvalid ERR@ArrowInvalid ERR@ArrowInvalid ERR@ArrowTypeError ERR@ArrowTypeError ERR@ArrowTypeError ERR@ArrowTypeError [hello, world, ]@string [b'hello', b'world', b'']@binary @@ -8,7 +8,7 @@ date:standard ERR@ArrowInvalid ERR@ArrowInvalid ERR@ArrowInvalid ERR@ArrowTypeEr object:datetime ERR@ArrowInvalid ERR@ArrowInvalid ERR@ArrowInvalid [2024-06-15 18:30:00]@timestamp[us] [2024-06-15]@date32[day] ERR@ArrowTypeError ERR@ArrowTypeError ERR@ArrowTypeError ERR@ArrowTypeError object:timedelta ERR@ArrowInvalid ERR@ArrowInvalid ERR@ArrowInvalid ERR@ArrowTypeError ERR@ArrowTypeError [1 day, 2:00:00]@duration[us] ERR@ArrowTypeError ERR@ArrowTypeError ERR@ArrowTypeError time:standard ERR@ArrowInvalid ERR@ArrowInvalid ERR@ArrowInvalid ERR@ArrowTypeError ERR@ArrowTypeError ERR@ArrowTypeError [18:30:45]@time64[ns] ERR@ArrowTypeError ERR@ArrowTypeError -Int64:standard ERR@ArrowInvalid [0, 1, 9223372036854775807, -9223372036854775808]@int64 ERR@ArrowInvalid [1970-01-01 00:00:00, 1970-01-01 00:00:00.000001, temporal overflow, temporal overflow]@timestamp[us] ERR@ArrowNotImplementedError [0:00:00, 0:00:00.000001, 106751991 days, 4:00:54.775807, -106751992 days, 19:59:05.224192]@duration[us] ERR@ValueError ERR@ArrowTypeError [b'', b'\x01', b'\xff\xff\xff\xff\xff\xff\xff\x7f', b'']@binary +Int64:standard ERR@ArrowInvalid [0, 1, 9223372036854775807, -9223372036854775808]@int64 ERR@ArrowInvalid [1970-01-01 00:00:00, 1970-01-01 00:00:00.000001, temporal overflow, temporal overflow]@timestamp[us] ERR@ArrowNotImplementedError [0:00:00, 0:00:00.000001, 106751991 days, 4:00:54.775807, -106751992 days, 19:59:05.224192]@duration[us] [00:00:00, 00:00:00, 23:47:16.854775, NaT collision]@time64[ns] ERR@ArrowTypeError [b'', b'\x01', b'\xff\xff\xff\xff\xff\xff\xff\x7f', b'']@binary string[pyarrow]:standard ERR@ArrowInvalid ERR@ArrowInvalid ERR@ArrowInvalid ERR@ArrowInvalid ERR@ArrowInvalid ERR@ArrowNotImplementedError ERR@ArrowNotImplementedError [hello, world]@string [b'hello', b'world']@binary large_binary[pyarrow]:standard ERR@ArrowInvalid ERR@ArrowInvalid ERR@ArrowInvalid ERR@ArrowNotImplementedError ERR@ArrowNotImplementedError ERR@ArrowNotImplementedError ERR@ArrowNotImplementedError [hello, world]@string [b'hello', b'world']@binary int64:overflow ERR@ArrowInvalid [300, 1]@int64 [300.0, 1.0]@float32 [1970-01-01 00:00:00.000300, 1970-01-01 00:00:00.000001]@timestamp[us] ERR@ArrowNotImplementedError [0:00:00.000300, 0:00:00.000001]@duration[us] [00:00:00, 00:00:00]@time64[ns] ERR@ArrowTypeError [b',\x01', b'\x01']@binary diff --git a/python/pyspark/tests/upstream/pyarrow/golden_pyarrow_array_from_pandas_type_scalar_safe.md b/python/pyspark/tests/upstream/pyarrow/golden_pyarrow_array_from_pandas_type_scalar_safe.md index 65476ad0e9e99..2157f49343ac9 100644 --- a/python/pyspark/tests/upstream/pyarrow/golden_pyarrow_array_from_pandas_type_scalar_safe.md +++ b/python/pyspark/tests/upstream/pyarrow/golden_pyarrow_array_from_pandas_type_scalar_safe.md @@ -1,21 +1,21 @@ -| source \ target | int8 | int64 | float32 | timestamp[us] | date32[day] | duration[us] | time64[ns] | string | binary | -|--------------------------------|------------------------------|-------------------------------------------------------------|------------------------------|-----------------------------------------------------------------------------------------------------------------------------------|--------------------------------------|-----------------------------------------------------------------------------------------------------------------------------------|---------------------------------|-------------------------|------------------------------------------------------------------------------------------------------| -| int64:standard | ERR@ArrowInvalid | [0, 1, -1, 9223372036854775807, -9223372036854775808]@int64 | ERR@ArrowInvalid | [1970-01-01 00:00:00, 1970-01-01 00:00:00.000001, 1969-12-31 23:59:59.999999, temporal overflow, temporal overflow]@timestamp[us] | ERR@ArrowNotImplementedError | [0:00:00, 0:00:00.000001, -1 day, 23:59:59.999999, 106751991 days, 4:00:54.775807, -106751992 days, 19:59:05.224192]@duration[us] | ERR@ValueError | ERR@ArrowTypeError | [b'', b'\x01', b'\xff\xff\xff\xff\xff\xff\xff\xff', b'\xff\xff\xff\xff\xff\xff\xff\x7f', b'']@binary | -| float64:standard | ERR@ArrowInvalid | ERR@ArrowInvalid | [0.0, 1.5, -1.5]@float32 | ERR@ArrowNotImplementedError | ERR@ArrowNotImplementedError | ERR@ArrowNotImplementedError | ERR@ArrowNotImplementedError | ERR@ArrowTypeError | [b'', b'', b'']@binary | -| bool:standard | [1, 0, 1]@int8 | [1, 0, 1]@int64 | [1.0, 0.0, 1.0]@float32 | ERR@ArrowNotImplementedError | ERR@ArrowNotImplementedError | ERR@ArrowNotImplementedError | ERR@ArrowNotImplementedError | ERR@ArrowTypeError | [b'\x01', b'', b'\x01']@binary | -| object:string | ERR@ArrowInvalid | ERR@ArrowInvalid | ERR@ArrowInvalid | ERR@ArrowTypeError | ERR@ArrowTypeError | ERR@ArrowTypeError | ERR@ArrowTypeError | [hello, world, ]@string | [b'hello', b'world', b'']@binary | -| object:bytes | ERR@ArrowInvalid | ERR@ArrowInvalid | ERR@ArrowInvalid | ERR@ArrowTypeError | ERR@ArrowTypeError | ERR@ArrowTypeError | ERR@ArrowTypeError | [hello, world]@string | [b'hello', b'world']@binary | -| date:standard | ERR@ArrowInvalid | ERR@ArrowInvalid | ERR@ArrowInvalid | ERR@ArrowTypeError | [2024-06-15]@date32[day] | ERR@ArrowTypeError | ERR@ArrowTypeError | ERR@ArrowTypeError | ERR@ArrowTypeError | -| object:datetime | ERR@ArrowInvalid | ERR@ArrowInvalid | ERR@ArrowInvalid | [2024-06-15 18:30:00]@timestamp[us] | [2024-06-15]@date32[day] | ERR@ArrowTypeError | ERR@ArrowTypeError | ERR@ArrowTypeError | ERR@ArrowTypeError | -| object:timedelta | ERR@ArrowInvalid | ERR@ArrowInvalid | ERR@ArrowInvalid | ERR@ArrowTypeError | ERR@ArrowTypeError | [1 day, 2:00:00]@duration[us] | ERR@ArrowTypeError | ERR@ArrowTypeError | ERR@ArrowTypeError | -| time:standard | ERR@ArrowInvalid | ERR@ArrowInvalid | ERR@ArrowInvalid | ERR@ArrowTypeError | ERR@ArrowTypeError | ERR@ArrowTypeError | [18:30:45]@time64[ns] | ERR@ArrowTypeError | ERR@ArrowTypeError | -| Int64:standard | ERR@ArrowInvalid | [0, 1, 9223372036854775807, -9223372036854775808]@int64 | ERR@ArrowInvalid | [1970-01-01 00:00:00, 1970-01-01 00:00:00.000001, temporal overflow, temporal overflow]@timestamp[us] | ERR@ArrowNotImplementedError | [0:00:00, 0:00:00.000001, 106751991 days, 4:00:54.775807, -106751992 days, 19:59:05.224192]@duration[us] | ERR@ValueError | ERR@ArrowTypeError | [b'', b'\x01', b'\xff\xff\xff\xff\xff\xff\xff\x7f', b'']@binary | -| string[pyarrow]:standard | ERR@ArrowInvalid | ERR@ArrowInvalid | ERR@ArrowInvalid | ERR@ArrowInvalid | ERR@ArrowInvalid | ERR@ArrowNotImplementedError | ERR@ArrowNotImplementedError | [hello, world]@string | [b'hello', b'world']@binary | -| large_binary[pyarrow]:standard | ERR@ArrowInvalid | ERR@ArrowInvalid | ERR@ArrowInvalid | ERR@ArrowNotImplementedError | ERR@ArrowNotImplementedError | ERR@ArrowNotImplementedError | ERR@ArrowNotImplementedError | [hello, world]@string | [b'hello', b'world']@binary | -| int64:overflow | ERR@ArrowInvalid | [300, 1]@int64 | [300.0, 1.0]@float32 | [1970-01-01 00:00:00.000300, 1970-01-01 00:00:00.000001]@timestamp[us] | ERR@ArrowNotImplementedError | [0:00:00.000300, 0:00:00.000001]@duration[us] | [00:00:00, 00:00:00]@time64[ns] | ERR@ArrowTypeError | [b',\x01', b'\x01']@binary | -| float64:fractional | ERR@ArrowInvalid | ERR@ArrowInvalid | [1.5, 2.5]@float32 | ERR@ArrowNotImplementedError | ERR@ArrowNotImplementedError | ERR@ArrowNotImplementedError | ERR@ArrowNotImplementedError | ERR@ArrowTypeError | [b'', b'']@binary | -| float64:infinity | ERR@ArrowInvalid | ERR@ArrowInvalid | [inf, 1.0]@float32 | ERR@ArrowNotImplementedError | ERR@ArrowNotImplementedError | ERR@ArrowNotImplementedError | ERR@ArrowNotImplementedError | ERR@ArrowTypeError | [b'', b'']@binary | -| float64:precision | ERR@ArrowInvalid | ERR@ArrowInvalid | [1.1234568357467651]@float32 | ERR@ArrowNotImplementedError | ERR@ArrowNotImplementedError | ERR@ArrowNotImplementedError | ERR@ArrowNotImplementedError | ERR@ArrowTypeError | [b'\x98nt\xd3\xad\xf9\xf1?']@binary | -| datetime64[ns]:sub-us | ERR@ArrowNotImplementedError | [1704067200000000123]@int64 | ERR@ArrowNotImplementedError | ERR@ArrowInvalid | [2024-01-01]@date32[day] | ERR@ArrowNotImplementedError | [00:00:00]@time64[ns] | ERR@ArrowTypeError | [b'{']@binary | -| object:date-then-datetime | ERR@ArrowInvalid | ERR@ArrowInvalid | ERR@ArrowInvalid | ERR@ArrowTypeError | [2024-01-01, 2024-01-01]@date32[day] | ERR@ArrowTypeError | ERR@ArrowTypeError | ERR@ArrowTypeError | ERR@ArrowTypeError | -| object:datetime-then-date | ERR@ArrowInvalid | ERR@ArrowInvalid | ERR@ArrowInvalid | ERR@ArrowTypeError | [2024-01-01, 2024-01-01]@date32[day] | ERR@ArrowTypeError | ERR@ArrowTypeError | ERR@ArrowTypeError | ERR@ArrowTypeError | \ No newline at end of file +| source \ target | int8 | int64 | float32 | timestamp[us] | date32[day] | duration[us] | time64[ns] | string | binary | +|--------------------------------|------------------------------|-------------------------------------------------------------|------------------------------|-----------------------------------------------------------------------------------------------------------------------------------|--------------------------------------|-----------------------------------------------------------------------------------------------------------------------------------|----------------------------------------------------------------------------------|-------------------------|------------------------------------------------------------------------------------------------------| +| int64:standard | ERR@ArrowInvalid | [0, 1, -1, 9223372036854775807, -9223372036854775808]@int64 | ERR@ArrowInvalid | [1970-01-01 00:00:00, 1970-01-01 00:00:00.000001, 1969-12-31 23:59:59.999999, temporal overflow, temporal overflow]@timestamp[us] | ERR@ArrowNotImplementedError | [0:00:00, 0:00:00.000001, -1 day, 23:59:59.999999, 106751991 days, 4:00:54.775807, -106751992 days, 19:59:05.224192]@duration[us] | [00:00:00, 00:00:00, 23:59:59.999999, 23:47:16.854775, NaT collision]@time64[ns] | ERR@ArrowTypeError | [b'', b'\x01', b'\xff\xff\xff\xff\xff\xff\xff\xff', b'\xff\xff\xff\xff\xff\xff\xff\x7f', b'']@binary | +| float64:standard | ERR@ArrowInvalid | ERR@ArrowInvalid | [0.0, 1.5, -1.5]@float32 | ERR@ArrowNotImplementedError | ERR@ArrowNotImplementedError | ERR@ArrowNotImplementedError | ERR@ArrowNotImplementedError | ERR@ArrowTypeError | [b'', b'', b'']@binary | +| bool:standard | [1, 0, 1]@int8 | [1, 0, 1]@int64 | [1.0, 0.0, 1.0]@float32 | ERR@ArrowNotImplementedError | ERR@ArrowNotImplementedError | ERR@ArrowNotImplementedError | ERR@ArrowNotImplementedError | ERR@ArrowTypeError | [b'\x01', b'', b'\x01']@binary | +| object:string | ERR@ArrowInvalid | ERR@ArrowInvalid | ERR@ArrowInvalid | ERR@ArrowTypeError | ERR@ArrowTypeError | ERR@ArrowTypeError | ERR@ArrowTypeError | [hello, world, ]@string | [b'hello', b'world', b'']@binary | +| object:bytes | ERR@ArrowInvalid | ERR@ArrowInvalid | ERR@ArrowInvalid | ERR@ArrowTypeError | ERR@ArrowTypeError | ERR@ArrowTypeError | ERR@ArrowTypeError | [hello, world]@string | [b'hello', b'world']@binary | +| date:standard | ERR@ArrowInvalid | ERR@ArrowInvalid | ERR@ArrowInvalid | ERR@ArrowTypeError | [2024-06-15]@date32[day] | ERR@ArrowTypeError | ERR@ArrowTypeError | ERR@ArrowTypeError | ERR@ArrowTypeError | +| object:datetime | ERR@ArrowInvalid | ERR@ArrowInvalid | ERR@ArrowInvalid | [2024-06-15 18:30:00]@timestamp[us] | [2024-06-15]@date32[day] | ERR@ArrowTypeError | ERR@ArrowTypeError | ERR@ArrowTypeError | ERR@ArrowTypeError | +| object:timedelta | ERR@ArrowInvalid | ERR@ArrowInvalid | ERR@ArrowInvalid | ERR@ArrowTypeError | ERR@ArrowTypeError | [1 day, 2:00:00]@duration[us] | ERR@ArrowTypeError | ERR@ArrowTypeError | ERR@ArrowTypeError | +| time:standard | ERR@ArrowInvalid | ERR@ArrowInvalid | ERR@ArrowInvalid | ERR@ArrowTypeError | ERR@ArrowTypeError | ERR@ArrowTypeError | [18:30:45]@time64[ns] | ERR@ArrowTypeError | ERR@ArrowTypeError | +| Int64:standard | ERR@ArrowInvalid | [0, 1, 9223372036854775807, -9223372036854775808]@int64 | ERR@ArrowInvalid | [1970-01-01 00:00:00, 1970-01-01 00:00:00.000001, temporal overflow, temporal overflow]@timestamp[us] | ERR@ArrowNotImplementedError | [0:00:00, 0:00:00.000001, 106751991 days, 4:00:54.775807, -106751992 days, 19:59:05.224192]@duration[us] | [00:00:00, 00:00:00, 23:47:16.854775, NaT collision]@time64[ns] | ERR@ArrowTypeError | [b'', b'\x01', b'\xff\xff\xff\xff\xff\xff\xff\x7f', b'']@binary | +| string[pyarrow]:standard | ERR@ArrowInvalid | ERR@ArrowInvalid | ERR@ArrowInvalid | ERR@ArrowInvalid | ERR@ArrowInvalid | ERR@ArrowNotImplementedError | ERR@ArrowNotImplementedError | [hello, world]@string | [b'hello', b'world']@binary | +| large_binary[pyarrow]:standard | ERR@ArrowInvalid | ERR@ArrowInvalid | ERR@ArrowInvalid | ERR@ArrowNotImplementedError | ERR@ArrowNotImplementedError | ERR@ArrowNotImplementedError | ERR@ArrowNotImplementedError | [hello, world]@string | [b'hello', b'world']@binary | +| int64:overflow | ERR@ArrowInvalid | [300, 1]@int64 | [300.0, 1.0]@float32 | [1970-01-01 00:00:00.000300, 1970-01-01 00:00:00.000001]@timestamp[us] | ERR@ArrowNotImplementedError | [0:00:00.000300, 0:00:00.000001]@duration[us] | [00:00:00, 00:00:00]@time64[ns] | ERR@ArrowTypeError | [b',\x01', b'\x01']@binary | +| float64:fractional | ERR@ArrowInvalid | ERR@ArrowInvalid | [1.5, 2.5]@float32 | ERR@ArrowNotImplementedError | ERR@ArrowNotImplementedError | ERR@ArrowNotImplementedError | ERR@ArrowNotImplementedError | ERR@ArrowTypeError | [b'', b'']@binary | +| float64:infinity | ERR@ArrowInvalid | ERR@ArrowInvalid | [inf, 1.0]@float32 | ERR@ArrowNotImplementedError | ERR@ArrowNotImplementedError | ERR@ArrowNotImplementedError | ERR@ArrowNotImplementedError | ERR@ArrowTypeError | [b'', b'']@binary | +| float64:precision | ERR@ArrowInvalid | ERR@ArrowInvalid | [1.1234568357467651]@float32 | ERR@ArrowNotImplementedError | ERR@ArrowNotImplementedError | ERR@ArrowNotImplementedError | ERR@ArrowNotImplementedError | ERR@ArrowTypeError | [b'\x98nt\xd3\xad\xf9\xf1?']@binary | +| datetime64[ns]:sub-us | ERR@ArrowNotImplementedError | [1704067200000000123]@int64 | ERR@ArrowNotImplementedError | ERR@ArrowInvalid | [2024-01-01]@date32[day] | ERR@ArrowNotImplementedError | [00:00:00]@time64[ns] | ERR@ArrowTypeError | [b'{']@binary | +| object:date-then-datetime | ERR@ArrowInvalid | ERR@ArrowInvalid | ERR@ArrowInvalid | ERR@ArrowTypeError | [2024-01-01, 2024-01-01]@date32[day] | ERR@ArrowTypeError | ERR@ArrowTypeError | ERR@ArrowTypeError | ERR@ArrowTypeError | +| object:datetime-then-date | ERR@ArrowInvalid | ERR@ArrowInvalid | ERR@ArrowInvalid | ERR@ArrowTypeError | [2024-01-01, 2024-01-01]@date32[day] | ERR@ArrowTypeError | ERR@ArrowTypeError | ERR@ArrowTypeError | ERR@ArrowTypeError | \ No newline at end of file diff --git a/python/pyspark/tests/upstream/pyarrow/golden_pyarrow_array_from_pandas_type_scalar_unsafe.csv b/python/pyspark/tests/upstream/pyarrow/golden_pyarrow_array_from_pandas_type_scalar_unsafe.csv index b04f68c6e346a..32a24d6a661cf 100644 --- a/python/pyspark/tests/upstream/pyarrow/golden_pyarrow_array_from_pandas_type_scalar_unsafe.csv +++ b/python/pyspark/tests/upstream/pyarrow/golden_pyarrow_array_from_pandas_type_scalar_unsafe.csv @@ -1,5 +1,5 @@ source \ target int8 int64 float32 timestamp[us] date32[day] duration[us] time64[ns] string binary -int64:standard [0, 1, -1, -1, 0]@int8 [0, 1, -1, 9223372036854775807, -9223372036854775808]@int64 [0.0, 1.0, -1.0, 9.223372036854776e+18, -9.223372036854776e+18]@float32 [1970-01-01 00:00:00, 1970-01-01 00:00:00.000001, 1969-12-31 23:59:59.999999, temporal overflow, temporal overflow]@timestamp[us] ERR@ArrowNotImplementedError [0:00:00, 0:00:00.000001, -1 day, 23:59:59.999999, 106751991 days, 4:00:54.775807, -106751992 days, 19:59:05.224192]@duration[us] ERR@ValueError ERR@ArrowTypeError [b'', b'\x01', b'\xff\xff\xff\xff\xff\xff\xff\xff', b'\xff\xff\xff\xff\xff\xff\xff\x7f', b'']@binary +int64:standard [0, 1, -1, -1, 0]@int8 [0, 1, -1, 9223372036854775807, -9223372036854775808]@int64 [0.0, 1.0, -1.0, 9.223372036854776e+18, -9.223372036854776e+18]@float32 [1970-01-01 00:00:00, 1970-01-01 00:00:00.000001, 1969-12-31 23:59:59.999999, temporal overflow, temporal overflow]@timestamp[us] ERR@ArrowNotImplementedError [0:00:00, 0:00:00.000001, -1 day, 23:59:59.999999, 106751991 days, 4:00:54.775807, -106751992 days, 19:59:05.224192]@duration[us] [00:00:00, 00:00:00, 23:59:59.999999, 23:47:16.854775, NaT collision]@time64[ns] ERR@ArrowTypeError [b'', b'\x01', b'\xff\xff\xff\xff\xff\xff\xff\xff', b'\xff\xff\xff\xff\xff\xff\xff\x7f', b'']@binary float64:standard [0, 1, -1]@int8 [0, 1, -1]@int64 [0.0, 1.5, -1.5]@float32 ERR@ArrowNotImplementedError ERR@ArrowNotImplementedError ERR@ArrowNotImplementedError ERR@ArrowNotImplementedError ERR@ArrowTypeError [b'', b'', b'']@binary bool:standard [1, 0, 1]@int8 [1, 0, 1]@int64 [1.0, 0.0, 1.0]@float32 ERR@ArrowNotImplementedError ERR@ArrowNotImplementedError ERR@ArrowNotImplementedError ERR@ArrowNotImplementedError ERR@ArrowTypeError [b'\x01', b'', b'\x01']@binary object:string ERR@ArrowInvalid ERR@ArrowInvalid ERR@ArrowInvalid ERR@ArrowTypeError ERR@ArrowTypeError ERR@ArrowTypeError ERR@ArrowTypeError [hello, world, ]@string [b'hello', b'world', b'']@binary @@ -8,7 +8,7 @@ date:standard ERR@ArrowInvalid ERR@ArrowInvalid ERR@ArrowInvalid ERR@ArrowTypeEr object:datetime ERR@ArrowInvalid ERR@ArrowInvalid ERR@ArrowInvalid [2024-06-15 18:30:00]@timestamp[us] [2024-06-15]@date32[day] ERR@ArrowTypeError ERR@ArrowTypeError ERR@ArrowTypeError ERR@ArrowTypeError object:timedelta ERR@ArrowInvalid ERR@ArrowInvalid ERR@ArrowInvalid ERR@ArrowTypeError ERR@ArrowTypeError [1 day, 2:00:00]@duration[us] ERR@ArrowTypeError ERR@ArrowTypeError ERR@ArrowTypeError time:standard ERR@ArrowInvalid ERR@ArrowInvalid ERR@ArrowInvalid ERR@ArrowTypeError ERR@ArrowTypeError ERR@ArrowTypeError [18:30:45]@time64[ns] ERR@ArrowTypeError ERR@ArrowTypeError -Int64:standard ERR@ArrowInvalid [0, 1, 9223372036854775807, -9223372036854775808]@int64 ERR@ArrowInvalid [1970-01-01 00:00:00, 1970-01-01 00:00:00.000001, temporal overflow, temporal overflow]@timestamp[us] ERR@ArrowNotImplementedError [0:00:00, 0:00:00.000001, 106751991 days, 4:00:54.775807, -106751992 days, 19:59:05.224192]@duration[us] ERR@ValueError ERR@ArrowTypeError [b'', b'\x01', b'\xff\xff\xff\xff\xff\xff\xff\x7f', b'']@binary +Int64:standard ERR@ArrowInvalid [0, 1, 9223372036854775807, -9223372036854775808]@int64 ERR@ArrowInvalid [1970-01-01 00:00:00, 1970-01-01 00:00:00.000001, temporal overflow, temporal overflow]@timestamp[us] ERR@ArrowNotImplementedError [0:00:00, 0:00:00.000001, 106751991 days, 4:00:54.775807, -106751992 days, 19:59:05.224192]@duration[us] [00:00:00, 00:00:00, 23:47:16.854775, NaT collision]@time64[ns] ERR@ArrowTypeError [b'', b'\x01', b'\xff\xff\xff\xff\xff\xff\xff\x7f', b'']@binary string[pyarrow]:standard ERR@ArrowInvalid ERR@ArrowInvalid ERR@ArrowInvalid ERR@ArrowInvalid ERR@ArrowInvalid ERR@ArrowNotImplementedError ERR@ArrowNotImplementedError [hello, world]@string [b'hello', b'world']@binary large_binary[pyarrow]:standard ERR@ArrowInvalid ERR@ArrowInvalid ERR@ArrowInvalid ERR@ArrowNotImplementedError ERR@ArrowNotImplementedError ERR@ArrowNotImplementedError ERR@ArrowNotImplementedError [hello, world]@string [b'hello', b'world']@binary int64:overflow [44, 1]@int8 [300, 1]@int64 [300.0, 1.0]@float32 [1970-01-01 00:00:00.000300, 1970-01-01 00:00:00.000001]@timestamp[us] ERR@ArrowNotImplementedError [0:00:00.000300, 0:00:00.000001]@duration[us] [00:00:00, 00:00:00]@time64[ns] ERR@ArrowTypeError [b',\x01', b'\x01']@binary diff --git a/python/pyspark/tests/upstream/pyarrow/golden_pyarrow_array_from_pandas_type_scalar_unsafe.md b/python/pyspark/tests/upstream/pyarrow/golden_pyarrow_array_from_pandas_type_scalar_unsafe.md index 05ddfff08f593..a6f12040dc1de 100644 --- a/python/pyspark/tests/upstream/pyarrow/golden_pyarrow_array_from_pandas_type_scalar_unsafe.md +++ b/python/pyspark/tests/upstream/pyarrow/golden_pyarrow_array_from_pandas_type_scalar_unsafe.md @@ -1,21 +1,21 @@ -| source \ target | int8 | int64 | float32 | timestamp[us] | date32[day] | duration[us] | time64[ns] | string | binary | -|--------------------------------|------------------------------|-------------------------------------------------------------|-------------------------------------------------------------------------|-----------------------------------------------------------------------------------------------------------------------------------|--------------------------------------|-----------------------------------------------------------------------------------------------------------------------------------|---------------------------------|-------------------------|------------------------------------------------------------------------------------------------------| -| int64:standard | [0, 1, -1, -1, 0]@int8 | [0, 1, -1, 9223372036854775807, -9223372036854775808]@int64 | [0.0, 1.0, -1.0, 9.223372036854776e+18, -9.223372036854776e+18]@float32 | [1970-01-01 00:00:00, 1970-01-01 00:00:00.000001, 1969-12-31 23:59:59.999999, temporal overflow, temporal overflow]@timestamp[us] | ERR@ArrowNotImplementedError | [0:00:00, 0:00:00.000001, -1 day, 23:59:59.999999, 106751991 days, 4:00:54.775807, -106751992 days, 19:59:05.224192]@duration[us] | ERR@ValueError | ERR@ArrowTypeError | [b'', b'\x01', b'\xff\xff\xff\xff\xff\xff\xff\xff', b'\xff\xff\xff\xff\xff\xff\xff\x7f', b'']@binary | -| float64:standard | [0, 1, -1]@int8 | [0, 1, -1]@int64 | [0.0, 1.5, -1.5]@float32 | ERR@ArrowNotImplementedError | ERR@ArrowNotImplementedError | ERR@ArrowNotImplementedError | ERR@ArrowNotImplementedError | ERR@ArrowTypeError | [b'', b'', b'']@binary | -| bool:standard | [1, 0, 1]@int8 | [1, 0, 1]@int64 | [1.0, 0.0, 1.0]@float32 | ERR@ArrowNotImplementedError | ERR@ArrowNotImplementedError | ERR@ArrowNotImplementedError | ERR@ArrowNotImplementedError | ERR@ArrowTypeError | [b'\x01', b'', b'\x01']@binary | -| object:string | ERR@ArrowInvalid | ERR@ArrowInvalid | ERR@ArrowInvalid | ERR@ArrowTypeError | ERR@ArrowTypeError | ERR@ArrowTypeError | ERR@ArrowTypeError | [hello, world, ]@string | [b'hello', b'world', b'']@binary | -| object:bytes | ERR@ArrowInvalid | ERR@ArrowInvalid | ERR@ArrowInvalid | ERR@ArrowTypeError | ERR@ArrowTypeError | ERR@ArrowTypeError | ERR@ArrowTypeError | [hello, world]@string | [b'hello', b'world']@binary | -| date:standard | ERR@ArrowInvalid | ERR@ArrowInvalid | ERR@ArrowInvalid | ERR@ArrowTypeError | [2024-06-15]@date32[day] | ERR@ArrowTypeError | ERR@ArrowTypeError | ERR@ArrowTypeError | ERR@ArrowTypeError | -| object:datetime | ERR@ArrowInvalid | ERR@ArrowInvalid | ERR@ArrowInvalid | [2024-06-15 18:30:00]@timestamp[us] | [2024-06-15]@date32[day] | ERR@ArrowTypeError | ERR@ArrowTypeError | ERR@ArrowTypeError | ERR@ArrowTypeError | -| object:timedelta | ERR@ArrowInvalid | ERR@ArrowInvalid | ERR@ArrowInvalid | ERR@ArrowTypeError | ERR@ArrowTypeError | [1 day, 2:00:00]@duration[us] | ERR@ArrowTypeError | ERR@ArrowTypeError | ERR@ArrowTypeError | -| time:standard | ERR@ArrowInvalid | ERR@ArrowInvalid | ERR@ArrowInvalid | ERR@ArrowTypeError | ERR@ArrowTypeError | ERR@ArrowTypeError | [18:30:45]@time64[ns] | ERR@ArrowTypeError | ERR@ArrowTypeError | -| Int64:standard | ERR@ArrowInvalid | [0, 1, 9223372036854775807, -9223372036854775808]@int64 | ERR@ArrowInvalid | [1970-01-01 00:00:00, 1970-01-01 00:00:00.000001, temporal overflow, temporal overflow]@timestamp[us] | ERR@ArrowNotImplementedError | [0:00:00, 0:00:00.000001, 106751991 days, 4:00:54.775807, -106751992 days, 19:59:05.224192]@duration[us] | ERR@ValueError | ERR@ArrowTypeError | [b'', b'\x01', b'\xff\xff\xff\xff\xff\xff\xff\x7f', b'']@binary | -| string[pyarrow]:standard | ERR@ArrowInvalid | ERR@ArrowInvalid | ERR@ArrowInvalid | ERR@ArrowInvalid | ERR@ArrowInvalid | ERR@ArrowNotImplementedError | ERR@ArrowNotImplementedError | [hello, world]@string | [b'hello', b'world']@binary | -| large_binary[pyarrow]:standard | ERR@ArrowInvalid | ERR@ArrowInvalid | ERR@ArrowInvalid | ERR@ArrowNotImplementedError | ERR@ArrowNotImplementedError | ERR@ArrowNotImplementedError | ERR@ArrowNotImplementedError | [hello, world]@string | [b'hello', b'world']@binary | -| int64:overflow | [44, 1]@int8 | [300, 1]@int64 | [300.0, 1.0]@float32 | [1970-01-01 00:00:00.000300, 1970-01-01 00:00:00.000001]@timestamp[us] | ERR@ArrowNotImplementedError | [0:00:00.000300, 0:00:00.000001]@duration[us] | [00:00:00, 00:00:00]@time64[ns] | ERR@ArrowTypeError | [b',\x01', b'\x01']@binary | -| float64:fractional | [1, 2]@int8 | [1, 2]@int64 | [1.5, 2.5]@float32 | ERR@ArrowNotImplementedError | ERR@ArrowNotImplementedError | ERR@ArrowNotImplementedError | ERR@ArrowNotImplementedError | ERR@ArrowTypeError | [b'', b'']@binary | -| float64:infinity | [0, 1]@int8 | [-9223372036854775808, 1]@int64 | [inf, 1.0]@float32 | ERR@ArrowNotImplementedError | ERR@ArrowNotImplementedError | ERR@ArrowNotImplementedError | ERR@ArrowNotImplementedError | ERR@ArrowTypeError | [b'', b'']@binary | -| float64:precision | [1]@int8 | [1]@int64 | [1.1234568357467651]@float32 | ERR@ArrowNotImplementedError | ERR@ArrowNotImplementedError | ERR@ArrowNotImplementedError | ERR@ArrowNotImplementedError | ERR@ArrowTypeError | [b'\x98nt\xd3\xad\xf9\xf1?']@binary | -| datetime64[ns]:sub-us | ERR@ArrowNotImplementedError | [1704067200000000123]@int64 | ERR@ArrowNotImplementedError | [2024-01-01 00:00:00]@timestamp[us] | [2024-01-01]@date32[day] | ERR@ArrowNotImplementedError | [00:00:00]@time64[ns] | ERR@ArrowTypeError | [b'{']@binary | -| object:date-then-datetime | ERR@ArrowInvalid | ERR@ArrowInvalid | ERR@ArrowInvalid | ERR@ArrowTypeError | [2024-01-01, 2024-01-01]@date32[day] | ERR@ArrowTypeError | ERR@ArrowTypeError | ERR@ArrowTypeError | ERR@ArrowTypeError | -| object:datetime-then-date | ERR@ArrowInvalid | ERR@ArrowInvalid | ERR@ArrowInvalid | ERR@ArrowTypeError | [2024-01-01, 2024-01-01]@date32[day] | ERR@ArrowTypeError | ERR@ArrowTypeError | ERR@ArrowTypeError | ERR@ArrowTypeError | \ No newline at end of file +| source \ target | int8 | int64 | float32 | timestamp[us] | date32[day] | duration[us] | time64[ns] | string | binary | +|--------------------------------|------------------------------|-------------------------------------------------------------|-------------------------------------------------------------------------|-----------------------------------------------------------------------------------------------------------------------------------|--------------------------------------|-----------------------------------------------------------------------------------------------------------------------------------|----------------------------------------------------------------------------------|-------------------------|------------------------------------------------------------------------------------------------------| +| int64:standard | [0, 1, -1, -1, 0]@int8 | [0, 1, -1, 9223372036854775807, -9223372036854775808]@int64 | [0.0, 1.0, -1.0, 9.223372036854776e+18, -9.223372036854776e+18]@float32 | [1970-01-01 00:00:00, 1970-01-01 00:00:00.000001, 1969-12-31 23:59:59.999999, temporal overflow, temporal overflow]@timestamp[us] | ERR@ArrowNotImplementedError | [0:00:00, 0:00:00.000001, -1 day, 23:59:59.999999, 106751991 days, 4:00:54.775807, -106751992 days, 19:59:05.224192]@duration[us] | [00:00:00, 00:00:00, 23:59:59.999999, 23:47:16.854775, NaT collision]@time64[ns] | ERR@ArrowTypeError | [b'', b'\x01', b'\xff\xff\xff\xff\xff\xff\xff\xff', b'\xff\xff\xff\xff\xff\xff\xff\x7f', b'']@binary | +| float64:standard | [0, 1, -1]@int8 | [0, 1, -1]@int64 | [0.0, 1.5, -1.5]@float32 | ERR@ArrowNotImplementedError | ERR@ArrowNotImplementedError | ERR@ArrowNotImplementedError | ERR@ArrowNotImplementedError | ERR@ArrowTypeError | [b'', b'', b'']@binary | +| bool:standard | [1, 0, 1]@int8 | [1, 0, 1]@int64 | [1.0, 0.0, 1.0]@float32 | ERR@ArrowNotImplementedError | ERR@ArrowNotImplementedError | ERR@ArrowNotImplementedError | ERR@ArrowNotImplementedError | ERR@ArrowTypeError | [b'\x01', b'', b'\x01']@binary | +| object:string | ERR@ArrowInvalid | ERR@ArrowInvalid | ERR@ArrowInvalid | ERR@ArrowTypeError | ERR@ArrowTypeError | ERR@ArrowTypeError | ERR@ArrowTypeError | [hello, world, ]@string | [b'hello', b'world', b'']@binary | +| object:bytes | ERR@ArrowInvalid | ERR@ArrowInvalid | ERR@ArrowInvalid | ERR@ArrowTypeError | ERR@ArrowTypeError | ERR@ArrowTypeError | ERR@ArrowTypeError | [hello, world]@string | [b'hello', b'world']@binary | +| date:standard | ERR@ArrowInvalid | ERR@ArrowInvalid | ERR@ArrowInvalid | ERR@ArrowTypeError | [2024-06-15]@date32[day] | ERR@ArrowTypeError | ERR@ArrowTypeError | ERR@ArrowTypeError | ERR@ArrowTypeError | +| object:datetime | ERR@ArrowInvalid | ERR@ArrowInvalid | ERR@ArrowInvalid | [2024-06-15 18:30:00]@timestamp[us] | [2024-06-15]@date32[day] | ERR@ArrowTypeError | ERR@ArrowTypeError | ERR@ArrowTypeError | ERR@ArrowTypeError | +| object:timedelta | ERR@ArrowInvalid | ERR@ArrowInvalid | ERR@ArrowInvalid | ERR@ArrowTypeError | ERR@ArrowTypeError | [1 day, 2:00:00]@duration[us] | ERR@ArrowTypeError | ERR@ArrowTypeError | ERR@ArrowTypeError | +| time:standard | ERR@ArrowInvalid | ERR@ArrowInvalid | ERR@ArrowInvalid | ERR@ArrowTypeError | ERR@ArrowTypeError | ERR@ArrowTypeError | [18:30:45]@time64[ns] | ERR@ArrowTypeError | ERR@ArrowTypeError | +| Int64:standard | ERR@ArrowInvalid | [0, 1, 9223372036854775807, -9223372036854775808]@int64 | ERR@ArrowInvalid | [1970-01-01 00:00:00, 1970-01-01 00:00:00.000001, temporal overflow, temporal overflow]@timestamp[us] | ERR@ArrowNotImplementedError | [0:00:00, 0:00:00.000001, 106751991 days, 4:00:54.775807, -106751992 days, 19:59:05.224192]@duration[us] | [00:00:00, 00:00:00, 23:47:16.854775, NaT collision]@time64[ns] | ERR@ArrowTypeError | [b'', b'\x01', b'\xff\xff\xff\xff\xff\xff\xff\x7f', b'']@binary | +| string[pyarrow]:standard | ERR@ArrowInvalid | ERR@ArrowInvalid | ERR@ArrowInvalid | ERR@ArrowInvalid | ERR@ArrowInvalid | ERR@ArrowNotImplementedError | ERR@ArrowNotImplementedError | [hello, world]@string | [b'hello', b'world']@binary | +| large_binary[pyarrow]:standard | ERR@ArrowInvalid | ERR@ArrowInvalid | ERR@ArrowInvalid | ERR@ArrowNotImplementedError | ERR@ArrowNotImplementedError | ERR@ArrowNotImplementedError | ERR@ArrowNotImplementedError | [hello, world]@string | [b'hello', b'world']@binary | +| int64:overflow | [44, 1]@int8 | [300, 1]@int64 | [300.0, 1.0]@float32 | [1970-01-01 00:00:00.000300, 1970-01-01 00:00:00.000001]@timestamp[us] | ERR@ArrowNotImplementedError | [0:00:00.000300, 0:00:00.000001]@duration[us] | [00:00:00, 00:00:00]@time64[ns] | ERR@ArrowTypeError | [b',\x01', b'\x01']@binary | +| float64:fractional | [1, 2]@int8 | [1, 2]@int64 | [1.5, 2.5]@float32 | ERR@ArrowNotImplementedError | ERR@ArrowNotImplementedError | ERR@ArrowNotImplementedError | ERR@ArrowNotImplementedError | ERR@ArrowTypeError | [b'', b'']@binary | +| float64:infinity | [0, 1]@int8 | [-9223372036854775808, 1]@int64 | [inf, 1.0]@float32 | ERR@ArrowNotImplementedError | ERR@ArrowNotImplementedError | ERR@ArrowNotImplementedError | ERR@ArrowNotImplementedError | ERR@ArrowTypeError | [b'', b'']@binary | +| float64:precision | [1]@int8 | [1]@int64 | [1.1234568357467651]@float32 | ERR@ArrowNotImplementedError | ERR@ArrowNotImplementedError | ERR@ArrowNotImplementedError | ERR@ArrowNotImplementedError | ERR@ArrowTypeError | [b'\x98nt\xd3\xad\xf9\xf1?']@binary | +| datetime64[ns]:sub-us | ERR@ArrowNotImplementedError | [1704067200000000123]@int64 | ERR@ArrowNotImplementedError | [2024-01-01 00:00:00]@timestamp[us] | [2024-01-01]@date32[day] | ERR@ArrowNotImplementedError | [00:00:00]@time64[ns] | ERR@ArrowTypeError | [b'{']@binary | +| object:date-then-datetime | ERR@ArrowInvalid | ERR@ArrowInvalid | ERR@ArrowInvalid | ERR@ArrowTypeError | [2024-01-01, 2024-01-01]@date32[day] | ERR@ArrowTypeError | ERR@ArrowTypeError | ERR@ArrowTypeError | ERR@ArrowTypeError | +| object:datetime-then-date | ERR@ArrowInvalid | ERR@ArrowInvalid | ERR@ArrowInvalid | ERR@ArrowTypeError | [2024-01-01, 2024-01-01]@date32[day] | ERR@ArrowTypeError | ERR@ArrowTypeError | ERR@ArrowTypeError | ERR@ArrowTypeError | \ No newline at end of file