diff --git a/doc/source/whatsnew/v3.2.0.rst b/doc/source/whatsnew/v3.2.0.rst index cb3c3b9c6a873..ad0f9896f16d8 100644 --- a/doc/source/whatsnew/v3.2.0.rst +++ b/doc/source/whatsnew/v3.2.0.rst @@ -36,6 +36,7 @@ Other enhancements - :meth:`DataFrame.plot.area` and :meth:`Series.plot.area` now accept keyword arguments that only the lines or only the filled areas support, such as ``hatch``, ``edgecolor`` and ``marker``, instead of raising ``AttributeError``; ``hatch`` also accepts a list with one hatch per column (:issue:`12354`) - :meth:`DataFrame.to_csv` and other writers now accept an ``os.PathLike`` such as :class:`pathlib.Path` for the ``archive_name`` entry of a zip or tar ``compression`` dict (:issue:`31934`) - :meth:`DataFrame.to_json`, :meth:`Series.to_json` and :meth:`DataFrame.to_xml` now warn when a compression method is requested but no path is given, since the returned string is not compressed, and raise for an unrecognized ``compression`` as they do with a path (:issue:`32326`) +- :meth:`Series.astype` and :func:`array` with ``dtype="boolean"`` now convert values the same way as ``astype(bool)`` instead of raising for values other than booleans, 0 and 1 (e.g. ``"False"`` becomes ``True``), with missing values becoming ``NA`` (:issue:`37614`, :issue:`40566`) - :meth:`Series.groupby` now accepts ``as_index=False``, returning aggregation results as a :class:`DataFrame` with the group labels as columns (:issue:`36507`) - Added :meth:`Categorical.to_dense` and :meth:`Series.cat.to_dense` to get the values with the dtype of the categories, including extension dtypes (:issue:`26410`) - Added :meth:`DatetimeIndex.set_freq` and :meth:`TimedeltaIndex.set_freq`, which return a new index with the given ``freq`` instead of modifying it in place (:issue:`61094`) diff --git a/pandas/core/arrays/boolean.py b/pandas/core/arrays/boolean.py index f74df715984d0..baf69fa181c83 100644 --- a/pandas/core/arrays/boolean.py +++ b/pandas/core/arrays/boolean.py @@ -5,12 +5,13 @@ TYPE_CHECKING, ClassVar, Self, - cast, ) import warnings import numpy as np +from pandas._config import is_nan_na + from pandas._libs import ( lib, missing as libmissing, @@ -20,16 +21,21 @@ from pandas.util._exceptions import find_stack_level from pandas.core.dtypes.common import is_list_like -from pandas.core.dtypes.dtypes import register_extension_dtype +from pandas.core.dtypes.dtypes import ( + CategoricalDtype, + register_extension_dtype, +) from pandas.core.dtypes.missing import isna from pandas.core import ops from pandas.core.array_algos import masked_accumulations +from pandas.core.array_algos.take import take_nd from pandas.core.arrays import ExtensionArray from pandas.core.arrays.masked import ( BaseMaskedArray, BaseMaskedDtype, ) +from pandas.core.construction import extract_array if TYPE_CHECKING: import pyarrow @@ -200,45 +206,49 @@ def coerce_to_array( mask = mask.copy() return values, mask + # Values are converted as by astype(bool), except that missing values + # (NA, None, NaT, and NaN unless future.distinguish_nan_and_na) are masked + values = extract_array(values, extract_numpy=True) mask_values = None if isinstance(values, np.ndarray) and values.dtype == np.bool_: if copy: values = values.copy() - elif isinstance(values, np.ndarray) and values.dtype.kind in "iufcb": - mask_values = isna(values) - + elif isinstance(values, np.ndarray) and values.dtype.kind in "iufcmM": + if values.dtype.kind in "fc" and not is_nan_na(): + mask_values = np.zeros(values.shape, dtype=bool) + else: + mask_values = isna(values) + values = values.astype(bool) + values[mask_values] = False + elif isinstance(getattr(values, "dtype", None), CategoricalDtype): + # convert each category once; Categorical.astype(bool) can warn here + cat_values, _ = coerce_to_array(values.categories._values) + mask_values = values.codes == -1 + values = take_nd(cat_values, values.codes, fill_value=False) + elif isinstance(values, ExtensionArray): + mask_values = np.asarray(values.isna(), dtype=bool) values_bool = np.zeros(len(values), dtype=bool) values_bool[~mask_values] = values[~mask_values].astype(bool) - - if not np.all( - values_bool[~mask_values].astype(values.dtype) == values[~mask_values] - ): - raise TypeError("Need to pass bool-like values") - values = values_bool else: values_object = np.asarray(values, dtype=object) - - inferred_dtype = lib.infer_dtype(values_object, skipna=True) - integer_like = ("floating", "integer", "mixed-integer-float") - if inferred_dtype not in ("boolean", "empty", *integer_like): - raise TypeError("Need to pass bool-like values") - - # mypy does not narrow the type of mask_values to npt.NDArray[np.bool_] - # within this branch, it assumes it can also be None - mask_values = cast("npt.NDArray[np.bool_]", isna(values_object)) # type: ignore[redundant-cast] + if values_object.ndim != 1: + raise TypeError("values must be a 1D list-like") + mask_values = isna(values_object) + if not is_nan_na(): + # NaN is a value rather than missing, so it converts to True + idx = mask_values.nonzero()[0] + is_nan = np.array( + [ + lib.is_float(x) or lib.is_complex(x) or lib.is_decimal(x) + for x in values_object[idx] + ], + dtype=bool, + ) + mask_values[idx[is_nan]] = False values = np.zeros(len(values), dtype=bool) values[~mask_values] = values_object[~mask_values].astype(bool) - # if the values were integer-like, validate it were actually 0/1's - if (inferred_dtype in integer_like) and not ( - np.all( - values[~mask_values].astype(float) - == values_object[~mask_values].astype(float) - ) - ): - raise TypeError("Need to pass bool-like values") - if mask is None and mask_values is None: mask = np.zeros(values.shape, dtype=bool) elif mask is None: diff --git a/pandas/core/reshape/encoding.py b/pandas/core/reshape/encoding.py index 40252569fca56..fad59a8f4ea27 100644 --- a/pandas/core/reshape/encoding.py +++ b/pandas/core/reshape/encoding.py @@ -547,11 +547,9 @@ def from_dummies( f"Dummy DataFrame contains NA value in column: '{col_isna_mask.idxmax()}'" ) - # index data with a list of all columns that are dummies - try: - data_to_decode = data.astype("boolean") - except TypeError as err: - raise TypeError("Passed DataFrame contains non-dummy data") from err + if not data.isin([0, 1]).all().all(): + raise TypeError("Passed DataFrame contains non-dummy data") + data_to_decode = data.astype("boolean") # collect prefixes and get lists to slice data for each prefix variables_slice = defaultdict(list) diff --git a/pandas/tests/arrays/boolean/test_astype.py b/pandas/tests/arrays/boolean/test_astype.py index 8c2672218f273..1733e26319e63 100644 --- a/pandas/tests/arrays/boolean/test_astype.py +++ b/pandas/tests/arrays/boolean/test_astype.py @@ -50,6 +50,37 @@ def test_astype_to_boolean_array(): tm.assert_extension_array_equal(result, arr) +@pytest.mark.parametrize( + "values, dtype, expected", + [ + ([0, 1, 2], "int64", [False, True, True]), + ([0.0, 0.5, np.nan], "float64", [False, True, None]), + (["", "False", None], "object", [False, True, None]), + (["", "False", None], "string", [False, True, None]), + (["", "False", None], pd.StringDtype(na_value=np.nan), [False, True, None]), + ([0, 2, None], "category", [False, True, None]), + (["", "x", None], "category", [False, True, None]), + ([None, None], "category", [None, None]), + ([0, 5, None], "timedelta64[ns]", [False, True, None]), + (["1970-01-01", "2000-01-01", None], "M8[ns]", [False, True, None]), + (["1970-01-01", "2000-01-01", None], "M8[ns, UTC]", [False, True, None]), + (["1970-01-01", None], "period[D]", [True, None]), + ([0, 2, None], "int64[pyarrow]", [False, True, None]), + ], +) +def test_astype_from_other_dtypes_matches_astype_bool(values, dtype, expected): + # GH#37614, GH#40566 + if dtype == "int64[pyarrow]": + pytest.importorskip("pyarrow") + ser = pd.Series(values, dtype=dtype) + result = ser.astype("boolean") + expected = pd.Series(expected, dtype="boolean") + tm.assert_series_equal(result, expected) + + notna = ser.notna() + tm.assert_series_equal(result[notna], ser[notna].astype(bool).astype("boolean")) + + def test_astype_to_integer_array(): # astype to IntegerArray arr = pd.array([True, False, None], dtype="boolean") diff --git a/pandas/tests/arrays/boolean/test_construction.py b/pandas/tests/arrays/boolean/test_construction.py index 45634aa278176..e63b7800ebd20 100644 --- a/pandas/tests/arrays/boolean/test_construction.py +++ b/pandas/tests/arrays/boolean/test_construction.py @@ -1,3 +1,5 @@ +from decimal import Decimal + import numpy as np import pytest @@ -96,25 +98,65 @@ def test_to_boolean_array_missing_indicators(a, b): @pytest.mark.parametrize( - "values", + "values, expected", + [ + (["foo", "", "False", None], [True, False, True, None]), + ([0, 1, 2, -1, None], [False, True, True, True, None]), + ([0.0, 0.5, np.nan], [False, True, None]), + ([0, "a", 1.5, pd.NA, pd.NaT], [False, True, True, None, None]), + ([np.nan, {"a": 1}, {}], [None, True, False]), + # a list is treated as object dtype, where every Timestamp is truthy + ([pd.Timestamp(0), pd.NaT], [True, None]), + (np.array(["foo", ""]), [True, False]), + (np.array([0, 1, 2]), [False, True, True]), + (np.array([0.0, 0.5, np.nan]), [False, True, None]), + (np.array([0j, 1j, complex(np.nan, 0)]), [False, True, None]), + (np.array([0, 5, "NaT"], dtype="m8[ns]"), [False, True, None]), + ( + np.array(["1970-01-01", "2000-01-01", "NaT"], dtype="M8[ns]"), + [False, True, None], + ), + (pd.date_range("1970-01-01", periods=2, tz="UTC"), [False, True]), + (pd.period_range("2000", periods=2), [True, True]), + ], +) +def test_to_boolean_array_matches_astype_bool(values, expected): + # GH#37614 values are converted as by astype(bool), with missing values masked + result = pd.array(values, dtype="boolean") + expected = pd.array(expected, dtype="boolean") + tm.assert_extension_array_equal(result, expected) + + +@pytest.mark.parametrize( + "values, expected", [ - ["foo", "bar"], - ["1", "2"], - # "foo", - [1, 2], - [1.0, 2.0], - pd.date_range("20130101", periods=2), - np.array(["foo"]), - np.array([1, 2]), - np.array([1.0, 2.0]), - [np.nan, {"a": 1}], + ( + [0.0, np.nan, None, pd.NA, pd.NaT, Decimal("nan")], + [False, True, None, None, None, True], + ), + (np.array([0.0, np.nan]), [False, True]), + (np.array([0j, complex(np.nan, 0)]), [False, True]), ], ) -def test_to_boolean_array_error(values): - # error in converting existing arrays to BooleanArray - msg = "Need to pass bool-like value" - with pytest.raises(TypeError, match=msg): - pd.array(values, dtype="boolean") +def test_to_boolean_array_distinguish_nan_and_na(values, expected): + # with NaN distinct from NA, NaN is a value and converts as bool(np.nan) + with pd.option_context("future.distinguish_nan_and_na", True): + result = pd.array(values, dtype="boolean") + expected = pd.array(expected, dtype="boolean") + tm.assert_extension_array_equal(result, expected) + + +def test_float_ea_astype_boolean_distinguish_nan_and_na(): + with pd.option_context("future.distinguish_nan_and_na", True): + arr = pd.array([0.0, np.nan, None], dtype="Float64") + result = arr.astype("boolean") + expected = pd.array([False, True, None], dtype="boolean") + tm.assert_extension_array_equal(result, expected) + + +def test_to_boolean_array_2d_raises(): + with pytest.raises(TypeError, match="values must be a 1D list-like"): + pd.array([[True, False]], dtype="boolean") def test_to_boolean_array_from_integer_array(): diff --git a/pandas/tests/reshape/test_from_dummies.py b/pandas/tests/reshape/test_from_dummies.py index 6705d38936ba7..96678ef8c1fa5 100644 --- a/pandas/tests/reshape/test_from_dummies.py +++ b/pandas/tests/reshape/test_from_dummies.py @@ -95,6 +95,14 @@ def test_error_contains_non_dummies(): pd.from_dummies(dummies) +@pytest.mark.parametrize("values", [[0, 0.5], ["0", "1"]]) +def test_error_contains_non_dummies_bool_castable(values): + # values that astype("boolean") accepts but are not 0 or 1 + dummies = pd.DataFrame({"a": values, "b": [1, 0]}) + with pytest.raises(TypeError, match=r"Passed DataFrame contains non-dummy data"): + pd.from_dummies(dummies) + + def test_error_with_prefix_multiple_separators(): dummies = pd.DataFrame( { @@ -250,6 +258,20 @@ def test_no_prefix_string_cats_basic_mixed_bool_values(): tm.assert_frame_equal(result, expected) +@pytest.mark.parametrize( + "dtype", ["int64", "uint8", "float64", "Int64", "boolean", "int64[pyarrow]"] +) +def test_no_prefix_string_cats_basic_dtypes(dtype): + if dtype == "int64[pyarrow]": + pytest.importorskip("pyarrow") + dummies = pd.DataFrame( + {"a": [1, 0, 0], "b": [0, 1, 0], "c": [0, 0, 1]}, dtype=dtype + ) + expected = pd.DataFrame({"": ["a", "b", "c"]}) + result = pd.from_dummies(dummies) + tm.assert_frame_equal(result, expected) + + def test_no_prefix_int_cats_basic(): dummies = pd.DataFrame( {1: [1, 0, 0, 0], 25: [0, 1, 0, 0], 2: [0, 0, 1, 0], 5: [0, 0, 0, 1]} diff --git a/pandas/tests/series/methods/test_astype.py b/pandas/tests/series/methods/test_astype.py index 1589439d49f33..1e299790cd1cc 100644 --- a/pandas/tests/series/methods/test_astype.py +++ b/pandas/tests/series/methods/test_astype.py @@ -511,13 +511,8 @@ class TestAstypeString: ], ) def test_astype_string_to_extension_dtype_roundtrip( - self, data, dtype, request, nullable_string_dtype + self, data, dtype, nullable_string_dtype ): - if dtype == "boolean": - mark = pytest.mark.xfail( - reason="TODO StringArray.astype() with missing values #GH40566" - ) - request.applymarker(mark) # GH-40351 ser = pd.Series(data, dtype=dtype)