ENH: astype("boolean") converts values as astype(bool) (GH#37614)

doc/source/whatsnew/v3.2.0.rst

@@ -36,6 +36,7 @@ Other enhancements
 - :meth:`DataFrame.plot.area` and :meth:`Series.plot.area` now accept keyword arguments that only the lines or only the filled areas support, such as ``hatch``, ``edgecolor`` and ``marker``, instead of raising ``AttributeError``; ``hatch`` also accepts a list with one hatch per column (:issue:`12354`)
 - :meth:`DataFrame.to_csv` and other writers now accept an ``os.PathLike`` such as :class:`pathlib.Path` for the ``archive_name`` entry of a zip or tar ``compression`` dict (:issue:`31934`)
 - :meth:`DataFrame.to_json`, :meth:`Series.to_json` and :meth:`DataFrame.to_xml` now warn when a compression method is requested but no path is given, since the returned string is not compressed, and raise for an unrecognized ``compression`` as they do with a path (:issue:`32326`)
+- :meth:`Series.astype` and :func:`array` with ``dtype="boolean"`` now convert values the same way as ``astype(bool)`` instead of raising for values other than booleans, 0 and 1 (e.g. ``"False"`` becomes ``True``), with missing values becoming ``NA`` (:issue:`37614`, :issue:`40566`)
 - :meth:`Series.groupby` now accepts ``as_index=False``, returning aggregation results as a :class:`DataFrame` with the group labels as columns (:issue:`36507`)
 - Added :meth:`Categorical.to_dense` and :meth:`Series.cat.to_dense` to get the values with the dtype of the categories, including extension dtypes (:issue:`26410`)
 - Added :meth:`DatetimeIndex.set_freq` and :meth:`TimedeltaIndex.set_freq`, which return a new index with the given ``freq`` instead of modifying it in place (:issue:`61094`)

pandas/core/arrays/boolean.py

@@ -5,12 +5,13 @@
     TYPE_CHECKING,
     ClassVar,
     Self,
-    cast,
 )
 import warnings
 
 import numpy as np
 
+from pandas._config import is_nan_na
+
 from pandas._libs import (
     lib,
     missing as libmissing,
@@ -20,16 +21,21 @@
 from pandas.util._exceptions import find_stack_level
 
 from pandas.core.dtypes.common import is_list_like
-from pandas.core.dtypes.dtypes import register_extension_dtype
+from pandas.core.dtypes.dtypes import (
+    CategoricalDtype,
+    register_extension_dtype,
+)
 from pandas.core.dtypes.missing import isna
 
 from pandas.core import ops
 from pandas.core.array_algos import masked_accumulations
+from pandas.core.array_algos.take import take_nd
 from pandas.core.arrays import ExtensionArray
 from pandas.core.arrays.masked import (
     BaseMaskedArray,
     BaseMaskedDtype,
 )
+from pandas.core.construction import extract_array
 
 if TYPE_CHECKING:
     import pyarrow
@@ -200,45 +206,49 @@ def coerce_to_array(
             mask = mask.copy()
         return values, mask
 
+    # Values are converted as by astype(bool), except that missing values
+    #  (NA, None, NaT, and NaN unless future.distinguish_nan_and_na) are masked
+    values = extract_array(values, extract_numpy=True)
     mask_values = None
     if isinstance(values, np.ndarray) and values.dtype == np.bool_:
         if copy:
             values = values.copy()
-    elif isinstance(values, np.ndarray) and values.dtype.kind in "iufcb":
-        mask_values = isna(values)
-
+    elif isinstance(values, np.ndarray) and values.dtype.kind in "iufcmM":
+        if values.dtype.kind in "fc" and not is_nan_na():
+            mask_values = np.zeros(values.shape, dtype=bool)
+        else:
+            mask_values = isna(values)
+        values = values.astype(bool)
+        values[mask_values] = False
+    elif isinstance(getattr(values, "dtype", None), CategoricalDtype):
+        # convert each category once; Categorical.astype(bool) can warn here
+        cat_values, _ = coerce_to_array(values.categories._values)
+        mask_values = values.codes == -1
+        values = take_nd(cat_values, values.codes, fill_value=False)
+    elif isinstance(values, ExtensionArray):
+        mask_values = np.asarray(values.isna(), dtype=bool)
         values_bool = np.zeros(len(values), dtype=bool)
         values_bool[~mask_values] = values[~mask_values].astype(bool)
-
-        if not np.all(
-            values_bool[~mask_values].astype(values.dtype) == values[~mask_values]
-        ):
-            raise TypeError("Need to pass bool-like values")
-
         values = values_bool
     else:
         values_object = np.asarray(values, dtype=object)
-
-        inferred_dtype = lib.infer_dtype(values_object, skipna=True)
-        integer_like = ("floating", "integer", "mixed-integer-float")
-        if inferred_dtype not in ("boolean", "empty", *integer_like):
-            raise TypeError("Need to pass bool-like values")
-
-        # mypy does not narrow the type of mask_values to npt.NDArray[np.bool_]
-        # within this branch, it assumes it can also be None
-        mask_values = cast("npt.NDArray[np.bool_]", isna(values_object))  # type: ignore[redundant-cast]
+        if values_object.ndim != 1:
+            raise TypeError("values must be a 1D list-like")
+        mask_values = isna(values_object)
+        if not is_nan_na():
+            # NaN is a value rather than missing, so it converts to True
+            idx = mask_values.nonzero()[0]
+            is_nan = np.array(
+                [
+                    lib.is_float(x) or lib.is_complex(x) or lib.is_decimal(x)
+                    for x in values_object[idx]
+                ],
+                dtype=bool,
+            )
+            mask_values[idx[is_nan]] = False
         values = np.zeros(len(values), dtype=bool)
         values[~mask_values] = values_object[~mask_values].astype(bool)
 
-        # if the values were integer-like, validate it were actually 0/1's
-        if (inferred_dtype in integer_like) and not (
-            np.all(
-                values[~mask_values].astype(float)
-                == values_object[~mask_values].astype(float)
-            )
-        ):
-            raise TypeError("Need to pass bool-like values")
-
     if mask is None and mask_values is None:
         mask = np.zeros(values.shape, dtype=bool)
     elif mask is None:

pandas/core/reshape/encoding.py

@@ -547,11 +547,9 @@ def from_dummies(
             f"Dummy DataFrame contains NA value in column: '{col_isna_mask.idxmax()}'"
         )
 
-    # index data with a list of all columns that are dummies
-    try:
-        data_to_decode = data.astype("boolean")
-    except TypeError as err:
-        raise TypeError("Passed DataFrame contains non-dummy data") from err
+    if not data.isin([0, 1]).all().all():
+        raise TypeError("Passed DataFrame contains non-dummy data")
+    data_to_decode = data.astype("boolean")
 
     # collect prefixes and get lists to slice data for each prefix
     variables_slice = defaultdict(list)

pandas/tests/arrays/boolean/test_astype.py

@@ -50,6 +50,37 @@ def test_astype_to_boolean_array():
     tm.assert_extension_array_equal(result, arr)
 
 
+@pytest.mark.parametrize(
+    "values, dtype, expected",
+    [
+        ([0, 1, 2], "int64", [False, True, True]),
+        ([0.0, 0.5, np.nan], "float64", [False, True, None]),
+        (["", "False", None], "object", [False, True, None]),
+        (["", "False", None], "string", [False, True, None]),
+        (["", "False", None], pd.StringDtype(na_value=np.nan), [False, True, None]),
+        ([0, 2, None], "category", [False, True, None]),
+        (["", "x", None], "category", [False, True, None]),
+        ([None, None], "category", [None, None]),
+        ([0, 5, None], "timedelta64[ns]", [False, True, None]),
+        (["1970-01-01", "2000-01-01", None], "M8[ns]", [False, True, None]),
+        (["1970-01-01", "2000-01-01", None], "M8[ns, UTC]", [False, True, None]),
+        (["1970-01-01", None], "period[D]", [True, None]),
+        ([0, 2, None], "int64[pyarrow]", [False, True, None]),
+    ],
+)
+def test_astype_from_other_dtypes_matches_astype_bool(values, dtype, expected):
+    # GH#37614, GH#40566
+    if dtype == "int64[pyarrow]":
+        pytest.importorskip("pyarrow")
+    ser = pd.Series(values, dtype=dtype)
+    result = ser.astype("boolean")
+    expected = pd.Series(expected, dtype="boolean")
+    tm.assert_series_equal(result, expected)
+
+    notna = ser.notna()
+    tm.assert_series_equal(result[notna], ser[notna].astype(bool).astype("boolean"))
+
+
 def test_astype_to_integer_array():
     # astype to IntegerArray
     arr = pd.array([True, False, None], dtype="boolean")

pandas/tests/arrays/boolean/test_construction.py

@@ -1,3 +1,5 @@
+from decimal import Decimal
+
 import numpy as np
 import pytest
 
@@ -96,25 +98,65 @@ def test_to_boolean_array_missing_indicators(a, b):
 
 
 @pytest.mark.parametrize(
-    "values",
+    "values, expected",
+    [
+        (["foo", "", "False", None], [True, False, True, None]),
+        ([0, 1, 2, -1, None], [False, True, True, True, None]),
+        ([0.0, 0.5, np.nan], [False, True, None]),
+        ([0, "a", 1.5, pd.NA, pd.NaT], [False, True, True, None, None]),
+        ([np.nan, {"a": 1}, {}], [None, True, False]),
+        # a list is treated as object dtype, where every Timestamp is truthy
+        ([pd.Timestamp(0), pd.NaT], [True, None]),
+        (np.array(["foo", ""]), [True, False]),
+        (np.array([0, 1, 2]), [False, True, True]),
+        (np.array([0.0, 0.5, np.nan]), [False, True, None]),
+        (np.array([0j, 1j, complex(np.nan, 0)]), [False, True, None]),
+        (np.array([0, 5, "NaT"], dtype="m8[ns]"), [False, True, None]),
+        (
+            np.array(["1970-01-01", "2000-01-01", "NaT"], dtype="M8[ns]"),
+            [False, True, None],
+        ),
+        (pd.date_range("1970-01-01", periods=2, tz="UTC"), [False, True]),
+        (pd.period_range("2000", periods=2), [True, True]),
+    ],
+)
+def test_to_boolean_array_matches_astype_bool(values, expected):
+    # GH#37614 values are converted as by astype(bool), with missing values masked
+    result = pd.array(values, dtype="boolean")
+    expected = pd.array(expected, dtype="boolean")
+    tm.assert_extension_array_equal(result, expected)
+
+
+@pytest.mark.parametrize(
+    "values, expected",
     [
-        ["foo", "bar"],
-        ["1", "2"],
-        # "foo",
-        [1, 2],
-        [1.0, 2.0],
-        pd.date_range("20130101", periods=2),
-        np.array(["foo"]),
-        np.array([1, 2]),
-        np.array([1.0, 2.0]),
-        [np.nan, {"a": 1}],
+        (
+            [0.0, np.nan, None, pd.NA, pd.NaT, Decimal("nan")],
+            [False, True, None, None, None, True],
+        ),
+        (np.array([0.0, np.nan]), [False, True]),
+        (np.array([0j, complex(np.nan, 0)]), [False, True]),
     ],
 )
-def test_to_boolean_array_error(values):
-    # error in converting existing arrays to BooleanArray
-    msg = "Need to pass bool-like value"
-    with pytest.raises(TypeError, match=msg):
-        pd.array(values, dtype="boolean")
+def test_to_boolean_array_distinguish_nan_and_na(values, expected):
+    # with NaN distinct from NA, NaN is a value and converts as bool(np.nan)
+    with pd.option_context("future.distinguish_nan_and_na", True):
+        result = pd.array(values, dtype="boolean")
+    expected = pd.array(expected, dtype="boolean")
+    tm.assert_extension_array_equal(result, expected)
+
+
+def test_float_ea_astype_boolean_distinguish_nan_and_na():
+    with pd.option_context("future.distinguish_nan_and_na", True):
+        arr = pd.array([0.0, np.nan, None], dtype="Float64")
+        result = arr.astype("boolean")
+    expected = pd.array([False, True, None], dtype="boolean")
+    tm.assert_extension_array_equal(result, expected)
+
+
+def test_to_boolean_array_2d_raises():
+    with pytest.raises(TypeError, match="values must be a 1D list-like"):
+        pd.array([[True, False]], dtype="boolean")
 
 
 def test_to_boolean_array_from_integer_array():

pandas/tests/reshape/test_from_dummies.py

@@ -95,6 +95,14 @@ def test_error_contains_non_dummies():
         pd.from_dummies(dummies)
 
 
+@pytest.mark.parametrize("values", [[0, 0.5], ["0", "1"]])
+def test_error_contains_non_dummies_bool_castable(values):
+    # values that astype("boolean") accepts but are not 0 or 1
+    dummies = pd.DataFrame({"a": values, "b": [1, 0]})
+    with pytest.raises(TypeError, match=r"Passed DataFrame contains non-dummy data"):
+        pd.from_dummies(dummies)
+
+
 def test_error_with_prefix_multiple_separators():
     dummies = pd.DataFrame(
         {
@@ -250,6 +258,20 @@ def test_no_prefix_string_cats_basic_mixed_bool_values():
     tm.assert_frame_equal(result, expected)
 
 
+@pytest.mark.parametrize(
+    "dtype", ["int64", "uint8", "float64", "Int64", "boolean", "int64[pyarrow]"]
+)
+def test_no_prefix_string_cats_basic_dtypes(dtype):
+    if dtype == "int64[pyarrow]":
+        pytest.importorskip("pyarrow")
+    dummies = pd.DataFrame(
+        {"a": [1, 0, 0], "b": [0, 1, 0], "c": [0, 0, 1]}, dtype=dtype
+    )
+    expected = pd.DataFrame({"": ["a", "b", "c"]})
+    result = pd.from_dummies(dummies)
+    tm.assert_frame_equal(result, expected)
+
+
 def test_no_prefix_int_cats_basic():
     dummies = pd.DataFrame(
         {1: [1, 0, 0, 0], 25: [0, 1, 0, 0], 2: [0, 0, 1, 0], 5: [0, 0, 0, 1]}

pandas/tests/series/methods/test_astype.py

@@ -511,13 +511,8 @@ class TestAstypeString:
         ],
     )
     def test_astype_string_to_extension_dtype_roundtrip(
-        self, data, dtype, request, nullable_string_dtype
+        self, data, dtype, nullable_string_dtype
     ):
-        if dtype == "boolean":
-            mark = pytest.mark.xfail(
-                reason="TODO StringArray.astype() with missing values #GH40566"
-            )
-            request.applymarker(mark)
         # GH-40351
         ser = pd.Series(data, dtype=dtype)