ENH: say which DataFrame is missing the key in merge KeyError (GH#71819)

doc/source/whatsnew/v3.2.0.rst

@@ -28,6 +28,7 @@ enhancement2
 
 Other enhancements
 ^^^^^^^^^^^^^^^^^^
+- :func:`merge`, :meth:`DataFrame.merge` and :meth:`DataFrame.join` now raise a more informative ``KeyError`` when a key passed to ``on``, ``left_on`` or ``right_on`` is missing, saying whether the left DataFrame, the right DataFrame or both are missing it and listing the available columns and index levels (:issue:`71819`)
 - :func:`numpy.delete` on an :class:`ExtensionArray`, and :func:`numpy.concatenate` on :class:`ExtensionArray` objects of the same dtype, now return an :class:`ExtensionArray` of that dtype instead of a NumPy array (:issue:`26380`)
 - :func:`read_csv` and :func:`read_table` with ``sep=None`` now sniff the separator with the C engine instead of falling back to the slower python engine (:issue:`9645`)
 - :func:`read_csv` with the ``"c"`` and ``"python"`` engines now accepts a :class:`DatetimeTZDtype` in ``dtype`` (:issue:`24542`)

pandas/core/reshape/merge.py

@@ -42,6 +42,7 @@
     is_bool,
     is_bool_dtype,
     is_float_dtype,
+    is_hashable,
     is_integer,
     is_integer_dtype,
     is_list_like,
@@ -1569,6 +1570,48 @@ def _handle_anti_join(
 
         return join_index, left_indexer, right_indexer
 
+    @final
+    def _get_merge_key_values(
+        self, side: Literal["left", "right"], key: Hashable, shared: bool = False
+    ) -> ArrayLike:
+        """
+        Get the values of merge key ``key`` from the left or right DataFrame.
+
+        If ``key`` is not found, re-raise the ``KeyError`` with a message
+        saying which DataFrame is missing it and what keys are available.
+
+        Parameters
+        ----------
+        side : {"left", "right"}
+            Which DataFrame to look up ``key`` in.
+        key : Hashable
+            Column label or index level name.
+        shared : bool, default False
+            Whether ``key`` is used on both sides (e.g. ``on=``). If True and
+            ``key`` is missing from both DataFrames, the message says so.
+        """
+        obj = self.left if side == "left" else self.right
+        try:
+            return obj._get_label_or_level_values(key)
+        except KeyError as err:
+            if not is_hashable(key):
+                # e.g. a wrong-length array-like; keep the original error
+                raise
+            if shared:
+                other = self.right if side == "left" else self.left
+                if not other._is_label_or_level_reference(key):
+                    msg = (
+                        f"{key!r} not found in left or right DataFrame. "
+                        f"Left {_describe_merge_key_options(self.left)}. "
+                        f"Right {_describe_merge_key_options(self.right)}"
+                    )
+                    raise KeyError(msg) from err
+            msg = (
+                f"{key!r} not found in {side} DataFrame. "
+                f"Available {_describe_merge_key_options(obj)}"
+            )
+            raise KeyError(msg) from err
+
     @final
     def _get_merge_keys(
         self,
@@ -1622,7 +1665,7 @@ def _get_merge_keys(
                         #  the latter of which will raise
                         rk = cast("Hashable", rk)
                         if rk is not None:
-                            right_keys.append(right._get_label_or_level_values(rk))
+                            right_keys.append(self._get_merge_key_values("right", rk))
                             join_names.append(rk)
                         else:
                             # work-around for merge_asof(right_index=True)
@@ -1634,7 +1677,12 @@ def _get_merge_keys(
                         #  the latter of which will raise
                         rk = cast("Hashable", rk)
                         if rk is not None:
-                            right_keys.append(right._get_label_or_level_values(rk))
+                            # shared: the same key is looked up in left below
+                            #  (e.g. on=), so report if neither side has it
+                            shared = bool(is_hashable(lk) and lk == rk)
+                            right_keys.append(
+                                self._get_merge_key_values("right", rk, shared=shared)
+                            )
                         else:
                             # work-around for merge_asof(right_index=True)
                             right_keys.append(right.index._values)
@@ -1647,7 +1695,7 @@ def _get_merge_keys(
                         # Then we're either Hashable or a wrong-length arraylike,
                         #  the latter of which will raise
                         lk = cast("Hashable", lk)
-                        left_keys.append(left._get_label_or_level_values(lk))
+                        left_keys.append(self._get_merge_key_values("left", lk))
                         join_names.append(lk)
                     else:
                         # work-around for merge_asof(left_index=True)
@@ -1664,7 +1712,7 @@ def _get_merge_keys(
                     # Then we're either Hashable or a wrong-length arraylike,
                     #  the latter of which will raise
                     k = cast("Hashable", k)
-                    left_keys.append(left._get_label_or_level_values(k))
+                    left_keys.append(self._get_merge_key_values("left", k))
                     join_names.append(k)
             if isinstance(self.right.index, MultiIndex):
                 right_keys = [
@@ -1684,7 +1732,7 @@ def _get_merge_keys(
                     # Then we're either Hashable or a wrong-length arraylike,
                     #  the latter of which will raise
                     k = cast("Hashable", k)
-                    right_keys.append(right._get_label_or_level_values(k))
+                    right_keys.append(self._get_merge_key_values("right", k))
                     join_names.append(k)
             if isinstance(self.left.index, MultiIndex):
                 left_keys = [
@@ -3307,6 +3355,18 @@ def _any(x) -> bool:
     return x is not None and com.any_not_none(*x)
 
 
+def _describe_merge_key_options(obj: DataFrame) -> str:
+    """
+    Describe the column labels and named index levels of ``obj`` that can be
+    used as merge keys, for use in error messages.
+    """
+    desc = f"columns: {obj.columns.tolist()}"
+    levels = [name for name in obj.index.names if name is not None]
+    if levels:
+        desc += f", index levels: {levels}"
+    return desc
+
+
 def _validate_operand(obj: DataFrame | Series) -> DataFrame:
     if isinstance(obj, ABCDataFrame):
         return obj

pandas/tests/reshape/merge/test_join.py

@@ -139,8 +139,12 @@ def test_join_on(self, target_source, infer_string):
         assert np.isnan(joined["two"]["c"])
         assert np.isnan(joined["three"]["c"])
 
-        # merge column not p resent
-        with pytest.raises(KeyError, match="^'E'$"):
+        # merge column not present
+        msg = (
+            r"'E' not found in left DataFrame\. "
+            r"Available columns: \['A', 'B', 'C', 'D'\]"
+        )
+        with pytest.raises(KeyError, match=msg):
             target.join(source, on="E")
 
         # overlap

pandas/tests/reshape/merge/test_merge.py

@@ -3579,3 +3579,77 @@ def test_merge_sort_false_range_like_span_exceeds_int64_max(how):
     else:
         expected = pd.DataFrame({"k": other_k, "v": [2.0, np.nan, 1.0], "w": [0, 1, 2]})
     tm.assert_frame_equal(result, expected)
+
+
+class TestMergeMissingKeyError:
+    # GH#71819 - the KeyError for a missing merge key says which DataFrame
+    # is missing it and which keys are available
+
+    @pytest.fixture
+    def orders(self):
+        return pd.DataFrame({"customer_id": [1, 2, 3], "amount": [10, 20, 30]})
+
+    @pytest.fixture
+    def customers(self):
+        return pd.DataFrame({"id": [1, 2, 3], "name": ["A", "B", "C"]})
+
+    @pytest.mark.parametrize(
+        "kwargs, msg",
+        [
+            (
+                {"on": "customer_id"},
+                "'customer_id' not found in right DataFrame. "
+                "Available columns: ['id', 'name']",
+            ),
+            (
+                {"on": "id"},
+                "'id' not found in left DataFrame. "
+                "Available columns: ['customer_id', 'amount']",
+            ),
+            (
+                {"on": "zzz"},
+                "'zzz' not found in left or right DataFrame. "
+                "Left columns: ['customer_id', 'amount']. "
+                "Right columns: ['id', 'name']",
+            ),
+            (
+                {"left_on": "customr_id", "right_on": "id"},
+                "'customr_id' not found in left DataFrame. "
+                "Available columns: ['customer_id', 'amount']",
+            ),
+            (
+                {"left_on": "customer_id", "right_on": "idd"},
+                "'idd' not found in right DataFrame. Available columns: ['id', 'name']",
+            ),
+            (
+                {"left_on": "zzz", "right_index": True},
+                "'zzz' not found in left DataFrame. "
+                "Available columns: ['customer_id', 'amount']",
+            ),
+            (
+                {"left_index": True, "right_on": "zzz"},
+                "'zzz' not found in right DataFrame. Available columns: ['id', 'name']",
+            ),
+        ],
+    )
+    def test_merge_missing_key_names_side(self, orders, customers, kwargs, msg):
+        with pytest.raises(KeyError, match=re.escape(msg)) as err:
+            orders.merge(customers, **kwargs)
+        # the original KeyError is chained
+        assert isinstance(err.value.__cause__, KeyError)
+
+    def test_merge_missing_key_lists_index_levels(self, customers):
+        left = pd.DataFrame({"a": [1, 2]}, index=pd.Index([1, 2], name="lvl"))
+        msg = (
+            "'zzz' not found in left DataFrame. "
+            "Available columns: ['a'], index levels: ['lvl']"
+        )
+        with pytest.raises(KeyError, match=re.escape(msg)):
+            left.merge(customers, left_on="zzz", right_on="id")
+
+    def test_merge_missing_key_non_string_labels(self):
+        left = pd.DataFrame({0: [1, 2], 1: [3, 4]})
+        right = pd.DataFrame({0: [1, 2], 2: [5, 6]})
+        msg = "2 not found in left DataFrame. Available columns: [0, 1]"
+        with pytest.raises(KeyError, match=re.escape(msg)):
+            left.merge(right, on=2)