feat: Expose comparison objects upon failure in testing assertions to enable interactive debugging

diffly/testing.py

@@ -22,6 +22,67 @@
 from .metrics.change import ChangeMetric, ChangeMetricFn
 from .metrics.data import DataMetric, DataMetricFn
 
+# ------------------------------------ EXCEPTIONS ------------------------------------- #
+
+
+class ComparisonAssertionError(AssertionError):
+    """Base class for diffly assertion failures."""
+
+
+class FrameComparisonAssertionError(ComparisonAssertionError):
+    """Raised when :func:`assert_frame_equal` fails.
+
+    Access the underlying comparison from the exception using ``e.comparison``
+    or in a post-mortem debugger via the ``$_exception`` convenience variable
+    (requires Python >= 3.12)::
+
+        (Pdb) cmp = $_exception.comparison
+        (Pdb) cmp.joined_unequal()
+
+    Attributes:
+        comparison: The comparison between the two data frames.
+    """
+
+    def __init__(self, message: str, *, comparison: DataFrameComparison) -> None:
+        super().__init__(message)
+        self.comparison = comparison
+
+
+class CollectionComparisonAssertionError(ComparisonAssertionError):
+    """Raised when :func:`assert_collection_equal` fails.
+
+    Access the failing member comparisons from the exception using ``e.comparisons``
+    or from a post-mortem debugger via the ``$_exception`` convenience variable
+    (requires Python >= 3.12)::
+
+        (Pdb) cmps = $_exception.comparisons
+        (Pdb) cmps["some_member"].joined_unequal()
+
+    Attributes:
+        comparisons: A mapping from member name to comparison for the failing members.
+    """
+
+    def __init__(
+        self, message: str, *, comparisons: dict[str, DataFrameComparison]
+    ) -> None:
+        super().__init__(message)
+        self.comparisons = comparisons
+
+
+# ------------------------------------ ASSERTIONS ------------------------------------- #
+
+_DEBUG_HINT_FRAME = (
+    "To debug interactively (e.g. with `pytest --pdb`), access the comparison via "
+    "`$_exception.comparison` at the debugger prompt (requires Python >= 3.12) or via "
+    "the `.comparison` attribute of this error."
+)
+_DEBUG_HINT_COLLECTION = (
+    "To debug interactively (e.g. with `pytest --pdb`), access the failing member "
+    "comparisons via `$_exception.comparisons` (a mapping from member name to "
+    "comparison) at the debugger prompt (requires Python >= 3.12) or via the "
+    "`.comparisons` attribute of this error."
+)
+
 
 def assert_collection_equal(
     left: dy.Collection,
@@ -100,7 +161,7 @@ def assert_collection_equal(
             change metrics are computed.
 
     Raises:
-        AssertionError: If the collections are not equal.
+        CollectionComparisonAssertionError: If the collections are not equal.
 
     Examples:
         >>> import dataframely as dy
@@ -163,7 +224,10 @@ def assert_collection_equal(
             ),
             " " * 2,
         )
-        raise AssertionError(f"The following members are not equal:\n\n{text}")
+        raise CollectionComparisonAssertionError(
+            f"The following members are not equal:\n\n{text}\n\n{_DEBUG_HINT_COLLECTION}",
+            comparisons=failed_member_comparisons,
+        )
 
 
 def assert_frame_equal(
@@ -251,7 +315,7 @@ def assert_frame_equal(
             change metrics are computed.
 
     Raises:
-        AssertionError: If the data frames are not equal.
+        FrameComparisonAssertionError: If the data frames are not equal.
 
     Note:
         Contrary to :meth:`polars.testing.assert_frame_equal`, the data frames ``left``
@@ -298,7 +362,10 @@ def assert_frame_equal(
             change_metrics=change_metrics,
         )
         text = textwrap.indent(str(summary), " " * 2)
-        raise AssertionError(f"Data frames are not equal:\n\n{text}")
+        raise FrameComparisonAssertionError(
+            f"Data frames are not equal:\n\n{text}\n\n{_DEBUG_HINT_FRAME}",
+            comparison=comparison,
+        )
 
 
 def _get_heading(title: str) -> str:

tests/test_assert_collection_equal.py

@@ -4,7 +4,11 @@
 import polars as pl
 import pytest
 
-from diffly.testing import assert_collection_equal
+from diffly.comparison import DataFrameComparison
+from diffly.testing import (
+    CollectionComparisonAssertionError,
+    assert_collection_equal,
+)
 
 pytest.importorskip("dataframely", reason="requires dataframely")
 import dataframely as dy
@@ -37,89 +41,103 @@ class Quux(dy.Collection):
     bar: dy.LazyFrame[Bar]
 
 
-def test_identical() -> None:
+@pytest.fixture
+def matching() -> pl.DataFrame:
+    return pl.DataFrame({"index": [1, 2, 3], "value": [1.0, 2.0, 3.0]})
+
+
+@pytest.fixture
+def mismatching() -> pl.DataFrame:
+    return pl.DataFrame({"index": [1, 2, 3], "value": [1.0, 2.0, 4.0]})
+
+
+def test_identical(matching: pl.DataFrame) -> None:
     qux = Qux.validate(
         {
-            "foo": Foo.validate(
-                pl.DataFrame({"index": [1, 2, 3], "value": [1.0, 2.0, 3.0]}), cast=True
-            ),
-            "bar": Bar.validate(
-                pl.DataFrame({"index": [1, 2, 3], "value": [1.0, 2.0, 3.0]}), cast=True
-            ),
+            "foo": Foo.validate(matching, cast=True),
+            "bar": Bar.validate(matching, cast=True),
         }
     )
     assert_collection_equal(qux, qux)
 
 
-def test_different_types() -> None:
+def test_different_types(matching: pl.DataFrame) -> None:
     qux = Qux.validate(
         {
-            "foo": Foo.validate(
-                pl.DataFrame({"index": [1, 2, 3], "value": [1.0, 2.0, 3.0]}), cast=True
-            ),
-            "bar": Bar.validate(
-                pl.DataFrame({"index": [1, 2, 3], "value": [1.0, 2.0, 3.0]}), cast=True
-            ),
+            "foo": Foo.validate(matching, cast=True),
+            "bar": Bar.validate(matching, cast=True),
         }
     )
     quux = Quux.validate(qux.to_dict())
     with pytest.raises(AssertionError, match="The collection definitions do not match"):
         assert_collection_equal(qux, quux)
 
 
-def test_missing_member() -> None:
+def test_missing_member(matching: pl.DataFrame) -> None:
     qux1 = Qux.validate(
         {
-            "foo": Foo.validate(
-                pl.DataFrame({"index": [1, 2, 3], "value": [1.0, 2.0, 3.0]}), cast=True
-            ),
-            "bar": Bar.validate(
-                pl.DataFrame({"index": [1, 2, 3], "value": [1.0, 2.0, 3.0]}), cast=True
-            ),
-            "baz": Baz.validate(
-                pl.DataFrame({"index": [1, 2, 3], "value": [1.0, 2.0, 3.0]}), cast=True
-            ),
+            "foo": Foo.validate(matching, cast=True),
+            "bar": Bar.validate(matching, cast=True),
+            "baz": Baz.validate(matching, cast=True),
         }
     )
     qux2 = Qux.validate(
         {
-            "foo": Foo.validate(
-                pl.DataFrame({"index": [1, 2, 3], "value": [1.0, 2.0, 3.0]}), cast=True
-            ),
-            "bar": Bar.validate(
-                pl.DataFrame({"index": [1, 2, 3], "value": [1.0, 2.0, 3.0]}), cast=True
-            ),
+            "foo": Foo.validate(matching, cast=True),
+            "bar": Bar.validate(matching, cast=True),
         }
     )
     with pytest.raises(AssertionError, match="The collections have different members"):
         assert_collection_equal(qux1, qux2)
 
 
-def test_unequal_members() -> None:
+def test_unequal_members(matching: pl.DataFrame, mismatching: pl.DataFrame) -> None:
     qux1 = Qux.validate(
         {
-            "foo": Foo.validate(
-                pl.DataFrame({"index": [1, 2, 3], "value": [1.0, 2.0, 3.0]}), cast=True
-            ),
-            "bar": Bar.validate(
-                pl.DataFrame({"index": [1, 2, 3], "value": [1.0, 2.0, 3.0]}), cast=True
-            ),
+            "foo": Foo.validate(matching, cast=True),
+            "bar": Bar.validate(matching, cast=True),
         }
     )
     qux2 = Qux.validate(
         {
-            "foo": Foo.validate(
-                pl.DataFrame({"index": [1, 2, 3], "value": [1.0, 2.0, 4.0]}), cast=True
-            ),
-            "bar": Bar.validate(
-                pl.DataFrame({"index": [1, 2, 3], "value": [1.0, 2.0, 4.0]}), cast=True
-            ),
+            "foo": Foo.validate(mismatching, cast=True),
+            "bar": Bar.validate(mismatching, cast=True),
         }
     )
     with pytest.raises(AssertionError, match="The following members are not equal"):
         assert_collection_equal(qux1, qux2)
 
 
+def test_error_exposes_comparisons(
+    matching: pl.DataFrame, mismatching: pl.DataFrame
+) -> None:
+    # Arrange
+    qux1 = Qux.validate(
+        {
+            "foo": Foo.validate(matching, cast=True),
+            "bar": Bar.validate(matching, cast=True),
+        }
+    )
+    qux2 = Qux.validate(
+        {
+            "foo": Foo.validate(mismatching, cast=True),
+            "bar": Bar.validate(matching, cast=True),
+        }
+    )
+
+    # Act
+    with pytest.raises(CollectionComparisonAssertionError) as exc_info:
+        assert_collection_equal(qux1, qux2)
+
+    # Assert
+    assert isinstance(exc_info.value, CollectionComparisonAssertionError)
+    # Only the failing member is exposed, and its comparison is usable.
+    comparisons = exc_info.value.comparisons
+    assert set(comparisons) == {"foo"}
+    assert isinstance(comparisons["foo"], DataFrameComparison)
+    assert comparisons["foo"].fraction_same("value") == pytest.approx(2 / 3)
+
+
 def test_no_primary_key() -> None:
     no_pk_schema = create_schema("NoPKSchema", {"a": dy.Integer(nullable=True)})
     collection = create_collection("Test", {"first": Foo, "second": no_pk_schema})

tests/test_assert_frame_equal.py

@@ -7,7 +7,11 @@
 import pytest
 
 from diffly import compare_frames
-from diffly.testing import assert_frame_equal
+from diffly.comparison import DataFrameComparison
+from diffly.testing import (
+    FrameComparisonAssertionError,
+    assert_frame_equal,
+)
 
 
 def test_success_equal() -> None:
@@ -60,3 +64,19 @@ def test_success_with_nan() -> None:
     df = pl.DataFrame({"id": [1, 2], "value": [1.0, float("nan")]})
     assert_frame_equal(df, df)
     assert_frame_equal(df, df, primary_key="id")
+
+
+def test_error_exposes_comparison() -> None:
+    # Arrange
+    left = pl.DataFrame({"id": [1, 2], "value": [10.0, 20.0]})
+    right = pl.DataFrame({"id": [1, 2], "value": [10.0, 25.0]})
+
+    # Act
+    with pytest.raises(FrameComparisonAssertionError) as exc_info:
+        assert_frame_equal(left, right, primary_key="id")
+
+    # Assert
+    assert isinstance(exc_info.value, FrameComparisonAssertionError)
+    comparison = exc_info.value.comparison
+    assert isinstance(comparison, DataFrameComparison)
+    assert comparison.fraction_same("value") == 0.5