diff --git a/diffly/testing.py b/diffly/testing.py index 337d5e4..8b8bf66 100644 --- a/diffly/testing.py +++ b/diffly/testing.py @@ -22,6 +22,67 @@ from .metrics.change import ChangeMetric, ChangeMetricFn from .metrics.data import DataMetric, DataMetricFn +# ------------------------------------ EXCEPTIONS ------------------------------------- # + + +class ComparisonAssertionError(AssertionError): + """Base class for diffly assertion failures.""" + + +class FrameComparisonAssertionError(ComparisonAssertionError): + """Raised when :func:`assert_frame_equal` fails. + + Access the underlying comparison from the exception using ``e.comparison`` + or in a post-mortem debugger via the ``$_exception`` convenience variable + (requires Python >= 3.12):: + + (Pdb) cmp = $_exception.comparison + (Pdb) cmp.joined_unequal() + + Attributes: + comparison: The comparison between the two data frames. + """ + + def __init__(self, message: str, *, comparison: DataFrameComparison) -> None: + super().__init__(message) + self.comparison = comparison + + +class CollectionComparisonAssertionError(ComparisonAssertionError): + """Raised when :func:`assert_collection_equal` fails. + + Access the failing member comparisons from the exception using ``e.comparisons`` + or from a post-mortem debugger via the ``$_exception`` convenience variable + (requires Python >= 3.12):: + + (Pdb) cmps = $_exception.comparisons + (Pdb) cmps["some_member"].joined_unequal() + + Attributes: + comparisons: A mapping from member name to comparison for the failing members. + """ + + def __init__( + self, message: str, *, comparisons: dict[str, DataFrameComparison] + ) -> None: + super().__init__(message) + self.comparisons = comparisons + + +# ------------------------------------ ASSERTIONS ------------------------------------- # + +_DEBUG_HINT_FRAME = ( + "To debug interactively (e.g. with `pytest --pdb`), access the comparison via " + "`$_exception.comparison` at the debugger prompt (requires Python >= 3.12) or via " + "the `.comparison` attribute of this error." +) +_DEBUG_HINT_COLLECTION = ( + "To debug interactively (e.g. with `pytest --pdb`), access the failing member " + "comparisons via `$_exception.comparisons` (a mapping from member name to " + "comparison) at the debugger prompt (requires Python >= 3.12) or via the " + "`.comparisons` attribute of this error." +) + def assert_collection_equal( left: dy.Collection, @@ -100,7 +161,7 @@ def assert_collection_equal( change metrics are computed. Raises: - AssertionError: If the collections are not equal. + CollectionComparisonAssertionError: If the collections are not equal. Examples: >>> import dataframely as dy @@ -163,7 +224,10 @@ def assert_collection_equal( ), " " * 2, ) - raise AssertionError(f"The following members are not equal:\n\n{text}") + raise CollectionComparisonAssertionError( + f"The following members are not equal:\n\n{text}\n\n{_DEBUG_HINT_COLLECTION}", + comparisons=failed_member_comparisons, + ) def assert_frame_equal( @@ -251,7 +315,7 @@ def assert_frame_equal( change metrics are computed. Raises: - AssertionError: If the data frames are not equal. + FrameComparisonAssertionError: If the data frames are not equal. Note: Contrary to :meth:`polars.testing.assert_frame_equal`, the data frames ``left`` @@ -298,7 +362,10 @@ def assert_frame_equal( change_metrics=change_metrics, ) text = textwrap.indent(str(summary), " " * 2) - raise AssertionError(f"Data frames are not equal:\n\n{text}") + raise FrameComparisonAssertionError( + f"Data frames are not equal:\n\n{text}\n\n{_DEBUG_HINT_FRAME}", + comparison=comparison, + ) def _get_heading(title: str) -> str: diff --git a/tests/test_assert_collection_equal.py b/tests/test_assert_collection_equal.py index 6edc6fa..9d8195e 100644 --- a/tests/test_assert_collection_equal.py +++ b/tests/test_assert_collection_equal.py @@ -4,7 +4,11 @@ import polars as pl import pytest -from diffly.testing import assert_collection_equal +from diffly.comparison import DataFrameComparison +from diffly.testing import ( + CollectionComparisonAssertionError, + assert_collection_equal, +) pytest.importorskip("dataframely", reason="requires dataframely") import dataframely as dy @@ -37,29 +41,31 @@ class Quux(dy.Collection): bar: dy.LazyFrame[Bar] -def test_identical() -> None: +@pytest.fixture +def matching() -> pl.DataFrame: + return pl.DataFrame({"index": [1, 2, 3], "value": [1.0, 2.0, 3.0]}) + + +@pytest.fixture +def mismatching() -> pl.DataFrame: + return pl.DataFrame({"index": [1, 2, 3], "value": [1.0, 2.0, 4.0]}) + + +def test_identical(matching: pl.DataFrame) -> None: qux = Qux.validate( { - "foo": Foo.validate( - pl.DataFrame({"index": [1, 2, 3], "value": [1.0, 2.0, 3.0]}), cast=True - ), - "bar": Bar.validate( - pl.DataFrame({"index": [1, 2, 3], "value": [1.0, 2.0, 3.0]}), cast=True - ), + "foo": Foo.validate(matching, cast=True), + "bar": Bar.validate(matching, cast=True), } ) assert_collection_equal(qux, qux) -def test_different_types() -> None: +def test_different_types(matching: pl.DataFrame) -> None: qux = Qux.validate( { - "foo": Foo.validate( - pl.DataFrame({"index": [1, 2, 3], "value": [1.0, 2.0, 3.0]}), cast=True - ), - "bar": Bar.validate( - pl.DataFrame({"index": [1, 2, 3], "value": [1.0, 2.0, 3.0]}), cast=True - ), + "foo": Foo.validate(matching, cast=True), + "bar": Bar.validate(matching, cast=True), } ) quux = Quux.validate(qux.to_dict()) @@ -67,59 +73,71 @@ def test_different_types() -> None: assert_collection_equal(qux, quux) -def test_missing_member() -> None: +def test_missing_member(matching: pl.DataFrame) -> None: qux1 = Qux.validate( { - "foo": Foo.validate( - pl.DataFrame({"index": [1, 2, 3], "value": [1.0, 2.0, 3.0]}), cast=True - ), - "bar": Bar.validate( - pl.DataFrame({"index": [1, 2, 3], "value": [1.0, 2.0, 3.0]}), cast=True - ), - "baz": Baz.validate( - pl.DataFrame({"index": [1, 2, 3], "value": [1.0, 2.0, 3.0]}), cast=True - ), + "foo": Foo.validate(matching, cast=True), + "bar": Bar.validate(matching, cast=True), + "baz": Baz.validate(matching, cast=True), } ) qux2 = Qux.validate( { - "foo": Foo.validate( - pl.DataFrame({"index": [1, 2, 3], "value": [1.0, 2.0, 3.0]}), cast=True - ), - "bar": Bar.validate( - pl.DataFrame({"index": [1, 2, 3], "value": [1.0, 2.0, 3.0]}), cast=True - ), + "foo": Foo.validate(matching, cast=True), + "bar": Bar.validate(matching, cast=True), } ) with pytest.raises(AssertionError, match="The collections have different members"): assert_collection_equal(qux1, qux2) -def test_unequal_members() -> None: +def test_unequal_members(matching: pl.DataFrame, mismatching: pl.DataFrame) -> None: qux1 = Qux.validate( { - "foo": Foo.validate( - pl.DataFrame({"index": [1, 2, 3], "value": [1.0, 2.0, 3.0]}), cast=True - ), - "bar": Bar.validate( - pl.DataFrame({"index": [1, 2, 3], "value": [1.0, 2.0, 3.0]}), cast=True - ), + "foo": Foo.validate(matching, cast=True), + "bar": Bar.validate(matching, cast=True), } ) qux2 = Qux.validate( { - "foo": Foo.validate( - pl.DataFrame({"index": [1, 2, 3], "value": [1.0, 2.0, 4.0]}), cast=True - ), - "bar": Bar.validate( - pl.DataFrame({"index": [1, 2, 3], "value": [1.0, 2.0, 4.0]}), cast=True - ), + "foo": Foo.validate(mismatching, cast=True), + "bar": Bar.validate(mismatching, cast=True), } ) with pytest.raises(AssertionError, match="The following members are not equal"): assert_collection_equal(qux1, qux2) +def test_error_exposes_comparisons( + matching: pl.DataFrame, mismatching: pl.DataFrame +) -> None: + # Arrange + qux1 = Qux.validate( + { + "foo": Foo.validate(matching, cast=True), + "bar": Bar.validate(matching, cast=True), + } + ) + qux2 = Qux.validate( + { + "foo": Foo.validate(mismatching, cast=True), + "bar": Bar.validate(matching, cast=True), + } + ) + + # Act + with pytest.raises(CollectionComparisonAssertionError) as exc_info: + assert_collection_equal(qux1, qux2) + + # Assert + assert isinstance(exc_info.value, CollectionComparisonAssertionError) + # Only the failing member is exposed, and its comparison is usable. + comparisons = exc_info.value.comparisons + assert set(comparisons) == {"foo"} + assert isinstance(comparisons["foo"], DataFrameComparison) + assert comparisons["foo"].fraction_same("value") == pytest.approx(2 / 3) + + def test_no_primary_key() -> None: no_pk_schema = create_schema("NoPKSchema", {"a": dy.Integer(nullable=True)}) collection = create_collection("Test", {"first": Foo, "second": no_pk_schema}) diff --git a/tests/test_assert_frame_equal.py b/tests/test_assert_frame_equal.py index 3e59fc2..a34e0d0 100644 --- a/tests/test_assert_frame_equal.py +++ b/tests/test_assert_frame_equal.py @@ -7,7 +7,11 @@ import pytest from diffly import compare_frames -from diffly.testing import assert_frame_equal +from diffly.comparison import DataFrameComparison +from diffly.testing import ( + FrameComparisonAssertionError, + assert_frame_equal, +) def test_success_equal() -> None: @@ -60,3 +64,19 @@ def test_success_with_nan() -> None: df = pl.DataFrame({"id": [1, 2], "value": [1.0, float("nan")]}) assert_frame_equal(df, df) assert_frame_equal(df, df, primary_key="id") + + +def test_error_exposes_comparison() -> None: + # Arrange + left = pl.DataFrame({"id": [1, 2], "value": [10.0, 20.0]}) + right = pl.DataFrame({"id": [1, 2], "value": [10.0, 25.0]}) + + # Act + with pytest.raises(FrameComparisonAssertionError) as exc_info: + assert_frame_equal(left, right, primary_key="id") + + # Assert + assert isinstance(exc_info.value, FrameComparisonAssertionError) + comparison = exc_info.value.comparison + assert isinstance(comparison, DataFrameComparison) + assert comparison.fraction_same("value") == 0.5