Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
42 commits
Select commit Hold shift + click to select a range
bff2594
init
MoritzPotthoffQC Jul 17, 2026
51fbda2
add test
MoritzPotthoffQC Jul 17, 2026
cb086ef
refactor
MoritzPotthoffQC Jul 17, 2026
526c944
simplify
MoritzPotthoffQC Jul 17, 2026
5732c37
fix
MoritzPotthoffQC Jul 17, 2026
5867021
review
MoritzPotthoffQC Jul 22, 2026
6b705e7
review
MoritzPotthoffQC Jul 22, 2026
6c868d6
review
MoritzPotthoffQC Jul 22, 2026
6802592
review
MoritzPotthoffQC Jul 23, 2026
d827967
refactor
MoritzPotthoffQC Jul 23, 2026
2a49d96
simplify
MoritzPotthoffQC Jul 23, 2026
eef3d3b
simplify
MoritzPotthoffQC Jul 23, 2026
e918bbf
simplify
MoritzPotthoffQC Jul 23, 2026
65d80d6
fix
MoritzPotthoffQC Jul 23, 2026
ffeb05a
cleanup
MoritzPotthoffQC Jul 23, 2026
c116fdf
improve
MoritzPotthoffQC Jul 23, 2026
a57dc0f
docs
MoritzPotthoffQC Jul 23, 2026
1adcf78
review
MoritzPotthoffQC Jul 23, 2026
4000181
fix
MoritzPotthoffQC Jul 23, 2026
a52e24c
simplify
MoritzPotthoffQC Jul 23, 2026
2d3f152
fix
MoritzPotthoffQC Jul 23, 2026
a0955c0
fix
MoritzPotthoffQC Jul 23, 2026
0d9be70
add test
MoritzPotthoffQC Jul 23, 2026
7ecd908
Potential fix for pull request finding
MoritzPotthoffQC Jul 23, 2026
f44fc4f
review
MoritzPotthoffQC Jul 25, 2026
8c7c45a
Potential fix for pull request finding
MoritzPotthoffQC Jul 25, 2026
a6bb6d8
review
MoritzPotthoffQC Jul 27, 2026
ddc31f8
coverage
MoritzPotthoffQC Jul 27, 2026
69e5893
coverage
MoritzPotthoffQC Jul 27, 2026
42787c7
fix
MoritzPotthoffQC Jul 27, 2026
b1ac321
review
MoritzPotthoffQC Jul 27, 2026
cab9baf
Apply suggestions from code review
MoritzPotthoffQC Jul 28, 2026
ee99ec1
Update tests/summary/fixtures/metrics_long_labels/test_metrics_long_l…
MoritzPotthoffQC Jul 28, 2026
41f8637
Update tests/summary/fixtures/metrics_presets_few/test_metrics_preset…
MoritzPotthoffQC Jul 28, 2026
0958d6c
Update tests/summary/test_summary.py
MoritzPotthoffQC Jul 28, 2026
aeb3cd8
Update tests/test_metrics.py
MoritzPotthoffQC Jul 28, 2026
1d93a59
Update tests/summary/test_summary.py
MoritzPotthoffQC Jul 28, 2026
e5fd01a
Update tests/summary/fixtures/metrics_presets_many/test_metrics_prese…
MoritzPotthoffQC Jul 28, 2026
045ba31
review
MoritzPotthoffQC Jul 28, 2026
d150ba4
fix
MoritzPotthoffQC Jul 28, 2026
7835e7b
review
MoritzPotthoffQC Jul 28, 2026
6a09e68
review
MoritzPotthoffQC Jul 28, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
41 changes: 26 additions & 15 deletions diffly/cli.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,18 +12,11 @@

from ._compat import typer
from ._utils import ABS_TOL_DEFAULT, ABS_TOL_TEMPORAL_DEFAULT, REL_TOL_DEFAULT
from .metrics import Metric, MetricFn
from .metrics.change import DEFAULT_CHANGE_METRICS
from .metrics.data import DEFAULT_DATA_METRICS

app = typer.Typer()

#: All metric presets selectable via ``--metric``, combining the change and data sets.
AVAILABLE_METRICS: dict[str, MetricFn | Metric] = {
**DEFAULT_CHANGE_METRICS,
**DEFAULT_DATA_METRICS,
}


@app.command()
def main(
Expand Down Expand Up @@ -143,12 +136,21 @@ def main(
list[str],
typer.Option(hidden=True),
] = [],
metric: Annotated[
data_metric: Annotated[
list[str],
typer.Option(
help=(
"Metric presets to display per column. Repeatable. "
f"Available: {', '.join(AVAILABLE_METRICS)}."
"Data metric presets to display in the Data Inspection section. "
f"Repeatable. Available: {', '.join(DEFAULT_DATA_METRICS)}."
)
),
] = [],
change_metric: Annotated[
list[str],
typer.Option(
help=(
"Change metric presets to display as extra columns in the Columns "
f"table. Repeatable. Available: {', '.join(DEFAULT_CHANGE_METRICS)}."
)
),
Comment thread
MoritzPotthoffQC marked this conversation as resolved.
] = [],
Expand All @@ -161,12 +163,20 @@ def main(
)
hidden_column = [*hidden_column, *hidden_columns]

for name in metric:
if name not in AVAILABLE_METRICS:
for name in data_metric:
if name not in DEFAULT_DATA_METRICS:
raise typer.BadParameter(
f"Unknown data metric: {name!r}. "
f"Available: {', '.join(DEFAULT_DATA_METRICS)}."
)
for name in change_metric:
if name not in DEFAULT_CHANGE_METRICS:
raise typer.BadParameter(
f"Unknown metric: {name!r}. Available: {', '.join(AVAILABLE_METRICS)}."
f"Unknown change metric: {name!r}. "
f"Available: {', '.join(DEFAULT_CHANGE_METRICS)}."
)
metrics = {name: AVAILABLE_METRICS[name] for name in metric}
data_metrics = {name: DEFAULT_DATA_METRICS[name] for name in data_metric}
change_metrics = {name: DEFAULT_CHANGE_METRICS[name] for name in change_metric}

comparison = compare_frames(
pl.scan_parquet(left),
Expand All @@ -185,7 +195,8 @@ def main(
right_name=right_name,
slim=slim,
hidden_columns=hidden_column,
metrics=metrics,
data_metrics=data_metrics,
change_metrics=change_metrics,
)
if output_json:
typer.echo(summary.to_json())
Expand Down
67 changes: 46 additions & 21 deletions diffly/comparison.py
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,8 @@
lazy_len,
make_and_validate_mapping,
)
from .metrics import Metric, MetricFn, _make_numeric_metric
from .metrics.change import ChangeMetric, ChangeMetricFn
from .metrics.data import DataMetric, DataMetricFn

if TYPE_CHECKING: # pragma: no cover
# NOTE: We cannot import at runtime as we're otherwise running into circular
Expand Down Expand Up @@ -920,7 +921,8 @@ def summary(
right_name: str = Side.RIGHT,
slim: bool = False,
hidden_columns: list[str] | None = None,
metrics: Mapping[str, MetricFn | Metric] | None = None,
data_metrics: Mapping[str, DataMetricFn | DataMetric] | None = None,
change_metrics: Mapping[str, ChangeMetricFn | ChangeMetric] | None = None,
) -> Summary:
"""Generate a summary of all aspects of the comparison.

Expand Down Expand Up @@ -950,18 +952,30 @@ def summary(
advanced users who are familiar with the summary format.
hidden_columns: Columns for which no values are printed, e.g. because they
contain sensitive information.
metrics: Optional mapping from display label to a metric. A value may be a
callable ``(left_expr, right_expr) -> pl.Expr`` or a
:class:`~diffly.metrics.Metric`. Each callable receives two
:class:`polars.Expr` referring to the left and right values of a single
column across all joined rows, and must return a scalar aggregation
expression. Bare callables are only computed for numerical columns; wrap
one in a :class:`~diffly.metrics.Metric` with a column selector to target
other column types (e.g. ``Metric(fn, selector=cs.all())``).
See :doc:`/api/metrics` for the full list of presets and the
:data:`~diffly.metrics.MetricFn` type. When ``None`` (default), no metrics
are computed; presets are not applied automatically. Prefer short labels —
the summary has a fixed width and many or long labels degrade rendering.
data_metrics: Optional mapping from display label to a data metric,
describing each dataset individually and rendered in the "Data
Inspection" section. A value may be a
:class:`~diffly.metrics.data.DataMetric` or a bare callable taking a
single column expression, which is wrapped in a
:class:`~diffly.metrics.data.DataMetric` applying to all columns. To
target other column types, construct the metric explicitly with a
column selector (e.g. ``DataMetric(fn, selector=cs.numeric())``).
Comment thread
MoritzPotthoffQC marked this conversation as resolved.
See :doc:`/api/metrics` for the full list of presets. When ``None``
(default), no metrics are computed; presets are not applied
automatically. Prefer short labels — the summary has a fixed width and
many or long labels degrade rendering.
change_metrics: Optional mapping from display label to a change metric,
quantifying the change between the two sides and rendered as extra
columns in the "Columns" table. A value may be a
:class:`~diffly.metrics.change.ChangeMetric` or a bare callable taking a
pair of column expressions, which is wrapped in a
:class:`~diffly.metrics.change.ChangeMetric` applying to numerical
columns. To target other column types, construct the metric explicitly
with a column selector (e.g. ``ChangeMetric(fn, selector=cs.numeric())``).
See :doc:`/api/metrics` for the full list of presets. When ``None``
(default), no metrics are computed; presets are not applied
automatically. Prefer short labels — the summary has a fixed width and
many or long labels degrade rendering.

Returns:
A summary which can be printed or written to a file.
Expand All @@ -977,12 +991,14 @@ def summary(
# NOTE: We're importing here to prevent circular imports
from .summary import Summary

resolved_metrics = (
{
label: v if isinstance(v, Metric) else _make_numeric_metric(v)
for label, v in metrics.items()
}
if metrics is not None
resolved_data_metrics = (
{label: _resolve_data_metric(v) for label, v in data_metrics.items()}
if data_metrics is not None
else None
)
resolved_change_metrics = (
{label: _resolve_change_metric(v) for label, v in change_metrics.items()}
if change_metrics is not None
else None
)

Expand All @@ -996,7 +1012,8 @@ def summary(
right_name=right_name,
slim=slim,
hidden_columns=hidden_columns,
metrics=resolved_metrics,
data_metrics=resolved_data_metrics,
change_metrics=resolved_change_metrics,
)

# ----------------------------------- UTILITIES ----------------------------------- #
Expand Down Expand Up @@ -1239,3 +1256,11 @@ def _list_length_exprs(
for e in _list_length_exprs(expr.struct[field.name], field.dtype)
]
return []


def _resolve_data_metric(v: DataMetricFn | DataMetric) -> DataMetric:
return v if isinstance(v, DataMetric) else DataMetric(fn=v)


def _resolve_change_metric(v: ChangeMetricFn | ChangeMetric) -> ChangeMetric:
return v if isinstance(v, ChangeMetric) else ChangeMetric(fn=v)
43 changes: 5 additions & 38 deletions diffly/metrics/__init__.py
Original file line number Diff line number Diff line change
@@ -1,50 +1,17 @@
# Copyright (c) QuantCo 2025-2026
# SPDX-License-Identifier: BSD-3-Clause

"""Metrics computed per column when generating a summary.

Two families are provided:

- Metrics in :mod:`~diffly.metrics.change` describe the change between numeric
columns itself by aggregating over ``right - left``.
- Metrics in :mod:`~diffly.metrics.data` describe the left and right datasets
individually, explaining how a change affects the data.
- :class:`~diffly.metrics.change.ChangeMetric`s in :mod:`~diffly.metrics.change` describe the change between
columns itself by aggregating over a combination of the columns (e.g., ``right - left``).
- :class:`~diffly.metrics.data.DataMetric`s in :mod:`~diffly.metrics.data` describe the left and right
datasets individually, explaining how a change affects the data.
"""

from __future__ import annotations

from . import change, data
from ._common import Metric, MetricFn
from .change import (
_make_numeric_metric,
max,
mean,
mean_absolute_deviation,
mean_relative_deviation,
median,
min,
quantile,
std,
)

DEFAULT_METRICS: dict[str, MetricFn | Metric] = {
**change.DEFAULT_CHANGE_METRICS,
}
"""The default preset metrics, consisting of the change default set."""

__all__ = [
"DEFAULT_METRICS",
"Metric",
"MetricFn",
"change",
"data",
"max",
"mean",
"mean_absolute_deviation",
"mean_relative_deviation",
"median",
"min",
"quantile",
"std",
"_make_numeric_metric",
]
__all__ = ["change", "data"]
27 changes: 0 additions & 27 deletions diffly/metrics/_common.py

This file was deleted.

50 changes: 33 additions & 17 deletions diffly/metrics/change.py
Original file line number Diff line number Diff line change
@@ -1,21 +1,37 @@
# Copyright (c) QuantCo 2025-2026
# SPDX-License-Identifier: BSD-3-Clause

"""Metrics describing the change between numeric columns.

These aggregate over ``right - left`` to characterize the change itself.
"""

from __future__ import annotations

from collections.abc import Callable
from dataclasses import dataclass, field

import polars as pl
import polars.selectors as cs

from ._common import Metric, MetricFn
ChangeMetricFn = Callable[[pl.Expr, pl.Expr], pl.Expr]
"""A `ChangeMetricFn` maps a pair of column expressions to a scalar aggregation
expression."""


@dataclass(frozen=True)
class ChangeMetric:
"""A metric quantifying the *change* in a column between the two sides of a
comparison.

Change metrics are rendered as extra columns in the "Columns" table, alongside the
match rate.
"""

fn: ChangeMetricFn
"""Aggregates over ``right - left`` (e.g. the mean delta) to describe the change
itself."""

selector: cs.Selector = field(default_factory=cs.numeric)
"""Selects the columns the metric applies to; defaults to numeric columns."""


def _make_numeric_metric(fn: MetricFn) -> Metric:
return Metric(fn=fn, selector=cs.numeric())
# ---------------------------------- CHANGE METRICS ---------------------------------- #


def mean(left: pl.Expr, right: pl.Expr) -> pl.Expr:
Expand Down Expand Up @@ -54,7 +70,7 @@ def mean_relative_deviation(left: pl.Expr, right: pl.Expr) -> pl.Expr:
return ((right - left) / left).abs().mean()


def quantile(q: float) -> MetricFn:
def quantile(q: float) -> ChangeMetricFn:
"""Factory returning a metric that computes the ``q``-quantile of
``right - left``."""
if not 0 <= q <= 1:
Expand All @@ -66,13 +82,13 @@ def _quantile(left: pl.Expr, right: pl.Expr) -> pl.Expr:
return _quantile


DEFAULT_CHANGE_METRICS: dict[str, MetricFn] = {
"Mean": mean,
"Median": median,
"Min": min,
"Max": max,
"Std": std,
"Mean absolute deviation": mean_absolute_deviation,
"Mean relative deviation": mean_relative_deviation,
DEFAULT_CHANGE_METRICS: dict[str, ChangeMetric] = {
"Mean diff": ChangeMetric(fn=mean),
"Median diff": ChangeMetric(fn=median),
"Min diff": ChangeMetric(fn=min),
"Max diff": ChangeMetric(fn=max),
"Std diff": ChangeMetric(fn=std),
"Mean absolute diff": ChangeMetric(fn=mean_absolute_deviation),
"Mean relative diff": ChangeMetric(fn=mean_relative_deviation),
}
"""Preset metrics describing the change between numeric columns."""
Loading