From 9d4872bf83aa9932d5530498bf7cc149c86cc352 Mon Sep 17 00:00:00 2001 From: LoopX Agent <337587101+loopx-agent@users.noreply.github.com> Date: Sat, 21 Nov 2026 14:54:50 +0800 Subject: [PATCH] feat(benchmark): add Kendall tau-b and frontier cliff diagnostics to proxy_order Signed-off-by: LoopX Agent <337587101+loopx-agent@users.noreply.github.com> --- benchmark/proxy_order.py | 79 ++++++- benchmark/tests/test_proxy_order.py | 207 +++++++++++++++++- docs/reference/benchmark-behavior-findings.md | 19 +- 3 files changed, 301 insertions(+), 4 deletions(-) diff --git a/benchmark/proxy_order.py b/benchmark/proxy_order.py index 1d94ed2a5f..65608b31ef 100644 --- a/benchmark/proxy_order.py +++ b/benchmark/proxy_order.py @@ -3,6 +3,7 @@ import argparse from dataclasses import dataclass import json +import math import re from decimal import Decimal from itertools import combinations @@ -60,6 +61,61 @@ def order(left, right, direction): return sign(left, right) if direction == "maximize" else sign(right, left) +def _frontier_summary(matched, proxy_direction, native_direction): + """Diagnose top-frontier alignment and high-end proxy cliffs. + + Global pairwise agreement and ``kendall_tau_b`` can stay positive even when + an over-optimizing candidate wins the proxy frontier while regressing below + the native best (such as a reward hack or tail timeout cliff). A matched + candidate counts toward ``false_improvements_over_best_native`` iff it is + not native-optimal (``candidate.native != best_native_score``) and strictly + beats at least one native-optimal matched candidate on ``proxy`` + (``order(candidate.proxy, best.proxy, proxy_direction) > 0``). + """ + if not matched: + return dict( + best_native_count=0, + best_proxy_matches_best_native=None, + false_improvements_over_best_native=0, + ) + + best_native_score = (max if native_direction == "maximize" else min)( + candidate.native for candidate in matched + ) + best_proxy_score = (max if proxy_direction == "maximize" else min)( + candidate.proxy for candidate in matched + ) + + # Track indices so distinct candidates sharing (proxy, native, accepted_at) stay separate. + best_native_indices = { + index + for index, candidate in enumerate(matched) + if candidate.native == best_native_score + } + best_proxy_indices = { + index + for index, candidate in enumerate(matched) + if candidate.proxy == best_proxy_score + } + best_proxy_matches_best_native = bool(best_native_indices & best_proxy_indices) + + # Strictly beating any native-optimal candidate on proxy equals beating the worst proxy among them. + worst_best_native_proxy = (min if proxy_direction == "maximize" else max)( + matched[index].proxy for index in best_native_indices + ) + false_improvements = sum( + 1 + for index, candidate in enumerate(matched) + if index not in best_native_indices + and order(candidate.proxy, worst_best_native_proxy, proxy_direction) > 0 + ) + return dict( + best_native_count=len(best_native_indices), + best_proxy_matches_best_native=best_proxy_matches_best_native, + false_improvements_over_best_native=false_improvements, + ) + + def compare(proxy_path, native_path): cohort, proxy_direction, predictions = load_panel( proxy_path, clock="observed_at", allow_unscored=False @@ -98,13 +154,17 @@ def compare(proxy_path, native_path): for rule in ("proxy", "later_submission") } native_ties = 0 + proxy_ties = 0 for left, right in combinations(matched, 2): actual = order(left.native, right.native, native_direction) + proxy_decision = order(left.proxy, right.proxy, proxy_direction) + if proxy_decision == 0: + proxy_ties += 1 if actual == 0: native_ties += 1 continue decisions = { - "proxy": order(left.proxy, right.proxy, proxy_direction), + "proxy": proxy_decision, "later_submission": sign(left.accepted_at, right.accepted_at), } for rule, prediction in decisions.items(): @@ -114,8 +174,23 @@ def compare(proxy_path, native_path): rules[rule]["agrees"] += 1 else: rules[rule]["reverses"] += 1 + + pair_count = len(matched) * (len(matched) - 1) // 2 + tau_denominator = (pair_count - proxy_ties) * (pair_count - native_ties) + kendall_tau_b = ( + (rules["proxy"]["agrees"] - rules["proxy"]["reverses"]) + / math.sqrt(tau_denominator) + if tau_denominator > 0 + else None + ) + frontier = _frontier_summary(matched, proxy_direction, native_direction) return dict( - cohort=cohort, coverage=coverage, native_tied_pairs=native_ties, rules=rules + cohort=cohort, + coverage=coverage, + native_tied_pairs=native_ties, + kendall_tau_b=kendall_tau_b, + frontier=frontier, + rules=rules, ) diff --git a/benchmark/tests/test_proxy_order.py b/benchmark/tests/test_proxy_order.py index c347436b7d..f9b9af42a9 100644 --- a/benchmark/tests/test_proxy_order.py +++ b/benchmark/tests/test_proxy_order.py @@ -1,4 +1,8 @@ import json +import math +import os +import subprocess +import sys import pytest @@ -93,4 +97,205 @@ def test_scores_below_float_precision_are_ordered(tmp_path): .replace('"score": 0', '"score": 1.0000000000000000002', 1) .replace('"score": 0', '"score": 1.0000000000000000001', 1) ) - assert compare(proxy, native)["rules"]["proxy"]["reverses"] == 1 + result = compare(proxy, native) + assert result["rules"]["proxy"]["reverses"] == 1 + assert result["kendall_tau_b"] == pytest.approx(-1.0) + assert result["frontier"] == dict( + best_native_count=1, + best_proxy_matches_best_native=False, + false_improvements_over_best_native=1, + ) + + +def test_kendall_tau_b_monotonic_and_pure_reversal(tmp_path): + proxy = panel(tmp_path, "proxy", [(1, 30, 1), (2, 20, 1), (3, 10, 1)], "minimize") + native_agree = panel( + tmp_path, "native", [(1, 1, 2), (2, 2, 3), (3, 3, 4)], "maximize" + ) + agree_result = compare(proxy, native_agree) + assert agree_result["kendall_tau_b"] == pytest.approx(1.0) + assert agree_result["frontier"] == dict( + best_native_count=1, + best_proxy_matches_best_native=True, + false_improvements_over_best_native=0, + ) + + native_reverse = panel( + tmp_path, "native", [(1, 3, 2), (2, 2, 3), (3, 1, 4)], "maximize" + ) + reverse_result = compare(proxy, native_reverse) + assert reverse_result["kendall_tau_b"] == pytest.approx(-1.0) + assert reverse_result["frontier"] == dict( + best_native_count=1, + best_proxy_matches_best_native=False, + false_improvements_over_best_native=2, + ) + + +def test_kendall_tau_b_with_proxy_only_and_joint_ties(tmp_path): + proxy_only_tie = panel( + tmp_path, "proxy", [(1, 1, 1), (2, 1, 1), (3, 2, 1)], "maximize" + ) + native_strict = panel( + tmp_path, "native", [(1, 1, 2), (2, 2, 3), (3, 3, 4)], "maximize" + ) + proxy_tie_result = compare(proxy_only_tie, native_strict) + assert proxy_tie_result["kendall_tau_b"] == pytest.approx(2 / math.sqrt(2 * 3)) + + # Joint tie (1, 1) has abstains == 0, yet still enters T_p in tau-b. + joint_tie_proxy = panel( + tmp_path, "proxy", [(1, 1, 1), (2, 1, 1), (3, 2, 1)], "maximize" + ) + joint_tie_native = panel( + tmp_path, "native", [(1, 1, 2), (2, 1, 3), (3, 2, 4)], "maximize" + ) + joint_result = compare(joint_tie_proxy, joint_tie_native) + assert joint_result["native_tied_pairs"] == 1 + assert joint_result["rules"]["proxy"]["abstains"] == 0 + assert joint_result["kendall_tau_b"] == pytest.approx(1.0) + + +@pytest.mark.parametrize( + "proxy_rows,native_rows,expected_frontier", + [ + ( + [(1, 1, 5)], + [(1, 1, 2)], + dict( + best_native_count=0, + best_proxy_matches_best_native=None, + false_improvements_over_best_native=0, + ), + ), + ( + [(1, 2, 1)], + [(1, 5, 2)], + dict( + best_native_count=1, + best_proxy_matches_best_native=True, + false_improvements_over_best_native=0, + ), + ), + ( + [(1, 1, 1), (2, 1, 1), (3, 1, 1)], + [(1, 1, 2), (2, 2, 3), (3, 3, 4)], + dict( + best_native_count=1, + best_proxy_matches_best_native=True, + false_improvements_over_best_native=0, + ), + ), + ( + [(1, 1, 1), (2, 2, 1), (3, 3, 1)], + [(1, 5, 2), (2, 5, 3), (3, 5, 4)], + dict( + best_native_count=3, + best_proxy_matches_best_native=True, + false_improvements_over_best_native=0, + ), + ), + ], +) +def test_degenerate_tau_b_and_frontier_cases( + tmp_path, proxy_rows, native_rows, expected_frontier +): + result = compare( + panel(tmp_path, "proxy", proxy_rows, "maximize"), + panel(tmp_path, "native", native_rows, "maximize"), + ) + assert result["kendall_tau_b"] is None + assert result["frontier"] == expected_frontier + + +def test_high_end_proxy_cliff_counterexample(tmp_path): + proxy = panel( + tmp_path, + "proxy", + [(1, 1, 1), (2, 2, 1), (3, 3, 1), (4, 4, 1), (5, 5, 1)], + "maximize", + ) + native = panel( + tmp_path, + "native", + [(1, 1, 2), (2, 2, 3), (3, 3, 4), (4, 10, 5), (5, 0, 6)], + "maximize", + ) + result = compare(proxy, native) + assert result["rules"]["proxy"] == dict(agrees=6, reverses=4, abstains=0) + assert result["kendall_tau_b"] == pytest.approx(0.2) + assert result["frontier"] == dict( + best_native_count=1, + best_proxy_matches_best_native=False, + false_improvements_over_best_native=1, + ) + + +def test_frontier_multiple_native_optima_and_strict_tie_boundary(tmp_path): + # Minimize both: native-optimal candidates tie at native=1 with proxies 2 and 5. + proxy = panel( + tmp_path, + "proxy", + [(1, 2, 1), (2, 5, 1), (3, 4, 1), (4, 2, 1), (5, 6, 1)], + "minimize", + ) + native = panel( + tmp_path, + "native", + [(1, 1, 2), (2, 1, 3), (3, 3, 4), (4, 4, 5), (5, 2, 6)], + "minimize", + ) + result = compare(proxy, native) + assert result["frontier"] == dict( + best_native_count=2, + best_proxy_matches_best_native=True, + false_improvements_over_best_native=2, + ) + + # Tying the sole native-optimal candidate on proxy is not a strict false improvement. + tie_proxy = panel(tmp_path, "proxy", [(1, 10, 1), (2, 10, 1)], "maximize") + tie_native = panel(tmp_path, "native", [(1, 10, 2), (2, 5, 3)], "maximize") + tie_result = compare(tie_proxy, tie_native) + assert tie_result["frontier"] == dict( + best_native_count=1, + best_proxy_matches_best_native=True, + false_improvements_over_best_native=0, + ) + + +def test_cli_entrypoint_outputs_json_diagnostics(tmp_path): + proxy = panel( + tmp_path, + "proxy", + [(1, 1, 1), (2, 2, 1), (3, 3, 1), (4, 4, 1), (5, 5, 1)], + "maximize", + ) + native = panel( + tmp_path, + "native", + [(1, 1, 2), (2, 2, 3), (3, 3, 4), (4, 10, 5), (5, 0, 6)], + "maximize", + ) + env = {**os.environ, "LOOPX_USAGE_PING": "0"} + proc = subprocess.run( + [sys.executable, "-m", "benchmark.proxy_order", str(proxy), str(native)], + check=False, + capture_output=True, + text=True, + env=env, + ) + assert proc.returncode == 0, proc.stderr + payload = json.loads(proc.stdout) + assert set(payload) == { + "cohort", + "coverage", + "native_tied_pairs", + "kendall_tau_b", + "frontier", + "rules", + } + assert payload["kendall_tau_b"] == pytest.approx(0.2) + assert payload["frontier"] == dict( + best_native_count=1, + best_proxy_matches_best_native=False, + false_improvements_over_best_native=1, + ) diff --git a/docs/reference/benchmark-behavior-findings.md b/docs/reference/benchmark-behavior-findings.md index 87df154950..85954d3fa6 100644 --- a/docs/reference/benchmark-behavior-findings.md +++ b/docs/reference/benchmark-behavior-findings.md @@ -104,7 +104,24 @@ duplicate source rows. A matching cohort name records the analyst's selection; the analyst must verify comparable inputs, source digests and timestamps. The output distinguishes missing coverage, native ties, abstentions, agreements -and reversals. Each local result must precede acceptance of its own candidate. +and reversals, and adds two rank/cliff diagnostics across matched candidates: + +- `kendall_tau_b`: tie-adjusted Kendall rank correlation (tau-b) across all + matched pairs (`null` when fewer than two candidates match or all matched + candidates tie on either score), counting joint ties in the tie denominators. +- `frontier`: top-frontier alignment and high-end proxy cliff counts: + - `best_native_count`: number of matched candidates achieving the best native + score. + - `best_proxy_matches_best_native`: whether at least one matched candidate + achieving the best proxy score is also native-optimal (`null` when zero + candidates match). + - `false_improvements_over_best_native`: count of non-native-optimal matched + candidates that strictly beat at least one native-optimal matched candidate + on the proxy score. A cohort can show positive `kendall_tau_b` and + `agrees > reverses` while still failing at the frontier when an + over-optimizing candidate triggers a proxy cliff or tail timeout regression. + +Each local result must precede acceptance of its own candidate. Pairs share candidates and the solver may already have learned from older feedback. Counts describe the selected sequence. They give neither independent trials nor a population accuracy estimate. Close scores can change order under