Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
79 changes: 77 additions & 2 deletions benchmark/proxy_order.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
import argparse
from dataclasses import dataclass
import json
import math
import re
from decimal import Decimal
from itertools import combinations
Expand Down Expand Up @@ -60,6 +61,61 @@ def order(left, right, direction):
return sign(left, right) if direction == "maximize" else sign(right, left)


def _frontier_summary(matched, proxy_direction, native_direction):
"""Diagnose top-frontier alignment and high-end proxy cliffs.

Global pairwise agreement and ``kendall_tau_b`` can stay positive even when
an over-optimizing candidate wins the proxy frontier while regressing below
the native best (such as a reward hack or tail timeout cliff). A matched
candidate counts toward ``false_improvements_over_best_native`` iff it is
not native-optimal (``candidate.native != best_native_score``) and strictly
beats at least one native-optimal matched candidate on ``proxy``
(``order(candidate.proxy, best.proxy, proxy_direction) > 0``).
"""
if not matched:
return dict(
best_native_count=0,
best_proxy_matches_best_native=None,
false_improvements_over_best_native=0,
)

best_native_score = (max if native_direction == "maximize" else min)(
candidate.native for candidate in matched
)
best_proxy_score = (max if proxy_direction == "maximize" else min)(
candidate.proxy for candidate in matched
)

# Track indices so distinct candidates sharing (proxy, native, accepted_at) stay separate.
best_native_indices = {
index
for index, candidate in enumerate(matched)
if candidate.native == best_native_score
}
best_proxy_indices = {
index
for index, candidate in enumerate(matched)
if candidate.proxy == best_proxy_score
}
best_proxy_matches_best_native = bool(best_native_indices & best_proxy_indices)

# Strictly beating any native-optimal candidate on proxy equals beating the worst proxy among them.
worst_best_native_proxy = (min if proxy_direction == "maximize" else max)(
matched[index].proxy for index in best_native_indices
)
false_improvements = sum(
1
for index, candidate in enumerate(matched)
if index not in best_native_indices
and order(candidate.proxy, worst_best_native_proxy, proxy_direction) > 0
)
return dict(
best_native_count=len(best_native_indices),
best_proxy_matches_best_native=best_proxy_matches_best_native,
false_improvements_over_best_native=false_improvements,
)


def compare(proxy_path, native_path):
cohort, proxy_direction, predictions = load_panel(
proxy_path, clock="observed_at", allow_unscored=False
Expand Down Expand Up @@ -98,13 +154,17 @@ def compare(proxy_path, native_path):
for rule in ("proxy", "later_submission")
}
native_ties = 0
proxy_ties = 0
for left, right in combinations(matched, 2):
actual = order(left.native, right.native, native_direction)
proxy_decision = order(left.proxy, right.proxy, proxy_direction)
if proxy_decision == 0:
proxy_ties += 1
if actual == 0:
native_ties += 1
continue
decisions = {
"proxy": order(left.proxy, right.proxy, proxy_direction),
"proxy": proxy_decision,
"later_submission": sign(left.accepted_at, right.accepted_at),
}
for rule, prediction in decisions.items():
Expand All @@ -114,8 +174,23 @@ def compare(proxy_path, native_path):
rules[rule]["agrees"] += 1
else:
rules[rule]["reverses"] += 1

pair_count = len(matched) * (len(matched) - 1) // 2
tau_denominator = (pair_count - proxy_ties) * (pair_count - native_ties)
kendall_tau_b = (
(rules["proxy"]["agrees"] - rules["proxy"]["reverses"])
/ math.sqrt(tau_denominator)
if tau_denominator > 0
else None
)
frontier = _frontier_summary(matched, proxy_direction, native_direction)
return dict(
cohort=cohort, coverage=coverage, native_tied_pairs=native_ties, rules=rules
cohort=cohort,
coverage=coverage,
native_tied_pairs=native_ties,
kendall_tau_b=kendall_tau_b,
frontier=frontier,
rules=rules,
)


Expand Down
207 changes: 206 additions & 1 deletion benchmark/tests/test_proxy_order.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,8 @@
import json
import math
import os
import subprocess
import sys

import pytest

Expand Down Expand Up @@ -93,4 +97,205 @@ def test_scores_below_float_precision_are_ordered(tmp_path):
.replace('"score": 0', '"score": 1.0000000000000000002', 1)
.replace('"score": 0', '"score": 1.0000000000000000001', 1)
)
assert compare(proxy, native)["rules"]["proxy"]["reverses"] == 1
result = compare(proxy, native)
assert result["rules"]["proxy"]["reverses"] == 1
assert result["kendall_tau_b"] == pytest.approx(-1.0)
assert result["frontier"] == dict(
best_native_count=1,
best_proxy_matches_best_native=False,
false_improvements_over_best_native=1,
)


def test_kendall_tau_b_monotonic_and_pure_reversal(tmp_path):
proxy = panel(tmp_path, "proxy", [(1, 30, 1), (2, 20, 1), (3, 10, 1)], "minimize")
native_agree = panel(
tmp_path, "native", [(1, 1, 2), (2, 2, 3), (3, 3, 4)], "maximize"
)
agree_result = compare(proxy, native_agree)
assert agree_result["kendall_tau_b"] == pytest.approx(1.0)
assert agree_result["frontier"] == dict(
best_native_count=1,
best_proxy_matches_best_native=True,
false_improvements_over_best_native=0,
)

native_reverse = panel(
tmp_path, "native", [(1, 3, 2), (2, 2, 3), (3, 1, 4)], "maximize"
)
reverse_result = compare(proxy, native_reverse)
assert reverse_result["kendall_tau_b"] == pytest.approx(-1.0)
assert reverse_result["frontier"] == dict(
best_native_count=1,
best_proxy_matches_best_native=False,
false_improvements_over_best_native=2,
)


def test_kendall_tau_b_with_proxy_only_and_joint_ties(tmp_path):
proxy_only_tie = panel(
tmp_path, "proxy", [(1, 1, 1), (2, 1, 1), (3, 2, 1)], "maximize"
)
native_strict = panel(
tmp_path, "native", [(1, 1, 2), (2, 2, 3), (3, 3, 4)], "maximize"
)
proxy_tie_result = compare(proxy_only_tie, native_strict)
assert proxy_tie_result["kendall_tau_b"] == pytest.approx(2 / math.sqrt(2 * 3))

# Joint tie (1, 1) has abstains == 0, yet still enters T_p in tau-b.
joint_tie_proxy = panel(
tmp_path, "proxy", [(1, 1, 1), (2, 1, 1), (3, 2, 1)], "maximize"
)
joint_tie_native = panel(
tmp_path, "native", [(1, 1, 2), (2, 1, 3), (3, 2, 4)], "maximize"
)
joint_result = compare(joint_tie_proxy, joint_tie_native)
assert joint_result["native_tied_pairs"] == 1
assert joint_result["rules"]["proxy"]["abstains"] == 0
assert joint_result["kendall_tau_b"] == pytest.approx(1.0)


@pytest.mark.parametrize(
"proxy_rows,native_rows,expected_frontier",
[
(
[(1, 1, 5)],
[(1, 1, 2)],
dict(
best_native_count=0,
best_proxy_matches_best_native=None,
false_improvements_over_best_native=0,
),
),
(
[(1, 2, 1)],
[(1, 5, 2)],
dict(
best_native_count=1,
best_proxy_matches_best_native=True,
false_improvements_over_best_native=0,
),
),
(
[(1, 1, 1), (2, 1, 1), (3, 1, 1)],
[(1, 1, 2), (2, 2, 3), (3, 3, 4)],
dict(
best_native_count=1,
best_proxy_matches_best_native=True,
false_improvements_over_best_native=0,
),
),
(
[(1, 1, 1), (2, 2, 1), (3, 3, 1)],
[(1, 5, 2), (2, 5, 3), (3, 5, 4)],
dict(
best_native_count=3,
best_proxy_matches_best_native=True,
false_improvements_over_best_native=0,
),
),
],
)
def test_degenerate_tau_b_and_frontier_cases(
tmp_path, proxy_rows, native_rows, expected_frontier
):
result = compare(
panel(tmp_path, "proxy", proxy_rows, "maximize"),
panel(tmp_path, "native", native_rows, "maximize"),
)
assert result["kendall_tau_b"] is None
assert result["frontier"] == expected_frontier


def test_high_end_proxy_cliff_counterexample(tmp_path):
proxy = panel(
tmp_path,
"proxy",
[(1, 1, 1), (2, 2, 1), (3, 3, 1), (4, 4, 1), (5, 5, 1)],
"maximize",
)
native = panel(
tmp_path,
"native",
[(1, 1, 2), (2, 2, 3), (3, 3, 4), (4, 10, 5), (5, 0, 6)],
"maximize",
)
result = compare(proxy, native)
assert result["rules"]["proxy"] == dict(agrees=6, reverses=4, abstains=0)
assert result["kendall_tau_b"] == pytest.approx(0.2)
assert result["frontier"] == dict(
best_native_count=1,
best_proxy_matches_best_native=False,
false_improvements_over_best_native=1,
)


def test_frontier_multiple_native_optima_and_strict_tie_boundary(tmp_path):
# Minimize both: native-optimal candidates tie at native=1 with proxies 2 and 5.
proxy = panel(
tmp_path,
"proxy",
[(1, 2, 1), (2, 5, 1), (3, 4, 1), (4, 2, 1), (5, 6, 1)],
"minimize",
)
native = panel(
tmp_path,
"native",
[(1, 1, 2), (2, 1, 3), (3, 3, 4), (4, 4, 5), (5, 2, 6)],
"minimize",
)
result = compare(proxy, native)
assert result["frontier"] == dict(
best_native_count=2,
best_proxy_matches_best_native=True,
false_improvements_over_best_native=2,
)

# Tying the sole native-optimal candidate on proxy is not a strict false improvement.
tie_proxy = panel(tmp_path, "proxy", [(1, 10, 1), (2, 10, 1)], "maximize")
tie_native = panel(tmp_path, "native", [(1, 10, 2), (2, 5, 3)], "maximize")
tie_result = compare(tie_proxy, tie_native)
assert tie_result["frontier"] == dict(
best_native_count=1,
best_proxy_matches_best_native=True,
false_improvements_over_best_native=0,
)


def test_cli_entrypoint_outputs_json_diagnostics(tmp_path):
proxy = panel(
tmp_path,
"proxy",
[(1, 1, 1), (2, 2, 1), (3, 3, 1), (4, 4, 1), (5, 5, 1)],
"maximize",
)
native = panel(
tmp_path,
"native",
[(1, 1, 2), (2, 2, 3), (3, 3, 4), (4, 10, 5), (5, 0, 6)],
"maximize",
)
env = {**os.environ, "LOOPX_USAGE_PING": "0"}
proc = subprocess.run(
[sys.executable, "-m", "benchmark.proxy_order", str(proxy), str(native)],
check=False,
capture_output=True,
text=True,
env=env,
)
assert proc.returncode == 0, proc.stderr
payload = json.loads(proc.stdout)
assert set(payload) == {
"cohort",
"coverage",
"native_tied_pairs",
"kendall_tau_b",
"frontier",
"rules",
}
assert payload["kendall_tau_b"] == pytest.approx(0.2)
assert payload["frontier"] == dict(
best_native_count=1,
best_proxy_matches_best_native=False,
false_improvements_over_best_native=1,
)
19 changes: 18 additions & 1 deletion docs/reference/benchmark-behavior-findings.md
Original file line number Diff line number Diff line change
Expand Up @@ -104,7 +104,24 @@ duplicate source rows. A matching cohort name records the analyst's selection;
the analyst must verify comparable inputs, source digests and timestamps.

The output distinguishes missing coverage, native ties, abstentions, agreements
and reversals. Each local result must precede acceptance of its own candidate.
and reversals, and adds two rank/cliff diagnostics across matched candidates:

- `kendall_tau_b`: tie-adjusted Kendall rank correlation (tau-b) across all
matched pairs (`null` when fewer than two candidates match or all matched
candidates tie on either score), counting joint ties in the tie denominators.
- `frontier`: top-frontier alignment and high-end proxy cliff counts:
- `best_native_count`: number of matched candidates achieving the best native
score.
- `best_proxy_matches_best_native`: whether at least one matched candidate
achieving the best proxy score is also native-optimal (`null` when zero
candidates match).
- `false_improvements_over_best_native`: count of non-native-optimal matched
candidates that strictly beat at least one native-optimal matched candidate
on the proxy score. A cohort can show positive `kendall_tau_b` and
`agrees > reverses` while still failing at the frontier when an
over-optimizing candidate triggers a proxy cliff or tail timeout regression.

Each local result must precede acceptance of its own candidate.
Pairs share candidates and the solver may already have learned from older
feedback. Counts describe the selected sequence. They give neither independent
trials nor a population accuracy estimate. Close scores can change order under
Expand Down
Loading