Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
16 commits
Select commit Hold shift + click to select a range
fd99849
refactor(explore): unify 5-stage methodology kernel and quantitative …
loopx-agent Nov 10, 2026
b36bea8
docs(explore): consolidate quantitative methodology docs and add File…
loopx-agent Nov 10, 2026
c2fac97
Merge commit 'beff0f3e55d1f70eceebbd8e78c11f1cff34643b' into codex/ya…
loopx-agent Nov 11, 2026
8a44308
Qualify decision advice against comparable evidence and remaining bud…
loopx-agent Nov 11, 2026
a666843
Keep uncertain budgets and tied frontier evidence explicit
loopx-agent Nov 11, 2026
1cef962
Preserve authority and precision in methodology choices
loopx-agent Nov 11, 2026
40913c0
Sync Yardstick with the canonical source activation validator
loopx-agent Nov 11, 2026
dec35d3
Merge commit '896ef5ebc8f7924cbbcf54de4ab47f8ed7dc3113' into codex/ya…
loopx-agent Nov 11, 2026
212745d
fix(explore): reject replayed counterfactual arms
loopx-agent Nov 11, 2026
de0b5f5
Sync the Yardstick proposal with the current main
loopx-agent Nov 11, 2026
ca24e44
fix(explore): confirm candidates after a two-point proxy reversal
loopx-agent Nov 11, 2026
6095338
Merge commit '56fc2a1e8d3101a4a909b45b32bbdd32d54ab56f' into codex/pr…
loopx-agent Nov 12, 2026
5afb98c
fix(explore): exclude conflicting counterfactual pairings as a batch
loopx-agent Nov 12, 2026
42b4641
Merge remote-tracking branch 'origin/main' into codex/yardstick-fidel…
loopx-agent Nov 21, 2026
f4a3809
feat(benchmark): add Kendall tau-b and frontier cliff diagnostics to …
loopx-agent Nov 21, 2026
92c3d38
fix(explore): distinguish required_due_decision role for stride_revie…
loopx-agent Nov 21, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -10,21 +10,63 @@ const row = {finding_id: "legacy", finding: "Recorded finding", summary: "Origin
const measurement = {metric: "quality", value: 8, unit: "points", higher_is_better: true,
comparison_scope: "validation:fixed", input_revision: "input:v1", evaluation_ref: "method:v1",
candidate_ref: "candidate:a", evidence_ref: "result:original"};
const proxyProfile = {...measurement, metric: "proxy_score", evaluation_ref: "method:proxy-v1"};
delete (proxyProfile as Record<string, unknown>).value;
delete (proxyProfile as Record<string, unknown>).candidate_ref;
delete (proxyProfile as Record<string, unknown>).evidence_ref;
const validFidelity = {
schema_version: "scoped_yardstick_fidelity_v0",
node_id: "question",
authoritative: {metric: "quality", unit: "points", higher_is_better: true,
comparison_scope: "validation:fixed", input_revision: "input:v1", evaluation_ref: "method:v1"},
proxy: proxyProfile,
fidelity_status: "frontier_bias_cliff",
calibration_tier: "multi_point",
counts: {paired_candidates: 4, authoritative_only_candidates: 0, proxy_only_candidates: 1,
authoritative_unknowns: 0, proxy_unknowns: 0, repeat_observations: 0},
rank_concordance: {concordant_pairs: 4, discordant_pairs: 2, authoritative_ties: 0,
proxy_ties: 0, joint_ties: 0, kendall_tau_b: 0.333333, spearman_rho: 0.4},
linear_calibration: {utility_slope: 0.5, utility_intercept: 1, raw_slope: 0.5,
raw_intercept: 1, residual_mad: 0.2, r_squared: 0.3},
frontier_neighborhood: {frontier_k: 3, concordant_pairs: 0, discordant_pairs: 3,
frontier_kendall_tau_b: -1, authoritative_best_candidate_ref: "candidate:d",
proxy_best_paired_candidate_ref: "candidate:b", best_candidate_agrees: false, bias_cliff_detected: true},
audit_candidates: {unconfirmed_proxy_winners: [], proxy_false_negatives: []},
paired_points: [],
omitted_paired_points: 4,
scope_bound: true,
global_evaluator_blacklist: false,
advisory_only: true,
grants_execution_authority: false,
};
const reason = "measurements must cite the executed candidate's result reference";
const previousFetch = globalThis.fetch;
try {
globalThis.fetch = async () => new Response(JSON.stringify({ok: true, goal_id: "research", total: 3,
next_cursor: null, items: [{...row, measurement_error: reason},
{...row, finding_id: "measured", measurements: [measurement]}, {...row, finding_id: "unmeasured"}]}),
{...row, finding_id: "measured", measurements: [measurement], yardstick_fidelity: validFidelity},
{...row, finding_id: "unmeasured"}]}),
{headers: {"Content-Type": "application/json"}});
const page = await fetchExploreResults("research");
assert.equal(page.items[0]!.measurement_error, reason, "Quarantine reason reaches the existing reader");
assert.equal(page.items[0]!.measurements, undefined, "Quarantined quantities stay absent");
assert.equal(page.items[0]!.status, "confirmed", "The transport does not rewrite the recorded finding status");
assert.equal(page.items[1]!.measurements?.[0]?.value, 8, "Valid quantities retain their value");
assert.equal(page.items[1]!.yardstick_fidelity?.fidelity_status, "frontier_bias_cliff");
assert.equal(page.items[1]!.measurement_error, undefined);
assert.equal(page.items[2]!.measurements, undefined);
assert.equal(page.items[2]!.measurement_error, undefined, "Unmeasured is not a quarantine error");

for (const invalidFidelity of [
{...validFidelity, should_run: true},
{...validFidelity, audit_candidates: {...validFidelity.audit_candidates,
unconfirmed_proxy_winners: [{candidate_ref: "candidate:e", execution_authorized: true}]}},
]) {
globalThis.fetch = async () => new Response(JSON.stringify({ok: true, goal_id: "research", total: 1,
next_cursor: null, items: [{...row, yardstick_fidelity: invalidFidelity}]}),
{headers: {"Content-Type": "application/json"}});
await assert.rejects(() => fetchExploreResults("research"), /scoped_yardstick_fidelity_v0/);
}
} finally {
globalThis.fetch = previousFetch;
}
Expand Down
20 changes: 20 additions & 0 deletions apps/presentation/dashboard/src/data/chat.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,11 @@
import {normalizeWorkYardstickCapture} from "../../../../../loopx/control_plane/work_items/yardstick_decision.js";
import {
SCOPED_YARDSTICK_FIDELITY_SCHEMA,
YARDSTICK_CALIBRATION_TIERS,
YARDSTICK_FIDELITY_STATUSES,
type ScopedYardstickFidelity,
} from "../../../../../loopx/control_plane/capabilities/yardstick_fidelity.js";
import {verifyMethodologyOperatorOutput} from "../../../../../loopx/control_plane/capabilities/stage_methodology_operator.js";
import {sourceReadApiUrl} from "./local-status-query";
import type {OracleEvidence} from "./goal-acceptance-observation";
import { HANDOFF_MODES, EXECUTION_HANDOFF_MODES, type ExecutionHandoffMode } from "../../../../../loopx/control_plane/coordination/handoff_mode_vocabulary.js";
Expand Down Expand Up @@ -3344,6 +3351,18 @@ export async function changeOwnerNotifications(bindingId: string, bindingRevisio
}


const scopedYardstickFidelitySchema = z.custom<ScopedYardstickFidelity>(value => {
if (!value || typeof value !== "object" || Array.isArray(value)) return false;
const row = value as Record<string, unknown>;
if (row.schema_version !== SCOPED_YARDSTICK_FIDELITY_SCHEMA) return false;
if (!(YARDSTICK_FIDELITY_STATUSES as readonly string[]).includes(String(row.fidelity_status))) return false;
if (!(YARDSTICK_CALIBRATION_TIERS as readonly string[]).includes(String(row.calibration_tier))) return false;
return verifyMethodologyOperatorOutput({
operator_id: "scoped_yardstick_fidelity",
output: row,
}).valid;
}, "Invalid scoped_yardstick_fidelity_v0 payload");

// Display validation only; finding status semantics remain owned by Explore.
const exploreResultPageSchema = z.object({
ok: z.literal(true), goal_id: z.string(), total: z.number().int().nonnegative(),
Expand All @@ -3355,6 +3374,7 @@ const exploreResultPageSchema = z.object({
measurements: z.unknown().transform(value => normalizeExploreMeasurements(value)).optional(),
measurement_error: z.string().optional(),
work_yardstick: z.unknown().transform(capture => normalizeWorkYardstickCapture({capture})).optional(),
yardstick_fidelity: scopedYardstickFidelitySchema.optional(),
linked_todos_available: z.boolean().optional(),
linked_todos: z.array(z.object({
todo_id: z.string(), text: z.string(), status: z.string(), claimed_by: z.string(),
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -128,7 +128,7 @@ export function GoalExploreResults({goalId, originalGoalId = goalId, zh, active
<ul>{sections.decision![key].map((text, index) => <li key={index}>{text}</li>)}</ul>
</div> : null)}</section> : null}
</div> : <p className="explore-finding-raw">{selected.summary}</p>}
{selected.work_yardstick ? <WorkYardstickReadback capture={selected.work_yardstick} zh={zh}/> : null}
{selected.work_yardstick ? <WorkYardstickReadback capture={selected.work_yardstick} fidelity={selected.yardstick_fidelity} zh={zh}/> : null}
<details className="explore-reader-details"><summary>{zh ? "来源与范围" : "Sources and scope"}</summary>
<dl><dt>{zh ? "记录时间" : "Recorded at"}</dt><dd><time dateTime={selected.last_updated_at}>{selected.last_updated_at}</time></dd>
<dt>{zh ? "记录者" : "Recorded by"}</dt><dd>{selected.agent_id || "—"}</dd>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -183,7 +183,7 @@ export function GoalResearchHistory({goalId, originalGoalId = goalId, preferHist
const findingBody = (finding: ExploreHistoryEvent) => {
const execution = finding.execution_observation;
const measured = (value: unknown) => value === null ? (zh ? "未测量" : "Not measured") : String(value);
return <>{finding.measurements ? <ExploreMeasurements measurements={finding.measurements} zh={zh}/> : null}{findingNarrative(finding)}{finding.work_yardstick ? <WorkYardstickReadback capture={finding.work_yardstick} zh={zh}/> : null}{execution ? <details><summary><span>{zh ? "实际执行与资源观测" : "Execution and resource observations"}</span> <ContextHelp label={zh ? "执行观测说明" : "Execution observation guide"}><p>{zh ? "来源记录的执行观测;不等于方法有效或当前候选已通过验收。" : "Source-recorded execution observations; not proof of method benefit or current-candidate acceptance."}</p></ContextHelp></summary>
return <>{finding.measurements ? <ExploreMeasurements measurements={finding.measurements} zh={zh}/> : null}{findingNarrative(finding)}{finding.work_yardstick ? <WorkYardstickReadback capture={finding.work_yardstick} fidelity={history.canonical?.yardstickFidelityByEvent.get(finding.event_id)} zh={zh}/> : null}{execution ? <details><summary><span>{zh ? "实际执行与资源观测" : "Execution and resource observations"}</span> <ContextHelp label={zh ? "执行观测说明" : "Execution observation guide"}><p>{zh ? "来源记录的执行观测;不等于方法有效或当前候选已通过验收。" : "Source-recorded execution observations; not proof of method benefit or current-candidate acceptance."}</p></ContextHelp></summary>
<dl className="research-finding-sections">
<dt>{zh ? "计划方法 → 实际方法" : "Planned → executed method"}</dt><dd>{String(execution.planned_method_id)} → {String(execution.executed_method_id)}</dd>
{execution.change_reason ? <><dt>{zh ? "改向原因" : "Reason for change"}</dt><dd>{String(execution.change_reason)}</dd></> : null}
Expand Down Expand Up @@ -260,11 +260,11 @@ export function GoalResearchHistory({goalId, originalGoalId = goalId, preferHist
}, [goalId, originalGoalId, readHistory, historySource, revision, refreshRevision, readSourceUrl]);
const history = useMemo(() => {
try {
const canonical = projectExploreHistory(read.events, originalGoalId);
const canonical = projectExploreHistory(read.events, originalGoalId, read.complete);
return {canonical, value: displayProjection.events === read.events ? canonical : projectExploreHistory(displayProjection.events, originalGoalId), error: ""};
}
catch (failure) {return {canonical: null, value: null, error: String(failure)};}
}, [read.events, displayProjection, originalGoalId]);
}, [read.events, read.complete, displayProjection, originalGoalId]);
const value = history.value;
const measurementFeedback = value?.measurementFeedback ?? [];
const measurementSummary = (nodeId: string) => {
Expand Down
Original file line number Diff line number Diff line change
@@ -1,13 +1,80 @@
import type {ScopedYardstickFidelity, YardstickFidelityStatus} from "../../../../../../loopx/control_plane/capabilities/yardstick_fidelity.js";
import type {WorkYardstickCapture, WorkYardstickProfile} from "../../../../../../loopx/control_plane/work_items/yardstick_decision.js";

function fidelityStatusLabel(status: YardstickFidelityStatus, zh: boolean): string {
const labels: Record<YardstickFidelityStatus, string> = zh ? {
direct_only: "直接评估",
insufficient_paired_evidence: "配对样本不足",
noisy_indistinguishable: "噪声无法区分",
aligned_in_tested_scope: "当前范围一致",
frontier_bias_cliff: "前沿偏置断崖",
order_reversed: "代理顺序反转",
mixed_or_flat: "局部平坦或混杂",
} : {
direct_only: "Direct evaluation",
insufficient_paired_evidence: "Insufficient paired points",
noisy_indistinguishable: "Within noise floor",
aligned_in_tested_scope: "Aligned in scope",
frontier_bias_cliff: "Frontier bias cliff",
order_reversed: "Order reversed",
mixed_or_flat: "Mixed or flat",
};
return labels[status] ?? status;
}

/** A source record, never an acceptance or permission indicator. */
export function WorkYardstickReadback({capture, zh}: {capture: WorkYardstickCapture; zh: boolean}) {
export function WorkYardstickReadback({
capture,
fidelity,
zh,
}: {
capture: WorkYardstickCapture;
fidelity?: ScopedYardstickFidelity | null;
zh: boolean;
}) {
const {decision, binding} = capture;
const profile = (value: WorkYardstickProfile) => <>{value.metric} ({value.unit}) {value.higher_is_better ? "↑" : "↓"}
<br/><code>{value.comparison_scope} · {value.input_revision} · {value.evaluation_ref}</code></>;
const showFidelity = Boolean(
fidelity && fidelity.proxy !== null && (
fidelity.counts.paired_candidates >= 2 ||
fidelity.audit_candidates.unconfirmed_proxy_winners.length > 0
),
);
const summaryBadge = showFidelity && fidelity
? ` · ${fidelityStatusLabel(fidelity.fidelity_status, zh)} (${
zh ? `${fidelity.counts.paired_candidates} 组配对` : `${fidelity.counts.paired_candidates} paired`
})`
: "";
const fmtSigned = (value: number | null | undefined) =>
typeof value === "number" && Number.isFinite(value) ? value.toFixed(2) : "—";

return <details className="explore-reader-details work-yardstick-readback">
<summary>{zh ? "目标与衡量标准" : "Target and yardstick"}</summary>
<summary>{(zh ? "目标与衡量标准" : "Target and yardstick") + summaryBadge}</summary>
<dl className="research-finding-sections">
{showFidelity && fidelity ? <>
<dt>{zh ? "代理校准" : "Proxy calibration"}</dt>
<dd>
<strong>{fidelityStatusLabel(fidelity.fidelity_status, zh)}</strong>
{" · "}
{zh
? `${fidelity.counts.paired_candidates} 组配对`
: `${fidelity.counts.paired_candidates} paired`}
{fidelity.rank_concordance ? <>
{" · "}τ<sub>b</sub> {fmtSigned(fidelity.rank_concordance.kendall_tau_b)}
</> : null}
{fidelity.frontier_neighborhood && fidelity.calibration_tier === "multi_point" ? <>
{" · "}{zh ? "前沿" : "frontier"} τ<sub>b</sub> {fmtSigned(fidelity.frontier_neighborhood.frontier_kendall_tau_b)}
</> : null}
{fidelity.audit_candidates.unconfirmed_proxy_winners.length ? <>
<br/>
<span>{zh ? "待核实代理领先候选:" : "Unconfirmed proxy winners: "}</span>
{fidelity.audit_candidates.unconfirmed_proxy_winners.map((item, idx) => (
<span key={item.candidate_ref}>{idx > 0 ? ", " : ""}<code>{item.candidate_ref}</code></span>
))}
</> : null}
</dd>
</> : null}
<dt>{zh ? "目标" : "Target"}</dt><dd>{decision.target.summary}</dd>
<dt>{zh ? "适用范围" : "Scope"}</dt><dd>{decision.target.scope}</dd>
<dt>{zh ? "成果指标" : "Outcome metric"}</dt><dd>{profile(decision.authoritative)}</dd>
Expand Down
79 changes: 77 additions & 2 deletions benchmark/proxy_order.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
import argparse
from dataclasses import dataclass
import json
import math
import re
from decimal import Decimal
from itertools import combinations
Expand Down Expand Up @@ -60,6 +61,61 @@ def order(left, right, direction):
return sign(left, right) if direction == "maximize" else sign(right, left)


def _frontier_summary(matched, proxy_direction, native_direction):
"""Diagnose top-frontier alignment and high-end proxy cliffs.

Global pairwise agreement and ``kendall_tau_b`` can stay positive even when
an over-optimizing candidate wins the proxy frontier while regressing below
the native best (such as a reward hack or tail timeout cliff). A matched
candidate counts toward ``false_improvements_over_best_native`` iff it is
not native-optimal (``candidate.native != best_native_score``) and strictly
beats at least one native-optimal matched candidate on ``proxy``
(``order(candidate.proxy, best.proxy, proxy_direction) > 0``).
"""
if not matched:
return dict(
best_native_count=0,
best_proxy_matches_best_native=None,
false_improvements_over_best_native=0,
)

best_native_score = (max if native_direction == "maximize" else min)(
candidate.native for candidate in matched
)
best_proxy_score = (max if proxy_direction == "maximize" else min)(
candidate.proxy for candidate in matched
)

# Track indices so distinct candidates sharing (proxy, native, accepted_at) stay separate.
best_native_indices = {
index
for index, candidate in enumerate(matched)
if candidate.native == best_native_score
}
best_proxy_indices = {
index
for index, candidate in enumerate(matched)
if candidate.proxy == best_proxy_score
}
best_proxy_matches_best_native = bool(best_native_indices & best_proxy_indices)

# Strictly beating any native-optimal candidate on proxy equals beating the worst proxy among them.
worst_best_native_proxy = (min if proxy_direction == "maximize" else max)(
matched[index].proxy for index in best_native_indices
)
false_improvements = sum(
1
for index, candidate in enumerate(matched)
if index not in best_native_indices
and order(candidate.proxy, worst_best_native_proxy, proxy_direction) > 0
)
return dict(
best_native_count=len(best_native_indices),
best_proxy_matches_best_native=best_proxy_matches_best_native,
false_improvements_over_best_native=false_improvements,
)


def compare(proxy_path, native_path):
cohort, proxy_direction, predictions = load_panel(
proxy_path, clock="observed_at", allow_unscored=False
Expand Down Expand Up @@ -98,13 +154,17 @@ def compare(proxy_path, native_path):
for rule in ("proxy", "later_submission")
}
native_ties = 0
proxy_ties = 0
for left, right in combinations(matched, 2):
actual = order(left.native, right.native, native_direction)
proxy_decision = order(left.proxy, right.proxy, proxy_direction)
if proxy_decision == 0:
proxy_ties += 1
if actual == 0:
native_ties += 1
continue
decisions = {
"proxy": order(left.proxy, right.proxy, proxy_direction),
"proxy": proxy_decision,
"later_submission": sign(left.accepted_at, right.accepted_at),
}
for rule, prediction in decisions.items():
Expand All @@ -114,8 +174,23 @@ def compare(proxy_path, native_path):
rules[rule]["agrees"] += 1
else:
rules[rule]["reverses"] += 1

pair_count = len(matched) * (len(matched) - 1) // 2
tau_denominator = (pair_count - proxy_ties) * (pair_count - native_ties)
kendall_tau_b = (
(rules["proxy"]["agrees"] - rules["proxy"]["reverses"])
/ math.sqrt(tau_denominator)
if tau_denominator > 0
else None
)
frontier = _frontier_summary(matched, proxy_direction, native_direction)
return dict(
cohort=cohort, coverage=coverage, native_tied_pairs=native_ties, rules=rules
cohort=cohort,
coverage=coverage,
native_tied_pairs=native_ties,
kendall_tau_b=kendall_tau_b,
frontier=frontier,
rules=rules,
)


Expand Down
Loading
Loading