diff --git a/apps/presentation/dashboard/smoke/explore-finding-diagnostics-smoke.ts b/apps/presentation/dashboard/smoke/explore-finding-diagnostics-smoke.ts index f9ea7bbca2..e08a4f5aa8 100644 --- a/apps/presentation/dashboard/smoke/explore-finding-diagnostics-smoke.ts +++ b/apps/presentation/dashboard/smoke/explore-finding-diagnostics-smoke.ts @@ -10,21 +10,63 @@ const row = {finding_id: "legacy", finding: "Recorded finding", summary: "Origin const measurement = {metric: "quality", value: 8, unit: "points", higher_is_better: true, comparison_scope: "validation:fixed", input_revision: "input:v1", evaluation_ref: "method:v1", candidate_ref: "candidate:a", evidence_ref: "result:original"}; +const proxyProfile = {...measurement, metric: "proxy_score", evaluation_ref: "method:proxy-v1"}; +delete (proxyProfile as Record).value; +delete (proxyProfile as Record).candidate_ref; +delete (proxyProfile as Record).evidence_ref; +const validFidelity = { + schema_version: "scoped_yardstick_fidelity_v0", + node_id: "question", + authoritative: {metric: "quality", unit: "points", higher_is_better: true, + comparison_scope: "validation:fixed", input_revision: "input:v1", evaluation_ref: "method:v1"}, + proxy: proxyProfile, + fidelity_status: "frontier_bias_cliff", + calibration_tier: "multi_point", + counts: {paired_candidates: 4, authoritative_only_candidates: 0, proxy_only_candidates: 1, + authoritative_unknowns: 0, proxy_unknowns: 0, repeat_observations: 0}, + rank_concordance: {concordant_pairs: 4, discordant_pairs: 2, authoritative_ties: 0, + proxy_ties: 0, joint_ties: 0, kendall_tau_b: 0.333333, spearman_rho: 0.4}, + linear_calibration: {utility_slope: 0.5, utility_intercept: 1, raw_slope: 0.5, + raw_intercept: 1, residual_mad: 0.2, r_squared: 0.3}, + frontier_neighborhood: {frontier_k: 3, concordant_pairs: 0, discordant_pairs: 3, + frontier_kendall_tau_b: -1, authoritative_best_candidate_ref: "candidate:d", + proxy_best_paired_candidate_ref: "candidate:b", best_candidate_agrees: false, bias_cliff_detected: true}, + audit_candidates: {unconfirmed_proxy_winners: [], proxy_false_negatives: []}, + paired_points: [], + omitted_paired_points: 4, + scope_bound: true, + global_evaluator_blacklist: false, + advisory_only: true, + grants_execution_authority: false, +}; const reason = "measurements must cite the executed candidate's result reference"; const previousFetch = globalThis.fetch; try { globalThis.fetch = async () => new Response(JSON.stringify({ok: true, goal_id: "research", total: 3, next_cursor: null, items: [{...row, measurement_error: reason}, - {...row, finding_id: "measured", measurements: [measurement]}, {...row, finding_id: "unmeasured"}]}), + {...row, finding_id: "measured", measurements: [measurement], yardstick_fidelity: validFidelity}, + {...row, finding_id: "unmeasured"}]}), {headers: {"Content-Type": "application/json"}}); const page = await fetchExploreResults("research"); assert.equal(page.items[0]!.measurement_error, reason, "Quarantine reason reaches the existing reader"); assert.equal(page.items[0]!.measurements, undefined, "Quarantined quantities stay absent"); assert.equal(page.items[0]!.status, "confirmed", "The transport does not rewrite the recorded finding status"); assert.equal(page.items[1]!.measurements?.[0]?.value, 8, "Valid quantities retain their value"); + assert.equal(page.items[1]!.yardstick_fidelity?.fidelity_status, "frontier_bias_cliff"); assert.equal(page.items[1]!.measurement_error, undefined); assert.equal(page.items[2]!.measurements, undefined); assert.equal(page.items[2]!.measurement_error, undefined, "Unmeasured is not a quarantine error"); + + for (const invalidFidelity of [ + {...validFidelity, should_run: true}, + {...validFidelity, audit_candidates: {...validFidelity.audit_candidates, + unconfirmed_proxy_winners: [{candidate_ref: "candidate:e", execution_authorized: true}]}}, + ]) { + globalThis.fetch = async () => new Response(JSON.stringify({ok: true, goal_id: "research", total: 1, + next_cursor: null, items: [{...row, yardstick_fidelity: invalidFidelity}]}), + {headers: {"Content-Type": "application/json"}}); + await assert.rejects(() => fetchExploreResults("research"), /scoped_yardstick_fidelity_v0/); + } } finally { globalThis.fetch = previousFetch; } diff --git a/apps/presentation/dashboard/src/data/chat.ts b/apps/presentation/dashboard/src/data/chat.ts index 451aaebd1e..fbdce91e1a 100644 --- a/apps/presentation/dashboard/src/data/chat.ts +++ b/apps/presentation/dashboard/src/data/chat.ts @@ -1,4 +1,11 @@ import {normalizeWorkYardstickCapture} from "../../../../../loopx/control_plane/work_items/yardstick_decision.js"; +import { + SCOPED_YARDSTICK_FIDELITY_SCHEMA, + YARDSTICK_CALIBRATION_TIERS, + YARDSTICK_FIDELITY_STATUSES, + type ScopedYardstickFidelity, +} from "../../../../../loopx/control_plane/capabilities/yardstick_fidelity.js"; +import {verifyMethodologyOperatorOutput} from "../../../../../loopx/control_plane/capabilities/stage_methodology_operator.js"; import {sourceReadApiUrl} from "./local-status-query"; import type {OracleEvidence} from "./goal-acceptance-observation"; import { HANDOFF_MODES, EXECUTION_HANDOFF_MODES, type ExecutionHandoffMode } from "../../../../../loopx/control_plane/coordination/handoff_mode_vocabulary.js"; @@ -3344,6 +3351,18 @@ export async function changeOwnerNotifications(bindingId: string, bindingRevisio } +const scopedYardstickFidelitySchema = z.custom(value => { + if (!value || typeof value !== "object" || Array.isArray(value)) return false; + const row = value as Record; + if (row.schema_version !== SCOPED_YARDSTICK_FIDELITY_SCHEMA) return false; + if (!(YARDSTICK_FIDELITY_STATUSES as readonly string[]).includes(String(row.fidelity_status))) return false; + if (!(YARDSTICK_CALIBRATION_TIERS as readonly string[]).includes(String(row.calibration_tier))) return false; + return verifyMethodologyOperatorOutput({ + operator_id: "scoped_yardstick_fidelity", + output: row, + }).valid; +}, "Invalid scoped_yardstick_fidelity_v0 payload"); + // Display validation only; finding status semantics remain owned by Explore. const exploreResultPageSchema = z.object({ ok: z.literal(true), goal_id: z.string(), total: z.number().int().nonnegative(), @@ -3355,6 +3374,7 @@ const exploreResultPageSchema = z.object({ measurements: z.unknown().transform(value => normalizeExploreMeasurements(value)).optional(), measurement_error: z.string().optional(), work_yardstick: z.unknown().transform(capture => normalizeWorkYardstickCapture({capture})).optional(), + yardstick_fidelity: scopedYardstickFidelitySchema.optional(), linked_todos_available: z.boolean().optional(), linked_todos: z.array(z.object({ todo_id: z.string(), text: z.string(), status: z.string(), claimed_by: z.string(), diff --git a/apps/presentation/dashboard/src/features/personal-workspace/goal-explore-results.tsx b/apps/presentation/dashboard/src/features/personal-workspace/goal-explore-results.tsx index f0ba8725ff..0e778d813d 100644 --- a/apps/presentation/dashboard/src/features/personal-workspace/goal-explore-results.tsx +++ b/apps/presentation/dashboard/src/features/personal-workspace/goal-explore-results.tsx @@ -128,7 +128,7 @@ export function GoalExploreResults({goalId, originalGoalId = goalId, zh, active
    {sections.decision![key].map((text, index) =>
  • {text}
  • )}
: null)} : null} :

{selected.summary}

} - {selected.work_yardstick ? : null} + {selected.work_yardstick ? : null}
{zh ? "来源与范围" : "Sources and scope"}
{zh ? "记录时间" : "Recorded at"}
{zh ? "记录者" : "Recorded by"}
{selected.agent_id || "—"}
diff --git a/apps/presentation/dashboard/src/features/personal-workspace/goal-research-history.tsx b/apps/presentation/dashboard/src/features/personal-workspace/goal-research-history.tsx index 7bb83d289b..d713c0ff7b 100644 --- a/apps/presentation/dashboard/src/features/personal-workspace/goal-research-history.tsx +++ b/apps/presentation/dashboard/src/features/personal-workspace/goal-research-history.tsx @@ -183,7 +183,7 @@ export function GoalResearchHistory({goalId, originalGoalId = goalId, preferHist const findingBody = (finding: ExploreHistoryEvent) => { const execution = finding.execution_observation; const measured = (value: unknown) => value === null ? (zh ? "未测量" : "Not measured") : String(value); - return <>{finding.measurements ? : null}{findingNarrative(finding)}{finding.work_yardstick ? : null}{execution ?
{zh ? "实际执行与资源观测" : "Execution and resource observations"}

{zh ? "来源记录的执行观测;不等于方法有效或当前候选已通过验收。" : "Source-recorded execution observations; not proof of method benefit or current-candidate acceptance."}

+ return <>{finding.measurements ? : null}{findingNarrative(finding)}{finding.work_yardstick ? : null}{execution ?
{zh ? "实际执行与资源观测" : "Execution and resource observations"}

{zh ? "来源记录的执行观测;不等于方法有效或当前候选已通过验收。" : "Source-recorded execution observations; not proof of method benefit or current-candidate acceptance."}

{zh ? "计划方法 → 实际方法" : "Planned → executed method"}
{String(execution.planned_method_id)} → {String(execution.executed_method_id)}
{execution.change_reason ? <>
{zh ? "改向原因" : "Reason for change"}
{String(execution.change_reason)}
: null} @@ -260,11 +260,11 @@ export function GoalResearchHistory({goalId, originalGoalId = goalId, preferHist }, [goalId, originalGoalId, readHistory, historySource, revision, refreshRevision, readSourceUrl]); const history = useMemo(() => { try { - const canonical = projectExploreHistory(read.events, originalGoalId); + const canonical = projectExploreHistory(read.events, originalGoalId, read.complete); return {canonical, value: displayProjection.events === read.events ? canonical : projectExploreHistory(displayProjection.events, originalGoalId), error: ""}; } catch (failure) {return {canonical: null, value: null, error: String(failure)};} - }, [read.events, displayProjection, originalGoalId]); + }, [read.events, read.complete, displayProjection, originalGoalId]); const value = history.value; const measurementFeedback = value?.measurementFeedback ?? []; const measurementSummary = (nodeId: string) => { diff --git a/apps/presentation/dashboard/src/features/personal-workspace/work-yardstick-readback.tsx b/apps/presentation/dashboard/src/features/personal-workspace/work-yardstick-readback.tsx index 017cc0b494..3d36633e4a 100644 --- a/apps/presentation/dashboard/src/features/personal-workspace/work-yardstick-readback.tsx +++ b/apps/presentation/dashboard/src/features/personal-workspace/work-yardstick-readback.tsx @@ -1,13 +1,80 @@ +import type {ScopedYardstickFidelity, YardstickFidelityStatus} from "../../../../../../loopx/control_plane/capabilities/yardstick_fidelity.js"; import type {WorkYardstickCapture, WorkYardstickProfile} from "../../../../../../loopx/control_plane/work_items/yardstick_decision.js"; +function fidelityStatusLabel(status: YardstickFidelityStatus, zh: boolean): string { + const labels: Record = zh ? { + direct_only: "直接评估", + insufficient_paired_evidence: "配对样本不足", + noisy_indistinguishable: "噪声无法区分", + aligned_in_tested_scope: "当前范围一致", + frontier_bias_cliff: "前沿偏置断崖", + order_reversed: "代理顺序反转", + mixed_or_flat: "局部平坦或混杂", + } : { + direct_only: "Direct evaluation", + insufficient_paired_evidence: "Insufficient paired points", + noisy_indistinguishable: "Within noise floor", + aligned_in_tested_scope: "Aligned in scope", + frontier_bias_cliff: "Frontier bias cliff", + order_reversed: "Order reversed", + mixed_or_flat: "Mixed or flat", + }; + return labels[status] ?? status; +} + /** A source record, never an acceptance or permission indicator. */ -export function WorkYardstickReadback({capture, zh}: {capture: WorkYardstickCapture; zh: boolean}) { +export function WorkYardstickReadback({ + capture, + fidelity, + zh, +}: { + capture: WorkYardstickCapture; + fidelity?: ScopedYardstickFidelity | null; + zh: boolean; +}) { const {decision, binding} = capture; const profile = (value: WorkYardstickProfile) => <>{value.metric} ({value.unit}) {value.higher_is_better ? "↑" : "↓"}
{value.comparison_scope} · {value.input_revision} · {value.evaluation_ref}; + const showFidelity = Boolean( + fidelity && fidelity.proxy !== null && ( + fidelity.counts.paired_candidates >= 2 || + fidelity.audit_candidates.unconfirmed_proxy_winners.length > 0 + ), + ); + const summaryBadge = showFidelity && fidelity + ? ` · ${fidelityStatusLabel(fidelity.fidelity_status, zh)} (${ + zh ? `${fidelity.counts.paired_candidates} 组配对` : `${fidelity.counts.paired_candidates} paired` + })` + : ""; + const fmtSigned = (value: number | null | undefined) => + typeof value === "number" && Number.isFinite(value) ? value.toFixed(2) : "—"; + return
- {zh ? "目标与衡量标准" : "Target and yardstick"} + {(zh ? "目标与衡量标准" : "Target and yardstick") + summaryBadge}
+ {showFidelity && fidelity ? <> +
{zh ? "代理校准" : "Proxy calibration"}
+
+ {fidelityStatusLabel(fidelity.fidelity_status, zh)} + {" · "} + {zh + ? `${fidelity.counts.paired_candidates} 组配对` + : `${fidelity.counts.paired_candidates} paired`} + {fidelity.rank_concordance ? <> + {" · "}τb {fmtSigned(fidelity.rank_concordance.kendall_tau_b)} + : null} + {fidelity.frontier_neighborhood && fidelity.calibration_tier === "multi_point" ? <> + {" · "}{zh ? "前沿" : "frontier"} τb {fmtSigned(fidelity.frontier_neighborhood.frontier_kendall_tau_b)} + : null} + {fidelity.audit_candidates.unconfirmed_proxy_winners.length ? <> +
+ {zh ? "待核实代理领先候选:" : "Unconfirmed proxy winners: "} + {fidelity.audit_candidates.unconfirmed_proxy_winners.map((item, idx) => ( + {idx > 0 ? ", " : ""}{item.candidate_ref} + ))} + : null} +
+ : null}
{zh ? "目标" : "Target"}
{decision.target.summary}
{zh ? "适用范围" : "Scope"}
{decision.target.scope}
{zh ? "成果指标" : "Outcome metric"}
{profile(decision.authoritative)}
diff --git a/benchmark/proxy_order.py b/benchmark/proxy_order.py index 1d94ed2a5f..65608b31ef 100644 --- a/benchmark/proxy_order.py +++ b/benchmark/proxy_order.py @@ -3,6 +3,7 @@ import argparse from dataclasses import dataclass import json +import math import re from decimal import Decimal from itertools import combinations @@ -60,6 +61,61 @@ def order(left, right, direction): return sign(left, right) if direction == "maximize" else sign(right, left) +def _frontier_summary(matched, proxy_direction, native_direction): + """Diagnose top-frontier alignment and high-end proxy cliffs. + + Global pairwise agreement and ``kendall_tau_b`` can stay positive even when + an over-optimizing candidate wins the proxy frontier while regressing below + the native best (such as a reward hack or tail timeout cliff). A matched + candidate counts toward ``false_improvements_over_best_native`` iff it is + not native-optimal (``candidate.native != best_native_score``) and strictly + beats at least one native-optimal matched candidate on ``proxy`` + (``order(candidate.proxy, best.proxy, proxy_direction) > 0``). + """ + if not matched: + return dict( + best_native_count=0, + best_proxy_matches_best_native=None, + false_improvements_over_best_native=0, + ) + + best_native_score = (max if native_direction == "maximize" else min)( + candidate.native for candidate in matched + ) + best_proxy_score = (max if proxy_direction == "maximize" else min)( + candidate.proxy for candidate in matched + ) + + # Track indices so distinct candidates sharing (proxy, native, accepted_at) stay separate. + best_native_indices = { + index + for index, candidate in enumerate(matched) + if candidate.native == best_native_score + } + best_proxy_indices = { + index + for index, candidate in enumerate(matched) + if candidate.proxy == best_proxy_score + } + best_proxy_matches_best_native = bool(best_native_indices & best_proxy_indices) + + # Strictly beating any native-optimal candidate on proxy equals beating the worst proxy among them. + worst_best_native_proxy = (min if proxy_direction == "maximize" else max)( + matched[index].proxy for index in best_native_indices + ) + false_improvements = sum( + 1 + for index, candidate in enumerate(matched) + if index not in best_native_indices + and order(candidate.proxy, worst_best_native_proxy, proxy_direction) > 0 + ) + return dict( + best_native_count=len(best_native_indices), + best_proxy_matches_best_native=best_proxy_matches_best_native, + false_improvements_over_best_native=false_improvements, + ) + + def compare(proxy_path, native_path): cohort, proxy_direction, predictions = load_panel( proxy_path, clock="observed_at", allow_unscored=False @@ -98,13 +154,17 @@ def compare(proxy_path, native_path): for rule in ("proxy", "later_submission") } native_ties = 0 + proxy_ties = 0 for left, right in combinations(matched, 2): actual = order(left.native, right.native, native_direction) + proxy_decision = order(left.proxy, right.proxy, proxy_direction) + if proxy_decision == 0: + proxy_ties += 1 if actual == 0: native_ties += 1 continue decisions = { - "proxy": order(left.proxy, right.proxy, proxy_direction), + "proxy": proxy_decision, "later_submission": sign(left.accepted_at, right.accepted_at), } for rule, prediction in decisions.items(): @@ -114,8 +174,23 @@ def compare(proxy_path, native_path): rules[rule]["agrees"] += 1 else: rules[rule]["reverses"] += 1 + + pair_count = len(matched) * (len(matched) - 1) // 2 + tau_denominator = (pair_count - proxy_ties) * (pair_count - native_ties) + kendall_tau_b = ( + (rules["proxy"]["agrees"] - rules["proxy"]["reverses"]) + / math.sqrt(tau_denominator) + if tau_denominator > 0 + else None + ) + frontier = _frontier_summary(matched, proxy_direction, native_direction) return dict( - cohort=cohort, coverage=coverage, native_tied_pairs=native_ties, rules=rules + cohort=cohort, + coverage=coverage, + native_tied_pairs=native_ties, + kendall_tau_b=kendall_tau_b, + frontier=frontier, + rules=rules, ) diff --git a/benchmark/tests/test_proxy_order.py b/benchmark/tests/test_proxy_order.py index c347436b7d..f9b9af42a9 100644 --- a/benchmark/tests/test_proxy_order.py +++ b/benchmark/tests/test_proxy_order.py @@ -1,4 +1,8 @@ import json +import math +import os +import subprocess +import sys import pytest @@ -93,4 +97,205 @@ def test_scores_below_float_precision_are_ordered(tmp_path): .replace('"score": 0', '"score": 1.0000000000000000002', 1) .replace('"score": 0', '"score": 1.0000000000000000001', 1) ) - assert compare(proxy, native)["rules"]["proxy"]["reverses"] == 1 + result = compare(proxy, native) + assert result["rules"]["proxy"]["reverses"] == 1 + assert result["kendall_tau_b"] == pytest.approx(-1.0) + assert result["frontier"] == dict( + best_native_count=1, + best_proxy_matches_best_native=False, + false_improvements_over_best_native=1, + ) + + +def test_kendall_tau_b_monotonic_and_pure_reversal(tmp_path): + proxy = panel(tmp_path, "proxy", [(1, 30, 1), (2, 20, 1), (3, 10, 1)], "minimize") + native_agree = panel( + tmp_path, "native", [(1, 1, 2), (2, 2, 3), (3, 3, 4)], "maximize" + ) + agree_result = compare(proxy, native_agree) + assert agree_result["kendall_tau_b"] == pytest.approx(1.0) + assert agree_result["frontier"] == dict( + best_native_count=1, + best_proxy_matches_best_native=True, + false_improvements_over_best_native=0, + ) + + native_reverse = panel( + tmp_path, "native", [(1, 3, 2), (2, 2, 3), (3, 1, 4)], "maximize" + ) + reverse_result = compare(proxy, native_reverse) + assert reverse_result["kendall_tau_b"] == pytest.approx(-1.0) + assert reverse_result["frontier"] == dict( + best_native_count=1, + best_proxy_matches_best_native=False, + false_improvements_over_best_native=2, + ) + + +def test_kendall_tau_b_with_proxy_only_and_joint_ties(tmp_path): + proxy_only_tie = panel( + tmp_path, "proxy", [(1, 1, 1), (2, 1, 1), (3, 2, 1)], "maximize" + ) + native_strict = panel( + tmp_path, "native", [(1, 1, 2), (2, 2, 3), (3, 3, 4)], "maximize" + ) + proxy_tie_result = compare(proxy_only_tie, native_strict) + assert proxy_tie_result["kendall_tau_b"] == pytest.approx(2 / math.sqrt(2 * 3)) + + # Joint tie (1, 1) has abstains == 0, yet still enters T_p in tau-b. + joint_tie_proxy = panel( + tmp_path, "proxy", [(1, 1, 1), (2, 1, 1), (3, 2, 1)], "maximize" + ) + joint_tie_native = panel( + tmp_path, "native", [(1, 1, 2), (2, 1, 3), (3, 2, 4)], "maximize" + ) + joint_result = compare(joint_tie_proxy, joint_tie_native) + assert joint_result["native_tied_pairs"] == 1 + assert joint_result["rules"]["proxy"]["abstains"] == 0 + assert joint_result["kendall_tau_b"] == pytest.approx(1.0) + + +@pytest.mark.parametrize( + "proxy_rows,native_rows,expected_frontier", + [ + ( + [(1, 1, 5)], + [(1, 1, 2)], + dict( + best_native_count=0, + best_proxy_matches_best_native=None, + false_improvements_over_best_native=0, + ), + ), + ( + [(1, 2, 1)], + [(1, 5, 2)], + dict( + best_native_count=1, + best_proxy_matches_best_native=True, + false_improvements_over_best_native=0, + ), + ), + ( + [(1, 1, 1), (2, 1, 1), (3, 1, 1)], + [(1, 1, 2), (2, 2, 3), (3, 3, 4)], + dict( + best_native_count=1, + best_proxy_matches_best_native=True, + false_improvements_over_best_native=0, + ), + ), + ( + [(1, 1, 1), (2, 2, 1), (3, 3, 1)], + [(1, 5, 2), (2, 5, 3), (3, 5, 4)], + dict( + best_native_count=3, + best_proxy_matches_best_native=True, + false_improvements_over_best_native=0, + ), + ), + ], +) +def test_degenerate_tau_b_and_frontier_cases( + tmp_path, proxy_rows, native_rows, expected_frontier +): + result = compare( + panel(tmp_path, "proxy", proxy_rows, "maximize"), + panel(tmp_path, "native", native_rows, "maximize"), + ) + assert result["kendall_tau_b"] is None + assert result["frontier"] == expected_frontier + + +def test_high_end_proxy_cliff_counterexample(tmp_path): + proxy = panel( + tmp_path, + "proxy", + [(1, 1, 1), (2, 2, 1), (3, 3, 1), (4, 4, 1), (5, 5, 1)], + "maximize", + ) + native = panel( + tmp_path, + "native", + [(1, 1, 2), (2, 2, 3), (3, 3, 4), (4, 10, 5), (5, 0, 6)], + "maximize", + ) + result = compare(proxy, native) + assert result["rules"]["proxy"] == dict(agrees=6, reverses=4, abstains=0) + assert result["kendall_tau_b"] == pytest.approx(0.2) + assert result["frontier"] == dict( + best_native_count=1, + best_proxy_matches_best_native=False, + false_improvements_over_best_native=1, + ) + + +def test_frontier_multiple_native_optima_and_strict_tie_boundary(tmp_path): + # Minimize both: native-optimal candidates tie at native=1 with proxies 2 and 5. + proxy = panel( + tmp_path, + "proxy", + [(1, 2, 1), (2, 5, 1), (3, 4, 1), (4, 2, 1), (5, 6, 1)], + "minimize", + ) + native = panel( + tmp_path, + "native", + [(1, 1, 2), (2, 1, 3), (3, 3, 4), (4, 4, 5), (5, 2, 6)], + "minimize", + ) + result = compare(proxy, native) + assert result["frontier"] == dict( + best_native_count=2, + best_proxy_matches_best_native=True, + false_improvements_over_best_native=2, + ) + + # Tying the sole native-optimal candidate on proxy is not a strict false improvement. + tie_proxy = panel(tmp_path, "proxy", [(1, 10, 1), (2, 10, 1)], "maximize") + tie_native = panel(tmp_path, "native", [(1, 10, 2), (2, 5, 3)], "maximize") + tie_result = compare(tie_proxy, tie_native) + assert tie_result["frontier"] == dict( + best_native_count=1, + best_proxy_matches_best_native=True, + false_improvements_over_best_native=0, + ) + + +def test_cli_entrypoint_outputs_json_diagnostics(tmp_path): + proxy = panel( + tmp_path, + "proxy", + [(1, 1, 1), (2, 2, 1), (3, 3, 1), (4, 4, 1), (5, 5, 1)], + "maximize", + ) + native = panel( + tmp_path, + "native", + [(1, 1, 2), (2, 2, 3), (3, 3, 4), (4, 10, 5), (5, 0, 6)], + "maximize", + ) + env = {**os.environ, "LOOPX_USAGE_PING": "0"} + proc = subprocess.run( + [sys.executable, "-m", "benchmark.proxy_order", str(proxy), str(native)], + check=False, + capture_output=True, + text=True, + env=env, + ) + assert proc.returncode == 0, proc.stderr + payload = json.loads(proc.stdout) + assert set(payload) == { + "cohort", + "coverage", + "native_tied_pairs", + "kendall_tau_b", + "frontier", + "rules", + } + assert payload["kendall_tau_b"] == pytest.approx(0.2) + assert payload["frontier"] == dict( + best_native_count=1, + best_proxy_matches_best_native=False, + false_improvements_over_best_native=1, + ) diff --git a/docs/reference/benchmark-behavior-findings.md b/docs/reference/benchmark-behavior-findings.md index 87df154950..85954d3fa6 100644 --- a/docs/reference/benchmark-behavior-findings.md +++ b/docs/reference/benchmark-behavior-findings.md @@ -104,7 +104,24 @@ duplicate source rows. A matching cohort name records the analyst's selection; the analyst must verify comparable inputs, source digests and timestamps. The output distinguishes missing coverage, native ties, abstentions, agreements -and reversals. Each local result must precede acceptance of its own candidate. +and reversals, and adds two rank/cliff diagnostics across matched candidates: + +- `kendall_tau_b`: tie-adjusted Kendall rank correlation (tau-b) across all + matched pairs (`null` when fewer than two candidates match or all matched + candidates tie on either score), counting joint ties in the tie denominators. +- `frontier`: top-frontier alignment and high-end proxy cliff counts: + - `best_native_count`: number of matched candidates achieving the best native + score. + - `best_proxy_matches_best_native`: whether at least one matched candidate + achieving the best proxy score is also native-optimal (`null` when zero + candidates match). + - `false_improvements_over_best_native`: count of non-native-optimal matched + candidates that strictly beat at least one native-optimal matched candidate + on the proxy score. A cohort can show positive `kendall_tau_b` and + `agrees > reverses` while still failing at the frontier when an + over-optimizing candidate triggers a proxy cliff or tail timeout regression. + +Each local result must precede acceptance of its own candidate. Pairs share candidates and the solver may already have learned from older feedback. Counts describe the selected sequence. They give neither independent trials nor a population accuracy estimate. Close scores can change order under diff --git a/docs/reference/protocols/README.md b/docs/reference/protocols/README.md index bb3a1ff16e..992229d44c 100644 --- a/docs/reference/protocols/README.md +++ b/docs/reference/protocols/README.md @@ -30,6 +30,7 @@ until their documented migration is qualified. - [`quota_cli_hot_path_compaction_v0`](quota-cli-hot-path-compaction-v0.md): Quota CLI hot-path compaction v0 - [`quota_planning_horizon_v0`](quota-planning-horizon-v0.md): Bounded agent planning horizon v0 - [`rollback_packet_v0`](rollback-packet-v0.md): Rollback packet v0 +- [`stage_methodology_operator_catalog_v0`](stage-methodology-operator-contract-v0.md): Five-stage composable methodology operator contract, anti-entropy invariants, and advisory quantitative/search schemas (`scoped_yardstick_fidelity_v0`, `budgeted_continuation_plan_v0`, `counterfactual_continuation_eval_v0`, `evolutionary_population_plan_v0`) - [`task_graph_projection_v0`](task-graph-projection-v0.md): Task graph projection v0 - [`typed_date_resume_trigger_v0`](typed-date-resume-trigger-v0.md): timezone-aware one-shot Todo resume and stable due receipt v0 - [`typed_date_resume_trigger_v0`](typed-date-resume-trigger-v0.zh-CN.md): timezone-aware one-shot Todo resume and stable due receipt v0 (中文) diff --git a/docs/reference/protocols/stage-methodology-operator-contract-v0.md b/docs/reference/protocols/stage-methodology-operator-contract-v0.md new file mode 100644 index 0000000000..4511943c0f --- /dev/null +++ b/docs/reference/protocols/stage-methodology-operator-contract-v0.md @@ -0,0 +1,197 @@ +# Stage Methodology Operator Contract v0 + +Status: supported TypeScript control-plane contract for composable long-horizon +decision methodologies and advisory quantitative planning operators. + +As LoopX grows beyond pairwise work comparisons into multi-point proxy +calibration, 18h+ confirmation-budgeted continuation, same-checkpoint +counterfactual attribution, and evolutionary population search, implementing +each methodology as bespoke stage glue across `admission`, `compile_context`, +`execute_verify`, `writeback_settle`, and `feedback_evolve` repeatedly produces +five recurring failure classes: + +1. **Feature-off and quiet-turn leakage**: truthy flag coercion or untriggered + cues inflate prompt entry cost and control-plane overhead. +2. **Frozen-prefix and late-replay drift**: mixing append order with wall-clock + timestamps, tied commit clocks, or post-prefix repeated citations mutates + historical receipts on retry or replay. +3. **Unbounded compact projections**: variable-length findings or measurement + tables leak into hot-path turn context or Effect RPC payloads without + explicit item caps, omitted counts, or a cold-path `details_command`. +4. **Cross-scope contamination and silent zero coercion**: comparing across + different `node_id`, `comparison_scope`, `input_revision`, `evaluation_ref`, + `todo_id`, or `target` bindings—or coercing unknown `null` measurements and + resource bounds to `0`—corrupts quantitative direction choices. +5. **Advisory-to-authority escalation**: advisory planning cues accidentally + emit admission, lease, gate-override, or execution-authorization fields. + +`stage_methodology_operator_catalog_v0` (`loopx/control_plane/capabilities/stage_methodology_operator.ts`) +establishes one browser-safe TypeScript kernel for all methodology operators. +Existing control-plane mechanisms (`work_yardstick` in `yardstick_decision.ts` / +`yardstick_source.ts` and `stride_review` in `stride_policy.ts`) and quantitative +research operators (`yardstick_fidelity.ts`, `budgeted_continuation.ts`, +`counterfactual_continuation.ts`, `evolutionary_search.ts`) share the same +activation, replay-window, scope-tuple, digest, bounded-projection, and +non-escalation primitives. Python CLI/API surfaces act strictly as transport and +canonical state loaders; dashboard readers reuse the same output verifier. + +## Five Canonical Control Stages (`LOOPX_CONTROL_STAGES`) + +| Stage ID | Lifecycle Responsibility | Methodology Operator Contract | +| --- | --- | --- | +| `admission` | `quota should-run`, trigger matching, lease/claim eligibility, stop/wait boundaries | Strict boolean opt-in (`evaluateMethodologyActivation`); zero silent spend or claim creation | +| `compile_context` | `turn-context`, progressive hot-path cues, cold-path `details_command` | Bounded advisory cues (`projectBoundedMethodologyItems`) over frozen canonical prefixes | +| `execute_verify` | Agent work execution, candidate probes, evaluator runs, isolated branches | Exact scope-tuple matching (`matchesExactScopeTuple`) and checkpoint/workspace isolation | +| `writeback_settle` | `explore finding`, `refresh-state`, replay-deterministic delivery receipts | Canonical append-prefix & committed-clock windowing (`selectCanonicalEvidenceWindow`, `isHistoricalReplayPrecedingCurrent`) | +| `feedback_evolve` | Proxy fidelity, confirmation back-scheduling, counterfactual attribution, evolutionary selection | Read-only typed schemas verified at runtime by `assertMethodologyOperatorOutput` | + +## Shared Anti-Entropy Primitives + +1. **Strict Feature-Off & Quiet-Turn Isolation (`evaluateMethodologyActivation`)**: + Requires `enabled === true` (never truthy `"true"` or `1`). Suppresses cues + when `quiet_turn_policy === "silent_when_untriggered"` and `trigger_active === false`, + when `quiet_turn_policy === "due_obligation_only"` and + (`due_obligation_active === false` or `trigger_active === false`), + or when any `required_scope` key is blank. +2. **Canonical Prefix & Replay-Precedence Guards (`normalizeCanonicalSourceSnapshot`, `selectCanonicalEvidenceWindow`, `isHistoricalReplayPrecedingCurrent`)**: + Validates lowercase 64-hex SHA-256 digests (`isValidContentSha256`) through + the single `content_digest.ts` owner. Uncited work observations enter a + post-decision window only when both `recorded_order >= workObservationStart` + and committed timestamp strictly exceeds the decision timestamp + (`committed > decisionTimeMicros`). Tied or missing clocks never promote a + delayed replay into new evidence. +3. **Exact Scope-Tuple & Work-Identity Isolation (`matchesExactScopeTuple`, `matchesWorkIdentity`)**: + Compares exact values across every declared scope key (`metric`, `unit`, + `higher_is_better`, `comparison_scope`, `input_revision`, `evaluation_ref`) + and work binding (`goal_id`, `agent_id`, `todo_id`). Unknown measurements and + resource bounds remain `null`. +4. **Bounded Hot-Path Projections (`projectBoundedMethodologyItems`)**: + Slices hot-path arrays to `maxItems`, returning exact `omitted_count` and an + optional `details_command`. +5. **Runtime Non-Escalation Output Verifier (`verifyMethodologyOperatorOutput`, `assertMethodologyOperatorOutput`)**: + Separates `operator_role` across `advisory_cue` (`advisory_only: true`, + requiring `output.advisory_only === true` and `output.grants_execution_authority === false`) + and `required_due_decision` (`stride_review`, `advisory_only: false`, + `grants_execution_authority: false` — accepting Core-persisted `stride_decision_v2` + decisions without synthetic advisory flags while still rejecting + `grants_execution_authority: true`). Always rejects forbidden authority + fields (`should_run`, `normal_delivery_allowed`, `lease_granted`, + `gate_override`, `admin_bypass`, `execution_authorized`) at any nesting depth, + non-finite numbers (`NaN`/`Infinity`), control characters, and payloads + exceeding the 64 KiB UTF-8 cap. + +## Built-In Operator Catalog (`stage_methodology_operator_catalog_v0`) + +Inspect registered operators via `loopx --format json explore methodology-operators [--stage ]`: + +| `operator_id` | `operator_role` | Primary Stage | Companion Stages | Transition Schema | +| --- | --- | --- | --- | --- | +| `work_yardstick` | `advisory_cue` | `execute_verify` | `compile_context`, `writeback_settle`, `feedback_evolve` | `work_yardstick_decision_v0` | +| `stride_review` | `required_due_decision` | `feedback_evolve` | `compile_context`, `writeback_settle` | `stride_decision_v2` | +| `scoped_yardstick_fidelity` | `advisory_cue` | `execute_verify` | `compile_context`, `feedback_evolve` | `scoped_yardstick_fidelity_v0` | +| `budgeted_continuation` | `advisory_cue` | `feedback_evolve` | `admission`, `compile_context` | `budgeted_continuation_plan_v0` | +| `counterfactual_continuation` | `advisory_cue` | `feedback_evolve` | `execute_verify` | `counterfactual_continuation_eval_v0` | +| `evolutionary_population` | `advisory_cue` | `feedback_evolve` | `compile_context`, `execute_verify` | `evolutionary_population_plan_v0` | + +## Quantitative & Search Operator Schemas + +### 1. `scoped_yardstick_fidelity_v0` (`explore yardstick-fidelity`) + +Evaluates multi-point agreement between a fast `proxy` profile and an +`authoritative` outcome profile on one `node_id`: + +- Pairs confirmed, finite measurements per `candidate_ref` under exact + scope-tuple equality (`metric`, `unit`, `higher_is_better`, `comparison_scope`, + `input_revision`, `evaluation_ref`). +- Computes direction-normalized `rank_concordance` (`kendall_tau_b`, + `spearman_rho`), `linear_calibration` (`utility_slope`, `residual_mad`, + `r_squared`), and top-$K$ `frontier_neighborhood` (`frontier_k`, + `frontier_kendall_tau_b`, `bias_cliff_detected`). +- Emits `fidelity_status`: `direct_only`, `insufficient_paired_evidence`, + `noisy_indistinguishable`, `aligned_in_tested_scope`, `frontier_bias_cliff`, + `order_reversed`, or `mixed_or_flat`. +- Audits `unconfirmed_proxy_winners` (proxy-only candidates exceeding the best + paired proxy score) and `proxy_false_negatives` without globally blacklisting + any evaluator (`scope_bound: true`, `global_evaluator_blacklist: false`). +- Includes every candidate tied at the authoritative top-K cutoff. The returned + `frontier_k` is the number of candidates analyzed after including those ties. + +### 2. `budgeted_continuation_plan_v0` (`explore budgeted-plan`) + +Back-schedules authoritative confirmation reserve from `remaining_budget` +(`wall_minutes`, `token_budget`, `authoritative_eval_slots`; keeping unknown +bounds `null`) and evaluates candidate continuation routes: + +- Computes `latest_confirmation_start_minute` and `remaining_work_window_minutes` + from the remaining time and the duration of confirmation. The remaining + cooldown starts at the planning boundary and can overlap useful work. + `pending_in_flight_minutes` reserves time for a result the planned route + depends on. Confirmation starts at the later of candidate readiness and + cooldown expiry. +- Uses the sum of known nonnegative token costs as a lower bound. A route + above the ceiling is infeasible even when its full cost is unknown. +- Uses a baseline route's expected gain as a decision reference only when its + costs fit the known budget. Otherwise the measured incumbent supplies a + zero-gain reference. Without a measured incumbent the reference is `null`. +- Accepts explicit `null` gain estimates and checks the order of the available + estimates. A missing estimate leaves its corresponding break-even probability + unknown. Dominance requires a known upper gain bound and a known reference. +- Under LoopX's retained-incumbent rule, a regressed candidate does not replace + the verified incumbent, so floor post-confirmation gain is + $g_{\text{floor}} = \max(0, g_{\min})$ and break-even success probability + against `effective_safe_gain` is: + $$p^* = \operatorname{clamp}_{[0,1]}\!\left(\frac{g_{\text{safe}} - g_{\text{floor}}}{g_{\text{target}} - g_{\text{floor}}}\right)$$ +- Classifies each route's `feasibility_status` (`feasible`, + `feasible_with_unknown_bounds`, `deadline_infeasible`, `token_infeasible`, + `eval_slot_infeasible`) and `recommended_disposition` (`reject_infeasible`, + `dominated_by_baseline`, `probe_then_decide`, `viable_direct_continuation`, + `upside_tail_only`). + +### 3. `counterfactual_continuation_eval_v0` (`explore counterfactual-eval`) + +Evaluates matched `treatment` and `control` continuations forked from an +identical frozen `checkpoint` (`source_state_sha256`, `input_revision`, +`evaluation_ref`, `fork_recorded_at`). Use the same `terminal_boundary_ref` +on the checkpoint and both continuations. + +- Use each observed run's stable `arm_id`, rather than a strategy label, within + the same task, checkpoint and source digest. The two arms must identify + different runs, and a run can contribute to only one pair in that scope. + Changing a pair or workspace label does not make the run independent. +- Checks distinct, unreused `workspace_isolation_id` values, matching + checkpoint and terminal boundary, and an observed context time at or before + `fork_recorded_at` for each run. Strict mode rejects failed checks. + Audit mode excludes those pairs from the estimates and marginal costs. + These input checks do not attest physical isolation. +- Reports first-action divergence separately from score delta. Missing action + evidence is `action_unobserved`. Direction-normalized deltas combine only + within the same declared comparison scope, metric, unit, direction, input, + evaluator and terminal boundary. For one comparable scope, it computes the + mean and median delta, exact two-sided paired sign-flip $p$-value, and + `complete_marginal_costs` across `wall_seconds`, `total_tokens`, + `evaluator_runs`, `control_plane_requests`, and `recovery_events`. + +### 4. `evolutionary_population_plan_v0` (`explore evolutionary-plan`) + +Performs mechanism-diverse evolutionary selection over confirmed Explore Graph +measurements without collapsing into single-mechanism hyperparameter tweaks or +Goodharted proxy winners: + +- Groups confirmed `authoritative` candidates by the observed executed method + or explicit `mechanism_by_candidate` mapping. A candidate with unknown + mechanism keeps its score and can fill an elite slot. Crossovers and targeted + mutations use candidates with identified mechanisms. +- **Island elitism (`elites`)**: preserves the top champion of each distinct + mechanism island first (`selection_reason: "island_champion"`), filling + remaining slots up to `elite_limit` by utility (`"utility_fill"`). +- **Crossover proposals (`crossover_pairs`)**: pairs elites across distinct + identified mechanisms. The combination requires its own validation. +- **Bottleneck mutation (`mutation_targets`)**: emits `island_champion_refinement` + or `tie_breaking_bottleneck_probe` targets on top elites. +- **Goodhart proxy quarantine (`proxy_quarantine`)**: delegates proxy screening + to `evaluateScopedYardstickFidelity`; unconfirmed proxy winners are never + admitted into `elites` or `crossover_pairs`, receiving disposition + `quarantined_proxy_cliff` for a multi-point `frontier_bias_cliff` or + `order_reversed`. Two paired candidates show one ordering; keep that + observation in the fidelity report and use `confirm_before_breeding` in the + population plan, as with other unqualified proxy evidence. diff --git a/loopx/capabilities/explore/README.md b/loopx/capabilities/explore/README.md index 0717063e07..7ca167a18a 100644 --- a/loopx/capabilities/explore/README.md +++ b/loopx/capabilities/explore/README.md @@ -2,243 +2,6 @@ Status: supported optional capability; default-off harness execution contract. -Ordinary result writeback can include an optional `execution_observation` -(`explore_execution_observation_v0`): exact candidate SHA-256 and input revision, -planned/executed method ids and change reason, measured elapsed/limit seconds, -search iterations, stop reason and opaque evidence refs. Unknown measurements -are explicit nulls. Its refs and input revision must belong to the enclosing -finding. Released `result_ref` requires a matching `result_candidate_sha256`. -This records producer evidence, not verified execution or utility. Findings, -replan context and the existing research-history detail retain it; omission -preserves old events. See the [local probe producer](../../../benchmark/runtime/EXECUTION_OBSERVATIONS.md). - -When the observed execution used the finding's tested input revision, result -attachments can bind it once and select the evidence to reuse. Readback and -standalone observations keep the complete observation form. [Execution evidence authoring](../../../docs/reference/protocols/explore-result-from-path-delta-v0.md#execution-evidence-on-the-same-input) - -## Quantitative findings - -An ordinary result attachment can include `measurements`, saved on its existing -node-linked finding. The graph and research-history detail retain the values; -`explore turn-context` supplies bounded `measurement_feedback` to the next turn. -To use a new result during the current Turn, save the same ordinary attachment -as `result.json` and record it with the open claimed Todo: - -```sh -loopx --format json explore finding --goal-id GOAL --agent-id AGENT \ - --todo-id TODO --turn-instance-id TURN --result-json result.json -``` - -Supply the question and applicability for a new scoped question. For one -existing linked question, omit both; when there are several, name its -`node_id`. The same attachment works with `refresh-state --explore-result-json`. -The receipt returns `measurement_feedback` for confirmed measurements in this -write, including the earlier best, latest value and improvement under the -same profile. For an enabled Yardstick with declared outcome and proxy -profiles, it also returns `search_comparison` when this write contributes a -qualified comparison. A reversal requires two distinct candidates measured on -both profiles with different preferences. This uses the current Todo's -recorded evidence and doesn't settle the Turn. Exact retries within the same -Turn return the comparison at the original write. - -With Yardstick enabled, a quantitative or decision attachment on ordinary -`refresh-state` returns the same feedback in `explore_result_delivery` after -the graph and Todo association commit. A successful replay reads the original -committed prefix, including after later results arrive. Partial delivery still -retries through the existing owner; feedback creates no claim, spend or evaluation. -Disabled Yardstick and unmeasured refresh attachments retain their earlier wire. - -When no outcome/proxy decision has been selected, a quantitative receipt can -also return bounded `work_measurements` from this Todo's committed evidence. -These are the same scoped facts available in ordinary task context. The receipt -does not duplicate its current feedback: it supplements it with at most two -other groups, prioritizing this question and recent evidence, with a source-bound -`details_command` and an explicit omitted count for the remaining groups. -No metric is assigned an outcome or proxy role. The Agent can use the facts -without another declaration or probe; if its method changes, the existing -optional decision field fits the same normal result attachment. - -Manual capture uses `explore finding --node --evidence-ref result:a ---measurements-json measurements.json`. The file contains one to eight items: - -```json -[{"metric":"quality","value":8,"unit":"points","higher_is_better":true, - "comparison_scope":"validation:fixed-budget-v1","input_revision":"dataset-v1", - "evaluation_ref":"evaluator:v1","candidate_ref":"candidate:a","evidence_ref":"result:a"}] -``` - -Each result reference must belong to the finding's `evidence_refs`; ordinary -attachments also require the same `input_revision`. Use a new finding id per -evaluation or paired comparison, and reuse an id only to correct that finding. -Missing values are explicit `null`; zero and negative values remain valid measurements. - -When an `execution_observation` declares a `result_ref`, each measurement must -cite that same result. This applies to manual findings and writeback attachments, -including shared comparison contexts. Listing both receipts in `evidence_refs` -does not bind a different result to the executed candidate. No result is inferred -when execution has not declared one. These consistency checks do not independently -verify a producer's calculator or turn `confirmed` into an evaluator attestation. - -To reuse verified comparison conditions, save these three fields once in -`conditions.json`: - -```json -{"comparison_scope":"validation:fixed-budget-v1","input_revision":"dataset-v1", - "evaluation_ref":"evaluator:v1"} -``` - -Pass `--measurement-context-json conditions.json` alongside -`--measurements-json observations.json`. Each observation then omits those -three fields, for example: - -```json -[{"metric":"quality","value":9,"unit":"points","higher_is_better":true, - "candidate_ref":"candidate:b","evidence_ref":"result:b"}] -``` - -Ordinary and path-delta attachments support the same optional -`measurement_context` object alongside `measurements`. The writer expands it -into the existing complete records; it stores no mutable context reference. -Observation overrides, even equal ones, and incomplete or extra context fields -reject before recording. The original complete-item format remains supported. -Reusing a declaration does not verify comparability: select a different context -when task, data, evaluator, model or budget changes. Keep unknown conditions -separate. This convenience grants no evaluation, correction or execution authority. - -Keep comparison conditions separate from the candidate and its result: - -| Field | Meaning across observations | -| --- | --- | -| `input_revision` | Evaluated task/data revision; retain it when the evaluated inputs are verified unchanged. | -| `evaluation_ref` | Evaluator/scoring protocol version; retain it when the evaluation protocol is verified unchanged. | -| `candidate_ref` | Code/artifact version actually evaluated; change it when the candidate changes. | -| `evidence_ref` | Individual evaluation result/receipt; retain its exact provenance in the finding's `evidence_refs`. | - -For example, record candidate A's score 8 and candidate B's score 9 as **two -findings** on the same question. With the other comparison fields unchanged, -both use `input_revision=dataset-v1` and `evaluation_ref=evaluator:v1`, while -their candidate/result refs are `candidate:a`/`result:a` and -`candidate:b`/`result:b`. Feedback has two measured records and improvement +1. -Putting each candidate hash in `input_revision` or each submission id in -`evaluation_ref` instead creates singleton groups with null improvement. -That null is not zero gain. Check provenance before correcting a mistaken -finding through the existing correction path; historical records remain intact. -Changed or unknown evaluation conditions must stay separate. Do not invent a -shared revision merely to obtain a comparison. A growing local corpus does not -change the input revision for an unchanged evaluated subset; changed samples do. - -File-based producers may use `candidate_content.candidate_snapshot` and -`candidate_manifest_sha256` from this capability's host I/O neighborhood to -capture selected relative paths and byte digests. This preserves one content -encoding for execution probes and artifact adapters. It does not cover modes, -packaging, transitive inputs or execution conditions, certify an execution, or -require every measurement subject to use a content hash. Keep each producer's -actual subject, workload and evaluator; extra content evidence does not rewrite -earlier records or count as another evaluation. - -A paired comparison may instead record both candidates in **one finding** under -the same verified comparison conditions. Keep both `candidate_ref` and -`evidence_ref` values, and list both results in the finding's `evidence_refs`. -Distinct candidates do not require invented metrics or scopes. A repeated -candidate under the same metric, unit, direction and conditions is still a -duplicate, even with another result reference; record a repeat evaluation in a -new finding. Every measurement still requires a candidate reference. A paired -finding cannot claim two separate receipts as one execution observation's result. - -Comparisons require the same node, metric, unit, direction, comparison scope, -input revision and evaluator. Producers must change the scope when task, -model/resource budget or evaluation conditions change, and keep distinct workloads -or evaluation conditions in their own profiles. -A producer's location does not decide whether its measurement is an outcome or a -proxy; that role remains an explicit decision declaration. Explore validates the -declared comparison keys, -not their truth. It computes the latest recorded measurement, historical best -candidate/source and direction-adjusted improvement against the prior best, -before applying the context's display limit. Latest follows source-log append -order, including tied or backfilled timestamps. Within a finding, observations -follow their declared array order: latest is the last matching observation and -prior best includes earlier observations in that batch. This order does not -assert execution chronology or identify the retained candidate. The source ordinal -is derived, not another persisted fact. Corrected findings replace earlier -versions in comparisons; original versions remain in history. Counts are records, -not independent trials. Feedback pages reuse `--result-limit` (1..20, default 3), -`--result-offset`, `--result-revision` and the optional `--result-node` filter. -Follow `measurement_feedback_page.next_command` for additional groups, including -multiple metrics or conditions on one node. The shared revision binds all compared -sources, including manual findings; a changed source requires restarting the read. -The next-turn guidance uses this feedback to investigate -regressions or revisit better candidates before repeating work. Best does not -identify the current retained candidate, establish causality, satisfy acceptance -or authorize execution. Actual adoption and outcome benefit require subsequent -work and evidence; automatic branch rankings remain unchanged. - -When an enabled work Yardstick has two different measured candidates on its -declared goal profile, from findings on the same node and Goal/Agent/Todo -binding, Explore context names the observed better-scoring candidate as an -available comparison base and links the scoped evidence. A -tied goal outcome calls for checking headroom. The detail read shows a proxy -ordering only when its exact profile compares the same two candidates, with -separate result references and metric directions for each profile. Weigh -continuation, a distinct mechanism or a compatible combination by expected -end-to-end gain and full implementation and validation cost. Use a bounded -check that can change the choice while honoring the accepted constraints and -budget. The compact cue keeps the recorded next-action reason and adds a -separate comparison cue. The detail read retains the complete work decision. -The comparison uses findings from the current work, target, and acceptance -reference. A different target or acceptance reference needs its own eligible -pair, even when the measured metric, evaluator and inputs stay the same. -Uncited ordinary observations enter the current work window only when their -original server commit is provably later than that decision and their canonical -log position follows its original prefix. A delayed old delivery, tied time or -unknown time does not become new work evidence. Original results and explicitly -cited, verified prefix history remain available. -The detail read also connects confirmed `execution_observation` records to the -current or explicitly preceding decision through `previous_decision_followthrough`. -An ordinary result can omit `work_yardstick_decision`, including when it has no -quantitative measurements; the existing choice remains current. A later choice -keeps the preceding execution's original finding reference. Attribution requires -the same Goal/Agent/Todo, question and tested input, plus the original commit and -visible work window. Foreign, tentative, stale and late-visible records cannot -become that choice's execution. This reuses existing findings and readback; -the association does not independently verify method adoption or benefit. -A verified successor history prefix determines whether prior execution was -already visible even when the successor clock moves backward. Captures without -that prefix retain the strict timestamp bound; later delivery stays excluded. -Ordinary `turn run-once` accepts the same inline `explore_result` inside the -Host's existing `agent_vision_json`. It retains one decoded Vision packet, -then uses the original attachment parser and delivery owner. A failed delivery -leaves quota unspent and recovers from the original Run on the same Turn; - -Off mode with an omitted attachment creates no finding; supplied invalid or -disabled attachments reject before Todo completion and primary writeback. The -completion path uses the same capability preflight before lifecycle effects; -the primary writer still checks it again at commit. An omitted attachment adds -no capability preflight, and recovery retains already committed completion. -The quota writeback receipt is published only after requested attachment delivery -finishes in both `turn run-once` and `refresh-state`. The public quota CLI rejects -a debit while that receipt is missing. Ordinary refresh and -`refresh-state --settle-turn` recovery deliver the verified original Run before -repairing its receipt; a failed attachment leaves the same recovery pending without a debit. -Receipt repair preserves the original committed replay after Yardstick is disabled. -Finish that delivery and settlement before starting the next Turn. -The comparison is advisory and carries no retention or execution obligation. -On an admitted work turn, the progressive starting read includes the enabled -Yardstick's decision guidance and any compact cue for the selected Todo. Use -the Explore context reader for the attachment template and scoped evidence. -Before a first decision is recorded, the same opt-in entry shows up to three -exact-profile measurement groups from questions canonically linked to that -Todo, with omitted counts and the existing task reader. Distinct questions get -a place before additional metrics from one question. These are declared -comparisons, including shared findings; they do not establish which Todo ran a -measurement or select an outcome/proxy profile. Default-off input and the -benchmark's optional exact-field first read retain their existing behavior. - -Manually captured measured findings also appear in the existing result pages -and, when linked to eligible work, its exact-source comparison cards. They need -no `writeback-result` origin tag. Read the page before optionally binding a -`selected_candidate`; normal source-revision, Todo-link and followthrough checks -still apply. Unmeasured manual notes retain their existing summary/audit path. - ## At a Glance Explore Harness is a supported, default-off optional capability for @@ -566,6 +329,101 @@ journey is not delivered by this slice. Existing capability configuration remain the mode owner, and Lark publication remains separately authorized. Full research adoption and score benefit require a subsequent matched worker experiment. +## Quantitative Findings, Execution Observations & Stage Methodology Operators + +### Execution Observations (`explore_execution_observation_v0`) + +Ordinary result attachments and manual findings can attach an optional +`execution_observation`: exact `candidate_sha256` and `input_revision`, +`planned_method_id` vs. `executed_method_id` (requiring non-null `change_reason` +when they diverge), measured `elapsed_seconds` / `limit_seconds`, +`search_iterations`, `stop_reason`, and `1..8` opaque `evidence_refs`. Unknown +measurements are explicit `null` (never `0`). Its `input_revision` and +`evidence_refs` must belong to the enclosing finding, and any released +`result_ref` requires a matching `result_candidate_sha256` (see +[local probe producer](../../../benchmark/runtime/EXECUTION_OBSERVATIONS.md) and +[execution evidence authoring](../../../docs/reference/protocols/explore-result-from-path-delta-v0.md#execution-evidence-on-the-same-input)). +This records producer attribution, not verified execution or utility; omitting +it preserves legacy events. Detail reads also connect confirmed +`execution_observation` records to the current or explicitly preceding +`work_yardstick_decision` through `previous_decision_followthrough` (omitting +`work_yardstick_decision` retains the active choice). Attribution requires the +same Goal/Agent/Todo, question, tested input, original commit, and visible work +window; a verified successor history prefix determines visibility even across +clock rollback, while legacy captures without that prefix retain the strict +timestamp bound. + +### Quantitative Measurements & Scope-Tuple Comparability + +Findings and writeback attachments can attach `1..8` `measurements` (optionally +sharing a dry `measurement_context` supplying `comparison_scope`, +`input_revision`, and `evaluation_ref` that expands into complete records before +persistence, rejecting conflicting or partial overrides): + +```sh +# In-turn capture without settling the claimed Todo: +loopx --format json explore finding --goal-id GOAL --agent-id AGENT \ + --todo-id TODO --turn-instance-id TURN --result-json result.json + +# Manual capture with shared comparison context: +loopx explore finding --goal-id GOAL --node --evidence-ref result:b \ + --measurement-context-json conditions.json --measurements-json observations.json +``` + +Keep comparison conditions strictly separate from candidate and receipt identity: + +| Field | Meaning across observations | +| --- | --- | +| `input_revision` | Evaluated task/data revision; retain only when evaluated inputs are verified unchanged. | +| `evaluation_ref` | Evaluator/scoring protocol version; retain only when scoring protocol is verified unchanged. | +| `candidate_ref` | Code/artifact version actually evaluated (`candidate_content.candidate_snapshot` / `candidate_manifest_sha256` optional); update when candidate changes. | +| `evidence_ref` | Individual evaluation receipt in the finding's `evidence_refs` (must match `execution_observation.result_ref` when declared). | + +Key measurement invariants (canonicalized in `explore_measurements.ts`): + +- **Exact 7-key scope tuple:** Comparisons group strictly by `(node_id, metric, unit, higher_is_better, comparison_scope, input_revision, evaluation_ref)`. Putting candidate hashes in `input_revision` or submission IDs in `evaluation_ref` creates singleton groups with `null` improvement (`null` is unknown, never zero gain). Never invent a shared revision when conditions changed or are unknown. +- **Null vs. zero & paired findings:** Unmeasured values are explicit `null`; `0` and negative numbers are valid scores. A single finding may record paired distinct candidates under identical scope, while repeated evaluations of the same candidate belong in a new finding. Use a new finding ID per evaluation and reuse an ID only to correct a mistaken finding (corrections replace prior versions in active comparisons while preserving append-order history). +- **Append-order feedback & replay prefix isolation:** `explore turn-context`, `explore finding`, `turn run-once` (via inline `explore_result` inside `agent_vision_json`), and Yardstick-enabled `refresh-state` (`explore_result_delivery`) compute latest, historical best, and direction-adjusted delta along canonical log append order (and array order within a batch), paginated via `--result-limit` (`1..20`, default `3`), `--result-offset`, `--result-revision`, and `--result-node`. Successful settlement replays freeze and read only the original committed event prefix (even after Yardstick is disabled). Off mode with an omitted attachment creates no finding and adds no capability preflight, while supplied invalid or disabled attachments reject before Todo completion and primary writeback. In both `turn run-once` and `refresh-state`, the quota writeback receipt is published only after requested attachment delivery finishes (the public quota CLI rejects a debit while missing, and `refresh-state --settle-turn` recovery delivers the verified original Run before repairing its receipt). When no Yardstick decision is active, receipts supplement current feedback with at most two other `work_measurements` groups from the Todo's linked questions. Manually captured measured findings participate in result pages and exact-source comparison cards without needing a `writeback-result` tag. Scores and `best` markers never certify adoption, causality, acceptance, or execution authority. + +### Stage Methodology Operators & Quantitative Planning + +All six long-horizon decision and quantitative research methodologies — +`work_yardstick`, `stride_review`, `scoped_yardstick_fidelity`, +`budgeted_continuation`, `counterfactual_continuation`, and +`evolutionary_population` — share the single TypeScript control-plane kernel in +[`stage_methodology_operator_catalog_v0`](../../../docs/reference/protocols/stage-methodology-operator-contract-v0.md). +Every operator enforces strict boolean/invocation gating (zero cold-path prompt +tax when off or untriggered, including `due_obligation_only` silence when a +due obligation is inactive), canonical replay-prefix isolation (blocking late +replays and tied/unparseable clocks), exact scope-tuple isolation, explicit +`null` preservation, bounded hot-path cues with `omitted_count` + +`details_command`, and runtime non-escalation verification across distinct +`operator_role` values (`advisory_cue` requiring `advisory_only: true` / +`grants_execution_authority: false`, vs. `stride_review`'s +`required_due_decision` with `advisory_only: false` / +`grants_execution_authority: false` on Core-persisted `stride_decision_v2` +while still rejecting `grants_execution_authority: true` and forbidden +authority keys anywhere in the payload): + +```sh +# Inspect registered 5-stage methodology operators & anti-entropy invariants +loopx --format json explore methodology-operators [--stage feedback_evolve] + +# Multi-point proxy vs. outcome rank concordance (Kendall tau-b / Spearman rho), OLS calibration & Goodhart cliff check +loopx --format json explore yardstick-fidelity --goal-id GOAL [--node-id NODE] [--frontier-k 3] + +# Back-schedule authoritative confirmation reserve and break-even p* under retained-incumbent floor +loopx --format json explore budgeted-plan --document budgeted_plan.json + +# Same-checkpoint paired counterfactual evaluation (tau_B, exact sign-flip p-value, workspace & clock isolation) +loopx --format json explore counterfactual-eval --document counterfactual_pairs.json + +# Mechanism-diverse elites, candidate combinations to test, and proxy-cliff quarantine +loopx --format json explore evolutionary-plan --goal-id GOAL [--node-id NODE] [--elite-limit 4] +``` + +When an enabled `work_yardstick` binds outcome and proxy profiles on a Todo/target/acceptance scope, `explore turn-context` surfaces 2-candidate reversals (`search_comparison`) and, once $\ge 3$ paired candidates exist, multi-point `yardstick_fidelity` (`aligned_in_tested_scope`, `frontier_bias_cliff`, `order_reversed`, `noisy_indistinguishable`, or `mixed_or_flat`) plus unconfirmed proxy winners — rendered directly in Personal Workspace Saved Findings and Research History. + ## Projection And Topology `loopx explore summary` folds the log into @@ -1451,6 +1309,11 @@ loopx explore presentation --goal-id loopx explore graph --goal-id [--graph-format mermaid|json] [--out ] loopx explore todo-branch-plan --goal-id [--agent-id ] [--width 3] loopx explore worker-branch-plan --goal-id [--agent-id ] [--harness-profile generic|adaptive-resilient|moe-router] [--worker-width 3] [--max-todos-per-branch 3] [--router-state ] [--load-profile ] +loopx explore methodology-operators [--stage admission|compile_context|execute_verify|writeback_settle|feedback_evolve] +loopx explore yardstick-fidelity --goal-id [--node-id ] [--frontier-k 3] +loopx explore budgeted-plan --document +loopx explore counterfactual-eval --document +loopx explore evolutionary-plan --goal-id [--node-id ] [--elite-limit 4] [--crossover-limit 3] [--mutation-limit 3] loopx explore feishu-setup [--base-url ...] [--execute] loopx explore feishu-visual-configure [--whiteboard-token ] --docx-token [--stage-whiteboard-token ...] [--stage-capacity 10..20] [--board-style auto_flow|semantic_lane_columns] [--view-role canonical|executive] [--projection-mode canonical_filtered|issue_fix_two_lane|canonical_full|executive_auto] [--tag ] [--status ] [--execute] loopx explore feishu-sync --goal-id [--sink-visibility owner-only|shared] [--execute] diff --git a/loopx/capabilities/explore/README.zh-CN.md b/loopx/capabilities/explore/README.zh-CN.md index eb489895d6..7dfc938bde 100644 --- a/loopx/capabilities/explore/README.zh-CN.md +++ b/loopx/capabilities/explore/README.zh-CN.md @@ -2,125 +2,6 @@ 状态:受支持的 optional capability;harness 执行契约默认关闭。 -普通结果写回可附带可选 `execution_observation`(`explore_execution_observation_v0`): -候选 SHA-256、输入版本、计划/实际方法标识、改向原因、实测耗时/上限、搜索次数、 -结束原因与证据引用。未测量项显式为 null;版本和引用必须与所属 finding 一致。 -已释放结果的 `result_ref` 必须有匹配的 `result_candidate_sha256`。该记录是来源证据, -不认证执行真实性或收益;finding、replan 和既有研究历程详情保留相同信息,缺省仍兼容 -旧事件。参见[本地探针生产路径](../../../benchmark/runtime/EXECUTION_OBSERVATIONS.md)。 - -## 量化结论 - -普通结果附件可附带 `measurements`,保存到既有节点所属的 finding。关系图和 -研究历程详情保留分数;`explore turn-context` 将有界的 `measurement_feedback` -交给下一轮。手动记录使用 `explore finding --node --evidence-ref result:a ---measurements-json measurements.json`,文件包含一至八项: - -```json -[{"metric":"quality","value":8,"unit":"points","higher_is_better":true, - "comparison_scope":"validation:fixed-budget-v1","input_revision":"dataset-v1", - "evaluation_ref":"evaluator:v1","candidate_ref":"candidate:a","evidence_ref":"result:a"}] -``` - -结果引用必须属于 finding 的 `evidence_refs`;普通附件还要求相同 `input_revision`。 -每个受评测候选使用新 finding id,仅纠正同一结论时复用 id。未测量显式记为 `null`, -零分和负分仍为有效测量。 - -开启 Yardstick 后,普通 `refresh-state` 的量化或决策附件会在关系图与 Todo -关联提交后,通过 `explore_result_delivery` 返回与普通 finding 相同的反馈。 -成功重放只读原始已提交前缀,后来的结果不会改变首次回执。部分失败仍沿原 owner -重试;反馈不产生 claim、spend 或评测。关闭 Yardstick 或不带量化信息的旧 refresh -附件保持原有回执形状。 - -尚未选择 outcome/proxy 决策时,量化回执还可返回当前 Todo 已提交证据的有界 -`work_measurements`,复用普通任务上下文的事实视图。不重复回传本次反馈,只补充至多 -两组其他比较,优先同一问题与近期证据;省略数量明确标注,原 registry/runtime 路由的 -`details_command` 可达其余组。系统不替 Agent 指定指标角色, -也不要求增加一次声明或 probe。方法有实质变化时,原有可选决策字段仍可放在同一次 -普通结果附件中;记录比较事实不代表验收、执行许可或效果收益。 - -若要复用已核实的比较条件,将三个字段一次保存到 `conditions.json`: - -```json -{"comparison_scope":"validation:fixed-budget-v1","input_revision":"dataset-v1", - "evaluation_ref":"evaluator:v1"} -``` - -同时传入 `--measurement-context-json conditions.json` 与 -`--measurements-json observations.json`。此时每项观测省略这三个字段,例如: - -```json -[{"metric":"quality","value":9,"unit":"points","higher_is_better":true, - "candidate_ref":"candidate:b","evidence_ref":"result:b"}] -``` - -普通附件及 path-delta 附件也支持在 `measurements` 旁提供可选 -`measurement_context` 对象。写入前会展开成原有完整记录,不保存可变上下文引用。 -观测重复填写比较字段(即使值相同)、上下文字段缺失或多余都会在记录前拒绝。 -原有完整条目格式继续可用。复用声明不认证可比性;任务、数据、评测器、模型或预算 -改变时另选上下文,条件未知时保持分开。 - -比较条件与候选、单次结果分别记录: - -| 字段 | 连续 finding 中的含义 | -| --- | --- | -| `input_revision` | 实际受评测的任务或数据版本;确认输入未变时保持一致。 | -| `evaluation_ref` | 评测器或计分协议版本;确认协议未变时保持一致。 | -| `candidate_ref` | 实际受评测的代码或工件版本;候选改变时更新。 | -| `evidence_ref` | 单次评测结果或回执;在 finding 的 `evidence_refs` 中保留准确来源。 | - -例如,在同一问题上将候选 A 的 8 分、候选 B 的 9 分记录为**两个 finding**。 -其他比较字段不变时,两者均使用 `input_revision=dataset-v1`、 -`evaluation_ref=evaluator:v1`,候选与结果引用分别为 `candidate:a`/`result:a`、 -`candidate:b`/`result:b`。反馈包含两条实测记录,改善幅度为 +1。 -若把各候选哈希填进 `input_revision`,或把各次提交编号填进 `evaluation_ref`, -则会被拆成只有一条记录的组,改善幅度为 null;它不表示零收益。 -确认来源后才通过既有纠正路径修正误填的 finding,原始记录保留在历史中。 -条件已变或可比性未知时必须分开,不得为得到差值而编造共同版本。 -本地样本库扩充不改变实际受评测的固定子集版本;受评测样本改变才更新该版本。 - -只有节点、指标、单位、方向、比较范围、输入版本和评测器全部一致时才比较。 -任务、模型或资源预算、评测条件改变时,生产者必须更换比较范围;原生评测与本地 -代理指标分别记录。Explore 校验声明的比较键,不认证声明真实性。它在上下文裁剪前 -计算最近记录的测量、历史最佳候选及来源、相对先前最佳的方向归一化改善幅度。 -最近记录按源日志追加顺序确定;时间相同或事后回填不改变此顺序,序号仅从日志 -派生,不另行持久化。 -纠正结论会在比较中替换旧版本,原始版本仍可在历史中读取;计数表示记录数, -不表示独立试验次数。反馈分页复用 `--result-limit`(1..20,默认 3)、 -`--result-offset`、`--result-revision` 和可选的 `--result-node` 过滤。 -沿 `measurement_feedback_page.next_command` 读取其余组,同一节点的多个指标或条件 -也可完整翻页。共同 revision 绑定全部比较来源,包含手动 finding;来源变化须从头读取。 -下一轮指导使用这些反馈排查退步、回看更好候选,再决定是否 -重复尝试。best 不代表当前保留版本、因果收益、验收通过或执行授权;是否实际采用 -并产生收益,仍需后续工作和证据确认。自动分支排序保持原有行为。 - -手动登记的量化 finding 同样进入既有结果分页;关联可执行工作时,也进入其精确来源 -比较卡片,无需补写 `writeback-result` 来源标签。先读取结果页,再按需绑定 -`selected_candidate`,仍核对来源修订、Todo 关联和后续实际工作。未带测量的手动笔记 -保持既有摘要/审计路径。读取或选择某条成绩不认证实际采用或执行。 - -开启 work Yardstick 后,详情读取通过既有 `previous_decision_followthrough`, -把 confirmed 的 `execution_observation` 关联到当前或显式前驱决策。普通结果可以 -省略 `work_yardstick_decision`,也可以没有量化测量;原选择继续有效。后续选择 -保留此前执行的原始 finding 引用。关联要求相同 Goal/Agent/Todo、问题和测试输入, -并核对原始提交时间及证据可见的工作窗口;其他归属、tentative、过时或迟到的 -记录不能成为该选择的执行。复用现有 finding 与回读入口;关联本身不独立认证 -方法采用或收益。已验证的后继历史前缀决定此前执行是否已可见,后继时钟回拨 -不再否决前缀内的结果;缺少此前缀的旧 capture 保留严格时间上界,迟到投递仍被排除。 - -普通 `turn run-once` 接受 Host 既有 `agent_vision_json` 内的同一 -`explore_result`,保留一个解码后的 Vision packet,再复用原附件解析及交付 owner。 -交付失败时不扣额,在同一 Turn 从原 Run 恢复;已提交重放不重复 Host 工作、 -finding 写入或扣额。关闭且省略附件时不产生 finding;非法或未启用的附件在 -Todo 完成及主写回前被拒绝。完成路径在生命周期效果前复用原能力预检,主写回 -提交时仍再次检查;省略附件时不增加能力预检,恢复保留已提交的完成记录。 -`turn run-once` 和 `refresh-state` 都在请求的附件交付完成后才发布 quota 所需的 -写回回执。该回执缺失时,公开 quota CLI 拒绝扣额。普通 refresh 与 -`refresh-state --settle-turn` 恢复先交付已验证的原 Run,再修复回执;附件失败时 -保留同一恢复状态,不扣额。 -关闭 Yardstick 后,回执修复仍保留原已提交捕获的重放语义。 -先完成该交付和结算,再开始下一 Turn。 - ## 概览 LoopX Explore 是面向长程探索目标(软件研究、安全攻击面测绘、领域调研)的受支持、默认关闭的可选能力。它把“到处看看”变成有界、可观测、有门控的流程,包含三根支柱: @@ -136,11 +17,11 @@ LoopX Explore 是面向长程探索目标(软件研究、安全攻击面测绘 ## 快速开始 -开启门控、记录证据、投影与规划: +选择模式、记录证据、投影与规划: ```bash -loopx configure-goal --goal-id --explore-graph-enabled \ - --explore-harness-enabled --explore-harness-profile adaptive-resilient --execute +loopx configure-goal --goal-id --explore-mode planning \ + --explore-harness-profile adaptive-resilient --execute loopx explore node --goal-id --title "攻击面 A" --status exploring loopx explore edge --goal-id --from A --to B --type leads_to @@ -151,7 +32,7 @@ loopx explore graph --goal-id --graph-format mermaid --out explore.mmd loopx explore worker-branch-plan --goal-id --harness-profile adaptive-resilient --worker-width 3 ``` -两个门控相互独立且默认关闭(见下文“每 goal 独立 Opt-In 门控”)。以下是详细契约。 +Explore 默认关闭(见下文 Explore 模式配置)。以下是详细契约。 当已闭环、有证据的多个面之间存在显式理由需要组合验证时,下一次 `quota should-run` / turn packet 会投影 composition gap,并可衍生 joint experiment 后继 todo(见下文“Composition Frontier”)。 长程探索目标(例如通过 LoopX 研究某个外部软件领域的 Codex loop)产生的结果,operator 希望读成**拓扑**。 @@ -301,6 +182,64 @@ Todo 关联保留既有可恢复边界与 claim/lease 证明;恢复旧批次 复盘上下文。关联变化、任务缺失/关闭或改由他人认领时,旧页面失效,应重新读取。 未指定任务的全局读取和引用保持原行为;来源正确不证明模型已采用或效果提升。 +## 量化结论、执行观测与五阶段方法论算子 + +### 执行观测(`explore_execution_observation_v0`) + +普通结果附件与手动 finding 可附带可选 `execution_observation`:精确 `candidate_sha256`、`input_revision`、`planned_method_id` 与 `executed_method_id`(不一致时必须提供 `change_reason`)、实测耗时/上限、搜索次数、`stop_reason` 与 `1..8` 条不透明证据引用。未测量项显式记为 `null`(绝不静默转零);`input_revision` 与引用必须归属所属 finding,已释放结果的 `result_ref` 必须匹配 `result_candidate_sha256`(参见[本地探针生产路径](../../../benchmark/runtime/EXECUTION_OBSERVATIONS.md))。缺省保持对旧事件的兼容,仅记录来源证据而不认证执行或收益。开启 Yardstick 后,详情读取还通过既有 `previous_decision_followthrough` 将 confirmed `execution_observation` 关联到当前或显式前驱决策(普通结果可省略 `work_yardstick_decision`,原选择继续有效)。关联要求相同 Goal/Agent/Todo、问题、测试输入与原始提交可见工作窗口;已验证的后继历史前缀在后继时钟回拨时仍可判定此前执行是否可见,缺少此前缀的旧 capture 保留严格时间上界,迟到投递仍被排除。 + +### 量化测量与七键比较域隔离 + +Finding 与写回附件可附带 `1..8` 条 `measurements`(亦可附带 `measurement_context` 一次性声明共享的 `comparison_scope`、`input_revision` 与 `evaluation_ref`,在落盘前展开为完整记录,冲突或残缺字段直接拒绝): + +```sh +# 轮内直接记录并获取量化反馈(不结算当前 Todo): +loopx --format json explore finding --goal-id GOAL --agent-id AGENT \ + --todo-id TODO --turn-instance-id TURN --result-json result.json + +# 手动记录并复用已核实的比较上下文: +loopx explore finding --goal-id GOAL --node --evidence-ref result:b \ + --measurement-context-json conditions.json --measurements-json observations.json +``` + +比较条件与受评测候选、单次结果严格分离: + +| 字段 | 连续 finding 中的含义 | +| --- | --- | +| `input_revision` | 实际受评测的任务或数据版本;确认输入未变时保持一致。 | +| `evaluation_ref` | 评测器或计分协议版本;确认协议未变时保持一致。 | +| `candidate_ref` | 实际受评测的代码或工件版本;候选改变时更新。 | +| `evidence_ref` | 单次评测结果或回执;在 finding 的 `evidence_refs` 中保留准确来源(若声明 `execution_observation.result_ref` 则须一致)。 | + +核心不变量(由 `explore_measurements.ts` 单一 owner 维护): + +- **精确七键比较域:** 仅当 `(node_id, metric, unit, higher_is_better, comparison_scope, input_revision, evaluation_ref)` 完全一致时才聚合比较;把候选哈希填入 `input_revision` 或把提交编号填入 `evaluation_ref` 会产生单元素组(改善幅度为 `null`,表示未知而非零收益)。条件变化或未知时保持分开,严禁编造共享版本。 +- **空值语义与成对记录:** 未测量显式记为 `null`,`0` 与负数仍为有效分数。单条 finding 可在相同比较条件下记录两个不同候选的成对比较;同一候选的重复评测须写入新 finding,仅在修正误填 finding 时复用同一 finding ID(修正替换比较中的旧版本,原始记录仍保留在追加历史中)。 +- **追加顺序反馈与重放前缀冻结:** `explore turn-context`、`explore finding`、普通 `turn run-once`(通过 `agent_vision_json` 内联 `explore_result`)及开启 Yardstick 后的 `refresh-state`(`explore_result_delivery`)按源日志追加顺序计算最新值、历史最佳与方向归一化改善幅度,并通过 `--result-limit`(`1..20`,默认 `3`)、`--result-offset`、`--result-revision` 与 `--result-node` 翻页。成功结算重放冻结并仅读取首次提交前缀(关闭 Yardstick 后的回执修复仍保留原已提交重放语义)。关闭模式且省略附件时不产生 finding 也不增加能力预检,非法或未启用的附件则在 Todo 完成与主写回前被拒绝;`turn run-once` 与 `refresh-state` 均在请求附件交付完成后才发布 quota 写回回执(缺失时公开 quota CLI 拒绝扣额,`refresh-state --settle-turn` 先交付已验证原 Run 再修复回执)。尚未声明 outcome/proxy 决策时,回执补充当前 Todo 关联问题下至多两组 `work_measurements`。手动量化 finding 无需 `writeback-result` 标签即可进入结果分页与比较卡片;任何分数或 `best` 标记均不代表采纳、因果收益、验收通过或执行授权。 + +### 五阶段方法论算子与量化研究规划 + +LoopX 的六类长程决策与量化研究方法论(`work_yardstick`、`stride_review`、`scoped_yardstick_fidelity`、`budgeted_continuation`、`counterfactual_continuation` 与 `evolutionary_population`)统一构建在 [`stage_methodology_operator_catalog_v0`](../../../docs/reference/protocols/stage-methodology-operator-contract-v0.md) 单内核之上:严格布尔门控与静默轮零提示词税(含 `due_obligation_only` 在无到期义务或未触发时的静默)、canonical 重放前缀与延迟重放防污染、精确比较域元组隔离、未知值 `null` 不转零、热路径有界裁剪(附 `omitted_count` 与 `details_command`),以及按 `operator_role` 区分的运行时非提权校验(五类 `advisory_cue` 算子强制 `advisory_only: true` 与 `grants_execution_authority: false`;`stride_review` 标为 `required_due_decision`,`advisory_only: false`、`grants_execution_authority: false`,放行 Core 落盘的 `stride_decision_v2` 且同样在任意嵌套层级拒绝 `grants_execution_authority: true` 与禁止的权限字段): + +```sh +# 查看已注册的五阶段方法论算子及反熵不变量 +loopx --format json explore methodology-operators [--stage feedback_evolve] + +# 多点 Proxy vs. Outcome 秩相关(Kendall tau-b / Spearman rho)、线性校准与前沿 Goodhart 悬崖检测 +loopx --format json explore yardstick-fidelity --goal-id GOAL [--node-id NODE] [--frontier-k 3] + +# 基于剩余预算倒排权威评测确认窗口与保留当前最佳工件下的盈亏平衡成功率 p* +loopx --format json explore budgeted-plan --document budgeted_plan.json + +# 同检查点成对反事实延续评估(方向归一化 tau_B、精确符号翻转 p 值、工作区与时钟防泄漏) +loopx --format json explore counterfactual-eval --document counterfactual_pairs.json + +# 机制岛屿精英保留、正交机制交叉配对与 Goodhart 代理指标隔离(防超参微调塌缩) +loopx --format json explore evolutionary-plan --goal-id GOAL [--node-id NODE] [--elite-limit 4] +``` + +当已开启的 `work_yardstick` 在声明的权威指标与代理指标上积累 $\ge 3$ 个成对候选时,`explore turn-context` 与 Personal Workspace 的研究结论/历程界面会自动展示 `yardstick_fidelity` 校准状态(`aligned_in_tested_scope`、`frontier_bias_cliff`、`order_reversed`、`noisy_indistinguishable` 或 `mixed_or_flat`)及待权威确认的代理优胜候选。 + ## 可选 Todo 分支规划 `loopx explore todo-branch-plan` 是面向探索目标的窄 opt-in harness:一次尝试多个看起来都合理的下一个 todo。它使用 CPU 分支预测类比 + DSpark 启发调度器:对 open agent todos 排序,估计分支置信度与预期证据单位,选择置信度调度的验证前缀,选一个 `primary` 分支加安全的 `speculative` 分支,拒绝声明写作用域与已选分支重叠的分支。 @@ -636,6 +575,11 @@ loopx explore presentation --goal-id loopx explore graph --goal-id [--graph-format mermaid|json] [--out ] loopx explore todo-branch-plan --goal-id [--agent-id ] [--width 3] loopx explore worker-branch-plan --goal-id [--agent-id ] [--harness-profile generic|adaptive-resilient|moe-router] [--worker-width 3] [--max-todos-per-branch 3] [--router-state ] [--load-profile ] +loopx explore methodology-operators [--stage admission|compile_context|execute_verify|writeback_settle|feedback_evolve] +loopx explore yardstick-fidelity --goal-id [--node-id ] [--frontier-k 3] +loopx explore budgeted-plan --document +loopx explore counterfactual-eval --document +loopx explore evolutionary-plan --goal-id [--node-id ] [--elite-limit 4] [--crossover-limit 3] [--mutation-limit 3] loopx explore feishu-setup [--base-url ...] [--execute] loopx explore feishu-visual-configure [--whiteboard-token ] --docx-token [--stage-whiteboard-token ...] [--stage-capacity 10..20] [--board-style auto_flow|semantic_lane_columns] [--view-role canonical|executive] [--projection-mode canonical_filtered|issue_fix_two_lane|canonical_full|executive_auto] [--tag ] [--status ] [--execute] loopx explore feishu-sync --goal-id [--sink-visibility owner-only|shared] [--execute] diff --git a/loopx/capabilities/explore/quantitative_planning.py b/loopx/capabilities/explore/quantitative_planning.py new file mode 100644 index 0000000000..e4cfb01d89 --- /dev/null +++ b/loopx/capabilities/explore/quantitative_planning.py @@ -0,0 +1,423 @@ +"""Thin Python transport & canonical log reader for Explore quantitative planning +and the 5-stage methodology operator catalog. All math, validation, and +invariants live in TypeScript under loopx/control_plane/capabilities/.""" +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any, cast + +from ...control_plane.effect_runtime import EffectRuntimeRejected, effect_runtime_result +from ...control_plane.runtime.public_safety import validate_public_safe_value +from .result_log import ( + build_explore_result_projection, + explore_result_log_path, + load_explore_result_events_strict, + validate_explore_result_event, +) + + +def _as_dict(value: object) -> dict[str, Any]: + return value if isinstance(value, dict) else {} + + +def _as_list(value: object) -> list[Any]: + return value if isinstance(value, list) else [] + + +def _call_ts_owner( + method: str, + params: dict[str, Any], + *, + path: str, + large_local_snapshot: bool = False, +) -> dict[str, Any]: + validate_public_safe_value(params, path=path) + try: + result = effect_runtime_result( + method, + params, + large_local_snapshot=large_local_snapshot, + ) + except EffectRuntimeRejected as exc: + raise ValueError(str(exc)) from exc + return cast(dict[str, Any], result) + + +def evaluate_scoped_yardstick_fidelity(params: dict[str, Any]) -> dict[str, Any]: + return _call_ts_owner( + "explore.yardstick_fidelity.evaluate", + params, + path="yardstick_fidelity", + large_local_snapshot=True, + ) + + +def evaluate_evolutionary_population(params: dict[str, Any]) -> dict[str, Any]: + return _call_ts_owner( + "explore.evolutionary_population.evaluate", + params, + path="evolutionary_population", + large_local_snapshot=True, + ) + + +def _load_scoped_yardstick_targets( + *, + runtime_root: Path, + goal_id: str, + node_id: str | None = None, + todo_id: str | None = None, + agent_id: str | None = None, + profiles_override: dict[str, Any] | None = None, +) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]: + raw_events = load_explore_result_events_strict( + explore_result_log_path(runtime_root, goal_id), + goal_id=goal_id, + ) + validated = [ + validate_explore_result_event(event, expected_goal_id=goal_id) + for event in raw_events + ] + projection = build_explore_result_projection( + validated, + goal_id=goal_id, + finding_limit=max(1, len(validated)), + ) + findings = [ + row + for row in projection.get("findings") or [] + if isinstance(row, dict) + ] + + targets: list[dict[str, Any]] = [] + if profiles_override is not None: + if not isinstance(profiles_override, dict): + raise ValueError("--profiles-json must contain a JSON object") + resolved_node = str(node_id or profiles_override.get("node_id") or "").strip() + if not resolved_node: + raise ValueError("--node-id or profiles.node_id is required with --profiles-json") + targets.append( + { + "node_id": resolved_node, + "authoritative": profiles_override.get("authoritative"), + "proxy": profiles_override.get("proxy"), + "mechanism_by_candidate": profiles_override.get("mechanism_by_candidate"), + "source_finding_id": None, + "todo_id": todo_id, + } + ) + return findings, targets + + seen_keys: set[str] = set() + for row in findings: + capture = row.get("work_yardstick") + if not isinstance(capture, dict): + continue + row_node = str(row.get("node_id") or "").strip() + if node_id and row_node != node_id: + continue + binding = _as_dict(capture.get("binding")) + if todo_id and binding.get("todo_id") != todo_id: + continue + if agent_id and binding.get("agent_id") != agent_id: + continue + decision = capture.get("decision") + if not isinstance(decision, dict): + continue + dedupe_key = json.dumps( + [row_node, decision.get("authoritative"), decision.get("proxy")], + sort_keys=True, + ) + if dedupe_key in seen_keys: + continue + seen_keys.add(dedupe_key) + targets.append( + { + "node_id": row_node, + "authoritative": decision.get("authoritative"), + "proxy": decision.get("proxy"), + "mechanism_by_candidate": None, + "source_finding_id": row.get("finding_id"), + "todo_id": binding.get("todo_id"), + } + ) + return findings, targets + + +def build_goal_yardstick_fidelity( + *, + runtime_root: Path, + goal_id: str, + node_id: str | None = None, + todo_id: str | None = None, + agent_id: str | None = None, + frontier_k: int | None = None, + profiles_override: dict[str, Any] | None = None, +) -> dict[str, Any]: + findings, targets = _load_scoped_yardstick_targets( + runtime_root=runtime_root, + goal_id=goal_id, + node_id=node_id, + todo_id=todo_id, + agent_id=agent_id, + profiles_override=profiles_override, + ) + evaluations: list[dict[str, Any]] = [] + for target in targets: + req: dict[str, Any] = { + "node_id": target["node_id"], + "authoritative": target["authoritative"], + "proxy": target["proxy"], + "findings": [f for f in findings if f.get("node_id") == target["node_id"]], + } + if frontier_k is not None: + req["frontier_k"] = frontier_k + evaluated = evaluate_scoped_yardstick_fidelity(req) + evaluations.append( + { + **evaluated, + "source_finding_id": target["source_finding_id"], + "todo_id": target["todo_id"], + } + ) + + return { + "ok": True, + "schema_version": "loopx_explore_yardstick_fidelity_report_v0", + "goal_id": goal_id, + "evaluation_count": len(evaluations), + "evaluations": evaluations, + "advisory_only": True, + "grants_execution_authority": False, + } + + +def build_goal_evolutionary_population( + *, + runtime_root: Path, + goal_id: str, + node_id: str | None = None, + todo_id: str | None = None, + agent_id: str | None = None, + elite_limit: int | None = None, + crossover_limit: int | None = None, + mutation_limit: int | None = None, + profiles_override: dict[str, Any] | None = None, +) -> dict[str, Any]: + findings, targets = _load_scoped_yardstick_targets( + runtime_root=runtime_root, + goal_id=goal_id, + node_id=node_id, + todo_id=todo_id, + agent_id=agent_id, + profiles_override=profiles_override, + ) + plans: list[dict[str, Any]] = [] + for target in targets: + req: dict[str, Any] = { + "node_id": target["node_id"], + "authoritative": target["authoritative"], + "proxy": target["proxy"], + "findings": [f for f in findings if f.get("node_id") == target["node_id"]], + } + if elite_limit is not None: + req["elite_limit"] = elite_limit + if crossover_limit is not None: + req["crossover_limit"] = crossover_limit + if mutation_limit is not None: + req["mutation_limit"] = mutation_limit + if target.get("mechanism_by_candidate") is not None: + req["mechanism_by_candidate"] = target["mechanism_by_candidate"] + evaluated = evaluate_evolutionary_population(req) + plans.append( + { + **evaluated, + "source_finding_id": target["source_finding_id"], + "todo_id": target["todo_id"], + } + ) + + return { + "ok": True, + "schema_version": "loopx_explore_evolutionary_population_report_v0", + "goal_id": goal_id, + "plan_count": len(plans), + "plans": plans, + "advisory_only": True, + "grants_execution_authority": False, + } + + +def evaluate_budgeted_continuation_plan(document: dict[str, Any]) -> dict[str, Any]: + if not isinstance(document, dict): + raise ValueError("budgeted continuation document must be a JSON object") + result = _call_ts_owner( + "explore.budgeted_continuation.evaluate", + document, + path="budgeted_continuation", + large_local_snapshot=False, + ) + return {"ok": True, **result} + + +def evaluate_counterfactual_continuations( + document: dict[str, Any], + *, + strict_identification: bool | None = None, +) -> dict[str, Any]: + if not isinstance(document, dict): + raise ValueError("counterfactual continuation document must be a JSON object") + req = dict(document) + if strict_identification is not None: + req["strict_identification"] = bool(strict_identification) + result = _call_ts_owner( + "explore.counterfactual_continuation.evaluate", + req, + path="counterfactual_continuation", + large_local_snapshot=True, + ) + return {"ok": True, **result} + + +def list_stage_methodology_operators(*, stage: str | None = None) -> dict[str, Any]: + params: dict[str, Any] = {} + if stage is not None: + params["stage"] = stage + result = _call_ts_owner( + "methodology.operators.catalog", + params, + path="methodology_operators", + large_local_snapshot=False, + ) + return {"ok": True, **result} + + +def verify_methodology_operator_output( + *, + operator_id: str, + output: dict[str, Any], + max_bytes: int | None = None, +) -> dict[str, Any]: + params: dict[str, Any] = {"operator_id": operator_id, "output": output} + if max_bytes is not None: + params["max_bytes"] = max_bytes + result = _call_ts_owner( + "methodology.operator.verify_output", + params, + path="methodology_verification", + large_local_snapshot=False, + ) + return {"ok": True, **result} + + +def render_quantitative_planning_markdown(payload: dict[str, object]) -> str | None: + schema = str(payload.get("schema_version") or "") + if schema == "loopx_explore_yardstick_fidelity_report_v0": + rows = _as_list(payload.get("evaluations")) + lines = [ + f"# Scoped Yardstick Fidelity: {payload.get('goal_id')}", + "", + f"- Evaluations: {len(rows)} (`advisory_only=true`)", + ] + for item in rows: + if not isinstance(item, dict): + continue + counts = _as_dict(item.get("counts")) + rc = _as_dict(item.get("rank_concordance")) + fn = _as_dict(item.get("frontier_neighborhood")) + lines.extend( + [ + "", + f"## Node `{item.get('node_id')}` -> `{item.get('fidelity_status')}`", + f"- Paired candidates: {counts.get('paired_candidates', 0)} " + f"(proxy-only: {counts.get('proxy_only_candidates', 0)}, " + f"authoritative-only: {counts.get('authoritative_only_candidates', 0)})", + f"- Kendall tau-b: {rc.get('kendall_tau_b')} | Frontier tau-b: {fn.get('frontier_kendall_tau_b')}", + ] + ) + lines.append("") + return "\n".join(lines) + + if schema == "budgeted_continuation_plan_v0": + conf = _as_dict(payload.get("confirmation")) + routes = _as_list(payload.get("candidate_routes")) + lines = [ + "# Budgeted Continuation Plan", + "", + f"- Latest confirmation start minute: `{conf.get('latest_confirmation_start_minute')}`", + f"- Remaining work window minutes: `{conf.get('remaining_work_window_minutes')}`", + f"- Effective safe baseline gain: `{payload.get('effective_safe_gain')}`", + "", + "## Candidate Routes", + "", + ] + for route in routes: + if isinstance(route, dict): + lines.append( + f"- `{route.get('route_id')}` (`{route.get('feasibility_status')}` -> " + f"`{route.get('recommended_disposition')}`): slack={route.get('slack_before_deadline_minutes')}m, " + f"p*={route.get('p_star_at_expected_gain')}" + ) + lines.append("") + return "\n".join(lines) + + if schema == "counterfactual_continuation_eval_v0": + counts = _as_dict(payload.get("counts")) + est = _as_dict(payload.get("estimand_summary")) + lines = [ + "# Same-Checkpoint Counterfactual Continuation Evaluation", + "", + f"- Pairs: {counts.get('valid_pairs', 0)} valid / {counts.get('total_pairs', 0)} total " + f"(contaminated: {counts.get('contaminated_pairs', 0)})", + f"- Action diverged: {counts.get('action_diverged_pairs', 0)} | " + f"Wins/Ties/Losses: {counts.get('wins', 0)}/{counts.get('ties', 0)}/{counts.get('losses', 0)}", + f"- Mean tau_B: `{est.get('mean_tau_b')}` | Sign-flip p-value: `{est.get('exact_sign_flip_p_value')}`", + "", + ] + return "\n".join(lines) + + if schema == "loopx_explore_evolutionary_population_report_v0": + rows = _as_list(payload.get("plans")) + lines = [ + f"# Evolutionary Population Search Plan: {payload.get('goal_id')}", + "", + f"- Plans: {len(rows)} (`advisory_only=true`)", + ] + for item in rows: + if not isinstance(item, dict): + continue + counts = _as_dict(item.get("counts")) + lines.extend( + [ + "", + f"## Node `{item.get('node_id')}` -> `{item.get('selection_status')}` " + f"(proxy=`{item.get('proxy_fidelity_status')}`)", + f"- Confirmed candidates: {counts.get('confirmed_candidates', 0)} across " + f"{counts.get('mechanism_islands', 0)} mechanism islands", + f"- Elites: {counts.get('selected_elites', 0)} | " + f"Crossovers: {counts.get('crossover_pairs', 0)} | " + f"Mutations: {counts.get('mutation_targets', 0)} | " + f"Quarantined proxy winners: {counts.get('quarantined_proxy_candidates', 0)}", + ] + ) + lines.append("") + return "\n".join(lines) + + if schema == "stage_methodology_operator_catalog_v0": + ops = _as_list(payload.get("operators")) + lines = [ + "# 5-Stage Methodology Operator Catalog", + "", + ] + for op in ops: + if isinstance(op, dict): + lines.append( + f"- `{op.get('operator_id')}` (role=`{op.get('operator_role')}`, " + f"stage=`{op.get('stage')}`, gate=`{op.get('enabled_gate')}`, " + f"quiet=`{op.get('quiet_turn_policy')}`): {op.get('lightweight_guidance')}" + ) + lines.append("") + return "\n".join(lines) + + return None diff --git a/loopx/cli_commands/explore.py b/loopx/cli_commands/explore.py index e5e938b46e..a53e80ab57 100644 --- a/loopx/cli_commands/explore.py +++ b/loopx/cli_commands/explore.py @@ -61,7 +61,16 @@ handle_explore_feishu_command, register_explore_feishu_commands, ) -from .explore_planning_commands import register_explore_planning_commands +from ..capabilities.explore.quantitative_planning import ( + build_goal_evolutionary_population, + build_goal_yardstick_fidelity, + render_quantitative_planning_markdown, +) +from .explore_planning_commands import ( + STATELESS_EXPLORE_PLANNING_COMMANDS, + handle_stateless_explore_planning_command, + register_explore_planning_commands, +) PrintPayload = Callable[ @@ -347,6 +356,10 @@ def _tree_lines(tree: object, *, indent: int = 0) -> list[str]: def render_explore_markdown(payload: dict[str, object]) -> str: + if payload.get("ok"): + rendered = render_quantitative_planning_markdown(payload) + if rendered is not None: + return rendered if "graph_enabled" in payload and "harness_enabled" in payload: return "# Explore Harness turn context\n\n```json\n" + json.dumps(payload, ensure_ascii=False, indent=2) + "\n```\n" lines = ["# LoopX Explore", ""] @@ -529,6 +542,10 @@ def handle_explore_command( fmt = output_format(args) activation: dict[str, object] | None = None try: + if args.explore_command in STATELESS_EXPLORE_PLANNING_COMMANDS: + payload = handle_stateless_explore_planning_command(args) + print_payload(payload, fmt, render_explore_markdown) + return 0 if payload.get("ok") else 1 registry = load_registry(registry_path) runtime_root = resolve_runtime_root(registry, runtime_root_arg) if args.explore_command in ( @@ -818,6 +835,38 @@ def handle_explore_command( "state_file": todo_payload.get("state_file"), "todo_count": todo_payload.get("todo_count"), } + elif args.explore_command == "yardstick-fidelity": + profiles_override = ( + json.loads(Path(args.profiles_json).expanduser().read_text(encoding="utf-8-sig")) + if getattr(args, "profiles_json", None) + else None + ) + payload = build_goal_yardstick_fidelity( + runtime_root=runtime_root, + goal_id=args.goal_id, + node_id=args.node_id, + todo_id=args.todo_id, + agent_id=args.agent_id, + frontier_k=args.frontier_k, + profiles_override=profiles_override, + ) + elif args.explore_command == "evolutionary-plan": + profiles_override = ( + json.loads(Path(args.profiles_json).expanduser().read_text(encoding="utf-8-sig")) + if getattr(args, "profiles_json", None) + else None + ) + payload = build_goal_evolutionary_population( + runtime_root=runtime_root, + goal_id=args.goal_id, + node_id=args.node_id, + todo_id=args.todo_id, + agent_id=args.agent_id, + elite_limit=args.elite_limit, + crossover_limit=args.crossover_limit, + mutation_limit=args.mutation_limit, + profiles_override=profiles_override, + ) elif args.explore_command in FEISHU_SINK_EXPLORE_COMMANDS: payload = handle_explore_feishu_command( args, diff --git a/loopx/cli_commands/explore_planning_commands.py b/loopx/cli_commands/explore_planning_commands.py index 1237a2887b..16f754c865 100644 --- a/loopx/cli_commands/explore_planning_commands.py +++ b/loopx/cli_commands/explore_planning_commands.py @@ -1,13 +1,28 @@ from __future__ import annotations import argparse -from typing import Callable +import json +from pathlib import Path +from typing import Any, Callable +from ..capabilities.explore.quantitative_planning import ( + evaluate_budgeted_continuation_plan, + evaluate_counterfactual_continuations, + list_stage_methodology_operators, +) from ..capabilities.explore.worker_branch_plan import ( DEFAULT_WORKER_HARNESS_PROFILE, worker_harness_profile_names, ) +STATELESS_EXPLORE_PLANNING_COMMANDS = frozenset( + { + "budgeted-plan", + "counterfactual-eval", + "methodology-operators", + } +) + def _add_resource_portfolio_args(parser: argparse.ArgumentParser) -> None: parser.add_argument( @@ -155,3 +170,110 @@ def register_explore_planning_commands( "calibrate the lane-admission load factor from measurements." ), ) + + yardstick_fidelity = subparsers.add_parser( + "yardstick-fidelity", + help=( + "Evaluate scoped multi-point Yardstick rank concordance, frontier bias cliffs, " + "and unconfirmed proxy winners from recorded Explore findings. Advisory only." + ), + ) + add_subcommand_format(yardstick_fidelity) + yardstick_fidelity.add_argument("--goal-id", required=True) + yardstick_fidelity.add_argument("--node-id", help="Filter to one Explore question node.") + yardstick_fidelity.add_argument("--todo-id", help="Filter declared Yardsticks to one Todo.") + yardstick_fidelity.add_argument("--agent-id", help="Filter declared Yardsticks to one Agent.") + yardstick_fidelity.add_argument( + "--frontier-k", + type=int, + help="Requested top-K cutoff (at least 2). Includes candidates tied at the cutoff.", + ) + yardstick_fidelity.add_argument( + "--profiles-json", + help="Optional JSON file specifying explicit {authoritative, proxy, node_id} profiles.", + ) + + evolutionary_plan = subparsers.add_parser( + "evolutionary-plan", + help=( + "Select mechanism-diverse evolutionary elites, orthogonal crossover pairs, " + "and bottleneck mutations while quarantining Goodharted proxy winners. Advisory only." + ), + ) + add_subcommand_format(evolutionary_plan) + evolutionary_plan.add_argument("--goal-id", required=True) + evolutionary_plan.add_argument("--node-id", help="Filter to one Explore question node.") + evolutionary_plan.add_argument("--todo-id", help="Filter declared Yardsticks to one Todo.") + evolutionary_plan.add_argument("--agent-id", help="Filter declared Yardsticks to one Agent.") + evolutionary_plan.add_argument("--elite-limit", type=int, help="Maximum confirmed elites (2..12).") + evolutionary_plan.add_argument("--crossover-limit", type=int, help="Maximum orthogonal crossover pairs (0..6).") + evolutionary_plan.add_argument("--mutation-limit", type=int, help="Maximum targeted mutations (0..6).") + evolutionary_plan.add_argument( + "--profiles-json", + help="Optional JSON file specifying explicit {authoritative, proxy, node_id, mechanism_by_candidate}.", + ) + + budgeted_plan = subparsers.add_parser( + "budgeted-plan", + help=( + "Evaluate confirmation-deadline back-scheduling, reversible probe slack, and " + "break-even thresholds across candidate continuation routes. Advisory only." + ), + ) + add_subcommand_format(budgeted_plan) + budgeted_plan.add_argument( + "--document", + required=True, + help="Path to a budgeted_continuation_plan_v0 request JSON file.", + ) + + counterfactual_eval = subparsers.add_parser( + "counterfactual-eval", + help=( + "Evaluate same-checkpoint paired counterfactual continuations (tau_B, " + "identification/non-interference guards, action divergence, complete marginal costs)." + ), + ) + add_subcommand_format(counterfactual_eval) + counterfactual_eval.add_argument( + "--document", + required=True, + help="Path to a counterfactual_continuation_eval_v0 request JSON file.", + ) + counterfactual_eval.add_argument( + "--strict-identification", + action=argparse.BooleanOptionalAction, + default=None, + help="Fail closed on checkpoint/isolation contamination (default true) or audit with --no-strict-identification.", + ) + + methodology_ops = subparsers.add_parser( + "methodology-operators", + help="List registered 5-stage methodology operators, enablement gates, and anti-entropy invariants.", + ) + add_subcommand_format(methodology_ops) + methodology_ops.add_argument( + "--stage", + choices=["admission", "compile_context", "execute_verify", "writeback_settle", "feedback_evolve"], + help="Filter operators participating in one control-loop stage.", + ) + + +def handle_stateless_explore_planning_command(args: argparse.Namespace) -> dict[str, Any]: + cmd = str(args.explore_command) + if cmd == "budgeted-plan": + doc = json.loads(Path(args.document).read_text(encoding="utf-8-sig")) + if not isinstance(doc, dict): + raise ValueError("budgeted-plan --document must contain a JSON object") + return evaluate_budgeted_continuation_plan(doc) + if cmd == "counterfactual-eval": + doc = json.loads(Path(args.document).read_text(encoding="utf-8-sig")) + if not isinstance(doc, dict): + raise ValueError("counterfactual-eval --document must contain a JSON object") + return evaluate_counterfactual_continuations( + doc, + strict_identification=args.strict_identification, + ) + if cmd == "methodology-operators": + return list_stage_methodology_operators(stage=getattr(args, "stage", None)) + raise ValueError(f"unsupported stateless explore command: {cmd}") diff --git a/loopx/control_plane/capabilities/budgeted_continuation.ts b/loopx/control_plane/capabilities/budgeted_continuation.ts new file mode 100644 index 0000000000..54acb71b1e --- /dev/null +++ b/loopx/control_plane/capabilities/budgeted_continuation.ts @@ -0,0 +1,391 @@ +/** Budgeted continuation & confirmation-deadline back-scheduling owner. + * Advisory research planning only: computes confirmation-deadline feasibility, + * reversible probe slack, and break-even thresholds without granting authority + * or coercing unknown resource bounds to zero. */ +import {EffectRuntimeRequestError} from "../effect_runtime_errors.ts"; +import {type JsonObject} from "../runtime_decode.ts"; +import { + assertMethodologyOperatorOutput, + optionalFiniteNumber, + requireBoundedSingleLine as singleLine, + requireExactFields as exactObject, + requireOpaqueMethodologyRef as opaqueId, + roundMethodologyMetric as round6, +} from "./stage_methodology_operator.ts"; + +export const BUDGETED_CONTINUATION_PLAN_SCHEMA = "budgeted_continuation_plan_v0"; +export const CONTINUATION_ROUTE_KINDS = [ + "continue_current", + "switch_mechanism", + "compatible_combination", + "bounded_probe", +] as const; +export type ContinuationRouteKind = (typeof CONTINUATION_ROUTE_KINDS)[number]; + +export const ROUTE_FEASIBILITY_STATUSES = [ + "feasible", + "feasible_with_unknown_bounds", + "deadline_infeasible", + "token_infeasible", + "eval_slot_infeasible", +] as const; +export type RouteFeasibilityStatus = (typeof ROUTE_FEASIBILITY_STATUSES)[number]; + +export const ROUTE_DISPOSITIONS = [ + "reject_infeasible", + "dominated_by_baseline", + "probe_then_decide", + "viable_direct_continuation", + "upside_tail_only", +] as const; +export type RouteDisposition = (typeof ROUTE_DISPOSITIONS)[number]; + +export interface EvaluatedContinuationRoute { + route_id: string; + summary: string; + kind: ContinuationRouteKind; + probe_minutes: number; + implementation_minutes: number; + confirmation_minutes: number; + total_wall_minutes: number; + slack_before_deadline_minutes: number; + total_expected_tokens: number | null; + requires_authoritative_eval: boolean; + feasibility_status: RouteFeasibilityStatus; + wall_feasible: boolean; + token_feasible: boolean | null; + eval_slot_feasible: boolean | null; + unknown_cost_dimensions: string[]; + reversible_probe_window_feasible: boolean; + expected_gain_min: number | null; + expected_gain_expected: number | null; + expected_gain_max: number | null; + dominated_by_baseline: boolean; + p_star_at_expected_gain: number | null; + p_star_optimistic: number | null; + recommended_disposition: RouteDisposition; +} + +export interface BudgetedContinuationPlan { + schema_version: typeof BUDGETED_CONTINUATION_PLAN_SCHEMA; + remaining_budget: { + wall_minutes: number; + token_budget: number | null; + authoritative_eval_slots: number | null; + }; + confirmation: { + eval_duration_minutes: number; + cooldown_minutes: number; + eval_tokens: number | null; + pending_in_flight_minutes: number; + latest_confirmation_start_minute: number; + remaining_work_window_minutes: number; + }; + incumbent: { + candidate_ref: string | null; + value: number | null; + higher_is_better: boolean; + metric: string; + unit: string; + }; + baseline_route: EvaluatedContinuationRoute; + effective_safe_gain: number | null; + candidate_routes: EvaluatedContinuationRoute[]; + counts: { + total_candidates: number; + feasible_candidates: number; + deadline_infeasible_candidates: number; + reversible_probe_candidates: number; + }; + advisory_only: true; + grants_execution_authority: false; +} + +function nonNegativeNumber(value: unknown, label: string, positive = false): number { + if (typeof value !== "number" || !Number.isFinite(value) || (positive ? value <= 0 : value < 0)) { + throw new EffectRuntimeRequestError(`${label} requires a finite ${positive ? "positive" : "non-negative"} number`); + } + return value; +} + +const nullableNonNegativeNumber = (value: unknown, label: string): number | null => + optionalFiniteNumber(value, label, {nonNegative: true}); + +function nullableNonNegativeInt(value: unknown, label: string): number | null { + if (value === null || value === undefined) return null; + if (typeof value !== "number" || !Number.isSafeInteger(value) || value < 0) { + throw new EffectRuntimeRequestError(`${label} requires a non-negative integer or null`); + } + return value; +} + +function roundNullableGain(value: number | null): number | null { + return value === null ? null : round6(value); +} + +function breakEvenProbability(reference: number, floor: number, target: number | null): number | null { + if (target === null) return null; + if (reference <= floor) return 0; + if (target < reference) return null; + return round6((reference - floor) / (target - floor)); +} + +interface NormalizedRouteInput { + route_id: string; + summary: string; + kind: ContinuationRouteKind; + probe_minutes: number; + implementation_minutes: number; + expected_tokens: number | null; + requires_authoritative_eval: boolean; + expected_gain_min: number | null; + expected_gain_expected: number | null; + expected_gain_max: number | null; +} + +function normalizeRoute(value: unknown, label: string, defaultKind?: ContinuationRouteKind): NormalizedRouteInput { + const allowed = [ + "route_id", "summary", "kind", "probe_minutes", "implementation_minutes", + "expected_tokens", "requires_authoritative_eval", + "expected_gain_min", "expected_gain_expected", "expected_gain_max", + ]; + const required = [ + "route_id", "summary", "probe_minutes", "implementation_minutes", + "requires_authoritative_eval", "expected_gain_min", "expected_gain_expected", "expected_gain_max", + ]; + const row = exactObject(value, allowed, defaultKind ? required : [...required, "kind"], label); + const kindRaw = row.kind === undefined ? defaultKind : row.kind; + if (typeof kindRaw !== "string" || !(CONTINUATION_ROUTE_KINDS as readonly string[]).includes(kindRaw)) { + throw new EffectRuntimeRequestError(`${label} kind is unsupported`); + } + if (typeof row.requires_authoritative_eval !== "boolean") { + throw new EffectRuntimeRequestError(`${label} requires explicit requires_authoritative_eval boolean`); + } + const gMin = optionalFiniteNumber(row.expected_gain_min, `${label} expected_gain_min`); + const gExp = optionalFiniteNumber(row.expected_gain_expected, `${label} expected_gain_expected`); + const gMax = optionalFiniteNumber(row.expected_gain_max, `${label} expected_gain_max`); + const knownGains = [gMin, gExp, gMax].filter(gain => gain !== null); + if (knownGains.some((gain, index) => index > 0 && gain < knownGains[index - 1]!)) { + throw new EffectRuntimeRequestError(`${label} requires expected_gain_min <= expected_gain_expected <= expected_gain_max`); + } + return { + route_id: opaqueId(row.route_id, `${label} route_id`), + summary: singleLine(row.summary, `${label} summary`, 240), + kind: kindRaw as ContinuationRouteKind, + probe_minutes: nonNegativeNumber(row.probe_minutes, `${label} probe_minutes`), + implementation_minutes: nonNegativeNumber(row.implementation_minutes, `${label} implementation_minutes`), + expected_tokens: nullableNonNegativeNumber(row.expected_tokens, `${label} expected_tokens`), + requires_authoritative_eval: row.requires_authoritative_eval, + expected_gain_min: gMin, + expected_gain_expected: gExp, + expected_gain_max: gMax, + }; +} + +export function evaluateBudgetedContinuationPlan(params: JsonObject): BudgetedContinuationPlan { + const root = exactObject( + params, + ["schema_version", "remaining_budget", "confirmation", "incumbent", "baseline_route", "candidate_routes"], + ["remaining_budget", "confirmation", "incumbent", "baseline_route", "candidate_routes"], + "budgeted continuation request", + ); + if (root.schema_version !== undefined && root.schema_version !== BUDGETED_CONTINUATION_PLAN_SCHEMA) { + throw new EffectRuntimeRequestError("unsupported budgeted continuation schema"); + } + + const remRaw = exactObject( + root.remaining_budget, + ["wall_minutes", "token_budget", "authoritative_eval_slots"], + ["wall_minutes"], + "remaining_budget", + ); + const wallMinutes = nonNegativeNumber(remRaw.wall_minutes, "remaining_budget.wall_minutes", true); + const tokenBudget = nullableNonNegativeNumber(remRaw.token_budget, "remaining_budget.token_budget"); + const evalSlots = nullableNonNegativeInt(remRaw.authoritative_eval_slots, "remaining_budget.authoritative_eval_slots"); + + const confRaw = exactObject( + root.confirmation, + ["eval_duration_minutes", "cooldown_minutes", "eval_tokens", "pending_in_flight_minutes"], + ["eval_duration_minutes"], + "confirmation", + ); + const evalDuration = nonNegativeNumber(confRaw.eval_duration_minutes, "confirmation.eval_duration_minutes"); + const cooldown = confRaw.cooldown_minutes === undefined ? 0 : nonNegativeNumber(confRaw.cooldown_minutes, "confirmation.cooldown_minutes"); + const evalTokens = nullableNonNegativeNumber(confRaw.eval_tokens, "confirmation.eval_tokens"); + const inFlight = confRaw.pending_in_flight_minutes === undefined ? 0 : nonNegativeNumber(confRaw.pending_in_flight_minutes, "confirmation.pending_in_flight_minutes"); + + const incRaw = exactObject( + root.incumbent, + ["candidate_ref", "value", "higher_is_better", "metric", "unit"], + ["candidate_ref", "value", "higher_is_better", "metric", "unit"], + "incumbent", + ); + if (typeof incRaw.higher_is_better !== "boolean") { + throw new EffectRuntimeRequestError("incumbent requires explicit higher_is_better boolean"); + } + if (incRaw.value !== null && (typeof incRaw.value !== "number" || !Number.isFinite(incRaw.value))) { + throw new EffectRuntimeRequestError("incumbent value must be a finite number or null"); + } + const incumbent = { + candidate_ref: incRaw.candidate_ref === null ? null : opaqueId(incRaw.candidate_ref, "incumbent.candidate_ref"), + value: incRaw.value as number | null, + higher_is_better: incRaw.higher_is_better, + metric: singleLine(incRaw.metric, "incumbent.metric", 80), + unit: singleLine(incRaw.unit, "incumbent.unit", 40), + }; + + const baselineInput = normalizeRoute(root.baseline_route, "baseline_route", "continue_current"); + if (!Array.isArray(root.candidate_routes) || root.candidate_routes.length < 1 || root.candidate_routes.length > 16) { + throw new EffectRuntimeRequestError("candidate_routes requires 1 to 16 routes"); + } + const candidateInputs = root.candidate_routes.map((r, i) => normalizeRoute(r, `candidate_routes[${i}]`)); + const seenIds = new Set([baselineInput.route_id]); + for (const c of candidateInputs) { + if (seenIds.has(c.route_id)) { + throw new EffectRuntimeRequestError(`duplicate route_id: ${c.route_id}`); + } + seenIds.add(c.route_id); + } + + const latestConfirmStart = round6(wallMinutes - evalDuration); + const remainingWorkWindow = round6(latestConfirmStart - (cooldown > latestConfirmStart + ? Math.max(inFlight, cooldown) : inFlight)); + + const confirmationMinutes = (route: NormalizedRouteInput, readyAt: number) => + route.requires_authoritative_eval ? Math.max(0, cooldown - readyAt) + evalDuration : 0; + + const evaluateOne = ( + route: NormalizedRouteInput, safeGain: number | null, baselineFeasible = false, + ): EvaluatedContinuationRoute => { + const readyAt = inFlight + route.probe_minutes + route.implementation_minutes; + const confirmMinutes = confirmationMinutes(route, readyAt); + const totalWall = round6(readyAt + confirmMinutes); + const slack = round6(wallMinutes - totalWall); + const wallFeasible = slack >= 0; + + const unknownDims: string[] = []; + const confirmationTokens = route.requires_authoritative_eval ? evalTokens : 0; + const knownTokenFloor = (route.expected_tokens ?? 0) + (confirmationTokens ?? 0); + let totalTokens: number | null = null; + if (route.expected_tokens === null || confirmationTokens === null) { + unknownDims.push("tokens"); + } else { + totalTokens = round6(knownTokenFloor); + } + let tokenFeasible: boolean | null = null; + if (tokenBudget === null) { + if (!unknownDims.includes("tokens")) unknownDims.push("token_budget"); + } else if (knownTokenFloor > tokenBudget) { + tokenFeasible = false; + } else if (totalTokens !== null) { + tokenFeasible = true; + } + + let slotFeasible: boolean | null = null; + if (!route.requires_authoritative_eval) { + slotFeasible = true; + } else if (evalSlots === null) { + unknownDims.push("authoritative_eval_slots"); + } else { + slotFeasible = evalSlots >= 1; + } + + let feasibilityStatus: RouteFeasibilityStatus; + if (!wallFeasible) feasibilityStatus = "deadline_infeasible"; + else if (slotFeasible === false) feasibilityStatus = "eval_slot_infeasible"; + else if (tokenFeasible === false) feasibilityStatus = "token_infeasible"; + else if (unknownDims.length > 0) feasibilityStatus = "feasible_with_unknown_bounds"; + else feasibilityStatus = "feasible"; + + const feasible = feasibilityStatus === "feasible" || feasibilityStatus === "feasible_with_unknown_bounds"; + const fallbackReadyAt = inFlight + route.probe_minutes + baselineInput.probe_minutes + baselineInput.implementation_minutes; + const fallbackFinishesAt = fallbackReadyAt + confirmationMinutes(baselineInput, fallbackReadyAt); + const reversibleProbe = route.probe_minutes > 0 && feasible && baselineFeasible + && round6(fallbackFinishesAt) <= wallMinutes; + + const dominated = feasible && safeGain !== null && route.expected_gain_max !== null + && route.expected_gain_max <= safeGain; + let pStarExp: number | null = null; + let pStarOpt: number | null = null; + if (feasible && safeGain !== null && !dominated) { + const retainedFloor = Math.max(0, route.expected_gain_min ?? 0); + pStarExp = breakEvenProbability(safeGain, retainedFloor, route.expected_gain_expected); + pStarOpt = breakEvenProbability(safeGain, retainedFloor, route.expected_gain_max); + } + + let disposition: RouteDisposition; + if (!feasible) disposition = "reject_infeasible"; + else if (dominated) disposition = "dominated_by_baseline"; + else if (reversibleProbe) disposition = "probe_then_decide"; + else if (safeGain === null || route.expected_gain_expected === null || route.expected_gain_max === null + || pStarExp !== null) disposition = "viable_direct_continuation"; + else disposition = "upside_tail_only"; + + return { + route_id: route.route_id, + summary: route.summary, + kind: route.kind, + probe_minutes: round6(route.probe_minutes), + implementation_minutes: round6(route.implementation_minutes), + confirmation_minutes: round6(confirmMinutes), + total_wall_minutes: totalWall, + slack_before_deadline_minutes: slack, + total_expected_tokens: totalTokens, + requires_authoritative_eval: route.requires_authoritative_eval, + feasibility_status: feasibilityStatus, + wall_feasible: wallFeasible, + token_feasible: tokenFeasible, + eval_slot_feasible: slotFeasible, + unknown_cost_dimensions: unknownDims, + reversible_probe_window_feasible: reversibleProbe, + expected_gain_min: roundNullableGain(route.expected_gain_min), + expected_gain_expected: roundNullableGain(route.expected_gain_expected), + expected_gain_max: roundNullableGain(route.expected_gain_max), + dominated_by_baseline: dominated, + p_star_at_expected_gain: pStarExp, + p_star_optimistic: pStarOpt, + recommended_disposition: disposition, + }; + }; + + const evaluatedBaseline = evaluateOne(baselineInput, null); + const baselineFeasible = evaluatedBaseline.feasibility_status === "feasible" + || evaluatedBaseline.feasibility_status === "feasible_with_unknown_bounds"; + const effectiveSafeGain = incumbent.value === null ? null + : Math.max(0, evaluatedBaseline.feasibility_status === "feasible" ? baselineInput.expected_gain_expected ?? 0 : 0); + + const evaluatedCandidates = candidateInputs.map(c => evaluateOne(c, effectiveSafeGain, baselineFeasible)); + + return assertMethodologyOperatorOutput("budgeted_continuation", { + schema_version: BUDGETED_CONTINUATION_PLAN_SCHEMA, + remaining_budget: { + wall_minutes: round6(wallMinutes), + token_budget: tokenBudget === null ? null : round6(tokenBudget), + authoritative_eval_slots: evalSlots, + }, + confirmation: { + eval_duration_minutes: round6(evalDuration), + cooldown_minutes: round6(cooldown), + eval_tokens: evalTokens === null ? null : round6(evalTokens), + pending_in_flight_minutes: round6(inFlight), + latest_confirmation_start_minute: latestConfirmStart, + remaining_work_window_minutes: remainingWorkWindow, + }, + incumbent, + baseline_route: evaluatedBaseline, + effective_safe_gain: roundNullableGain(effectiveSafeGain), + candidate_routes: evaluatedCandidates, + counts: { + total_candidates: evaluatedCandidates.length, + feasible_candidates: evaluatedCandidates.filter( + c => c.feasibility_status === "feasible" || c.feasibility_status === "feasible_with_unknown_bounds", + ).length, + deadline_infeasible_candidates: evaluatedCandidates.filter( + c => c.feasibility_status === "deadline_infeasible", + ).length, + reversible_probe_candidates: evaluatedCandidates.filter(c => c.reversible_probe_window_feasible).length, + }, + advisory_only: true, + grants_execution_authority: false, + } as unknown as JsonObject) as unknown as BudgetedContinuationPlan; +} diff --git a/loopx/control_plane/capabilities/counterfactual_continuation.ts b/loopx/control_plane/capabilities/counterfactual_continuation.ts new file mode 100644 index 0000000000..ba40f4eeab --- /dev/null +++ b/loopx/control_plane/capabilities/counterfactual_continuation.ts @@ -0,0 +1,526 @@ +/** Same-checkpoint counterfactual continuation pair evaluator. + * Enforces checkpoint lineage, workspace non-interference, and future-leakage + * guards; computes direction-normalized tau_B, action-adoption trace, and + * complete marginal costs while preserving unknown (null) cost dimensions. */ +import {EffectRuntimeRequestError} from "../effect_runtime_errors.ts"; +import {type JsonObject} from "../runtime_decode.ts"; +import {parseTodoTimestampMicros} from "../runtime_timestamp.ts"; +import { + assertMethodologyOperatorOutput, + isValidContentSha256, + optionalFiniteNumber, + requireBoundedSingleLine as singleLine, + requireExactFields as exactObject, + requireOpaqueMethodologyRef as opaqueRef, + roundMethodologyMetric as round6, +} from "./stage_methodology_operator.ts"; + +export const COUNTERFACTUAL_CONTINUATION_EVAL_SCHEMA = "counterfactual_continuation_eval_v0"; +export const COUNTERFACTUAL_COST_DIMENSIONS = [ + "wall_seconds", + "total_tokens", + "evaluator_runs", + "control_plane_requests", + "recovery_events", +] as const; +export type CounterfactualCostDimension = (typeof COUNTERFACTUAL_COST_DIMENSIONS)[number]; + +export const COUNTERFACTUAL_PAIR_OUTCOMES = [ + "diverged_with_gain", + "diverged_no_gain", + "diverged_regressed", + "action_identical", + "action_unobserved", + "unscored_or_incomplete", + "contaminated_excluded", +] as const; +export type CounterfactualPairOutcome = (typeof COUNTERFACTUAL_PAIR_OUTCOMES)[number]; + +export type CounterfactualCostVector = Record; + +type ComparisonScope = { + metric: string; + unit: string; + higher_is_better: boolean; + comparison_scope: string; + input_revision: string; + evaluation_ref: string; + terminal_boundary_ref: string; +}; + +export interface EvaluatedCounterfactualPair { + pair_id: string; + task_id: string; + checkpoint_id: string; + scope: ComparisonScope; + identification_valid: boolean; + identification_violations: string[]; + outcome_classification: CounterfactualPairOutcome; + action_diverged: boolean | null; + treatment_arm_id: string; + control_arm_id: string; + treatment_terminal_value: number | null; + control_terminal_value: number | null; + raw_terminal_delta: number | null; + tau_b_direction_normalized: number | null; + treatment_gain_over_incumbent: number | null; + control_gain_over_incumbent: number | null; + marginal_cost_delta: CounterfactualCostVector; + unknown_cost_dimensions: CounterfactualCostDimension[]; +} + +export interface CounterfactualContinuationEvaluation { + schema_version: typeof COUNTERFACTUAL_CONTINUATION_EVAL_SCHEMA; + strict_identification: boolean; + all_pairs_identification_valid: boolean; + counts: { + total_pairs: number; + valid_pairs: number; + contaminated_pairs: number; + scored_valid_pairs: number; + unscored_valid_pairs: number; + action_diverged_pairs: number; + action_identical_pairs: number; + action_unobserved_pairs: number; + wins: number; + ties: number; + losses: number; + }; + estimand_summary: { + comparable_scope_count: number; + mean_tau_b: number | null; + median_tau_b: number | null; + min_tau_b: number | null; + max_tau_b: number | null; + exact_sign_flip_p_value: number | null; + binomial_sign_p_value: number | null; + }; + complete_marginal_costs: { + observed_delta_sums: Record; + observed_pair_counts: Record; + missing_pair_counts: Record; + }; + by_task: Array<{ + task_id: string; + scope: ComparisonScope; + valid_pairs: number; + scored_pairs: number; + mean_tau_b: number | null; + wins: number; + ties: number; + losses: number; + }>; + pairs: EvaluatedCounterfactualPair[]; + advisory_only: true; + grants_execution_authority: false; +} + +function sha256Hex(value: unknown, label: string): string { + const text = singleLine(value, label, 64); + if (!isValidContentSha256(text)) { + throw new EffectRuntimeRequestError(`${label} requires a 64-character lowercase hex SHA-256 digest`); + } + return text; +} + +const nullableFinite = (value: unknown, label: string, nonNegative = false): number | null => + optionalFiniteNumber(value, label, {nonNegative}); + +function median(values: readonly number[]): number { + const sorted = [...values].sort((a, b) => a - b); + const mid = Math.floor(sorted.length / 2); + return sorted.length % 2 === 1 ? sorted[mid]! : (sorted[mid - 1]! + sorted[mid]!) / 2; +} + +function parseOptionalTimestamp(value: unknown, label: string): bigint | null { + if (value === null || value === undefined) return null; + const text = singleLine(value, label, 64); + const parsed = parseTodoTimestampMicros(text); + if (parsed === null) { + throw new EffectRuntimeRequestError(`${label} requires an explicit ISO-8601 UTC/offset timestamp`); + } + return parsed; +} + +function normalizeCostVector(value: unknown, label: string): CounterfactualCostVector { + const row = exactObject(value, COUNTERFACTUAL_COST_DIMENSIONS, [], label); + return { + wall_seconds: nullableFinite(row.wall_seconds, `${label}.wall_seconds`, true), + total_tokens: nullableFinite(row.total_tokens, `${label}.total_tokens`, true), + evaluator_runs: nullableFinite(row.evaluator_runs, `${label}.evaluator_runs`, true), + control_plane_requests: nullableFinite(row.control_plane_requests, `${label}.control_plane_requests`, true), + recovery_events: nullableFinite(row.recovery_events, `${label}.recovery_events`, true), + }; +} + +function exactSignFlipTwoSidedPValue(diffs: readonly number[]): number | null { + const n = diffs.length; + if (n < 1 || n > 18) return null; + const observedAbsSum = Math.abs(diffs.reduce((a, b) => a + b, 0)); + if (observedAbsSum === 0) return 1; + const totalConfigs = 1 << n; + let extreme = 0; + for (let mask = 0; mask < totalConfigs; mask++) { + let sum = 0; + for (let bit = 0; bit < n; bit++) { + sum += (mask & (1 << bit)) !== 0 ? -diffs[bit]! : diffs[bit]!; + } + if (Math.abs(sum) + 1e-12 >= observedAbsSum) extreme++; + } + return round6(Math.min(1, extreme / totalConfigs)); +} + +function binomialSignTwoSidedPValue(wins: number, losses: number): number | null { + const n = wins + losses; + if (n < 1) return null; + const k = Math.min(wins, losses); + let logHalfPowN = -n * Math.LN2; + let logChoose = 0; + let tail = Math.exp(logHalfPowN); + for (let i = 1; i <= k; i++) { + logChoose += Math.log(n - i + 1) - Math.log(i); + tail += Math.exp(logChoose + logHalfPowN); + } + return round6(Math.min(1, 2 * tail)); +} + +interface NormalizedBranch { + arm_id: string; + checkpoint_id: string; + source_state_sha256: string; + input_revision: string; + evaluation_ref: string; + workspace_isolation_id: string; + terminal_boundary_ref: string; + decision_context_latest_at: bigint | null; + first_action_signature: string | null; + terminal_value: number | null; + valid_submission: boolean; + costs: CounterfactualCostVector; +} + +function normalizeBranch(value: unknown, label: string): NormalizedBranch { + const allowed = [ + "arm_id", "checkpoint_id", "source_state_sha256", "input_revision", "evaluation_ref", + "workspace_isolation_id", "terminal_boundary_ref", "decision_context_latest_at", "first_action_signature", + "terminal_value", "valid_submission", "costs", + ]; + const required = [ + "arm_id", "checkpoint_id", "source_state_sha256", "input_revision", "evaluation_ref", + "workspace_isolation_id", "terminal_boundary_ref", "terminal_value", "valid_submission", "costs", + ]; + const row = exactObject(value, allowed, required, label); + if (typeof row.valid_submission !== "boolean") { + throw new EffectRuntimeRequestError(`${label}.valid_submission requires a boolean`); + } + return { + arm_id: opaqueRef(row.arm_id, `${label}.arm_id`), + checkpoint_id: opaqueRef(row.checkpoint_id, `${label}.checkpoint_id`), + source_state_sha256: sha256Hex(row.source_state_sha256, `${label}.source_state_sha256`), + input_revision: singleLine(row.input_revision, `${label}.input_revision`, 160), + evaluation_ref: opaqueRef(row.evaluation_ref, `${label}.evaluation_ref`), + workspace_isolation_id: opaqueRef(row.workspace_isolation_id, `${label}.workspace_isolation_id`), + terminal_boundary_ref: opaqueRef(row.terminal_boundary_ref, `${label}.terminal_boundary_ref`), + decision_context_latest_at: parseOptionalTimestamp(row.decision_context_latest_at, `${label}.decision_context_latest_at`), + first_action_signature: row.first_action_signature === null || row.first_action_signature === undefined + ? null + : opaqueRef(row.first_action_signature, `${label}.first_action_signature`), + terminal_value: nullableFinite(row.terminal_value, `${label}.terminal_value`), + valid_submission: row.valid_submission, + costs: normalizeCostVector(row.costs, `${label}.costs`), + }; +} + +export function evaluateCounterfactualContinuations(params: JsonObject): CounterfactualContinuationEvaluation { + const root = exactObject( + params, + ["schema_version", "strict_identification", "pairs"], + ["pairs"], + "counterfactual continuation evaluation request", + ); + if (root.schema_version !== undefined && root.schema_version !== COUNTERFACTUAL_CONTINUATION_EVAL_SCHEMA) { + throw new EffectRuntimeRequestError("unsupported counterfactual continuation schema"); + } + const strict = root.strict_identification === undefined ? true : Boolean(root.strict_identification); + if (root.strict_identification !== undefined && typeof root.strict_identification !== "boolean") { + throw new EffectRuntimeRequestError("strict_identification requires a boolean"); + } + if (!Array.isArray(root.pairs) || root.pairs.length < 1 || root.pairs.length > 64) { + throw new EffectRuntimeRequestError("pairs requires 1 to 64 paired continuations"); + } + + const seenPairIds = new Set(); + const declaredPairs = root.pairs.map((value, idx) => { + const pairRow = exactObject( + value, + ["pair_id", "task_id", "checkpoint", "treatment", "control"], + ["pair_id", "task_id", "checkpoint", "treatment", "control"], + `pairs[${idx}]`, + ); + const pairId = opaqueRef(pairRow.pair_id, `pairs[${idx}].pair_id`); + if (seenPairIds.has(pairId)) { + throw new EffectRuntimeRequestError(`duplicate pair_id: ${pairId}`); + } + seenPairIds.add(pairId); + const taskId = opaqueRef(pairRow.task_id, `pairs[${idx}].task_id`); + + const cpRaw = exactObject( + pairRow.checkpoint, + ["checkpoint_id", "source_state_sha256", "input_revision", "evaluation_ref", + "metric", "unit", "higher_is_better", "comparison_scope", "terminal_boundary_ref", + "incumbent_at_fork", "fork_recorded_at"], + ["checkpoint_id", "source_state_sha256", "input_revision", "evaluation_ref", + "metric", "unit", "higher_is_better", "comparison_scope", "terminal_boundary_ref"], + `pairs[${idx}].checkpoint`, + ); + if (typeof cpRaw.higher_is_better !== "boolean") { + throw new EffectRuntimeRequestError(`pairs[${idx}].checkpoint.higher_is_better requires a boolean`); + } + const cpId = opaqueRef(cpRaw.checkpoint_id, `pairs[${idx}].checkpoint.checkpoint_id`); + const cpSha = sha256Hex(cpRaw.source_state_sha256, `pairs[${idx}].checkpoint.source_state_sha256`); + const cpInputRev = singleLine(cpRaw.input_revision, `pairs[${idx}].checkpoint.input_revision`, 160); + const cpEvalRef = opaqueRef(cpRaw.evaluation_ref, `pairs[${idx}].checkpoint.evaluation_ref`); + const scope: ComparisonScope = { + metric: singleLine(cpRaw.metric, `pairs[${idx}].checkpoint.metric`, 80), + unit: singleLine(cpRaw.unit, `pairs[${idx}].checkpoint.unit`, 40), + higher_is_better: cpRaw.higher_is_better, + comparison_scope: opaqueRef(cpRaw.comparison_scope, `pairs[${idx}].checkpoint.comparison_scope`), + input_revision: cpInputRev, + evaluation_ref: cpEvalRef, + terminal_boundary_ref: opaqueRef(cpRaw.terminal_boundary_ref, `pairs[${idx}].checkpoint.terminal_boundary_ref`), + }; + const higherIsBetter = cpRaw.higher_is_better; + const incumbentAtFork = nullableFinite(cpRaw.incumbent_at_fork, `pairs[${idx}].checkpoint.incumbent_at_fork`); + const forkAt = parseOptionalTimestamp(cpRaw.fork_recorded_at, `pairs[${idx}].checkpoint.fork_recorded_at`); + + const treatment = normalizeBranch(pairRow.treatment, `pairs[${idx}].treatment`); + const control = normalizeBranch(pairRow.control, `pairs[${idx}].control`); + + // A row and its workspace labels can change while the underlying arm is the + // same observation. Arm IDs are execution identities within a frozen fork. + const executionKey = (armId: string) => JSON.stringify([taskId, cpId, cpSha, armId]); + const treatmentExecution = executionKey(treatment.arm_id); + const controlExecution = executionKey(control.arm_id); + const violations: string[] = []; + if (treatment.checkpoint_id !== cpId || control.checkpoint_id !== cpId) { + violations.push("checkpoint_id_mismatch"); + } + if (treatment.source_state_sha256 !== cpSha || control.source_state_sha256 !== cpSha) { + violations.push("source_state_sha256_mismatch"); + } + if (treatment.input_revision !== cpInputRev || control.input_revision !== cpInputRev) { + violations.push("input_revision_mismatch"); + } + if (treatment.evaluation_ref !== cpEvalRef || control.evaluation_ref !== cpEvalRef) { + violations.push("evaluation_ref_mismatch"); + } + if (treatment.workspace_isolation_id === control.workspace_isolation_id) { + violations.push("shared_workspace_interference"); + } + if (treatmentExecution === controlExecution) { + violations.push("reused_arm_execution"); + } + if (treatment.terminal_boundary_ref !== scope.terminal_boundary_ref || + control.terminal_boundary_ref !== scope.terminal_boundary_ref) { + violations.push("terminal_boundary_mismatch"); + } + if (forkAt === null || treatment.decision_context_latest_at === null || control.decision_context_latest_at === null) { + violations.push("missing_context_timing"); + } + if (forkAt !== null && + ((treatment.decision_context_latest_at !== null && treatment.decision_context_latest_at > forkAt) || + (control.decision_context_latest_at !== null && control.decision_context_latest_at > forkAt))) { + violations.push("future_timestamp_context_leakage"); + } + const signature = JSON.stringify([taskId, cpId, cpSha, scope, incumbentAtFork, forkAt, treatment, control], + (_key, value) => typeof value === "bigint" ? value.toString() : value); + return {pairId, taskId, cpId, scope, higherIsBetter, incumbentAtFork, treatment, control, violations, + signature, executionKeys: [treatmentExecution, controlExecution], + workspaceKeys: [treatment.workspace_isolation_id, control.workspace_isolation_id]}; + }); + + // Only individually valid declarations participate in the batch check. + // Identical normalized evidence has one deterministic representative. + // All distinct declarations sharing a run or workspace are excluded. + const representatives = new Map(); + for (const pair of declaredPairs) { + if (pair.violations.length) continue; + const prior = representatives.get(pair.signature); + if (!prior || pair.pairId < prior.pairId) representatives.set(pair.signature, pair); + } + const executions = new Map(); + const workspaces = new Map(); + for (const pair of representatives.values()) { + for (const key of pair.executionKeys) executions.set(key, (executions.get(key) ?? 0) + 1); + for (const key of pair.workspaceKeys) workspaces.set(key, (workspaces.get(key) ?? 0) + 1); + } + + const evaluatedPairs: EvaluatedCounterfactualPair[] = declaredPairs.map(pair => { + const {pairId, taskId, cpId, scope, higherIsBetter, incumbentAtFork, treatment, control, violations} = pair; + if (violations.length === 0) { + const replay = representatives.get(pair.signature) !== pair; + if (replay || pair.workspaceKeys.some(key => (workspaces.get(key) ?? 0) > 1)) { + violations.push("reused_run_workspace"); + } + if (replay || pair.executionKeys.some(key => (executions.get(key) ?? 0) > 1)) { + violations.push("reused_arm_execution"); + } + } + if (violations.length > 0 && strict) { + throw new EffectRuntimeRequestError( + `counterfactual identification failed for ${pairId}: ${violations.join(", ")}`, + ); + } + + const util = (v: number) => higherIsBetter ? v : -v; + const actionDiverged = treatment.first_action_signature !== null && control.first_action_signature !== null + ? treatment.first_action_signature !== control.first_action_signature + : null; + + const marginalDelta = {} as CounterfactualCostVector; + const unknownDims: CounterfactualCostDimension[] = []; + for (const dim of COUNTERFACTUAL_COST_DIMENSIONS) { + const tc = treatment.costs[dim], cc = control.costs[dim]; + if (tc === null || cc === null) { + marginalDelta[dim] = null; + unknownDims.push(dim); + } else { + marginalDelta[dim] = round6(tc - cc); + } + } + + const validPair = violations.length === 0; + const bothScored = validPair && treatment.valid_submission && control.valid_submission + && treatment.terminal_value !== null && control.terminal_value !== null; + + let rawDelta: number | null = null; + let tauB: number | null = null; + let tGain: number | null = null; + let cGain: number | null = null; + let outcomeClass: CounterfactualPairOutcome; + + if (!validPair) { + outcomeClass = violations.length === 1 && violations[0] === "missing_context_timing" + ? "unscored_or_incomplete" : "contaminated_excluded"; + } else if (!bothScored) { + outcomeClass = "unscored_or_incomplete"; + } else { + rawDelta = round6(treatment.terminal_value! - control.terminal_value!); + tauB = round6(util(treatment.terminal_value!) - util(control.terminal_value!)); + if (incumbentAtFork !== null) { + tGain = round6(util(treatment.terminal_value!) - util(incumbentAtFork)); + cGain = round6(util(control.terminal_value!) - util(incumbentAtFork)); + } + if (actionDiverged === null) outcomeClass = "action_unobserved"; + else if (actionDiverged === false) outcomeClass = "action_identical"; + else if (tauB > 0) outcomeClass = "diverged_with_gain"; + else if (tauB === 0) outcomeClass = "diverged_no_gain"; + else outcomeClass = "diverged_regressed"; + } + + return { + pair_id: pairId, + task_id: taskId, + checkpoint_id: cpId, + scope, + identification_valid: validPair, + identification_violations: violations, + outcome_classification: outcomeClass, + action_diverged: actionDiverged, + treatment_arm_id: treatment.arm_id, + control_arm_id: control.arm_id, + treatment_terminal_value: treatment.terminal_value, + control_terminal_value: control.terminal_value, + raw_terminal_delta: rawDelta, + tau_b_direction_normalized: tauB, + treatment_gain_over_incumbent: tGain, + control_gain_over_incumbent: cGain, + marginal_cost_delta: marginalDelta, + unknown_cost_dimensions: unknownDims, + }; + }); + + const validRows = evaluatedPairs.filter(p => p.identification_valid); + const scoredRows = validRows.filter(p => p.tau_b_direction_normalized !== null); + const tauValues = scoredRows.map(p => p.tau_b_direction_normalized!); + const wins = tauValues.filter(v => v > 0).length; + const ties = tauValues.filter(v => v === 0).length; + const losses = tauValues.filter(v => v < 0).length; + const scopeCount = new Set(scoredRows.map(row => JSON.stringify(row.scope))).size; + const comparableTaus = scopeCount === 1 ? tauValues : []; + + const deltaSums = Object.fromEntries(COUNTERFACTUAL_COST_DIMENSIONS.map(d => [d, 0])) as Record; + const observedCounts = Object.fromEntries(COUNTERFACTUAL_COST_DIMENSIONS.map(d => [d, 0])) as Record; + const missingCounts = Object.fromEntries(COUNTERFACTUAL_COST_DIMENSIONS.map(d => [d, 0])) as Record; + + for (const row of validRows) { + for (const dim of COUNTERFACTUAL_COST_DIMENSIONS) { + const val = row.marginal_cost_delta[dim]; + if (val === null) missingCounts[dim]++; + else { + deltaSums[dim] = round6(deltaSums[dim] + val); + observedCounts[dim]++; + } + } + } + + const taskMap = new Map(); + for (const row of validRows) { + const group = JSON.stringify([row.task_id, row.scope]); + const list = taskMap.get(group) ?? []; + list.push(row); + taskMap.set(group, list); + } + const byTask = [...taskMap.entries()] + .sort(([a], [b]) => a.localeCompare(b)) + .map(([, rows]) => { + const taus = rows.filter(r => r.tau_b_direction_normalized !== null).map(r => r.tau_b_direction_normalized!); + return { + task_id: rows[0]!.task_id, + scope: rows[0]!.scope, + valid_pairs: rows.length, + scored_pairs: taus.length, + mean_tau_b: taus.length ? round6(taus.reduce((a, b) => a + b, 0) / taus.length) : null, + wins: taus.filter(v => v > 0).length, + ties: taus.filter(v => v === 0).length, + losses: taus.filter(v => v < 0).length, + }; + }); + + return assertMethodologyOperatorOutput("counterfactual_continuation", { + schema_version: COUNTERFACTUAL_CONTINUATION_EVAL_SCHEMA, + strict_identification: strict, + all_pairs_identification_valid: evaluatedPairs.every(p => p.identification_valid), + counts: { + total_pairs: evaluatedPairs.length, + valid_pairs: validRows.length, + contaminated_pairs: evaluatedPairs.length - validRows.length, + scored_valid_pairs: scoredRows.length, + unscored_valid_pairs: validRows.length - scoredRows.length, + action_diverged_pairs: validRows.filter(p => p.action_diverged === true).length, + action_identical_pairs: validRows.filter(p => p.action_diverged === false).length, + action_unobserved_pairs: validRows.filter(p => p.action_diverged === null).length, + wins, + ties, + losses, + }, + estimand_summary: { + comparable_scope_count: scopeCount, + mean_tau_b: comparableTaus.length ? round6(comparableTaus.reduce((a, b) => a + b, 0) / comparableTaus.length) : null, + median_tau_b: comparableTaus.length ? round6(median(comparableTaus)) : null, + min_tau_b: comparableTaus.length ? round6(Math.min(...comparableTaus)) : null, + max_tau_b: comparableTaus.length ? round6(Math.max(...comparableTaus)) : null, + exact_sign_flip_p_value: exactSignFlipTwoSidedPValue(comparableTaus), + binomial_sign_p_value: scopeCount === 1 ? binomialSignTwoSidedPValue(wins, losses) : null, + }, + complete_marginal_costs: { + observed_delta_sums: deltaSums, + observed_pair_counts: observedCounts, + missing_pair_counts: missingCounts, + }, + by_task: byTask, + pairs: evaluatedPairs, + advisory_only: true, + grants_execution_authority: false, + }, 262_144) as unknown as CounterfactualContinuationEvaluation; +} diff --git a/loopx/control_plane/capabilities/evolutionary_search.ts b/loopx/control_plane/capabilities/evolutionary_search.ts new file mode 100644 index 0000000000..20058151fe --- /dev/null +++ b/loopx/control_plane/capabilities/evolutionary_search.ts @@ -0,0 +1,363 @@ +/** Mechanism-diverse evolutionary population search operator. + * Built directly on the 5-stage methodology operator kernel and scoped + * Yardstick fidelity: selects island-diverse confirmed elites, orthogonal + * mechanism crossover pairs, and targeted bottleneck mutations while + * quarantining unconfirmed or Goodhart-cliff proxy winners. + * Pure, deterministic, browser-safe, and advisory-only. */ +import {EffectRuntimeRequestError} from "../effect_runtime_errors.ts"; +import {requireJsonObject, type JsonObject} from "../runtime_decode.ts"; +import { + extractVerifiedFindingMeasurements, + matchesExploreMetricProfile, + normalizeExploreMetricProfile, + orderCanonicalExploreFindings, + type ExploreMetricProfile, +} from "./explore_measurements.ts"; +import { + assertMethodologyOperatorOutput, + projectBoundedMethodologyItems, + requireOpaqueMethodologyRef, +} from "./stage_methodology_operator.ts"; +import { + evaluateScopedYardstickFidelity, + type YardstickFidelityStatus, +} from "./yardstick_fidelity.ts"; + +export const EVOLUTIONARY_POPULATION_PLAN_SCHEMA = "evolutionary_population_plan_v0"; + +export const EVOLUTIONARY_SELECTION_STATUSES = [ + "empty_population", + "single_seed", + "mechanism_unresolved", + "single_island_refinement", + "multi_island_evolution", +] as const; +export type EvolutionarySelectionStatus = (typeof EVOLUTIONARY_SELECTION_STATUSES)[number]; + +export const PROXY_QUARANTINE_DISPOSITIONS = [ + "confirm_before_breeding", + "quarantined_proxy_cliff", +] as const; +export type ProxyQuarantineDisposition = (typeof PROXY_QUARANTINE_DISPOSITIONS)[number]; + +export interface EvolutionaryCandidateMember { + candidate_ref: string; + mechanism_id: string | null; + authoritative_value: number; + utility_score: number; + finding_id: string; + evidence_ref: string; + island_rank: number | null; + selection_reason: "island_champion" | "utility_fill"; +} + +export interface EvolutionaryCrossoverPair { + pair_id: string; + parent_a_candidate_ref: string; + parent_a_mechanism_id: string; + parent_a_value: number; + parent_b_candidate_ref: string; + parent_b_mechanism_id: string; + parent_b_value: number; + combined_utility_score: number; + synthesis_goal: "test_mechanism_combination"; +} + +export interface EvolutionaryMutationTarget { + candidate_ref: string; + mechanism_id: string | null; + authoritative_value: number; + mutation_kind: "island_champion_refinement" | "tie_breaking_bottleneck_probe"; +} + +export interface EvolutionaryPopulationPlan { + schema_version: typeof EVOLUTIONARY_POPULATION_PLAN_SCHEMA; + node_id: string; + authoritative: ExploreMetricProfile; + proxy: ExploreMetricProfile | null; + selection_status: EvolutionarySelectionStatus; + proxy_fidelity_status: YardstickFidelityStatus | null; + counts: { + confirmed_candidates: number; + unknown_mechanism_candidates: number; + mechanism_islands: number; + selected_elites: number; + omitted_candidates: number; + crossover_pairs: number; + mutation_targets: number; + quarantined_proxy_candidates: number; + authoritative_unknowns: number; + repeat_observations: number; + }; + islands: Array<{ + mechanism_id: string; + member_count: number; + champion_candidate_ref: string; + champion_value: number; + }>; + omitted_islands: number; + elites: EvolutionaryCandidateMember[]; + crossover_pairs: EvolutionaryCrossoverPair[]; + mutation_targets: EvolutionaryMutationTarget[]; + proxy_quarantine: Array<{ + candidate_ref: string; + finding_id: string; + evidence_ref: string; + proxy_value: number; + disposition: ProxyQuarantineDisposition; + }>; + omitted_proxy_quarantine: number; + advisory_only: true; + grants_execution_authority: false; +} + +function boundedInt(value: unknown, fallback: number, min: number, max: number, label: string): number { + if (value === undefined || value === null) return fallback; + if (typeof value !== "number" || !Number.isSafeInteger(value) || value < min || value > max) { + throw new EffectRuntimeRequestError(`${label} requires an integer in ${min}..${max}`); + } + return value; +} + +function normalizeMechanismOverrides(value: unknown): Map { + if (value === undefined || value === null) return new Map(); + const row = requireJsonObject(value, "mechanism_by_candidate"); + const out = new Map(); + for (const [cand, mech] of Object.entries(row)) { + out.set(requireOpaqueMethodologyRef(cand, "mechanism_by_candidate key"), requireOpaqueMethodologyRef(mech, "mechanism_id")); + } + return out; +} + +export function evaluateEvolutionaryPopulation(params: JsonObject): EvolutionaryPopulationPlan { + const nodeId = requireOpaqueMethodologyRef(params.node_id, "node_id"); + const authoritative = normalizeExploreMetricProfile(params.authoritative, "authoritative profile"); + const proxy = + params.proxy === null || params.proxy === undefined + ? null + : normalizeExploreMetricProfile(params.proxy, "proxy profile"); + const eliteLimit = boundedInt(params.elite_limit, 4, 2, 12, "elite_limit"); + const crossoverLimit = boundedInt(params.crossover_limit, 2, 0, 6, "crossover_limit"); + const mutationLimit = boundedInt(params.mutation_limit, 2, 0, 6, "mutation_limit"); + const mechanismOverrides = normalizeMechanismOverrides(params.mechanism_by_candidate); + + if (!Array.isArray(params.findings)) { + throw new EffectRuntimeRequestError("findings requires an array of Explore finding records"); + } + const rawFindings = (params.findings as unknown[]).map(row => requireJsonObject(row, "Explore finding")); + const ordered = orderCanonicalExploreFindings(rawFindings.filter(row => String(row.node_id ?? "") === nodeId)); + + type ObservedCandidate = { + candidate_ref: string; + mechanism_id: string | null; + value: number; + utility: number; + finding_id: string; + evidence_ref: string; + }; + const byCandidate = new Map(); + let authUnknowns = 0; + let repeats = 0; + + for (const finding of ordered) { + const findingId = String(finding.finding_id ?? finding.result_id ?? ""); + if (!findingId) continue; + const {measurements, methodId} = extractVerifiedFindingMeasurements(finding); + for (const m of measurements) { + if (!matchesExploreMetricProfile(m, authoritative)) { + continue; + } + if (m.value === null) { + authUnknowns++; + continue; + } + const previous = byCandidate.get(m.candidate_ref); + if (previous) repeats++; + const mechanismId = + mechanismOverrides.get(m.candidate_ref) ?? + methodId ?? + previous?.mechanism_id ?? + null; + const utility = authoritative.higher_is_better ? m.value : -m.value; + byCandidate.set(m.candidate_ref, { + candidate_ref: m.candidate_ref, + mechanism_id: mechanismId, + value: m.value, + utility, + finding_id: findingId, + evidence_ref: m.evidence_ref, + }); + } + } + + const fidelity = proxy + ? evaluateScopedYardstickFidelity({ + node_id: nodeId, + authoritative: authoritative as unknown as JsonObject, + proxy: proxy as unknown as JsonObject, + findings: rawFindings, + }) + : null; + + // Group confirmed candidates into mechanism islands. + const islandMembers = new Map(); + for (const cand of byCandidate.values()) { + if (cand.mechanism_id === null) continue; + const list = islandMembers.get(cand.mechanism_id) ?? []; + list.push(cand); + islandMembers.set(cand.mechanism_id, list); + } + for (const list of islandMembers.values()) { + list.sort((a, b) => b.utility - a.utility || a.candidate_ref.localeCompare(b.candidate_ref)); + } + const sortedIslands = [...islandMembers.entries()].sort(([, left], [, right]) => { + const diff = right[0]!.utility - left[0]!.utility; + return diff !== 0 ? diff : left[0]!.candidate_ref.localeCompare(right[0]!.candidate_ref); + }); + + // Pass 1: preserve one champion per distinct mechanism island (anti-premature-convergence). + const selectedIds = new Set(); + const elites: EvolutionaryCandidateMember[] = []; + const rankedCandidates = [...byCandidate.values()].sort( + (a, b) => b.utility - a.utility || a.candidate_ref.localeCompare(b.candidate_ref), + ); + const ranks = new Map(); + for (const [, members] of sortedIslands) { + members.forEach((member, index) => ranks.set(member.candidate_ref, index + 1)); + } + const keep = (candidate: ObservedCandidate, reason: EvolutionaryCandidateMember["selection_reason"]) => { + selectedIds.add(candidate.candidate_ref); + elites.push({ + candidate_ref: candidate.candidate_ref, + mechanism_id: candidate.mechanism_id, + authoritative_value: candidate.value, + utility_score: candidate.utility, + finding_id: candidate.finding_id, + evidence_ref: candidate.evidence_ref, + island_rank: ranks.get(candidate.candidate_ref) ?? null, + selection_reason: reason, + }); + }; + const best = rankedCandidates[0]; + if (best?.mechanism_id === null) keep(best, "utility_fill"); + for (const [, members] of sortedIslands) { + if (elites.length >= eliteLimit) break; + keep(members[0]!, "island_champion"); + } + + // Pass 2: fill remaining elite slots by highest confirmed utility across islands. + for (const candidate of rankedCandidates) { + if (elites.length >= eliteLimit) break; + if (!selectedIds.has(candidate.candidate_ref)) keep(candidate, "utility_fill"); + } + elites.sort((a, b) => b.utility_score - a.utility_score || a.candidate_ref.localeCompare(b.candidate_ref)); + + // Distinct mechanisms give a combination to test. Compatibility still needs evidence. + const rawCrossovers: EvolutionaryCrossoverPair[] = []; + for (let i = 0; i < elites.length; i++) { + for (let j = i + 1; j < elites.length; j++) { + const a = elites[i]!; + const b = elites[j]!; + if (a.mechanism_id === null || b.mechanism_id === null || a.mechanism_id === b.mechanism_id) continue; + rawCrossovers.push({ + pair_id: `${a.candidate_ref}+${b.candidate_ref}`, + parent_a_candidate_ref: a.candidate_ref, + parent_a_mechanism_id: a.mechanism_id, + parent_a_value: a.authoritative_value, + parent_b_candidate_ref: b.candidate_ref, + parent_b_mechanism_id: b.mechanism_id, + parent_b_value: b.authoritative_value, + combined_utility_score: a.utility_score + b.utility_score, + synthesis_goal: "test_mechanism_combination", + }); + } + } + rawCrossovers.sort( + (a, b) => b.combined_utility_score - a.combined_utility_score || a.pair_id.localeCompare(b.pair_id), + ); + const crossoverPairs = + crossoverLimit === 0 + ? [] + : projectBoundedMethodologyItems(rawCrossovers, crossoverLimit).visible; + + // Targeted mutations on top elites. + const knownElites = elites.filter(elite => elite.mechanism_id !== null); + const topTied = knownElites.length >= 2 && knownElites[0]!.utility_score === knownElites[1]!.utility_score; + const rawMutations: EvolutionaryMutationTarget[] = knownElites.map((elite, idx) => ({ + candidate_ref: elite.candidate_ref, + mechanism_id: elite.mechanism_id, + authoritative_value: elite.authoritative_value, + mutation_kind: + topTied && idx < 2 ? "tie_breaking_bottleneck_probe" : "island_champion_refinement", + })); + const mutationTargets = + mutationLimit === 0 + ? [] + : projectBoundedMethodologyItems(rawMutations, mutationLimit).visible; + + // Two paired candidates show a single ordering, not a calibrated frontier. + // Proxy-only candidates still need authoritative confirmation before breeding. + const proxyCliff = fidelity?.calibration_tier === "multi_point" + && (fidelity.fidelity_status === "frontier_bias_cliff" || fidelity.fidelity_status === "order_reversed"); + const quarantineDisposition: ProxyQuarantineDisposition = proxyCliff + ? "quarantined_proxy_cliff" + : "confirm_before_breeding"; + const rawQuarantine = (fidelity?.audit_candidates.unconfirmed_proxy_winners ?? []).map(item => ({ + candidate_ref: item.candidate_ref, + finding_id: item.finding_id, + evidence_ref: item.evidence_ref, + proxy_value: item.proxy_value, + disposition: quarantineDisposition, + })); + const boundedQuarantine = projectBoundedMethodologyItems(rawQuarantine, 8); + const boundedIslands = projectBoundedMethodologyItems( + sortedIslands.map(([mechId, members]) => ({ + mechanism_id: mechId, + member_count: members.length, + champion_candidate_ref: members[0]!.candidate_ref, + champion_value: members[0]!.value, + })), + 12, + ); + + const selectionStatus: EvolutionarySelectionStatus = + byCandidate.size === 0 + ? "empty_population" + : byCandidate.size === 1 + ? "single_seed" + : sortedIslands.length === 0 + ? "mechanism_unresolved" + : sortedIslands.length === 1 + ? "single_island_refinement" + : "multi_island_evolution"; + + return assertMethodologyOperatorOutput("evolutionary_population", { + schema_version: EVOLUTIONARY_POPULATION_PLAN_SCHEMA, + node_id: nodeId, + authoritative, + proxy, + selection_status: selectionStatus, + proxy_fidelity_status: fidelity?.fidelity_status ?? null, + counts: { + confirmed_candidates: byCandidate.size, + unknown_mechanism_candidates: rankedCandidates.filter(candidate => candidate.mechanism_id === null).length, + mechanism_islands: sortedIslands.length, + selected_elites: elites.length, + omitted_candidates: Math.max(0, byCandidate.size - elites.length), + crossover_pairs: crossoverPairs.length, + mutation_targets: mutationTargets.length, + quarantined_proxy_candidates: rawQuarantine.length, + authoritative_unknowns: authUnknowns, + repeat_observations: repeats, + }, + islands: boundedIslands.visible, + omitted_islands: boundedIslands.omitted_count, + elites, + crossover_pairs: crossoverPairs, + mutation_targets: mutationTargets, + proxy_quarantine: boundedQuarantine.visible, + omitted_proxy_quarantine: boundedQuarantine.omitted_count, + advisory_only: true, + grants_execution_authority: false, + } as unknown as JsonObject) as unknown as EvolutionaryPopulationPlan; +} diff --git a/loopx/control_plane/capabilities/explore_execution_observation.ts b/loopx/control_plane/capabilities/explore_execution_observation.ts index bd77857053..26aa56bced 100644 --- a/loopx/control_plane/capabilities/explore_execution_observation.ts +++ b/loopx/control_plane/capabilities/explore_execution_observation.ts @@ -3,44 +3,51 @@ */ import type {JsonObject} from "../runtime_decode.ts"; import {EffectRuntimeRequestError} from "../effect_runtime_errors.ts"; -import {requireJsonObject} from "../runtime_decode.ts"; -import {BARE_SHA256_PATTERN} from "../content_digest.ts"; import {isExploreEvidenceRef} from "./explore_evidence_ref.ts"; +import { + isValidContentSha256, + requireBoundedSingleLine, + requireExactFields, +} from "./stage_methodology_operator.ts"; export const EXPLORE_EXECUTION_OBSERVATION_SCHEMA = "explore_execution_observation_v0"; +const EXECUTION_OBSERVATION_REQUIRED_FIELDS = [ + "schema_version", "candidate_sha256", "input_revision", "planned_method_id", + "executed_method_id", "change_reason", "elapsed_seconds", "limit_seconds", + "search_iterations", "stop_reason", "evidence_refs", +] as const; +const EXECUTION_OBSERVATION_ALLOWED_FIELDS = [ + ...EXECUTION_OBSERVATION_REQUIRED_FIELDS, + "exit_code", "result_ref", "result_candidate_sha256", +] as const; + export function normalizeExploreExecutionObservation(params: JsonObject): JsonObject { - const row = requireJsonObject(params.observation, "execution observation"); - const required = ["schema_version", "candidate_sha256", "input_revision", "planned_method_id", - "executed_method_id", "change_reason", "elapsed_seconds", "limit_seconds", - "search_iterations", "stop_reason", "evidence_refs"]; - if (row.schema_version !== EXPLORE_EXECUTION_OBSERVATION_SCHEMA - || required.some(key => !Object.hasOwn(row, key)) - || Object.keys(row).some(key => ![...required, "exit_code", "result_ref", "result_candidate_sha256"].includes(key))) { + const row = requireExactFields( + params.observation, + EXECUTION_OBSERVATION_ALLOWED_FIELDS, + EXECUTION_OBSERVATION_REQUIRED_FIELDS, + "execution observation", + ); + if (row.schema_version !== EXPLORE_EXECUTION_OBSERVATION_SCHEMA) { throw new EffectRuntimeRequestError("execution observation fields are missing or unsupported"); } - const text = (value: unknown, field: string, max: number): string => { - if (typeof value !== "string" || !value.trim() || value.length > max || /[\u0000-\u001f]/.test(value)) { - throw new EffectRuntimeRequestError(`${field} requires bounded single-line text`); - } - return value.trim(); - }; const id = (value: unknown, field: string): string => { - const result = text(value, field, 128); + const result = requireBoundedSingleLine(value, field, 128); if (!/^[A-Za-z0-9][A-Za-z0-9_.:-]*$/.test(result)) { throw new EffectRuntimeRequestError(`${field} requires an opaque identifier`); } return result; }; const evidenceRef = (value: unknown, field: string): string => { - const result = text(value, field, 128); + const result = requireBoundedSingleLine(value, field, 128); if (!isExploreEvidenceRef(result)) { throw new EffectRuntimeRequestError(`${field} requires an opaque identifier`); } return result; }; const sha = (value: unknown): string => { - if (typeof value !== "string" || !BARE_SHA256_PATTERN.test(value)) { + if (!isValidContentSha256(value)) { throw new EffectRuntimeRequestError("execution candidate requires its exact SHA-256"); } return value; @@ -55,7 +62,7 @@ export function normalizeExploreExecutionObservation(params: JsonObject): JsonOb }; const planned = id(row.planned_method_id, "planned_method_id"); const executed = id(row.executed_method_id, "executed_method_id"); - const change = row.change_reason === null ? null : text(row.change_reason, "change_reason", 240); + const change = row.change_reason === null ? null : requireBoundedSingleLine(row.change_reason, "change_reason", 240); if (planned !== executed && change === null) { throw new EffectRuntimeRequestError("a different executed method requires its change_reason"); } @@ -63,12 +70,13 @@ export function normalizeExploreExecutionObservation(params: JsonObject): JsonOb throw new EffectRuntimeRequestError("execution observation requires one to eight evidence references"); } const result: JsonObject = {schema_version: row.schema_version, - candidate_sha256: sha(row.candidate_sha256), input_revision: text(row.input_revision, "input_revision", 160), + candidate_sha256: sha(row.candidate_sha256), + input_revision: requireBoundedSingleLine(row.input_revision, "input_revision", 160), planned_method_id: planned, executed_method_id: executed, change_reason: change, elapsed_seconds: measured(row.elapsed_seconds, "elapsed_seconds"), limit_seconds: measured(row.limit_seconds, "limit_seconds", false, true), search_iterations: measured(row.search_iterations, "search_iterations", true), - stop_reason: row.stop_reason === null ? null : text(row.stop_reason, "stop_reason", 160), + stop_reason: row.stop_reason === null ? null : requireBoundedSingleLine(row.stop_reason, "stop_reason", 160), evidence_refs: [...new Set(row.evidence_refs.map(ref => evidenceRef(ref, "execution evidence_ref")))].sort()}; if (Object.hasOwn(row, "exit_code")) { if (row.exit_code !== null && (typeof row.exit_code !== "number" || !Number.isSafeInteger(row.exit_code))) { diff --git a/loopx/control_plane/capabilities/explore_measurements.ts b/loopx/control_plane/capabilities/explore_measurements.ts index 682c4d1982..66b0ad23d5 100644 --- a/loopx/control_plane/capabilities/explore_measurements.ts +++ b/loopx/control_plane/capabilities/explore_measurements.ts @@ -1,79 +1,153 @@ -/** Explore result measurements. Producers own scores; this owner compares only - * explicitly matching scopes. No ranking of tasks, acceptance or launch authority. - */ +/** Explore result measurements and canonical metric-profile owner. + * Producers own scores; this owner normalizes profiles, validates execution- + * measurement consistency, orders canonical finding prefixes, and computes + * scoped improvement without ranking tasks or granting execution authority. */ import {EffectRuntimeRequestError} from "../effect_runtime_errors.ts"; import {jsonObject, requireJsonObject, type JsonObject} from "../runtime_decode.ts"; -import {isExploreEvidenceRef} from "./explore_evidence_ref.ts"; import {normalizeExploreExecutionObservation} from "./explore_execution_observation.ts"; +import { + matchesExactScopeTuple, + requireBoundedSingleLine, + requireExactFields, + requireOpaqueMethodologyRef, +} from "./stage_methodology_operator.ts"; -export interface ExploreMeasurement { - metric: string; value: number | null; unit: string; higher_is_better: boolean; - comparison_scope: string; input_revision: string; evaluation_ref: string; - candidate_ref: string; evidence_ref: string; +export const EXPLORE_METRIC_PROFILE_KEYS = [ + "metric", + "unit", + "higher_is_better", + "comparison_scope", + "input_revision", + "evaluation_ref", +] as const; + +export interface ExploreMetricProfile { + metric: string; + unit: string; + higher_is_better: boolean; + comparison_scope: string; + input_revision: string; + evaluation_ref: string; +} + +export interface ExploreMeasurement extends ExploreMetricProfile { + value: number | null; + candidate_ref: string; + evidence_ref: string; } + export interface ExploreMeasurementPoint { - finding_id: string; candidate_ref: string; evidence_ref: string; value: number; + finding_id: string; + candidate_ref: string; + evidence_ref: string; + value: number; } -export interface ExploreMeasurementFeedback { - node_id: string; metric: string; unit: string; higher_is_better: boolean; - comparison_scope: string; input_revision: string; evaluation_ref: string; + +export interface ExploreMeasurementFeedback extends ExploreMetricProfile { + node_id: string; latest: {finding_id: string; measurement: ExploreMeasurement}; - best: ExploreMeasurementPoint | null; previous_best: ExploreMeasurementPoint | null; - improvement: number | null; measured_count: number; unknown_count: number; + best: ExploreMeasurementPoint | null; + previous_best: ExploreMeasurementPoint | null; + improvement: number | null; + measured_count: number; + unknown_count: number; } -export function normalizeExploreMeasurements(value: unknown, context?: unknown, executionObservation?: unknown): ExploreMeasurement[] { +export function normalizeExploreMetricProfile( + value: unknown, + label = "yardstick profile", +): ExploreMetricProfile { + const row = requireExactFields(value, EXPLORE_METRIC_PROFILE_KEYS, EXPLORE_METRIC_PROFILE_KEYS, label); + if (typeof row.higher_is_better !== "boolean") { + throw new EffectRuntimeRequestError(`${label} requires explicit metric direction`); + } + return { + metric: requireBoundedSingleLine(row.metric, `${label} metric`, 80), + unit: requireBoundedSingleLine(row.unit, `${label} unit`, 40), + higher_is_better: row.higher_is_better, + comparison_scope: requireOpaqueMethodologyRef(row.comparison_scope, `${label} comparison_scope`), + input_revision: requireBoundedSingleLine(row.input_revision, `${label} input_revision`, 160), + evaluation_ref: requireOpaqueMethodologyRef(row.evaluation_ref, `${label} evaluation_ref`), + }; +} + +export function matchesExploreMetricProfile( + candidate: object, + profile: ExploreMetricProfile, +): boolean { + return matchesExactScopeTuple(candidate, profile, EXPLORE_METRIC_PROFILE_KEYS); +} + +const CONTEXT_KEYS = ["comparison_scope", "input_revision", "evaluation_ref"] as const; +const MEASUREMENT_FIELDS = [ + ...EXPLORE_METRIC_PROFILE_KEYS, + "value", + "candidate_ref", + "evidence_ref", +] as const; + +export function normalizeExploreMeasurements( + value: unknown, + context?: unknown, + executionObservation?: unknown, +): ExploreMeasurement[] { if (!Array.isArray(value) || !value.length || value.length > 8) { throw new EffectRuntimeRequestError("measurements requires one to eight explicit measurements"); } if (context !== undefined) { - const shared = requireJsonObject(context, "measurement_context"); - const keys = ["comparison_scope", "input_revision", "evaluation_ref"]; - if (keys.some(key => !Object.hasOwn(shared, key)) || Object.keys(shared).some(key => !keys.includes(key))) { - throw new EffectRuntimeRequestError("measurement_context requires only comparison_scope, input_revision and evaluation_ref"); - } - return normalizeExploreMeasurements(value.map(item => { - const observation = requireJsonObject(item, "measurement"); - if (keys.some(key => Object.hasOwn(observation, key))) { - throw new EffectRuntimeRequestError("measurement cannot override its explicit comparison context"); - } - // Authoring only: persist the existing complete measurement, never a - // mutable context reference or inferred comparison membership. - return {...shared, ...observation}; - }), undefined, executionObservation); + const shared = requireExactFields( + context, + CONTEXT_KEYS, + CONTEXT_KEYS, + "measurement_context", + "measurement_context requires only comparison_scope, input_revision and evaluation_ref", + ); + return normalizeExploreMeasurements( + value.map(item => { + const observation = requireJsonObject(item, "measurement"); + if (CONTEXT_KEYS.some(key => Object.hasOwn(observation, key))) { + throw new EffectRuntimeRequestError("measurement cannot override its explicit comparison context"); + } + return {...shared, ...observation}; + }), + undefined, + executionObservation, + ); } const seen = new Set(); const measurements = value.map(item => { - const row = requireJsonObject(item, "measurement"); - const fields = ["metric", "value", "unit", "higher_is_better", "comparison_scope", - "input_revision", "evaluation_ref", "candidate_ref", "evidence_ref"]; - if (fields.some(key => !Object.hasOwn(row, key)) || Object.keys(row).some(key => !fields.includes(key))) { - throw new EffectRuntimeRequestError("measurement fields are missing or unsupported"); + const row = requireExactFields( + item, + MEASUREMENT_FIELDS, + MEASUREMENT_FIELDS, + "measurement", + "measurement fields are missing or unsupported", + ); + if ( + typeof row.higher_is_better !== "boolean" || + (row.value !== null && (typeof row.value !== "number" || !Number.isFinite(row.value))) + ) { + throw new EffectRuntimeRequestError( + "measurement requires explicit direction and finite value or null; unknown is not zero", + ); } - const text = (key: string, max: number): string => { - const text = row[key]; - if (typeof text !== "string" || !text.trim() || text.length > max || /[\u0000-\u001f]/.test(text)) { - throw new EffectRuntimeRequestError(`measurement ${key} requires bounded single-line text`); - } - return text.trim(); - }; - const ref = (key: string): string => { - const result = text(key, 128); - if (!isExploreEvidenceRef(result) || result.toLowerCase().startsWith("file:") || result.split("/").includes("..")) { - throw new EffectRuntimeRequestError(`measurement ${key} requires an opaque reference`); - } - return result; + const profile = normalizeExploreMetricProfile( + { + metric: row.metric, + unit: row.unit, + higher_is_better: row.higher_is_better, + comparison_scope: row.comparison_scope, + input_revision: row.input_revision, + evaluation_ref: row.evaluation_ref, + }, + "measurement", + ); + const result: ExploreMeasurement = { + ...profile, + value: row.value as number | null, + candidate_ref: requireOpaqueMethodologyRef(row.candidate_ref, "measurement candidate_ref"), + evidence_ref: requireOpaqueMethodologyRef(row.evidence_ref, "measurement evidence_ref"), }; - if (typeof row.higher_is_better !== "boolean" || row.value !== null - && (typeof row.value !== "number" || !Number.isFinite(row.value))) { - throw new EffectRuntimeRequestError("measurement requires explicit direction and finite value or null; unknown is not zero"); - } - const result: ExploreMeasurement = {metric: text("metric", 80), unit: text("unit", 40), - value: row.value as number | null, higher_is_better: row.higher_is_better, - comparison_scope: ref("comparison_scope"), input_revision: text("input_revision", 160), - evaluation_ref: ref("evaluation_ref"), candidate_ref: ref("candidate_ref"), evidence_ref: ref("evidence_ref")}; - // A comparison batch can contain distinct candidates under identical - // conditions. Candidate identity distinguishes observations, not groups. const key = JSON.stringify([comparisonKey("", result), result.candidate_ref]); if (seen.has(key)) throw new EffectRuntimeRequestError("measurement candidate/metric/scope is duplicated"); seen.add(key); @@ -84,74 +158,144 @@ export function normalizeExploreMeasurements(value: unknown, context?: unknown, return measurements; } -function executionMeasurementError(measurements: readonly ExploreMeasurement[], observation: unknown): string | null { +export function executionMeasurementError( + measurements: readonly ExploreMeasurement[], + observation: unknown, +): string | null { if (observation == null) return null; const execution = normalizeExploreExecutionObservation({observation}); - if (measurements.some(item => item.input_revision !== execution.input_revision)) + if (measurements.some(item => item.input_revision !== execution.input_revision)) { return "measurements must retain the executed candidate's input revision"; - if (execution.result_ref !== undefined && measurements.some(item => item.evidence_ref !== execution.result_ref)) + } + if (execution.result_ref !== undefined && measurements.some(item => item.evidence_ref !== execution.result_ref)) { return "measurements must cite the executed candidate's result reference"; + } return null; } -function comparisonKey(node: string, metric: ExploreMeasurement): string { - return JSON.stringify([node, metric.metric, metric.unit, metric.higher_is_better, - metric.comparison_scope, metric.input_revision, metric.evaluation_ref]); +function comparisonKey(node: string, metric: ExploreMetricProfile): string { + return JSON.stringify([ + node, + metric.metric, + metric.unit, + metric.higher_is_better, + metric.comparison_scope, + metric.input_revision, + metric.evaluation_ref, + ]); +} + +/** Shared canonical finding ordering: append order via `recorded_order` when + * present, or reverse of newest-first projections for direct callers. */ +export function orderCanonicalExploreFindings( + findings: readonly JsonObject[], + options: {strictRecordedOrder?: boolean} = {}, +): JsonObject[] { + const ordered = [...findings]; + if (ordered.some(finding => finding.recorded_order !== undefined)) { + if (options.strictRecordedOrder) { + if ( + ordered.some( + finding => !Number.isSafeInteger(finding.recorded_order) || Number(finding.recorded_order) < 0, + ) || + new Set(ordered.map(finding => finding.recorded_order)).size !== ordered.length + ) { + throw new EffectRuntimeRequestError("measurement source order must be complete and unambiguous"); + } + } + ordered.sort((a, b) => Number(a.recorded_order ?? 0) - Number(b.recorded_order ?? 0)); + } else { + ordered.reverse(); + } + return ordered; +} + +/** Extracts confirmed, execution-consistent measurements and optional method id + * from a canonical Explore finding record; skips malformed or contradicted rows. */ +export function extractVerifiedFindingMeasurements( + finding: JsonObject, +): {measurements: ExploreMeasurement[]; methodId: string | null} { + if (finding.measurements === undefined) return {measurements: [], methodId: null}; + const status = finding.status === undefined ? "confirmed" : String(finding.status); + if (status !== "confirmed") return {measurements: [], methodId: null}; + let measurements: ExploreMeasurement[]; + try { + measurements = normalizeExploreMeasurements(finding.measurements); + if (executionMeasurementError(measurements, finding.execution_observation) !== null) { + return {measurements: [], methodId: null}; + } + } catch { + return {measurements: [], methodId: null}; + } + let methodId: string | null = null; + if (finding.execution_observation != null) { + try { + const exec = normalizeExploreExecutionObservation({observation: finding.execution_observation}); + methodId = + typeof exec.executed_method_id === "string" + ? exec.executed_method_id + : typeof exec.planned_method_id === "string" + ? exec.planned_method_id + : null; + } catch { + return {measurements: [], methodId: null}; + } + } + return {measurements, methodId}; } -/** Latest means the last declared observation in the latest appended finding, - * not execution chronology or the current/retained candidate. Within a batch, - * previous best includes earlier declared observations; order is not inferred. - * Keep all matching evidence when computing best; presentation limits apply later. - */ export function projectExploreMeasurementFeedback( findings: readonly JsonObject[], includeMeasurement?: (finding: JsonObject, measurement: ExploreMeasurement) => boolean, ): ExploreMeasurementFeedback[] { const groups = new Map(); const measuredEvidence = new Set(); - const ordered = findings.filter(finding => finding.measurements !== undefined); - // Canonical readers derive this ordinal from append order; it is not stored - // or inferred from source timestamps, which can tie or arrive out of order. - if (ordered.some(finding => finding.recorded_order !== undefined)) { - if (ordered.some(finding => !Number.isSafeInteger(finding.recorded_order) || Number(finding.recorded_order) < 0) - || new Set(ordered.map(finding => finding.recorded_order)).size !== ordered.length) { - throw new EffectRuntimeRequestError("measurement source order must be complete and unambiguous"); - } - ordered.sort((a, b) => Number(a.recorded_order) - Number(b.recorded_order)); - } else ordered.reverse(); // Existing direct callers supply newest-first rows. + const ordered = orderCanonicalExploreFindings( + findings.filter(finding => finding.measurements !== undefined), + {strictRecordedOrder: true}, + ); for (const finding of ordered) { const node = String(finding.node_id ?? ""), id = String(finding.finding_id ?? ""); if (!node || !id) throw new EffectRuntimeRequestError("measurements require their node and finding identities"); const measurements = normalizeExploreMeasurements(finding.measurements); - // Old canonical history is auditable but is not a quantitative comparison - // source when its declared execution contradicts its measurements. if (executionMeasurementError(measurements, finding.execution_observation) !== null) continue; for (const measurement of measurements) { if (includeMeasurement && !includeMeasurement(finding, measurement)) continue; const key = comparisonKey(node, measurement); const work = jsonObject(finding.work_binding); if (work && measurement.value !== null) { - // A later note can cite the same result but cannot turn it into a new evaluation. - const evidence = JSON.stringify([work.goal_id, work.agent_id, work.todo_id, key, - measurement.candidate_ref, measurement.evidence_ref]); + const evidence = JSON.stringify([ + work.goal_id, work.agent_id, work.todo_id, key, measurement.candidate_ref, measurement.evidence_ref, + ]); if (measuredEvidence.has(evidence)) continue; measuredEvidence.add(evidence); } const prior = groups.get(key); const previous = prior?.best ?? null; - const point = measurement.value === null ? null : {finding_id: id, - candidate_ref: measurement.candidate_ref, evidence_ref: measurement.evidence_ref, value: measurement.value}; - const difference = point && previous ? (measurement.higher_is_better - ? point.value - previous.value : previous.value - point.value) : null; - groups.set(key, {node_id: node, metric: measurement.metric, unit: measurement.unit, - higher_is_better: measurement.higher_is_better, comparison_scope: measurement.comparison_scope, - input_revision: measurement.input_revision, evaluation_ref: measurement.evaluation_ref, - latest: {finding_id: id, measurement}, previous_best: previous, + const point = measurement.value === null ? null : { + finding_id: id, + candidate_ref: measurement.candidate_ref, + evidence_ref: measurement.evidence_ref, + value: measurement.value, + }; + const difference = point && previous + ? (measurement.higher_is_better ? point.value - previous.value : previous.value - point.value) + : null; + groups.set(key, { + node_id: node, + metric: measurement.metric, + unit: measurement.unit, + higher_is_better: measurement.higher_is_better, + comparison_scope: measurement.comparison_scope, + input_revision: measurement.input_revision, + evaluation_ref: measurement.evaluation_ref, + latest: {finding_id: id, measurement}, + previous_best: previous, best: point && (!previous || (difference !== null && difference > 0)) ? point : previous, improvement: difference !== null && Number.isFinite(difference) ? (difference === 0 ? 0 : difference) : null, measured_count: (prior?.measured_count ?? 0) + (point ? 1 : 0), - unknown_count: (prior?.unknown_count ?? 0) + (point ? 0 : 1)}); + unknown_count: (prior?.unknown_count ?? 0) + (point ? 0 : 1), + }); } } const recency = new Map(ordered.map((finding, index) => [String(finding.finding_id), index])); @@ -159,6 +303,11 @@ export function projectExploreMeasurementFeedback( } export function normalizeExploreMeasurementsRequest(params: JsonObject): JsonObject { - return {measurements: normalizeExploreMeasurements(params.measurements, params.measurement_context, - params.execution_observation)}; + return { + measurements: normalizeExploreMeasurements( + params.measurements, + params.measurement_context, + params.execution_observation, + ), + }; } diff --git a/loopx/control_plane/capabilities/explore_result_page.ts b/loopx/control_plane/capabilities/explore_result_page.ts index a07816b008..8e83a810ca 100644 --- a/loopx/control_plane/capabilities/explore_result_page.ts +++ b/loopx/control_plane/capabilities/explore_result_page.ts @@ -2,8 +2,8 @@ import type {JsonObject} from "../effect_program.ts"; import {EffectRuntimeRequestError} from "../effect_runtime_errors.ts"; import {requireJsonObject} from "../runtime_decode.ts"; -import {BARE_SHA256_PATTERN} from "../content_digest.ts"; import {PLANNING_TODO_ID} from "../work_items/planning_relations.ts"; +import {isValidContentSha256} from "./stage_methodology_operator.ts"; export const EXPLORE_RESULT_PAGE_MAX = 20; @@ -18,7 +18,7 @@ export function normalizeExploreResultPages(value: unknown): JsonObject[] { throw new EffectRuntimeRequestError("Explore result page todo_id requires the task identity returned by that read"); } if (Object.keys(row).some(key => !["revision", "offset", "limit", "todo_id"].includes(key)) || - typeof row.revision !== "string" || !BARE_SHA256_PATTERN.test(row.revision) || + !isValidContentSha256(row.revision) || !Number.isSafeInteger(row.offset) || Number(row.offset) < 0 || !Number.isSafeInteger(row.limit) || Number(row.limit) < 1 || Number(row.limit) > EXPLORE_RESULT_PAGE_MAX) { throw new EffectRuntimeRequestError("Explore result page requires revision, nonnegative offset and limit 1..20; node-filtered pages are unsupported"); diff --git a/loopx/control_plane/capabilities/explore_turn_context.ts b/loopx/control_plane/capabilities/explore_turn_context.ts index 284ecca4a9..c85a01c513 100644 --- a/loopx/control_plane/capabilities/explore_turn_context.ts +++ b/loopx/control_plane/capabilities/explore_turn_context.ts @@ -10,6 +10,7 @@ import {EXPLORE_WRITEBACK_SUMMARY_LIMIT, exploreBeforeWorkGuidance, exploreInTur exploreResultWritebackAffordance, exploreTurnContextBoundary, resultWritebackFields} from "./explore_context_delivery.ts"; import {EXPLORE_RESULT_PAGE_MAX, normalizeExploreResultPages} from "./explore_result_page.ts"; import {projectExploreMeasurementFeedback, type ExploreMeasurementPoint} from "./explore_measurements.ts"; +import {evaluateMethodologyActivation, projectBoundedMethodologyItems} from "./stage_methodology_operator.ts"; type YardstickProjection = Pick; @@ -54,19 +55,24 @@ function branchContext(row: JsonObject, findingSources?: Map const nodes = rows(audit.nodes), findings = rows(audit.findings), edges = rows(audit.relevant_edges); const requested = requireStringArray(audit.requested_node_refs ?? [], "requested_node_refs"); const unknown = requireStringArray(audit.unknown_node_refs ?? [], "unknown_node_refs"); + const boundedRequested = projectBoundedMethodologyItems(requested, 8); + const boundedUnknown = projectBoundedMethodologyItems(unknown, 8); + const boundedNodes = projectBoundedMethodologyItems(nodes, 3); + const boundedFindings = projectBoundedMethodologyItems(findings, 3); + const boundedEdges = projectBoundedMethodologyItems(edges, 3); // Preserve the existing diagnostic owner: no score change, inferred links or // new execution gate. Full status counts include details omitted upstream. result.typed_evidence_audit = { ...compact(audit, ["mode", "score_delta", "hazards"]), - requested_node_refs: requested.slice(0, 8), - unknown_node_refs: unknown.slice(0, 8), - omitted_requested_node_refs: Math.max(0, requested.length - 8), - omitted_unknown_node_refs: Math.max(0, unknown.length - 8), - nodes: nodes.slice(0, 3).map(node => compact(node, ["node_id", "title", "status"])), - findings: findings.slice(0, 3).map(finding => ({...compact(finding, ["finding_id", "node_id", "finding", "status"]), + requested_node_refs: boundedRequested.visible, + unknown_node_refs: boundedUnknown.visible, + omitted_requested_node_refs: boundedRequested.omitted_count, + omitted_unknown_node_refs: boundedUnknown.omitted_count, + nodes: boundedNodes.visible.map(node => compact(node, ["node_id", "title", "status"])), + findings: boundedFindings.visible.map(finding => ({...compact(finding, ["finding_id", "node_id", "finding", "status"]), ...findingDigest(finding, findingSources)})), - relevant_edges: edges.slice(0, 3).map(edge => compact(edge, ["from_node", "to_node", "edge_type"])), - omitted_audit_nodes: Math.max(0, nodes.length - 3), + relevant_edges: boundedEdges.visible.map(edge => compact(edge, ["from_node", "to_node", "edge_type"])), + omitted_audit_nodes: boundedNodes.omitted_count, omitted_audit_findings: Math.max(0, evidenceCount(audit, "findings", findings.length) - 3), omitted_audit_edges: Math.max(0, evidenceCount(audit, "edges", edges.length) - 3), }; @@ -92,9 +98,9 @@ export function projectExploreTurnContext(params: JsonObject, yardstick?: Yardst const agent = requireNonEmptyString(params.agent_id, "agent_id"); const route = requireStringArray(params.route, "route"); const gate = requireJsonObject(params.harness_gate, "harness_gate"); - const graph = params.graph_enabled === true; - const harness = gate.enabled === true; - const yardstickEnabled = graph && params.yardstick_enabled === true; + const graph = evaluateMethodologyActivation({enabled: params.graph_enabled}).active; + const harness = evaluateMethodologyActivation({enabled: gate.enabled}).active; + const yardstickEnabled = graph && evaluateMethodologyActivation({enabled: params.yardstick_enabled}).active; const evaluation = yardstickEnabled ? yardstick : undefined; if (yardstickEnabled && !evaluation) { throw new EffectRuntimeRequestError("The selected Yardstick projection is unavailable"); @@ -176,11 +182,12 @@ export function projectExploreTurnContext(params: JsonObject, yardstick?: Yardst return outcomes.length ? [{todo_id: branch.todo_id, outcomes}] : []; }); if (!linkedCandidates.length) return {}; + const boundedCandidates = projectBoundedMethodologyItems(linkedCandidates, limit); return { - comparison_candidates: linkedCandidates.slice(0, limit).map(card => ({todo_id: card.todo_id, + comparison_candidates: boundedCandidates.visible.map(card => ({todo_id: card.todo_id, outcomes: card.outcomes.map(row => ({node_id: row.node_id, finding_id: row.finding_id, source_sha256: createHash("sha256").update(JSON.stringify(row)).digest("hex")}))})), - omitted_comparison_candidates: Math.max(0, linkedCandidates.length - limit), + omitted_comparison_candidates: boundedCandidates.omitted_count, }; }; const relationsByNode = new Map(); @@ -194,15 +201,16 @@ export function projectExploreTurnContext(params: JsonObject, yardstick?: Yardst const currentQuestion = (row: JsonObject): JsonObject => { const node = nodeById.get(row.node_id); const relations = relationsByNode.get(row.node_id) ?? []; + const boundedRelations = relations.length ? projectBoundedMethodologyItems(relations, 3) : null; return { ...(node ? {question_status: node.status, ...(node.blocked_reason ? {question_blocked_reason: node.blocked_reason} : {})} : {}), - ...(relations.length ? { - relations: relations.slice(0, 3).map(edge => ({...compact(edge, ["from_node", "to_node", "edge_type"]), + ...(boundedRelations ? { + relations: boundedRelations.visible.map(edge => ({...compact(edge, ["from_node", "to_node", "edge_type"]), summary: String(edge.summary ?? "").slice(0, 320), ...(String(edge.summary ?? "").length > 320 ? {summary_truncated: true} : {}), })), - omitted_relations: Math.max(0, relations.length - 3), + omitted_relations: boundedRelations.omitted_count, } : {}), }; }; @@ -284,6 +292,13 @@ export function projectExploreTurnContext(params: JsonObject, yardstick?: Yardst command("turn-context", "--agent-id", agent, "--todo-id", String(todo.todo_id), "--result-limit", "20")); return facts ? [facts] : []; }) : []; + const boundedWorkMeasurements = projectBoundedMethodologyItems(workMeasurements, 3); + const boundedWorkCues = projectBoundedMethodologyItems(workCues, 3); + const boundedRecentNodes = projectBoundedMethodologyItems(nodes, 3); + const boundedRecentFindings = projectBoundedMethodologyItems(findings, 3); + const boundedSelectedBranches = projectBoundedMethodologyItems(branches, 3); + const boundedRejectedBranches = projectBoundedMethodologyItems(rejected, 3); + const boundedFrontier = projectBoundedMethodologyItems(frontier, 3); return { ok: true, goal_id: goal, agent_id: agent, graph_enabled: graph, harness_enabled: harness, ...(todoId === null ? {} : {todo_id: todoId}), @@ -297,11 +312,12 @@ export function projectExploreTurnContext(params: JsonObject, yardstick?: Yardst target: {summary: "", scope: "", acceptance_ref: ""}, authoritative: {metric: "", unit: "", higher_is_better: true, comparison_scope: "", input_revision: "", evaluation_ref: ""}, proxy: null, counterevidence_refs: [], next_action: {method_id: "", reason: "", cost_bound: ""}, prior_finding_id: null}}, - ...(workMeasurements.length ? {work_measurements: workMeasurements.slice(0, 3), - omitted_work_measurements: Math.max(0, workMeasurements.length - 3)} : {}), - ...(workCues.length ? {work_yardsticks: workCues.slice(0, 3).map(cue => { + ...(workMeasurements.length ? {work_measurements: boundedWorkMeasurements.visible, + omitted_work_measurements: boundedWorkMeasurements.omitted_count} : {}), + ...(workCues.length ? {work_yardsticks: boundedWorkCues.visible.map(cue => { if (resultNode !== null) return cue; const comparison = cue.search_comparison as JsonObject | undefined; + const fidelity = cue.yardstick_fidelity as JsonObject | undefined; const goalPair = comparison?.goal as JsonObject | undefined; const proxyPair = comparison?.proxy as JsonObject | undefined; const supportingFindings = comparison?.proxy_order_reversed === true && goalPair && proxyPair @@ -319,13 +335,18 @@ export function projectExploreTurnContext(params: JsonObject, yardstick?: Yardst comparison_finding_ids: supportingFindings, proxy_comparison_scope: (cue.proxy as JsonObject).comparison_scope, } : {})} : {}), + ...(fidelity ? { + fidelity_status: fidelity.fidelity_status, + paired_candidates: (fidelity.counts as JsonObject).paired_candidates, + } : {}), + ...(cue.fidelity_guidance ? {fidelity_guidance: cue.fidelity_guidance} : {}), details_command: resultRead("--result-node", String(cue.node_id))}; - }), omitted_work_yardsticks: Math.max(0, workCues.length - 3), + }), omitted_work_yardsticks: boundedWorkCues.omitted_count, work_yardstick_command: command("turn-context", "--agent-id", agent, "--todo-id", "")} : {}), } : {}), - recent_nodes: nodes.slice(0, 3).map(row => compact(row, ["node_id", "title", "status", "blocked_reason"])), + recent_nodes: boundedRecentNodes.visible.map(row => compact(row, ["node_id", "title", "status", "blocked_reason"])), // The canonical evidence projection calls the finding's title `finding`. - recent_findings: findings.slice(0, 3).map(row => ({...compact({...row, title: row.finding}, ["finding_id", "node_id", "title", "status", "measurement_error"]), + recent_findings: boundedRecentFindings.visible.map(row => ({...compact({...row, title: row.finding}, ["finding_id", "node_id", "title", "status", "measurement_error"]), ...findingDigest(row, findingSources)})), // Full scoped summaries; progressive reads can raise the page size. // These retain the observation AND applicability; clipping away conditions @@ -344,7 +365,7 @@ export function projectExploreTurnContext(params: JsonObject, yardstick?: Yardst next_command: nextRead(remainingResults, visibleResults.length), node_command_template: resultRead("--result-node", "", "--result-limit", "10"), }, - omitted_nodes: Math.max(0, nodes.length - 3), + omitted_nodes: boundedRecentNodes.omitted_count, summary_command: command("summary"), ...resultWritebackFields(writeback), ...(measurementFeedback.length ? { @@ -366,14 +387,14 @@ export function projectExploreTurnContext(params: JsonObject, yardstick?: Yardst harness: harness ? { orchestration_gate: gate, candidate_count: plan.candidate_count ?? 0, - selected_branches: branches.slice(0, 3).map(branch => branchContext(branch, findingSources)), + selected_branches: boundedSelectedBranches.visible.map(branch => branchContext(branch, findingSources)), ...visibleComparisons, - omitted_selected_branches: Math.max(0, branches.length - 3), - rejected_candidates: rejected.slice(0, 3).map(branch => branchContext(branch)), + omitted_selected_branches: boundedSelectedBranches.omitted_count, + rejected_candidates: boundedRejectedBranches.visible.map(branch => branchContext(branch)), omitted_rejected_candidates: Math.max(0, requireInteger(plan.rejected_candidate_count ?? rejected.length, "rejected_candidate_count") - 3), - frontier: frontier.slice(0, 3).map(node => compact(node, ["node_id", "title", "status", "summary"])), - omitted_frontier: Math.max(0, frontier.length - 3), + frontier: boundedFrontier.visible.map(node => compact(node, ["node_id", "title", "status", "summary"])), + omitted_frontier: boundedFrontier.omitted_count, // Expand the same Todo audit shown above, not a different worker-lane plan. plan_command: command("todo-branch-plan", "--agent-id", agent), link_evidence_template: [...route, "todo", "update", "--goal-id", goal, "--agent-id", agent, "--todo-id", "", "--explore-result-node-ref", ""], diff --git a/loopx/control_plane/capabilities/stage_methodology_operator.ts b/loopx/control_plane/capabilities/stage_methodology_operator.ts new file mode 100644 index 0000000000..2b24b3ecfd --- /dev/null +++ b/loopx/control_plane/capabilities/stage_methodology_operator.ts @@ -0,0 +1,645 @@ +/** Composable 5-stage methodology operator kernel. + * Centralizes activation isolation, canonical replay-safe evidence windows, + * scope-tuple partitioning, bounded progressive cues, and non-escalation + * invariants so methodology operators (Yardstick, Stride, Fidelity, Budgeted + * Continuation, Counterfactual Continuation, and future search operators) + * share one verified engineering foundation instead of bespoke wiring. + * Pure and browser-safe: zero node:* imports and zero store/authority mutation. */ +import {BARE_SHA256_PATTERN} from "../content_digest.ts"; +import {EffectRuntimeRequestError} from "../effect_runtime_errors.ts"; +import {jsonObject, requireJsonObject, type JsonObject} from "../runtime_decode.ts"; +import {parseTodoTimestampMicros} from "../runtime_timestamp.ts"; +import {isExploreEvidenceRef} from "./explore_evidence_ref.ts"; + +export const STAGE_METHODOLOGY_CATALOG_SCHEMA = "stage_methodology_operator_catalog_v0"; +export const STAGE_METHODOLOGY_VERIFICATION_SCHEMA = "stage_methodology_operator_verification_v0"; + +export const LOOPX_CONTROL_STAGES = [ + "admission", + "compile_context", + "execute_verify", + "writeback_settle", + "feedback_evolve", +] as const; +export type LoopxControlStage = (typeof LOOPX_CONTROL_STAGES)[number]; + +export const QUIET_TURN_POLICIES = [ + "silent_when_untriggered", + "due_obligation_only", + "explicit_invocation_only", +] as const; +export type QuietTurnPolicy = (typeof QUIET_TURN_POLICIES)[number]; + +export const METHODOLOGY_OPERATOR_ROLES = [ + "advisory_cue", + "required_due_decision", +] as const; +export type MethodologyOperatorRole = (typeof METHODOLOGY_OPERATOR_ROLES)[number]; + +export interface StageMethodologyOperatorDescriptor { + operator_id: string; + operator_role: MethodologyOperatorRole; + stage: LoopxControlStage; + companion_stages: readonly LoopxControlStage[]; + enabled_gate: string; + quiet_turn_policy: QuietTurnPolicy; + scope_keys: readonly string[]; + typed_transition_schema: string; + lightweight_guidance: string; + rollback_contract: "stateless_disable_preserves_history"; + advisory_only: boolean; + grants_execution_authority: false; +} + +export interface CanonicalSourceSnapshot { + event_count: number; + event_ids_sha256: string; +} + +export interface WorkIdentityTuple { + goal_id: string; + agent_id: string; + todo_id: string; +} + +const FORBIDDEN_AUTHORITY_KEYS = new Set([ + "should_run", + "normal_delivery_allowed", + "lease_granted", + "gate_override", + "admin_bypass", + "execution_authorized", +]); + +export const BUILTIN_STAGE_METHODOLOGY_OPERATORS: readonly StageMethodologyOperatorDescriptor[] = [ + { + operator_id: "work_yardstick", + operator_role: "advisory_cue", + stage: "execute_verify", + companion_stages: ["compile_context", "writeback_settle", "feedback_evolve"], + enabled_gate: "explore.yardstick_enabled", + quiet_turn_policy: "silent_when_untriggered", + scope_keys: ["goal_id", "agent_id", "todo_id", "node_id", "comparison_scope", "input_revision", "evaluation_ref"], + typed_transition_schema: "work_yardstick_decision_v0", + lightweight_guidance: + "Match proxy screening to the accepted outcome under identical scope and input_revision before discarding a viable route.", + rollback_contract: "stateless_disable_preserves_history", + advisory_only: true, + grants_execution_authority: false, + }, + { + operator_id: "stride_review", + operator_role: "required_due_decision", + stage: "feedback_evolve", + companion_stages: ["compile_context", "writeback_settle"], + enabled_gate: "goal.stride_strategy", + quiet_turn_policy: "due_obligation_only", + scope_keys: ["goal_id", "reference_context_id", "reference_obligation_id"], + typed_transition_schema: "stride_decision_v2", + lightweight_guidance: + "At an admitted replan boundary, compare continuing vs. switching routes by complete marginal cost and decisive evidence.", + rollback_contract: "stateless_disable_preserves_history", + advisory_only: false, + grants_execution_authority: false, + }, + { + operator_id: "scoped_yardstick_fidelity", + operator_role: "advisory_cue", + stage: "execute_verify", + companion_stages: ["compile_context", "feedback_evolve"], + enabled_gate: "explore.yardstick_enabled", + quiet_turn_policy: "silent_when_untriggered", + scope_keys: ["node_id", "metric", "unit", "higher_is_better", "comparison_scope", "input_revision", "evaluation_ref"], + typed_transition_schema: "scoped_yardstick_fidelity_v0", + lightweight_guidance: + "Quantify multi-point rank concordance and frontier bias cliffs in this scope; audit unconfirmed proxy winners without global bans.", + rollback_contract: "stateless_disable_preserves_history", + advisory_only: true, + grants_execution_authority: false, + }, + { + operator_id: "budgeted_continuation", + operator_role: "advisory_cue", + stage: "feedback_evolve", + companion_stages: ["admission", "compile_context"], + enabled_gate: "explore.budgeted_continuation", + quiet_turn_policy: "explicit_invocation_only", + scope_keys: ["remaining_budget.wall_minutes", "confirmation.eval_duration_minutes", "incumbent.metric"], + typed_transition_schema: "budgeted_continuation_plan_v0", + lightweight_guidance: + "Back-schedule authoritative confirmation from remaining wall/token/slot budgets; keep unknown resource bounds null rather than zero.", + rollback_contract: "stateless_disable_preserves_history", + advisory_only: true, + grants_execution_authority: false, + }, + { + operator_id: "counterfactual_continuation", + operator_role: "advisory_cue", + stage: "feedback_evolve", + companion_stages: ["execute_verify"], + enabled_gate: "explore.counterfactual_continuation", + quiet_turn_policy: "explicit_invocation_only", + scope_keys: ["checkpoint_id", "source_state_sha256", "input_revision", "evaluation_ref", "workspace_isolation_id"], + typed_transition_schema: "counterfactual_continuation_eval_v0", + lightweight_guidance: + "Compare treatment and control from the exact frozen checkpoint digest in isolated workspaces; separate action divergence from outcome delta.", + rollback_contract: "stateless_disable_preserves_history", + advisory_only: true, + grants_execution_authority: false, + }, + { + operator_id: "evolutionary_population", + operator_role: "advisory_cue", + stage: "feedback_evolve", + companion_stages: ["compile_context", "execute_verify"], + enabled_gate: "explore.evolutionary_population", + quiet_turn_policy: "explicit_invocation_only", + scope_keys: ["node_id", "metric", "unit", "higher_is_better", "comparison_scope", "input_revision", "evaluation_ref"], + typed_transition_schema: "evolutionary_population_plan_v0", + lightweight_guidance: + "Select mechanism-diverse elites, orthogonal crossover pairs, and bottleneck mutations on confirmed outcomes while quarantining Goodharted proxy-only candidates.", + rollback_contract: "stateless_disable_preserves_history", + advisory_only: true, + grants_execution_authority: false, + }, +]; + +/** Canonical lowercase 64-hex SHA-256 validator shared across methodology operators. */ +export function isValidContentSha256(value: unknown): value is string { + return typeof value === "string" && BARE_SHA256_PATTERN.test(value); +} + +/** Shared strict object-shape validator across methodology codecs. */ +export function requireExactFields( + value: unknown, + allowedFields: readonly string[], + requiredFields: readonly string[], + label: string, + errorMessage?: string, +): JsonObject { + const row = requireJsonObject(value, label); + if ( + Object.keys(row).some(key => !allowedFields.includes(key)) || + requiredFields.some(key => !Object.hasOwn(row, key)) + ) { + throw new EffectRuntimeRequestError(errorMessage ?? `${label} fields are missing or unsupported`); + } + return row; +} + +/** Shared bounded single-line text validator (rejects control characters and empty strings). */ +export function requireBoundedSingleLine(value: unknown, label: string, maxChars = 160): string { + if ( + typeof value !== "string" || + !value.trim() || + value.length > maxChars || + /[\u0000-\u001f]/.test(value) + ) { + throw new EffectRuntimeRequestError(`${label} requires bounded single-line text`); + } + return value.trim(); +} + +/** Shared opaque reference validator (rejects file: URIs and path traversal). */ +export function requireOpaqueMethodologyRef(value: unknown, label: string, maxChars = 128): string { + const trimmed = requireBoundedSingleLine(value, label, maxChars); + if ( + !isExploreEvidenceRef(trimmed) || + trimmed.toLowerCase().startsWith("file:") || + trimmed.split("/").includes("..") + ) { + throw new EffectRuntimeRequestError(`${label} requires an opaque reference`); + } + return trimmed; +} + +/** Shared bounded unique reference array validator. */ +export function requireUniqueOpaqueRefs( + value: unknown, + label: string, + maxCount = 8, + options: {sort?: boolean; requireDistinctInput?: boolean} = {}, +): string[] { + if (!Array.isArray(value) || value.length > maxCount) { + throw new EffectRuntimeRequestError(`${label} requires at most ${maxCount} references`); + } + const normalized = value.map(item => requireOpaqueMethodologyRef(item, label)); + const unique = [...new Set(normalized)]; + if (options.requireDistinctInput && unique.length !== normalized.length) { + throw new EffectRuntimeRequestError(`${label} requires distinct opaque references`); + } + return options.sort === false ? unique : unique.sort(); +} + +/** Shared nullable finite number validator (never coerces null/undefined to 0). */ +export function optionalFiniteNumber( + value: unknown, + label: string, + options: {nonNegative?: boolean} = {}, +): number | null { + if (value === null || value === undefined) return null; + if ( + typeof value !== "number" || + !Number.isFinite(value) || + (options.nonNegative && value < 0) + ) { + throw new EffectRuntimeRequestError( + `${label} requires ${options.nonNegative ? "a non-negative " : "a "}finite number or null`, + ); + } + return value; +} + +/** Shared deterministic 6-decimal rounding avoiding -0. */ +export function roundMethodologyMetric(value: number): number { + const rounded = Math.round(value * 1e6) / 1e6; + return Object.is(rounded, -0) ? 0 : rounded; +} + +/** Replay precedence check: returns true only when a settlement replay carries + * a strictly older committed timestamp than the current committed record. */ +export function isHistoricalReplayPrecedingCurrent( + replayCommittedAt: unknown, + currentUpdatedAt: unknown, +): boolean { + const replayTime = + typeof replayCommittedAt === "string" ? parseTodoTimestampMicros(replayCommittedAt) : null; + const currentTime = + typeof currentUpdatedAt === "string" ? parseTodoTimestampMicros(currentUpdatedAt) : null; + return replayTime !== null && currentTime !== null && replayTime < currentTime; +} + +/** Bug Class 1 Guard: strict boolean feature-off & quiet-turn activation check. + * Rejects truthy strings/objects and suppresses untriggered cues on quiet turns. */ +export function evaluateMethodologyActivation(params: { + enabled: unknown; + quiet_turn_policy?: QuietTurnPolicy; + trigger_active?: boolean; + due_obligation_active?: boolean; + required_scope?: Record; +}): {active: boolean; reason: "disabled" | "missing_scope" | "quiet_turn" | "active"} { + if (params.enabled !== true) { + return {active: false, reason: "disabled"}; + } + if (params.required_scope) { + for (const value of Object.values(params.required_scope)) { + if (typeof value !== "string" || !value.trim()) { + return {active: false, reason: "missing_scope"}; + } + } + } + if (params.quiet_turn_policy === "silent_when_untriggered" && params.trigger_active === false) { + return {active: false, reason: "quiet_turn"}; + } + if ( + params.quiet_turn_policy === "due_obligation_only" && + (params.due_obligation_active === false || params.trigger_active === false) + ) { + return {active: false, reason: "quiet_turn"}; + } + return {active: true, reason: "active"}; +} + +export function normalizeCanonicalSourceSnapshot( + value: unknown, + label = "source snapshot", +): CanonicalSourceSnapshot { + const row = requireJsonObject(value, label); + const keys = Object.keys(row); + if ( + keys.length !== 2 || + !Object.hasOwn(row, "event_count") || + !Object.hasOwn(row, "event_ids_sha256") || + !Number.isSafeInteger(row.event_count) || + Number(row.event_count) < 0 || + typeof row.event_ids_sha256 !== "string" || + !BARE_SHA256_PATTERN.test(row.event_ids_sha256) + ) { + throw new EffectRuntimeRequestError(`${label} requires its original event count and digest`); + } + return {event_count: Number(row.event_count), event_ids_sha256: row.event_ids_sha256}; +} + +/** Bug Class 4 Guard: exact Goal/Agent/Todo work ownership check. */ +export function matchesWorkIdentity( + row: JsonObject, + binding: JsonObject, + fallbackBinding?: JsonObject, +): boolean { + const source = jsonObject(row.work_binding) ?? fallbackBinding; + return ( + !!source && + source.goal_id === binding.goal_id && + source.agent_id === binding.agent_id && + source.todo_id === binding.todo_id + ); +} + +/** Bug Class 2 Guard: prefix order check for explicitly cited earlier records. */ +export function recordedBeforePrefix(row: JsonObject, sourceEventCount: unknown): boolean { + const order = row.recorded_order; + return ( + typeof sourceEventCount === "number" && + Number.isSafeInteger(sourceEventCount) && + typeof order === "number" && + Number.isSafeInteger(order) && + order >= 0 && + order < sourceEventCount + ); +} + +/** Bug Class 2 Guard: append-order vs. committed-clock late-replay check. + * Delayed delivery of an older primary cannot masquerade as new evidence, and + * tied or missing timestamps never prove a newer uncited observation. */ +export function committedInWorkWindow( + row: JsonObject, + startOrder: number, + decisionTimeMicros: bigint | null, +): boolean { + const order = row.recorded_order; + const committed = parseTodoTimestampMicros(String(row.last_updated_at ?? "")); + return ( + Number.isSafeInteger(order) && + Number(order) >= startOrder && + committed !== null && + decisionTimeMicros !== null && + committed > decisionTimeMicros + ); +} + +/** Selects the newest committed record for a scoped work identity; late replays + * retain their original committed timestamp rather than winning by append order. */ +export function latestCommittedScopedRecord( + records: readonly JsonObject[], + predicate: (row: JsonObject) => boolean, +): JsonObject | undefined { + return records + .filter(predicate) + .sort((a, b) => { + const left = parseTodoTimestampMicros(String(a.last_updated_at ?? "")); + const right = parseTodoTimestampMicros(String(b.last_updated_at ?? "")); + if (left !== null && right !== null && left !== right) return left > right ? -1 : 1; + return Number(b.recorded_order ?? 0) - Number(a.recorded_order ?? 0); + })[0]; +} + +/** Shared canonical evidence window selector across live work observations and + * explicitly cited frozen source snapshots. Eliminates hand-rolled replay bugs. */ +export function selectCanonicalEvidenceWindow(params: { + findings: readonly JsonObject[]; + cited_sources?: unknown; + source_finding: JsonObject; + binding: WorkIdentityTuple; + source_snapshot?: CanonicalSourceSnapshot; + cited_refs: ReadonlySet; + has_decision_attachment: (row: JsonObject) => boolean; + matches_decision_target: (row: JsonObject) => boolean; +}): { + scoped_findings: JsonObject[]; + source_order: number | undefined; + work_observation_start: number; + decision_time_micros: bigint | null; +} { + const {findings, source_finding, binding, source_snapshot, cited_refs} = params; + const sourceOrder = source_snapshot?.event_count; + const workObservationStart = sourceOrder ?? Number(source_finding.recorded_order) + 1; + const decisionTime = parseTodoTimestampMicros(String(source_finding.last_updated_at ?? "")); + const priorSource = Array.isArray(params.cited_sources) + ? (params.cited_sources as JsonObject[]).find( + row => + row.source_finding_id === source_finding.finding_id && + source_snapshot !== undefined && + (row.source_snapshot as JsonObject | undefined)?.event_count === source_snapshot.event_count && + (row.source_snapshot as JsonObject | undefined)?.event_ids_sha256 === source_snapshot.event_ids_sha256, + ) + : undefined; + const plainFindings = Array.isArray(priorSource?.findings) ? (priorSource.findings as JsonObject[]) : []; + const scopedFindings = [ + ...findings.filter( + row => + params.has_decision_attachment(row) || + (row.work_binding !== undefined && committedInWorkWindow(row, workObservationStart, decisionTime)), + ), + ...plainFindings.filter(row => !params.has_decision_attachment(row)), + ].filter(row => { + if (row.node_id !== source_finding.node_id) return false; + const bound = jsonObject(row.work_binding) !== null; + if (bound && (!matchesWorkIdentity(row, binding as unknown as JsonObject) || row.status !== "confirmed")) { + return false; + } + if (!params.has_decision_attachment(row)) { + if (bound) return true; + return ( + cited_refs.size > 0 && + recordedBeforePrefix(row, sourceOrder) && + row.agent_id === binding.agent_id && + Array.isArray(row.evidence_refs) && + row.evidence_refs.some(ref => typeof ref === "string" && cited_refs.has(ref)) + ); + } + return params.matches_decision_target(row); + }); + + return { + scoped_findings: scopedFindings, + source_order: sourceOrder, + work_observation_start: workObservationStart, + decision_time_micros: decisionTime, + }; +} + +/** Bug Class 4 Guard: exact scope-tuple comparison (never substring/prose match). */ +export function matchesExactScopeTuple( + candidate: object, + expected: T, + keys: readonly (keyof T)[], +): boolean { + const row = candidate as Record; + const target = expected as Record; + return keys.every(key => row[key as string] === target[key as string]); +} + +/** Bug Class 3 Guard: bounded compact projection with omitted count & cold-path pointer. */ +export function projectBoundedMethodologyItems( + items: readonly T[], + maxItems: number, + detailsCommand?: readonly string[], +): {visible: T[]; omitted_count: number; details_command?: string[]} { + if (!Number.isSafeInteger(maxItems) || maxItems < 1) { + throw new EffectRuntimeRequestError("maxItems requires a positive integer"); + } + const visible = items.slice(0, maxItems); + return { + visible, + omitted_count: Math.max(0, items.length - visible.length), + ...(detailsCommand ? {details_command: [...detailsCommand]} : {}), + }; +} + +function utf8ByteLength(text: string): number { + return new TextEncoder().encode(text).byteLength; +} + +function inspectTreeForInvariants( + value: unknown, + path: string, + violations: string[], + depth = 0, +): void { + if (depth > 32) { + violations.push(`${path}: nesting depth exceeds 32`); + return; + } + if (typeof value === "number") { + if (!Number.isFinite(value)) { + violations.push(`${path}: non-finite number is forbidden`); + } + return; + } + if (typeof value === "string") { + if (/[\u0000-\u0008\u000b\u000c\u000e-\u001f]/.test(value)) { + violations.push(`${path}: control characters are forbidden`); + } + return; + } + if (Array.isArray(value)) { + for (let i = 0; i < value.length; i++) { + inspectTreeForInvariants(value[i], `${path}[${i}]`, violations, depth + 1); + } + return; + } + if (value !== null && typeof value === "object") { + const row = value as Record; + for (const [key, child] of Object.entries(row)) { + if (FORBIDDEN_AUTHORITY_KEYS.has(key)) { + violations.push(`${path}.${key}: methodology operator cannot emit authority/gate field`); + } else if (path !== "$" && key === "grants_execution_authority" && child === true) { + violations.push(`${path}.${key}: methodology operator cannot grant execution authority`); + } + inspectTreeForInvariants(child, `${path}.${key}`, violations, depth + 1); + } + } +} + +/** Bug Class 5 Guard: verifies non-escalation, finite numbers, and byte bounds. */ +export function verifyMethodologyOperatorOutput(params: { + operator_id: string; + output: JsonObject; + max_bytes?: number; + require_advisory_flags?: boolean; +}): { + schema_version: typeof STAGE_METHODOLOGY_VERIFICATION_SCHEMA; + operator_id: string; + valid: boolean; + byte_length: number; + violations: string[]; +} { + const violations: string[] = []; + const {operator_id, output} = params; + const maxBytes = params.max_bytes ?? 65_536; + const descriptor = BUILTIN_STAGE_METHODOLOGY_OPERATORS.find(op => op.operator_id === operator_id); + const requireAdvisory = params.require_advisory_flags ?? (descriptor ? descriptor.advisory_only : true); + + if (requireAdvisory) { + if (output.advisory_only !== true) { + violations.push("$.advisory_only: must be explicitly true"); + } + if (output.grants_execution_authority !== false) { + violations.push("$.grants_execution_authority: must be explicitly false"); + } + } else if (output.grants_execution_authority === true) { + violations.push("$.grants_execution_authority: methodology operator cannot grant execution authority"); + } + inspectTreeForInvariants(output, "$", violations, 0); + const encoded = JSON.stringify(output); + const byteLength = utf8ByteLength(encoded); + if (byteLength > maxBytes) { + violations.push(`$: payload ${byteLength} UTF-8 bytes exceeds cap ${maxBytes}`); + } + return { + schema_version: STAGE_METHODOLOGY_VERIFICATION_SCHEMA, + operator_id, + valid: violations.length === 0, + byte_length: byteLength, + violations, + }; +} + +export function assertMethodologyOperatorOutput( + operatorId: string, + output: T, + maxBytes = 65_536, +): T { + const report = verifyMethodologyOperatorOutput({ + operator_id: operatorId, + output, + max_bytes: maxBytes, + }); + if (!report.valid) { + throw new EffectRuntimeRequestError( + `methodology operator ${operatorId} violated output invariants: ${report.violations.join("; ")}`, + ); + } + return output; +} + +export function listStageMethodologyOperators(params: JsonObject = {}): JsonObject { + const stageFilter = params.stage; + if (stageFilter !== undefined) { + if (typeof stageFilter !== "string" || !(LOOPX_CONTROL_STAGES as readonly string[]).includes(stageFilter)) { + throw new EffectRuntimeRequestError("stage must be one of the 5 LoopX control stages"); + } + } + const operators = BUILTIN_STAGE_METHODOLOGY_OPERATORS.filter( + op => + stageFilter === undefined || + op.stage === stageFilter || + op.companion_stages.includes(stageFilter as LoopxControlStage), + ); + return { + schema_version: STAGE_METHODOLOGY_CATALOG_SCHEMA, + stages: [...LOOPX_CONTROL_STAGES], + operators: operators.map(op => ({ + ...op, + companion_stages: [...op.companion_stages], + scope_keys: [...op.scope_keys], + })), + invariants: [ + "strict_feature_off_and_quiet_turn_silence", + "canonical_prefix_and_late_replay_provenance", + "exact_scope_tuple_isolation_without_null_to_zero_coercion", + "bounded_compact_projection_with_cold_path_pointer", + "advisory_non_escalation_and_stateless_rollback", + ], + }; +} + +export function verifyMethodologyOperatorOutputHandler(params: JsonObject): JsonObject { + const row = requireJsonObject(params, "methodology verification request"); + if (typeof row.operator_id !== "string" || !row.operator_id.trim()) { + throw new EffectRuntimeRequestError("operator_id requires a non-empty string"); + } + const output = requireJsonObject(row.output, "operator output"); + const maxBytes = + row.max_bytes === undefined + ? 65_536 + : typeof row.max_bytes === "number" && Number.isSafeInteger(row.max_bytes) && row.max_bytes > 0 + ? row.max_bytes + : (() => { + throw new EffectRuntimeRequestError("max_bytes requires a positive integer"); + })(); + const requireAdvisoryFlags = + row.require_advisory_flags === undefined + ? undefined + : typeof row.require_advisory_flags === "boolean" + ? row.require_advisory_flags + : (() => { + throw new EffectRuntimeRequestError("require_advisory_flags requires a boolean"); + })(); + return verifyMethodologyOperatorOutput({ + operator_id: row.operator_id.trim(), + output, + max_bytes: maxBytes, + ...(requireAdvisoryFlags === undefined ? {} : {require_advisory_flags: requireAdvisoryFlags}), + }) as unknown as JsonObject; +} diff --git a/loopx/control_plane/capabilities/yardstick_fidelity.ts b/loopx/control_plane/capabilities/yardstick_fidelity.ts new file mode 100644 index 0000000000..38499cdae2 --- /dev/null +++ b/loopx/control_plane/capabilities/yardstick_fidelity.ts @@ -0,0 +1,457 @@ +/** Scoped multi-point Yardstick fidelity and frontier bias-cliff analysis. + * Pure, deterministic, browser-compatible owner; never blacklists an evaluator + * globally or grants acceptance/execution authority. */ +import {EffectRuntimeRequestError} from "../effect_runtime_errors.ts"; +import {requireJsonObject, type JsonObject} from "../runtime_decode.ts"; +import { + type ExploreMeasurement, + extractVerifiedFindingMeasurements, + matchesExploreMetricProfile, + normalizeExploreMetricProfile, + orderCanonicalExploreFindings, + type ExploreMetricProfile, +} from "./explore_measurements.ts"; +import { + assertMethodologyOperatorOutput, + projectBoundedMethodologyItems, + requireOpaqueMethodologyRef, + roundMethodologyMetric as round6, +} from "./stage_methodology_operator.ts"; + +export const SCOPED_YARDSTICK_FIDELITY_SCHEMA = "scoped_yardstick_fidelity_v0"; +export const YARDSTICK_FIDELITY_STATUSES = [ + "direct_only", + "insufficient_paired_evidence", + "noisy_indistinguishable", + "aligned_in_tested_scope", + "frontier_bias_cliff", + "order_reversed", + "mixed_or_flat", +] as const; +export type YardstickFidelityStatus = (typeof YARDSTICK_FIDELITY_STATUSES)[number]; +export const YARDSTICK_CALIBRATION_TIERS = ["none", "two_point", "multi_point"] as const; +export type YardstickCalibrationTier = (typeof YARDSTICK_CALIBRATION_TIERS)[number]; + +export type YardstickMetricProfile = ExploreMetricProfile; +export const normalizeYardstickMetricProfile = normalizeExploreMetricProfile; + +export interface PairedYardstickCandidate { + candidate_ref: string; + authoritative_value: number; + authoritative_finding_id: string; + authoritative_evidence_ref: string; + proxy_value: number; + proxy_finding_id: string; + proxy_evidence_ref: string; +} + +export interface ScopedYardstickFidelity { + schema_version: typeof SCOPED_YARDSTICK_FIDELITY_SCHEMA; + node_id: string; + authoritative: YardstickMetricProfile; + proxy: YardstickMetricProfile | null; + fidelity_status: YardstickFidelityStatus; + calibration_tier: YardstickCalibrationTier; + counts: { + paired_candidates: number; + authoritative_only_candidates: number; + proxy_only_candidates: number; + authoritative_unknowns: number; + proxy_unknowns: number; + repeat_observations: number; + }; + rank_concordance: { + concordant_pairs: number; + discordant_pairs: number; + authoritative_ties: number; + proxy_ties: number; + joint_ties: number; + kendall_tau_b: number | null; + spearman_rho: number | null; + } | null; + linear_calibration: { + utility_slope: number; + utility_intercept: number; + raw_slope: number; + raw_intercept: number; + residual_mad: number; + r_squared: number | null; + } | null; + frontier_neighborhood: { + frontier_k: number; + concordant_pairs: number; + discordant_pairs: number; + frontier_kendall_tau_b: number | null; + authoritative_best_candidate_ref: string | null; + proxy_best_paired_candidate_ref: string | null; + best_candidate_agrees: boolean | null; + bias_cliff_detected: boolean; + } | null; + audit_candidates: { + unconfirmed_proxy_winners: Array<{ + candidate_ref: string; + finding_id: string; + evidence_ref: string; + proxy_value: number; + }>; + proxy_false_negatives: Array<{ + candidate_ref: string; + authoritative_value: number; + proxy_value: number; + }>; + }; + paired_points: PairedYardstickCandidate[]; + omitted_paired_points: number; + scope_bound: true; + global_evaluator_blacklist: false; + advisory_only: true; + grants_execution_authority: false; +} + +const PAIRED_POINT_CAP = 24; + +function median(values: readonly number[]): number { + const sorted = [...values].sort((a, b) => a - b); + const mid = Math.floor(sorted.length / 2); + return sorted.length % 2 === 1 ? sorted[mid]! : (sorted[mid - 1]! + sorted[mid]!) / 2; +} + +function averageRanks(values: readonly number[]): number[] { + const indexed = values.map((v, idx) => ({v, idx})).sort((a, b) => a.v - b.v); + const ranks = new Array(values.length); + let i = 0; + while (i < indexed.length) { + let j = i; + while (j + 1 < indexed.length && indexed[j + 1]!.v === indexed[i]!.v) j++; + const avg = (i + j + 2) / 2; + for (let k = i; k <= j; k++) ranks[indexed[k]!.idx] = avg; + i = j + 1; + } + return ranks; +} + +function pearson(xs: readonly number[], ys: readonly number[]): number | null { + const n = xs.length; + if (n < 2) return null; + const mx = xs.reduce((a, b) => a + b, 0) / n; + const my = ys.reduce((a, b) => a + b, 0) / n; + let num = 0, dx2 = 0, dy2 = 0; + for (let i = 0; i < n; i++) { + const dx = xs[i]! - mx, dy = ys[i]! - my; + num += dx * dy; dx2 += dx * dx; dy2 += dy * dy; + } + const denom = Math.sqrt(dx2 * dy2); + return denom > 0 ? round6(Math.max(-1, Math.min(1, num / denom))) : null; +} + +function pairwiseConcordance(authUtil: readonly number[], proxyUtil: readonly number[]) { + let concordant = 0, discordant = 0, authTies = 0, proxyTies = 0, jointTies = 0; + for (let i = 0; i < authUtil.length; i++) { + for (let j = i + 1; j < authUtil.length; j++) { + const da = authUtil[i]! - authUtil[j]!; + const dp = proxyUtil[i]! - proxyUtil[j]!; + if (da === 0 && dp === 0) jointTies++; + else if (da === 0) authTies++; + else if (dp === 0) proxyTies++; + else if ((da > 0) === (dp > 0)) concordant++; + else discordant++; + } + } + const denom = Math.sqrt((concordant + discordant + authTies) * (concordant + discordant + proxyTies)); + const tauB = denom > 0 ? round6((concordant - discordant) / denom) : null; + const rho = pearson(averageRanks(authUtil), averageRanks(proxyUtil)); + return { + concordant_pairs: concordant, + discordant_pairs: discordant, + authoritative_ties: authTies, + proxy_ties: proxyTies, + joint_ties: jointTies, + kendall_tau_b: tauB, + spearman_rho: rho, + }; +} + +function linearFit(xs: readonly number[], ys: readonly number[]) { + const n = xs.length; + const mx = xs.reduce((a, b) => a + b, 0) / n; + const my = ys.reduce((a, b) => a + b, 0) / n; + let sxx = 0, sxy = 0, syy = 0; + for (let i = 0; i < n; i++) { + const dx = xs[i]! - mx, dy = ys[i]! - my; + sxx += dx * dx; sxy += dx * dy; syy += dy * dy; + } + if (sxx <= 0) return null; + const slope = sxy / sxx; + const intercept = my - slope * mx; + const residuals = xs.map((x, i) => Math.abs(ys[i]! - (slope * x + intercept))); + const sse = xs.reduce((acc, x, i) => { + const err = ys[i]! - (slope * x + intercept); + return acc + err * err; + }, 0); + const r2 = syy > 0 ? round6(Math.max(0, Math.min(1, 1 - sse / syy))) : null; + return {slope: round6(slope), intercept: round6(intercept), residualMad: round6(median(residuals)), r2}; +} + +export function evaluateScopedYardstickFidelity( + params: JsonObject, + includeMeasurement?: (finding: JsonObject, measurement: ExploreMeasurement) => boolean, +): ScopedYardstickFidelity { + const nodeId = requireOpaqueMethodologyRef(params.node_id, "node_id"); + const authoritative = normalizeYardstickMetricProfile(params.authoritative, "authoritative profile"); + const proxy = params.proxy === null || params.proxy === undefined + ? null + : normalizeYardstickMetricProfile(params.proxy, "proxy profile"); + if (proxy && JSON.stringify(proxy) === JSON.stringify(authoritative)) { + throw new EffectRuntimeRequestError("proxy and authoritative profiles must be distinct"); + } + if (!Array.isArray(params.findings)) { + throw new EffectRuntimeRequestError("findings requires an array of Explore finding records"); + } + const rawFindings = (params.findings as unknown[]).map(row => requireJsonObject(row, "Explore finding")); + const ordered = orderCanonicalExploreFindings(rawFindings.filter(row => String(row.node_id ?? "") === nodeId)); + + type Obs = {value: number; finding_id: string; evidence_ref: string}; + const authByCand = new Map(); + const proxyByCand = new Map(); + let authUnknowns = 0, proxyUnknowns = 0, repeats = 0; + + for (const finding of ordered) { + const findingId = String(finding.finding_id ?? finding.result_id ?? ""); + if (!findingId) continue; + for (const m of extractVerifiedFindingMeasurements(finding).measurements) { + if (includeMeasurement && !includeMeasurement(finding, m)) continue; + if (matchesExploreMetricProfile(m, authoritative)) { + if (m.value === null) authUnknowns++; + else { + if (authByCand.has(m.candidate_ref)) repeats++; + authByCand.set(m.candidate_ref, {value: m.value, finding_id: findingId, evidence_ref: m.evidence_ref}); + } + } else if (proxy && matchesExploreMetricProfile(m, proxy)) { + if (m.value === null) proxyUnknowns++; + else { + if (proxyByCand.has(m.candidate_ref)) repeats++; + proxyByCand.set(m.candidate_ref, {value: m.value, finding_id: findingId, evidence_ref: m.evidence_ref}); + } + } + } + } + + if (!proxy) { + return assertMethodologyOperatorOutput("scoped_yardstick_fidelity", { + schema_version: SCOPED_YARDSTICK_FIDELITY_SCHEMA, + node_id: nodeId, + authoritative, + proxy: null, + fidelity_status: "direct_only", + calibration_tier: "none", + counts: { + paired_candidates: 0, + authoritative_only_candidates: authByCand.size, + proxy_only_candidates: 0, + authoritative_unknowns: authUnknowns, + proxy_unknowns: 0, + repeat_observations: repeats, + }, + rank_concordance: null, + linear_calibration: null, + frontier_neighborhood: null, + audit_candidates: {unconfirmed_proxy_winners: [], proxy_false_negatives: []}, + paired_points: [], + omitted_paired_points: 0, + scope_bound: true, + global_evaluator_blacklist: false, + advisory_only: true, + grants_execution_authority: false, + } as unknown as JsonObject) as unknown as ScopedYardstickFidelity; + } + + const authUtil = (v: number) => authoritative.higher_is_better ? v : -v; + const proxyUtil = (v: number) => proxy.higher_is_better ? v : -v; + + const pairedAll: PairedYardstickCandidate[] = []; + let authOnly = 0; + for (const [cand, aObs] of authByCand.entries()) { + const pObs = proxyByCand.get(cand); + if (pObs) { + pairedAll.push({ + candidate_ref: cand, + authoritative_value: aObs.value, + authoritative_finding_id: aObs.finding_id, + authoritative_evidence_ref: aObs.evidence_ref, + proxy_value: pObs.value, + proxy_finding_id: pObs.finding_id, + proxy_evidence_ref: pObs.evidence_ref, + }); + } else { + authOnly++; + } + } + pairedAll.sort((a, b) => authUtil(b.authoritative_value) - authUtil(a.authoritative_value) + || a.candidate_ref.localeCompare(b.candidate_ref)); + + const proxyOnlyRows: Array<{candidate_ref: string; finding_id: string; evidence_ref: string; proxy_value: number}> = []; + for (const [cand, pObs] of proxyByCand.entries()) { + if (!authByCand.has(cand)) { + proxyOnlyRows.push({ + candidate_ref: cand, + finding_id: pObs.finding_id, + evidence_ref: pObs.evidence_ref, + proxy_value: pObs.value, + }); + } + } + proxyOnlyRows.sort((a, b) => proxyUtil(b.proxy_value) - proxyUtil(a.proxy_value) + || a.candidate_ref.localeCompare(b.candidate_ref)); + + const n = pairedAll.length; + const bestPairedProxyUtil = n > 0 + ? Math.max(...pairedAll.map(p => proxyUtil(p.proxy_value))) + : Number.NEGATIVE_INFINITY; + const unconfirmedWinners = n > 0 + ? proxyOnlyRows.filter(r => proxyUtil(r.proxy_value) > bestPairedProxyUtil).slice(0, 3) + : proxyOnlyRows.slice(0, 3); + + const tier: YardstickCalibrationTier = n >= 3 ? "multi_point" : n === 2 ? "two_point" : "none"; + if (n < 2) { + return assertMethodologyOperatorOutput("scoped_yardstick_fidelity", { + schema_version: SCOPED_YARDSTICK_FIDELITY_SCHEMA, + node_id: nodeId, + authoritative, + proxy, + fidelity_status: "insufficient_paired_evidence", + calibration_tier: tier, + counts: { + paired_candidates: n, + authoritative_only_candidates: authOnly, + proxy_only_candidates: proxyOnlyRows.length, + authoritative_unknowns: authUnknowns, + proxy_unknowns: proxyUnknowns, + repeat_observations: repeats, + }, + rank_concordance: null, + linear_calibration: null, + frontier_neighborhood: null, + audit_candidates: {unconfirmed_proxy_winners: unconfirmedWinners, proxy_false_negatives: []}, + paired_points: pairedAll, + omitted_paired_points: 0, + scope_bound: true, + global_evaluator_blacklist: false, + advisory_only: true, + grants_execution_authority: false, + } as unknown as JsonObject) as unknown as ScopedYardstickFidelity; + } + + const aUtils = pairedAll.map(p => authUtil(p.authoritative_value)); + const pUtils = pairedAll.map(p => proxyUtil(p.proxy_value)); + const concordance = pairwiseConcordance(aUtils, pUtils); + + let linearCal: ScopedYardstickFidelity["linear_calibration"] = null; + if (n >= 3) { + const uFit = linearFit(pUtils, aUtils); + const rFit = linearFit(pairedAll.map(p => p.proxy_value), pairedAll.map(p => p.authoritative_value)); + if (uFit && rFit) { + linearCal = { + utility_slope: uFit.slope, + utility_intercept: uFit.intercept, + raw_slope: rFit.slope, + raw_intercept: rFit.intercept, + residual_mad: uFit.residualMad, + r_squared: uFit.r2, + }; + } + } + + const requestedK = params.frontier_k === undefined + ? Math.min(n, Math.max(3, Math.ceil(n / 2))) + : Number(params.frontier_k); + if (!Number.isSafeInteger(requestedK) || requestedK < 2) { + throw new EffectRuntimeRequestError("frontier_k must be an integer >= 2"); + } + let frontierK = Math.min(n, requestedK); + const frontierCutoff = pairedAll[frontierK - 1]!.authoritative_value; + while (frontierK < n && pairedAll[frontierK]!.authoritative_value === frontierCutoff) frontierK++; + const frontierSlice = pairedAll.slice(0, frontierK); + const frontierConcordance = pairwiseConcordance( + frontierSlice.map(p => authUtil(p.authoritative_value)), + frontierSlice.map(p => proxyUtil(p.proxy_value)), + ); + const authBest = aUtils[0]! > aUtils[1]! ? pairedAll[0]!.candidate_ref : null; + const byProxyDesc = [...pairedAll].sort((a, b) => proxyUtil(b.proxy_value) - proxyUtil(a.proxy_value) + || a.candidate_ref.localeCompare(b.candidate_ref)); + const proxyBest = proxyUtil(byProxyDesc[0]!.proxy_value) > proxyUtil(byProxyDesc[1]!.proxy_value) + ? byProxyDesc[0]!.candidate_ref + : null; + const bestAgrees = authBest !== null && proxyBest !== null ? authBest === proxyBest : null; + + const topHalfCutoff = aUtils[Math.ceil(n / 2) - 1]!; + const biasCliff = n >= 4 + && concordance.kendall_tau_b !== null && concordance.kendall_tau_b > 0 + && ((frontierConcordance.kendall_tau_b !== null && frontierConcordance.kendall_tau_b < 0) + || (proxyBest !== null && authUtil(byProxyDesc[0]!.authoritative_value) < topHalfCutoff)); + + let status: YardstickFidelityStatus; + if (concordance.kendall_tau_b === null) { + status = "noisy_indistinguishable"; + } else if (biasCliff) { + status = "frontier_bias_cliff"; + } else if (concordance.kendall_tau_b < 0) { + status = "order_reversed"; + } else if (concordance.kendall_tau_b > 0) { + status = "aligned_in_tested_scope"; + } else { + status = "mixed_or_flat"; + } + + const authMedian = median(aUtils); + const proxyMedian = median(pUtils); + const falseNegatives = n >= 3 + ? pairedAll + .filter(p => authUtil(p.authoritative_value) > authMedian && proxyUtil(p.proxy_value) < proxyMedian) + .slice(0, 3) + .map(p => ({ + candidate_ref: p.candidate_ref, + authoritative_value: p.authoritative_value, + proxy_value: p.proxy_value, + })) + : []; + + const boundedPairs = projectBoundedMethodologyItems(pairedAll, PAIRED_POINT_CAP); + return assertMethodologyOperatorOutput("scoped_yardstick_fidelity", { + schema_version: SCOPED_YARDSTICK_FIDELITY_SCHEMA, + node_id: nodeId, + authoritative, + proxy, + fidelity_status: status, + calibration_tier: tier, + counts: { + paired_candidates: n, + authoritative_only_candidates: authOnly, + proxy_only_candidates: proxyOnlyRows.length, + authoritative_unknowns: authUnknowns, + proxy_unknowns: proxyUnknowns, + repeat_observations: repeats, + }, + rank_concordance: concordance, + linear_calibration: linearCal, + frontier_neighborhood: { + frontier_k: frontierK, + concordant_pairs: frontierConcordance.concordant_pairs, + discordant_pairs: frontierConcordance.discordant_pairs, + frontier_kendall_tau_b: frontierConcordance.kendall_tau_b, + authoritative_best_candidate_ref: authBest, + proxy_best_paired_candidate_ref: proxyBest, + best_candidate_agrees: bestAgrees, + bias_cliff_detected: biasCliff, + }, + audit_candidates: { + unconfirmed_proxy_winners: unconfirmedWinners, + proxy_false_negatives: falseNegatives, + }, + paired_points: boundedPairs.visible, + omitted_paired_points: boundedPairs.omitted_count, + scope_bound: true, + global_evaluator_blacklist: false, + advisory_only: true, + grants_execution_authority: false, + } as unknown as JsonObject) as unknown as ScopedYardstickFidelity; +} diff --git a/loopx/control_plane/effect_runtime_handlers.ts b/loopx/control_plane/effect_runtime_handlers.ts index 6fbe7ec4ab..32a5d6dead 100644 --- a/loopx/control_plane/effect_runtime_handlers.ts +++ b/loopx/control_plane/effect_runtime_handlers.ts @@ -871,6 +871,12 @@ export function createEffectRuntimeHandlers( ["explore.research.composition_successor", lazyHandler(() => import("./capabilities/explore_research_execution.ts"), ({validateResearchCompositionSuccessor}) => validateResearchCompositionSuccessor)], ["explore.research.completion", lazyHandler(() => import("./capabilities/explore_research_execution.ts"), ({qualifyResearchCompletion}) => qualifyResearchCompletion)], ["explore.research.frontier", lazyHandler(() => import("./capabilities/explore_research.ts"), ({projectResearchFrontier}) => projectResearchFrontier)], + ["explore.yardstick_fidelity.evaluate", lazyHandler(() => import("./capabilities/yardstick_fidelity.ts"), ({evaluateScopedYardstickFidelity}) => evaluateScopedYardstickFidelity)], + ["explore.budgeted_continuation.evaluate", lazyHandler(() => import("./capabilities/budgeted_continuation.ts"), ({evaluateBudgetedContinuationPlan}) => evaluateBudgetedContinuationPlan)], + ["explore.counterfactual_continuation.evaluate", lazyHandler(() => import("./capabilities/counterfactual_continuation.ts"), ({evaluateCounterfactualContinuations}) => evaluateCounterfactualContinuations)], + ["explore.evolutionary_population.evaluate", lazyHandler(() => import("./capabilities/evolutionary_search.ts"), ({evaluateEvolutionaryPopulation}) => evaluateEvolutionaryPopulation)], + ["methodology.operators.catalog", lazyHandler(() => import("./capabilities/stage_methodology_operator.ts"), ({listStageMethodologyOperators}) => listStageMethodologyOperators)], + ["methodology.operator.verify_output", lazyHandler(() => import("./capabilities/stage_methodology_operator.ts"), ({verifyMethodologyOperatorOutputHandler}) => verifyMethodologyOperatorOutputHandler)], ["work_item.replan_history.project", lazyHandler(() => import("./work_items/replan_history_settlement.ts"), ({projectSettledReplanHistory}) => projectSettledReplanHistory)], ["work_item.replan_history.project_snapshot", lazyHandler(() => import("./work_items/replan_history_snapshot.ts"), ({projectReplanHistorySnapshot}) => projectReplanHistorySnapshot)], diff --git a/loopx/control_plane/effect_runtime_snapshot_methods.json b/loopx/control_plane/effect_runtime_snapshot_methods.json index 1616ef0506..fa4fc2aff4 100644 --- a/loopx/control_plane/effect_runtime_snapshot_methods.json +++ b/loopx/control_plane/effect_runtime_snapshot_methods.json @@ -30,5 +30,9 @@ "progress_review.context.qualify", "state.recovery.archive.verify", "state.recovery.manifest.verify", - "runtime.checkpoint_artifacts" + "runtime.checkpoint_artifacts", + "explore.yardstick_fidelity.evaluate", + "explore.counterfactual_continuation.evaluate", + "explore.evolutionary_population.evaluate", + "work_item.yardstick.presentation" ] diff --git a/loopx/control_plane/presentation/explore_history.ts b/loopx/control_plane/presentation/explore_history.ts index e0f137e243..3600df01fb 100644 --- a/loopx/control_plane/presentation/explore_history.ts +++ b/loopx/control_plane/presentation/explore_history.ts @@ -4,7 +4,8 @@ */ import {normalizeExploreExecutionObservation} from "../capabilities/explore_execution_observation.ts"; import {normalizeExploreMeasurements, projectExploreMeasurementFeedback, type ExploreMeasurement} from "../capabilities/explore_measurements.ts"; -import {normalizeWorkYardstickCapture, type WorkYardstickCapture} from "../work_items/yardstick_decision.ts"; +import type {ScopedYardstickFidelity} from "../capabilities/yardstick_fidelity.ts"; +import {normalizeWorkYardstickCapture, projectWorkYardstickPresentations, type WorkYardstickCapture} from "../work_items/yardstick_decision.ts"; import type {JsonObject} from "../runtime_decode.ts"; export type ExploreHistoryEvent = { schema_version: string; event_id: string; goal_id: string; event_kind: string; @@ -89,7 +90,9 @@ export function mergeExploreHistoryEvents(unique: Map(); mergeExploreHistoryEvents(unique, events, goalId); const nodes = new Map(); @@ -113,11 +116,31 @@ export function projectExploreHistory(events: readonly ExploreHistoryEvent[], go const times = [...unique.values()].map(event => Date.parse(event.recorded_at)).filter(Number.isFinite); const links = [...edges.values()]; const recordedOrder = new Map([...unique.keys()].map((id, index) => [id, index])); - const measurementFeedback = projectExploreMeasurementFeedback([...findings.values()].map(event => ({ + const currentFindings = [...findings.values()].map(event => ({ ...event, finding_id: event.result_id, recorded_order: recordedOrder.get(event.event_id), - }))); + })); + const measurementFeedback = projectExploreMeasurementFeedback(currentFindings); + const yardstickFidelityByEvent = new Map(); + if (includeCurrentFidelity) { + const currentDecisions = currentFindings.filter(row => row.work_yardstick && row.status === "confirmed"); + if (currentDecisions.length) { + const projected = projectWorkYardstickPresentations({ + captures: currentDecisions.map(row => ({ + capture: row.work_yardstick, goal_id: goalId, agent_id: row.agent_id, + evidence_refs: row.evidence_refs, node_id: row.node_id, + })), + findings: currentFindings, + }); + const fidelity = projected.yardstick_fidelities as Array; + currentDecisions.forEach((row, index) => { + const entry = fidelity[index]; + if (entry) yardstickFidelityByEvent.set(row.event_id, entry); + }); + } + } return { events: [...unique.values()], nodes: [...nodes.values()], edges: links, findings: [...findings.values()], + yardstickFidelityByEvent, ...(measurementFeedback.length ? {measurementFeedback} : {}), startAt: times.length ? new Date(Math.min(...times)).toISOString() : null, endAt: times.length ? new Date(Math.max(...times)).toISOString() : null, diff --git a/loopx/control_plane/work_items/stride_policy.ts b/loopx/control_plane/work_items/stride_policy.ts index 58d5e4219e..f0537b00f8 100644 --- a/loopx/control_plane/work_items/stride_policy.ts +++ b/loopx/control_plane/work_items/stride_policy.ts @@ -1,12 +1,17 @@ /** One work and research stride strategy; all execution authority retains its owners. */ import {createHash} from "node:crypto"; -import {BARE_SHA256_PATTERN} from "../content_digest.ts"; import schema from "../stride_strategy_schema.json" with {type: "json"}; import decisionSchema from "../stride_decision_schema.json" with {type: "json"}; import {EffectRuntimeRequestError} from "../effect_runtime_errors.ts"; import type {JsonObject} from "../effect_program.ts"; import {requireJsonObject, requireNonEmptyString, requireStringLiteral} from "../runtime_decode.ts"; import {EXPLORE_RESULT_PAGE_MAX, normalizeExploreResultPages} from "../capabilities/explore_result_page.ts"; +import { + evaluateMethodologyActivation, + isValidContentSha256, + projectBoundedMethodologyItems, + requireExactFields, +} from "../capabilities/stage_methodology_operator.ts"; import {PLANNING_TODO_ID} from "./planning_relations.ts"; export function normalizeStrideStrategy(value: unknown): string { @@ -50,7 +55,7 @@ function text(field: string, value: unknown, limit: number): string { return result; } function exactFields(value: JsonObject, fields: string[], label: string): void { - if (Object.keys(value).some(key => !fields.includes(key))) throw new EffectRuntimeRequestError(`${label} contains unsupported fields`); + requireExactFields(value, fields, [], label, `${label} contains unsupported fields`); } function refs(value: unknown, label: string): string[] { if (!Array.isArray(value) || value.length > STRIDE_REVIEW_REFERENCE_LIMIT || value.some(ref => typeof ref !== "string" || @@ -67,7 +72,7 @@ function selectedCandidate(value: unknown): JsonObject { const node = refs([input.node_id], "selected_candidate.node_id")[0]; const finding = refs([input.finding_id], "selected_candidate.finding_id")[0]; const digest = requireNonEmptyString(input.source_sha256, "selected_candidate.source_sha256"); - if (!BARE_SHA256_PATTERN.test(digest)) throw new EffectRuntimeRequestError("selected_candidate requires the exact source digest"); + if (!isValidContentSha256(digest)) throw new EffectRuntimeRequestError("selected_candidate requires the exact source digest"); return {todo_id: todo, node_id: node, finding_id: finding, source_sha256: digest}; } function comparisonCandidates(value: unknown): JsonObject[] { @@ -168,7 +173,7 @@ export function normalizeStrideDecision(value: unknown, bounded: BoundedText = t if (input.explore_review != null) { const review = requireJsonObject(input.explore_review, "stride_decision.explore_review"); exactFields(review, ["revision", "disposition", "considered_refs", "reason", ...(!historical ? ["result_pages", "selected_candidate"] : [])], "explore_review"); - if (review.revision !== null && !BARE_SHA256_PATTERN.test(requireNonEmptyString(review.revision, "explore_review.revision"))) throw new EffectRuntimeRequestError("explore_review requires a delivered revision or null when unavailable"); + if (review.revision !== null && !isValidContentSha256(requireNonEmptyString(review.revision, "explore_review.revision"))) throw new EffectRuntimeRequestError("explore_review requires a delivered revision or null when unavailable"); requireStringLiteral(review.disposition, decisionSchema.explore_dispositions, "explore_review.disposition"); const considered = refs(review.considered_refs, "explore_review.considered_refs"); if (review.disposition === "used" && !considered.length) throw new EffectRuntimeRequestError("used Explore evidence requires a delivered reference"); @@ -240,9 +245,11 @@ export function projectStrideExploreBasis(value: unknown, goal: string, agent: s const harness = source.harness as JsonObject | undefined; const findings = Array.isArray(graph?.writeback_results) ? graph.writeback_results as JsonObject[] : []; const recent = Array.isArray(graph?.recent_findings) ? graph.recent_findings as JsonObject[] : []; - const measurementFeedback = (Array.isArray(graph?.measurement_feedback) ? graph.measurement_feedback : []) - .slice(0, 3).filter((item): item is JsonObject => Boolean(item && typeof item === "object" && !Array.isArray(item))) - .map(row => { + const measurementFeedback = projectBoundedMethodologyItems( + (Array.isArray(graph?.measurement_feedback) ? graph.measurement_feedback : []) + .filter((item): item is JsonObject => Boolean(item && typeof item === "object" && !Array.isArray(item))), + 3, + ).visible.map(row => { // The profile is already on the group. Keep candidate/value comparisons // inline; the existing source reader retains full point provenance. const latest = row.latest as JsonObject | undefined; @@ -252,8 +259,11 @@ export function projectStrideExploreBasis(value: unknown, goal: string, agent: s return {...row, best: point(row.best), previous_best: point(row.previous_best), ...(latest && measurement ? {latest: point(measurement)} : {})}; }); - const workYardsticks = (Array.isArray(graph?.work_yardsticks) ? graph.work_yardsticks : []) - .slice(0, 3).filter((item): item is JsonObject => Boolean(item && typeof item === "object" && !Array.isArray(item))); + const workYardsticks = projectBoundedMethodologyItems( + (Array.isArray(graph?.work_yardsticks) ? graph.work_yardsticks : []) + .filter((item): item is JsonObject => Boolean(item && typeof item === "object" && !Array.isArray(item))), + 3, + ).visible; const branches = Array.isArray(harness?.selected_branches) ? harness.selected_branches as JsonObject[] : []; const candidates = source.harness_enabled === true ? comparisonCandidates(harness?.comparison_candidates) : []; const audits = branches.flatMap(branch => { @@ -268,12 +278,14 @@ export function projectStrideExploreBasis(value: unknown, goal: string, agent: s source_sha256: typeof row.source_sha256 === "string" ? row.source_sha256 : createHash("sha256").update(JSON.stringify(row)).digest("hex")}); } + const boundedFindings = projectBoundedMethodologyItems([...byId.values()], 6); + const boundedBranches = projectBoundedMethodologyItems(branches, 3); return {availability: "available", graph_enabled: source.graph_enabled, harness_enabled: source.harness_enabled, // Planner scores/counts and unrelated pool churn do not alter a cited // finding's source. Bind the actual bounded evidence, not the whole view. revision: createHash("sha256").update(JSON.stringify([goal, agent, source.graph_enabled, - source.harness_enabled, [...byId.values()].slice(0, 6), ...(candidates.length ? [candidates] : [])])).digest("hex"), - finding_refs: [...byId.values()].slice(0, 6), + source.harness_enabled, boundedFindings.visible, ...(candidates.length ? [candidates] : [])])).digest("hex"), + finding_refs: boundedFindings.visible, ...(measurementFeedback.length ? {measurement_feedback: measurementFeedback} : {}), ...(graph?.measurement_feedback_page ? {measurement_feedback_page: graph.measurement_feedback_page} : {}), ...(typeof graph?.omitted_measurement_feedback === "number" && graph.omitted_measurement_feedback > 0 @@ -284,9 +296,9 @@ export function projectStrideExploreBasis(value: unknown, goal: string, agent: s ...(candidates.length ? {comparison_candidates: candidates} : {}), ...(harness?.omitted_comparison_candidates !== undefined ? { omitted_comparison_candidates: harness.omitted_comparison_candidates} : {}), - branches: branches.slice(0, 3).map(branch => ({todo_id: branch.todo_id, + branches: boundedBranches.visible.map(branch => ({todo_id: branch.todo_id, selection_status: branch.selection_status, actionable_open: branch.actionable_open})), - omitted_finding_refs: Math.max(0, byId.size - 6), + omitted_finding_refs: boundedFindings.omitted_count, result_page: graph?.result_page ?? null, rule: "Use the existing Explore view for scope/applicability, counterevidence and omitted-result reads. " + "For considered findings outside this list, copy revision/offset/limit from up to three global result pages you read into explore_review.result_pages; writeback replays them against the source. Keep explore_review.revision unchanged. " + @@ -385,7 +397,10 @@ export function strideReviewMismatch(decision: JsonObject, context: JsonObject, } export function strideControlContract(context: JsonObject, obligationId: unknown = null): JsonObject | null { - if (normalizeStrideStrategy(context.stride_strategy) === "off") return null; + const strategy = normalizeStrideStrategy(context.stride_strategy); + if (!evaluateMethodologyActivation({enabled: strategy !== "off", quiet_turn_policy: "due_obligation_only"}).active) { + return null; + } const follow = context.decision_followthrough as JsonObject | undefined; const previous = (follow?.previous_decision as JsonObject | undefined)?.evidence_ref ?? null; const basis = context.explore_basis as JsonObject | undefined; diff --git a/loopx/control_plane/work_items/yardstick_decision.py b/loopx/control_plane/work_items/yardstick_decision.py index ddfd99ed57..ef2f0db658 100644 --- a/loopx/control_plane/work_items/yardstick_decision.py +++ b/loopx/control_plane/work_items/yardstick_decision.py @@ -12,12 +12,17 @@ @lru_cache(maxsize=256) def _capture(encoded: str) -> str: try: - return json.dumps(effect_runtime_result("work_item.yardstick.capture", json.loads(encoded)), sort_keys=True) + return json.dumps( + effect_runtime_result("work_item.yardstick.capture", json.loads(encoded)), + sort_keys=True, + ) except EffectRuntimeRejected as error: raise ValueError(str(error)) from error -def normalize_work_yardstick_capture(capture: Any, *, goal_id: str, agent_id: str, evidence_refs: Any) -> dict[str, Any]: +def normalize_work_yardstick_capture( + capture: Any, *, goal_id: str, agent_id: str, evidence_refs: Any +) -> dict[str, Any]: params = dict(capture=capture, goal_id=goal_id, agent_id=agent_id, evidence_refs=evidence_refs) validate_public_safe_value(params, path="work_yardstick") normalized = json.loads(_capture(json.dumps(params, sort_keys=True, allow_nan=False))) @@ -25,15 +30,54 @@ def normalize_work_yardstick_capture(capture: Any, *, goal_id: str, agent_id: st def project_work_yardstick_presentations( - rows: list[dict[str, Any]], *, goal_id: str | None = None, + rows: list[dict[str, Any]], + *, + goal_id: str | None = None, + findings: list[dict[str, Any]] | None = None, ) -> list[dict[str, Any]]: - """One typed projection call for the captures in an entire read snapshot.""" - requests = [{"capture": row["work_yardstick"], "goal_id": goal_id or row["goal_id"], - "agent_id": row.get("agent_id", ""), "evidence_refs": row.get("evidence_refs", [])} - for row in rows if "work_yardstick" in row] + """One typed projection call for captures and optional scoped fidelity in a snapshot.""" + requests = [ + { + "capture": row["work_yardstick"], + "goal_id": goal_id or row["goal_id"], + "agent_id": row.get("agent_id", ""), + "evidence_refs": row.get("evidence_refs", []), + **({"node_id": row["node_id"]} if "node_id" in row else {}), + } + for row in rows + if "work_yardstick" in row + ] if not requests: return rows - params = {"captures": requests} + params: dict[str, Any] = {"captures": requests} + if findings is not None: + params["findings"] = [row for row in findings if row.get("measurements")] validate_public_safe_value(params, path="work_yardsticks") - projected = iter(effect_runtime_result("work_item.yardstick.presentation", params)["captures"]) - return [{**row, "work_yardstick": next(projected)} if "work_yardstick" in row else row for row in rows] + try: + response = effect_runtime_result("work_item.yardstick.presentation", params) + except EffectRuntimeRejected as error: + if error.diagnostic_code != "request_too_large": + raise + response = effect_runtime_result( + "work_item.yardstick.presentation", + params, + large_local_snapshot=True, + ) + projected_captures = iter(response["captures"]) + projected_fidelities = iter( + response.get("yardstick_fidelities") or [None] * len(requests) + ) + enriched: list[dict[str, Any]] = [] + for row in rows: + if "work_yardstick" not in row: + enriched.append(row) + continue + fidelity = next(projected_fidelities) + enriched.append( + { + **row, + "work_yardstick": next(projected_captures), + **({"yardstick_fidelity": fidelity} if fidelity is not None else {}), + } + ) + return enriched diff --git a/loopx/control_plane/work_items/yardstick_decision.ts b/loopx/control_plane/work_items/yardstick_decision.ts index 52ae634939..2f2a463b98 100644 --- a/loopx/control_plane/work_items/yardstick_decision.ts +++ b/loopx/control_plane/work_items/yardstick_decision.ts @@ -1,121 +1,204 @@ /** Optional current-work decision carried by Explore's existing result writeback. * The declaration records a research method, not acceptance or execution authority. * Stride's replan-only v2 contract remains unchanged. */ -import {parseTodoTimestampMicros} from "../runtime_timestamp.ts"; -import {BARE_SHA256_PATTERN} from "../content_digest.ts"; import {EffectRuntimeRequestError} from "../effect_runtime_errors.ts"; import {jsonObject, requireJsonObject, requireStringArray, type JsonObject} from "../runtime_decode.ts"; -import {isExploreEvidenceRef} from "../capabilities/explore_evidence_ref.ts"; -import {projectExploreMeasurementFeedback, type ExploreMeasurementFeedback, - type ExploreMeasurementPoint} from "../capabilities/explore_measurements.ts"; +import {parseTodoTimestampMicros} from "../runtime_timestamp.ts"; +import { + matchesExploreMetricProfile, + normalizeExploreMetricProfile, + projectExploreMeasurementFeedback, + type ExploreMeasurement, + type ExploreMeasurementFeedback, + type ExploreMeasurementPoint, + type ExploreMetricProfile, +} from "../capabilities/explore_measurements.ts"; +import { + committedInWorkWindow, + evaluateMethodologyActivation, + matchesWorkIdentity, + normalizeCanonicalSourceSnapshot, + projectBoundedMethodologyItems, + recordedBeforePrefix, + requireBoundedSingleLine, + requireExactFields, + requireOpaqueMethodologyRef, + requireUniqueOpaqueRefs, + selectCanonicalEvidenceWindow, +} from "../capabilities/stage_methodology_operator.ts"; +import {evaluateScopedYardstickFidelity} from "../capabilities/yardstick_fidelity.ts"; export const WORK_YARDSTICK_DECISION_SCHEMA = "work_yardstick_decision_v0"; -export type WorkYardstickProfile = {metric: string; unit: string; higher_is_better: boolean; - comparison_scope: string; input_revision: string; evaluation_ref: string}; -export type WorkYardstickDecision = {schema_version: typeof WORK_YARDSTICK_DECISION_SCHEMA; +export type WorkYardstickProfile = ExploreMetricProfile; +export type WorkYardstickDecision = { + schema_version: typeof WORK_YARDSTICK_DECISION_SCHEMA; target: {summary: string; scope: string; acceptance_ref: string}; - authoritative: WorkYardstickProfile; proxy: WorkYardstickProfile | null; - counterevidence_refs: string[]; prior_finding_id: string | null; - next_action: {method_id: string; reason: string; cost_bound: string}}; -export type WorkYardstickCapture = {decision: WorkYardstickDecision; + authoritative: WorkYardstickProfile; + proxy: WorkYardstickProfile | null; + counterevidence_refs: string[]; + prior_finding_id: string | null; + next_action: {method_id: string; reason: string; cost_bound: string}; +}; +export type WorkYardstickCapture = { + decision: WorkYardstickDecision; binding: {goal_id: string; agent_id: string; todo_id: string; turn_instance_id: string}; - source_snapshot?: {event_count: number; event_ids_sha256: string}}; -function object(value: unknown, fields: string[], label: string): JsonObject { - const row = requireJsonObject(value, label); - if (Object.keys(row).some(key => !fields.includes(key)) || fields.some(key => !Object.hasOwn(row, key))) { - throw new EffectRuntimeRequestError(`${label} fields are missing or unsupported`); - } - return row; -} -function text(value: unknown, label: string, limit = 240): string { - if (typeof value !== "string" || !value.trim() || value.length > limit || /[\u0000-\u001f]/.test(value)) { - throw new EffectRuntimeRequestError(`${label} requires bounded single-line text`); - } - return value.trim(); -} -function ref(value: unknown, label: string): string { - const result = text(value, label, 128); - if (!isExploreEvidenceRef(result) || result.startsWith("file:") || result.split("/").includes("..")) { + source_snapshot?: {event_count: number; event_ids_sha256: string}; +}; + +const DECISION_FIELDS = [ + "schema_version", "target", "authoritative", "proxy", "counterevidence_refs", "next_action", "prior_finding_id", +] as const; +const TARGET_FIELDS = ["summary", "scope", "acceptance_ref"] as const; +const ACTION_FIELDS = ["method_id", "reason", "cost_bound"] as const; +const BINDING_FIELDS = ["goal_id", "agent_id", "todo_id", "turn_instance_id"] as const; + +function strictOpaqueRef(value: unknown, label: string): string { + const raw = requireBoundedSingleLine(value, label, 128); + if (raw.startsWith("file:")) { throw new EffectRuntimeRequestError(`${label} requires an opaque reference`); } - return result; -} -function refs(value: unknown, label: string): string[] { - if (!Array.isArray(value) || value.length > 8) throw new EffectRuntimeRequestError(`${label} requires at most eight references`); - return [...new Set(value.map(item => ref(item, label)))].sort(); -} -/** The scope keys deliberately reuse Explore measurement comparison identity. */ -function profile(value: unknown): WorkYardstickProfile { - const row = object(value, ["metric", "unit", "higher_is_better", "comparison_scope", "input_revision", "evaluation_ref"], "yardstick profile"); - if (typeof row.higher_is_better !== "boolean") throw new EffectRuntimeRequestError("yardstick requires explicit metric direction"); - return {metric: text(row.metric, "metric", 80), unit: text(row.unit, "unit", 40), - higher_is_better: row.higher_is_better, comparison_scope: ref(row.comparison_scope, "comparison_scope"), - input_revision: text(row.input_revision, "input_revision", 160), evaluation_ref: ref(row.evaluation_ref, "evaluation_ref")}; + return requireOpaqueMethodologyRef(raw, label, 128); } + export function normalizeWorkYardstickDecision(value: unknown, evidenceRefs?: unknown): WorkYardstickDecision { - const row = object(value, ["schema_version", "target", "authoritative", "proxy", "counterevidence_refs", "next_action", "prior_finding_id"], "work yardstick decision"); - if (row.schema_version !== WORK_YARDSTICK_DECISION_SCHEMA) throw new EffectRuntimeRequestError("unsupported work yardstick decision schema"); - const target = object(row.target, ["summary", "scope", "acceptance_ref"], "research target"); - const next = object(row.next_action, ["method_id", "reason", "cost_bound"], "next research action"); - const counter = refs(row.counterevidence_refs, "counterevidence_refs"); + const row = requireExactFields(value, DECISION_FIELDS, DECISION_FIELDS, "work yardstick decision"); + if (row.schema_version !== WORK_YARDSTICK_DECISION_SCHEMA) { + throw new EffectRuntimeRequestError("unsupported work yardstick decision schema"); + } + const target = requireExactFields(row.target, TARGET_FIELDS, TARGET_FIELDS, "research target"); + const next = requireExactFields(row.next_action, ACTION_FIELDS, ACTION_FIELDS, "next research action"); + const counter = requireUniqueOpaqueRefs(row.counterevidence_refs, "counterevidence_refs", 8); if (evidenceRefs !== undefined && (!Array.isArray(evidenceRefs) || counter.some(item => !evidenceRefs.includes(item)))) { throw new EffectRuntimeRequestError("counterevidence must reference evidence captured by this finding"); } - const authoritative = profile(row.authoritative), proxy = row.proxy === null ? null : profile(row.proxy); - if (proxy && JSON.stringify(proxy) === JSON.stringify(authoritative)) throw new EffectRuntimeRequestError("proxy and authoritative profiles must be distinct"); - if (!proxy && counter.length) throw new EffectRuntimeRequestError("proxy counterevidence requires the affected proxy profile"); - return {schema_version: WORK_YARDSTICK_DECISION_SCHEMA, - target: {summary: text(target.summary, "target summary"), scope: text(target.scope, "target scope"), acceptance_ref: ref(target.acceptance_ref, "acceptance_ref")}, - authoritative, proxy, counterevidence_refs: counter, - next_action: {method_id: ref(next.method_id, "method_id"), reason: text(next.reason, "next action reason", 320), cost_bound: text(next.cost_bound, "cost_bound", 160)}, - prior_finding_id: row.prior_finding_id === null ? null : ref(row.prior_finding_id, "prior_finding_id")}; + const authoritative = normalizeExploreMetricProfile(row.authoritative, "yardstick"); + const proxy = row.proxy === null ? null : normalizeExploreMetricProfile(row.proxy, "yardstick"); + if (proxy && JSON.stringify(proxy) === JSON.stringify(authoritative)) { + throw new EffectRuntimeRequestError("proxy and authoritative profiles must be distinct"); + } + if (!proxy && counter.length) { + throw new EffectRuntimeRequestError("proxy counterevidence requires the affected proxy profile"); + } + return { + schema_version: WORK_YARDSTICK_DECISION_SCHEMA, + target: { + summary: requireBoundedSingleLine(target.summary, "target summary", 240), + scope: requireBoundedSingleLine(target.scope, "target scope", 240), + acceptance_ref: strictOpaqueRef(target.acceptance_ref, "acceptance_ref"), + }, + authoritative, + proxy, + counterevidence_refs: counter, + next_action: { + method_id: strictOpaqueRef(next.method_id, "method_id"), + reason: requireBoundedSingleLine(next.reason, "next action reason", 320), + cost_bound: requireBoundedSingleLine(next.cost_bound, "cost_bound", 160), + }, + prior_finding_id: row.prior_finding_id === null ? null : strictOpaqueRef(row.prior_finding_id, "prior_finding_id"), + }; } + /** Binding is supplied by the committed settlement, never by an attachment. */ export function normalizeWorkYardstickCapture(params: JsonObject): WorkYardstickCapture { const raw = requireJsonObject(params.capture, "yardstick capture"); - const capture = object(raw, ["decision", "binding", ...(raw.source_snapshot === undefined ? [] : ["source_snapshot"])], "yardstick capture"); - const binding = object(capture.binding, ["goal_id", "agent_id", "todo_id", "turn_instance_id"], "work binding"); - const normalized = Object.fromEntries(Object.entries(binding).map(([key, value]) => [key, ref(value, key)])); - if (!String(normalized.todo_id).startsWith("todo_")) throw new EffectRuntimeRequestError("yardstick capture requires a work Todo"); + const captureFields = ["decision", "binding", ...(raw.source_snapshot === undefined ? [] : ["source_snapshot"])]; + const capture = requireExactFields(raw, captureFields, captureFields, "yardstick capture"); + const binding = requireExactFields(capture.binding, BINDING_FIELDS, BINDING_FIELDS, "work binding"); + const normalized = Object.fromEntries( + Object.entries(binding).map(([key, val]) => [key, strictOpaqueRef(val, key)]), + ); + if (!String(normalized.todo_id).startsWith("todo_")) { + throw new EffectRuntimeRequestError("yardstick capture requires a work Todo"); + } for (const key of ["goal_id", "agent_id"]) { - if (params[key] !== undefined && normalized[key] !== params[key]) throw new EffectRuntimeRequestError("yardstick capture cannot cross Goal or Agent identity"); + if (params[key] !== undefined && normalized[key] !== params[key]) { + throw new EffectRuntimeRequestError("yardstick capture cannot cross Goal or Agent identity"); + } } - return {decision: normalizeWorkYardstickDecision(capture.decision, params.evidence_refs), binding: normalized as WorkYardstickCapture["binding"], - ...(capture.source_snapshot === undefined ? {} : {source_snapshot: normalizeWorkYardstickSourceSnapshot(capture.source_snapshot)})}; + return { + decision: normalizeWorkYardstickDecision(capture.decision, params.evidence_refs), + binding: normalized as WorkYardstickCapture["binding"], + ...(capture.source_snapshot === undefined + ? {} + : {source_snapshot: normalizeWorkYardstickSourceSnapshot(capture.source_snapshot)}), + }; } -/** Presentation readers are independently deployed and retain their v0 shape. - * The canonical capture and policy cue keep server-owned source provenance. */ -export function projectWorkYardstickPresentation(params: JsonObject): Pick { + +/** Presentation readers retain their v0 capture shape; when canonical findings + * are supplied, scoped multi-point fidelity is computed in the same single batch. */ +export function projectWorkYardstickPresentation( + params: JsonObject, +): Pick { const {decision, binding} = normalizeWorkYardstickCapture(params); return {decision, binding}; } + export function projectWorkYardstickPresentations(params: JsonObject): JsonObject { - if (!Array.isArray(params.captures)) throw new EffectRuntimeRequestError("yardstick presentation requires capture requests"); - return {captures: params.captures.map(row => projectWorkYardstickPresentation(requireJsonObject(row, "capture request")))}; -} -export function normalizeWorkYardstickSourceSnapshot(value: unknown): NonNullable { - const row = object(value, ["event_count", "event_ids_sha256"], "yardstick source snapshot"); - if (!Number.isSafeInteger(row.event_count) || Number(row.event_count) < 0 - || typeof row.event_ids_sha256 !== "string" || !BARE_SHA256_PATTERN.test(row.event_ids_sha256)) { - throw new EffectRuntimeRequestError("yardstick source snapshot requires its original event count and digest"); + if (!Array.isArray(params.captures)) { + throw new EffectRuntimeRequestError("yardstick presentation requires capture requests"); + } + const captures = params.captures.map(row => + projectWorkYardstickPresentation(requireJsonObject(row, "capture request")), + ); + if (!Array.isArray(params.findings)) { + return {captures}; } - return {event_count: Number(row.event_count), event_ids_sha256: row.event_ids_sha256}; + const byNode = new Map(); + for (const item of params.findings) { + const row = requireJsonObject(item, "Explore finding"); + const node = String(row.node_id ?? ""); + const list = byNode.get(node) ?? []; + list.push(row); + byNode.set(node, list); + } + const fidelityCache = new Map(); + const yardstickFidelities = params.captures.map((rawReq, idx) => { + const req = requireJsonObject(rawReq, "capture request"); + const nodeId = typeof req.node_id === "string" ? req.node_id : ""; + const {decision} = captures[idx]!; + if (!nodeId || decision.proxy === null) return null; + const key = JSON.stringify([nodeId, decision.authoritative, decision.proxy]); + let cached = fidelityCache.get(key); + if (!cached) { + cached = evaluateScopedYardstickFidelity({ + node_id: nodeId, + authoritative: decision.authoritative as unknown as JsonObject, + proxy: decision.proxy as unknown as JsonObject, + findings: byNode.get(nodeId) ?? [], + }) as unknown as JsonObject; + fidelityCache.set(key, cached); + } + return cached; + }); + return {captures, yardstick_fidelities: yardstickFidelities}; } + +export function normalizeWorkYardstickSourceSnapshot( + value: unknown, +): NonNullable { + return normalizeCanonicalSourceSnapshot(value, "yardstick source snapshot"); +} + /** Share current-work eligibility between the read plan and its final projection. */ export function workYardstickTodos(params: JsonObject): JsonObject[] { - const todos = params.yardstick_enabled === true && Array.isArray(params.work_todos) ? (params.work_todos as JsonObject[]) - .filter(todo => todo.role === "agent" && !todo.done && todo.claimed_by === params.agent_id - && ["open", "pending", "in_progress", "blocked"].includes(String(todo.status)) - && (params.todo_id == null || todo.todo_id === params.todo_id)) : []; + const todos = evaluateMethodologyActivation({enabled: params.yardstick_enabled}).active && Array.isArray(params.work_todos) + ? (params.work_todos as JsonObject[]).filter( + todo => + todo.role === "agent" && + !todo.done && + todo.claimed_by === params.agent_id && + ["open", "pending", "in_progress", "blocked"].includes(String(todo.status)) && + (params.todo_id == null || todo.todo_id === params.todo_id), + ) + : []; const preferred = todos.findIndex(todo => todo.todo_id === params.preferred_yardstick_todo_id); if (preferred > 0) todos.unshift(...todos.splice(preferred, 1)); return todos; } + function belongsToWork(row: JsonObject, binding: JsonObject): boolean { const capture = row.work_yardstick as WorkYardstickCapture | undefined; - const source = jsonObject(row.work_binding) ?? capture?.binding; - return !!source && source.goal_id === binding.goal_id && source.agent_id === binding.agent_id - && source.todo_id === binding.todo_id; + return matchesWorkIdentity(row, binding, capture?.binding as unknown as JsonObject | undefined); } function newestFindingFirst(a: JsonObject, b: JsonObject): number { @@ -150,58 +233,49 @@ function measuredPair(row: ExploreMeasurementFeedback | undefined) { const {value, candidate_ref, evidence_ref} = latest.measurement; if (value === null || !Number.isFinite(value) || candidate_ref === previous.candidate_ref) return null; const recent: ExploreMeasurementPoint = {finding_id: latest.finding_id, candidate_ref, evidence_ref, value}; - const preferred = value === previous.value ? null : (value > previous.value) === row.higher_is_better - ? candidate_ref : previous.candidate_ref; + const preferred = + value === previous.value + ? null + : (value > previous.value) === row.higher_is_better + ? candidate_ref + : previous.candidate_ref; return {previous_best: previous, latest: recent, preferred_candidate_ref: preferred}; } -function matchesProfile(row: ExploreMeasurementFeedback, profile: WorkYardstickProfile): boolean { - return Object.entries(profile).every(([key, value]) => row[key as keyof WorkYardstickProfile] === value); -} - -/** Both declared pairs and first measurements support the same local choice. - * This is advisory work context, never another review or evaluation gate. */ export function workYardstickChoiceGuidance(): string { - return "At a meaningful evidence or probe boundary, weigh continuing, a different mechanism or a compatible combination " + + return ( + "At a meaningful evidence or probe boundary, weigh continuing, a different mechanism or a compatible combination " + "by expected accepted-outcome gain within accepted constraints, remaining horizon and full implementation and validation cost, " + "including search, confirmation and model/control overhead. Keep unknowns as unknown; continuing a supported route is valid, " + "and local improvement alone does not establish the best next investment. Without a material change, continue the authorized slice; " + - "this cue requires no extra review, measurement or decision payload. Consider a bounded check when it could change the choice."; + "this cue requires no extra review, measurement or decision payload. Consider a bounded check when it could change the choice." + ); } /** Existing work facts, without guessing an outcome/proxy role or adding a gate. */ -export function projectWorkMeasurementFacts(todoId: string, available: ExploreMeasurementFeedback[], - detailsCommand?: string[], supplementaryReceipt = false): JsonObject | null { +export function projectWorkMeasurementFacts( + todoId: string, + available: ExploreMeasurementFeedback[], + detailsCommand?: string[], + supplementaryReceipt = false, +): JsonObject | null { if (!available.length && !supplementaryReceipt) return null; const first = new Map(); for (const row of available) if (!first.has(row.node_id)) first.set(row.node_id, row); const representatives = new Set(first.values()); - // A receipt already carries its current feedback. Keep at most two other - // groups here; the task context retains its existing three-group budget. - const visible = [...representatives, ...available.filter(row => !representatives.has(row))].slice(0, supplementaryReceipt ? 2 : 3); - return {todo_id: todoId, measurement_feedback: visible, - omitted_measurement_feedback: available.length - visible.length, - ...(detailsCommand ? {details_command: detailsCommand} : {}), - guidance: (supplementaryReceipt ? "This write's measurement_feedback is returned separately; these groups supplement it. " : "") + + const prioritized = [...representatives, ...available.filter(row => !representatives.has(row))]; + const bounded = projectBoundedMethodologyItems(prioritized, supplementaryReceipt ? 2 : 3, detailsCommand); + return { + todo_id: todoId, + measurement_feedback: bounded.visible, + omitted_measurement_feedback: bounded.omitted_count, + ...(bounded.details_command ? {details_command: bounded.details_command} : {}), + guidance: + (supplementaryReceipt ? "This write's measurement_feedback is returned separately; these groups supplement it. " : "") + "Recorded comparisons for questions linked by this Todo; shared findings do not prove this Todo executed them. Match each metric and recorded scope to the accepted outcome before choosing work. " + workYardstickChoiceGuidance() + - " No outcome/proxy profile has been selected here. Latest recorded is not current or retained, and best is not acceptance or permission. Use the existing task reader for omitted groups; a decision attachment remains optional."}; -} - -function recordedBefore(row: JsonObject, sourceOrder: unknown): boolean { - const order = row.recorded_order; - return typeof sourceOrder === "number" && Number.isSafeInteger(sourceOrder) - && typeof order === "number" && Number.isSafeInteger(order) && order >= 0 && order < sourceOrder; -} - -function committedInWorkWindow(row: JsonObject, startOrder: number, decisionTime: bigint | null): boolean { - const order = row.recorded_order; - const committed = parseTodoTimestampMicros(String(row.last_updated_at ?? "")); - // Append order says when a result became visible, not when its primary - // committed. A delayed old delivery cannot become a new decision's evidence. - // Tied or unknown original times do not prove a newer uncited observation. - return Number.isSafeInteger(order) && Number(order) >= startOrder - && committed !== null && decisionTime !== null && committed > decisionTime; + " No outcome/proxy profile has been selected here. Latest recorded is not current or retained, and best is not acceptance or permission. Use the existing task reader for omitted groups; a decision attachment remains optional.", + }; } /** Ordinary execution and a revised choice share the existing finding history. @@ -229,7 +303,7 @@ function workYardstickFollowthrough(findings: JsonObject[], source: JsonObject, // clock rolls back; legacy captures still use the strict time bound. if (choice !== source && (endTime === null || !Number.isSafeInteger(source.recorded_order) || Number(row.recorded_order) > Number(source.recorded_order) - || row !== source && current.source_snapshot !== undefined && !recordedBefore(row, current.source_snapshot.event_count) + || row !== source && current.source_snapshot !== undefined && !recordedBeforePrefix(row, current.source_snapshot.event_count) || current.source_snapshot === undefined && parseTodoTimestampMicros(String(row.last_updated_at))! > endTime)) continue; if (!result || newestFindingFirst(row, result) < 0) result = row; @@ -248,92 +322,127 @@ function workYardstickFollowthrough(findings: JsonObject[], source: JsonObject, function measuredSearchComparison( scopedFindings: readonly JsonObject[], decision: WorkYardstickDecision, - cited: ReadonlySet, - sourceOrder: unknown, - workObservationStart: number, - decisionTime: bigint | null, + includeMeasurement: (row: JsonObject, measurement: ExploreMeasurement) => boolean, ) { const pairFor = (profile: WorkYardstickProfile, candidates?: ReadonlySet) => { const feedback = projectExploreMeasurementFeedback(scopedFindings, (row, measurement) => - (row.work_yardstick !== undefined || (cited.has(measurement.evidence_ref) - ? recordedBefore(row, sourceOrder) : row.work_binding !== undefined - && committedInWorkWindow(row, workObservationStart, decisionTime))) - && (!candidates || candidates.has(measurement.candidate_ref))); - return measuredPair(feedback.find(row => matchesProfile(row, profile))); + includeMeasurement(row, measurement) && + (!candidates || candidates.has(measurement.candidate_ref)), + ); + return measuredPair(feedback.find(row => matchesExploreMetricProfile(row, profile))); }; const goal = pairFor(decision.authoritative); if (!goal) return null; - // Both comparisons have two distinct candidates. Scope the proxy lookup to - // the two goal candidates so a rejected sibling proxy false-positive or a - // later proxy-only screening row cannot displace their paired comparison. const goalCandidates = new Set([goal.previous_best.candidate_ref, goal.latest.candidate_ref]); const proxy = decision.proxy && pairFor(decision.proxy, goalCandidates); - const reversed = !!(goal.preferred_candidate_ref && proxy?.preferred_candidate_ref - && goal.preferred_candidate_ref !== proxy.preferred_candidate_ref); - const opening = goal.preferred_candidate_ref === null - ? "Observed goal results tie under this scope. Check remaining headroom before another evaluation. " - : (reversed ? "The proxy reverses the observed goal order. " : "") + - "Use the better-scoring candidate under this scope as a comparison base when available. "; - return {goal, ...(proxy ? {proxy, proxy_order_reversed: reversed} : {}), - guidance: opening + workYardstickChoiceGuidance()}; + const reversed = !!( + goal.preferred_candidate_ref && + proxy?.preferred_candidate_ref && + goal.preferred_candidate_ref !== proxy.preferred_candidate_ref + ); + const opening = + goal.preferred_candidate_ref === null + ? "Observed goal results tie under this scope. Check remaining headroom before another evaluation. " + : (reversed ? "The proxy reverses the observed goal order. " : "") + + "Use the better-scoring candidate under this scope as a comparison base when available. "; + return { + goal, + ...(proxy ? {proxy, proxy_order_reversed: reversed} : {}), + guidance: opening + workYardstickChoiceGuidance(), + }; } export function projectWorkYardstickCue(params: JsonObject): JsonObject | null { - if (params.enabled !== true || !params.todo_id) return null; + if (!evaluateMethodologyActivation({enabled: params.enabled, required_scope: {todo_id: params.todo_id}}).active) { + return null; + } const findings = (params.findings ?? []) as JsonObject[]; const source = currentWorkYardstickFinding(findings, params); if (!source) return null; - const capture = normalizeWorkYardstickCapture({capture: source.work_yardstick, evidence_refs: source.evidence_refs, - goal_id: params.goal_id, agent_id: params.agent_id}); + const capture = normalizeWorkYardstickCapture({ + capture: source.work_yardstick, + evidence_refs: source.evidence_refs, + goal_id: params.goal_id, + agent_id: params.agent_id, + }); const decision = capture.decision; const counter = decision.counterevidence_refs; const followthrough = workYardstickFollowthrough(findings, source, params); const profiles = decision.proxy ? [decision.authoritative, decision.proxy] : [decision.authoritative]; - const feedback = ((params.measurement_feedback ?? []) as ExploreMeasurementFeedback[]).filter(row => - row.node_id === source.node_id && profiles.some(p => matchesProfile(row, p))); - // An earlier target can share the same evaluator profile and work identity. - // Filter source findings before ranking so its best cannot displace a legal pair. + const feedback = ((params.measurement_feedback ?? []) as ExploreMeasurementFeedback[]).filter( + row => row.node_id === source.node_id && profiles.some(p => matchesExploreMetricProfile(row, p)), + ); const cited = new Set(counter); - const sourceOrder = capture.source_snapshot?.event_count; - // New observations belong to this current decision's work window. Earlier - // ordinary measurements require the original explicitly cited snapshot. - const workObservationStart = sourceOrder ?? Number(source.recorded_order) + 1; - const decisionTime = parseTodoTimestampMicros(String(source.last_updated_at ?? "")); - const priorSource = Array.isArray(params.cited_sources) ? (params.cited_sources as JsonObject[]).find(row => - row.source_finding_id === source.finding_id - && capture.source_snapshot !== undefined - && (row.source_snapshot as JsonObject | undefined)?.event_count === capture.source_snapshot.event_count - && (row.source_snapshot as JsonObject | undefined)?.event_ids_sha256 === capture.source_snapshot.event_ids_sha256) : undefined; - const plainFindings = Array.isArray(priorSource?.findings) ? priorSource.findings as JsonObject[] : []; - const scopedFindings = [...findings.filter(row => row.work_yardstick !== undefined - || row.work_binding !== undefined && committedInWorkWindow(row, workObservationStart, decisionTime)), - ...plainFindings.filter(row => row.work_yardstick === undefined)].filter(row => { - if (row.node_id !== source.node_id) return false; - const bound = jsonObject(row.work_binding) !== null; - if (bound && (!belongsToWork(row, params) || row.status !== "confirmed")) return false; - // A decision can cite the agent's earlier measurements on this linked - // question without changing their provenance or assigning earlier work. - if (row.work_yardstick === undefined) { - if (bound) return true; - // A repeated reference appended later cannot change the decision's evidence. - return cited.size > 0 && recordedBefore(row, sourceOrder) - && row.agent_id === capture.binding.agent_id && Array.isArray(row.evidence_refs) - && row.evidence_refs.some(ref => typeof ref === "string" && cited.has(ref)); - } - const target = (row.work_yardstick as WorkYardstickCapture | undefined)?.decision?.target; - return belongsToWork(row, params) && target !== undefined && target.summary === decision.target.summary - && target.scope === decision.target.scope && target.acceptance_ref === decision.target.acceptance_ref; + const evidenceWindow = selectCanonicalEvidenceWindow({ + findings, + cited_sources: params.cited_sources, + source_finding: source, + binding: capture.binding, + source_snapshot: capture.source_snapshot, + cited_refs: cited, + has_decision_attachment: row => row.work_yardstick !== undefined, + matches_decision_target: row => { + const target = (row.work_yardstick as WorkYardstickCapture | undefined)?.decision?.target; + return ( + belongsToWork(row, params) && + target !== undefined && + target.summary === decision.target.summary && + target.scope === decision.target.scope && + target.acceptance_ref === decision.target.acceptance_ref + ); + }, }); - const searchComparison = measuredSearchComparison(scopedFindings, decision, cited, sourceOrder, workObservationStart, decisionTime); - return {source_finding_id: source.finding_id, node_id: source.node_id, binding: capture.binding, ...decision, + const scopedFindings = evidenceWindow.scoped_findings; + const includeMeasurement = (row: JsonObject, measurement: ExploreMeasurement) => + row.work_yardstick !== undefined || + (cited.has(measurement.evidence_ref) + ? recordedBeforePrefix(row, evidenceWindow.source_order) + : row.work_binding !== undefined && committedInWorkWindow( + row, evidenceWindow.work_observation_start, evidenceWindow.decision_time_micros, + )); + const searchComparison = measuredSearchComparison( + scopedFindings, decision, includeMeasurement, + ); + const multiPointFidelity = decision.proxy + ? evaluateScopedYardstickFidelity({ + node_id: source.node_id, + findings: scopedFindings, + authoritative: decision.authoritative as unknown as JsonObject, + proxy: decision.proxy as unknown as JsonObject, + }, includeMeasurement) + : null; + const frontier = multiPointFidelity?.calibration_tier === "multi_point" + ? multiPointFidelity.frontier_neighborhood : null; + let fidelityGuidance: string | undefined; + if (frontier && (frontier.best_candidate_agrees === false || + multiPointFidelity?.fidelity_status === "frontier_bias_cliff" || + multiPointFidelity?.fidelity_status === "order_reversed")) { + fidelityGuidance = "This scoped proxy has misranked accepted results. Use it to generate candidates."; + } else if (frontier?.authoritative_best_candidate_ref && frontier.proxy_best_paired_candidate_ref === null) { + fidelityGuidance = "This scoped proxy leaves the best candidates tied."; + } + if (fidelityGuidance) { + fidelityGuidance += " If a candidate would change the next route, use one eligible authoritative comparison before selecting or discarding it."; + } + return { + source_finding_id: source.finding_id, + node_id: source.node_id, + binding: capture.binding, + ...decision, ...(capture.source_snapshot ? {source_snapshot: capture.source_snapshot} : {}), measurement_feedback: feedback, ...(searchComparison ? {search_comparison: searchComparison} : {}), + ...(multiPointFidelity && multiPointFidelity.calibration_tier === "multi_point" + ? {yardstick_fidelity: multiPointFidelity as unknown as JsonObject} + : {}), + ...(fidelityGuidance ? {fidelity_guidance: fidelityGuidance} : {}), ...(followthrough ? {previous_decision_followthrough: followthrough} : {}), guidance: counter.length ? "Counterevidence is recorded for this scoped proxy. Reassess its use at the next concrete choice; a local screening heuristic cannot veto an informative authoritative check by itself. Preserve accepted correctness, authorization and budget constraints." : "Use this current target and scoped yardstick when choosing the next useful action. Proxy fidelity remains unqualified here; preserve accepted constraints and bound confirmation cost.", - authority: "Recorded method and source references are Agent declarations, not independently verified acceptance or permission. Omission preserves this decision; revise it only when work or evidence changes. No extra Turn, replan or mandatory capture is required."}; + authority: + "Recorded method and source references are Agent declarations, not independently verified acceptance or permission. Omission preserves this decision; revise it only when work or evidence changes. No extra Turn, replan or mandatory capture is required.", + }; } /** Project the receipt from the committed evidence prefix, before another Turn. */ @@ -342,30 +451,56 @@ export function projectExploreFindingReceipt(params: JsonObject): JsonObject { const written = findings.find(row => row.finding_id === params.finding_id && belongsToWork(row, params)); if (!written) throw new EffectRuntimeRequestError("Explore receipt requires its committed work finding"); const question = findings.filter(row => row.node_id === written.node_id); - const feedback = projectExploreMeasurementFeedback(question.filter(row => - row.status === "confirmed" && belongsToWork(row, params))); + const feedback = projectExploreMeasurementFeedback( + question.filter(row => row.status === "confirmed" && belongsToWork(row, params)), + ); const latest = feedback.filter(group => group.latest.finding_id === written.finding_id); const cue = projectWorkYardstickCue({...params, findings: question, measurement_feedback: feedback}); const comparison = cue?.search_comparison as ReturnType | undefined; - const participates = comparison && [comparison.goal, comparison.proxy].some(pair => - pair?.latest.finding_id === written.finding_id || pair?.previous_best.finding_id === written.finding_id); + const participates = + comparison && + [comparison.goal, comparison.proxy].some( + pair => pair?.latest.finding_id === written.finding_id || pair?.previous_best.finding_id === written.finding_id, + ); const route = params.route === undefined ? undefined : requireStringArray(params.route, "Explore read route"); let workFacts: JsonObject | null = null; - if (params.enabled === true && Array.isArray(written.measurements) - && !currentWorkYardstickFinding(findings, params)) { + if (params.enabled === true && Array.isArray(written.measurements) && !currentWorkYardstickFinding(findings, params)) { const order = new Map(findings.map(row => [row.finding_id, Number(row.recorded_order ?? 0)])); - const workFeedback = projectExploreMeasurementFeedback(findings.filter(row => row.status === "confirmed" && belongsToWork(row, params))) + const workFeedback = projectExploreMeasurementFeedback( + findings.filter(row => row.status === "confirmed" && belongsToWork(row, params)), + ) .filter(row => row.latest.finding_id !== written.finding_id) - .sort((a, b) => Number(b.node_id === written.node_id) - Number(a.node_id === written.node_id) - || (order.get(b.latest.finding_id) ?? 0) - (order.get(a.latest.finding_id) ?? 0)); - workFacts = projectWorkMeasurementFacts(String(params.todo_id), workFeedback, - route ? [...route, "explore", "turn-context", "--agent-id", String(params.agent_id), - "--todo-id", String(params.todo_id), "--result-limit", "20", "--goal-id", String(params.goal_id)] : undefined, true); + .sort( + (a, b) => + Number(b.node_id === written.node_id) - Number(a.node_id === written.node_id) || + (order.get(b.latest.finding_id) ?? 0) - (order.get(a.latest.finding_id) ?? 0), + ); + workFacts = projectWorkMeasurementFacts( + String(params.todo_id), + workFeedback, + route + ? [ + ...route, + "explore", + "turn-context", + "--agent-id", + String(params.agent_id), + "--todo-id", + String(params.todo_id), + "--result-limit", + "20", + "--goal-id", + String(params.goal_id), + ] + : undefined, + true, + ); } return { measurement_feedback: latest, ...(comparison && (participates || cue?.source_finding_id === written.finding_id) - ? {search_comparison: comparison} : {}), + ? {search_comparison: comparison} + : {}), ...(workFacts ? {work_measurements: [workFacts]} : {}), }; } diff --git a/loopx/control_plane/work_items/yardstick_source.ts b/loopx/control_plane/work_items/yardstick_source.ts index f0be7da56d..1132ba35be 100644 --- a/loopx/control_plane/work_items/yardstick_source.ts +++ b/loopx/control_plane/work_items/yardstick_source.ts @@ -1,9 +1,12 @@ /** Server-owned canonical event-prefix capture and recovery. Browser readers * share the pure decision/capture normalizers without importing Node hashing. */ import {createHash} from "node:crypto"; +import { + evaluateMethodologyActivation, + isHistoricalReplayPrecedingCurrent, +} from "../capabilities/stage_methodology_operator.ts"; import {EffectRuntimeRequestError} from "../effect_runtime_errors.ts"; import {requireJsonObject, type JsonObject} from "../runtime_decode.ts"; -import {parseTodoTimestampMicros} from "../runtime_timestamp.ts"; import {currentWorkYardstickFinding, normalizeWorkYardstickCapture, normalizeWorkYardstickDecision, normalizeWorkYardstickSourceSnapshot, workYardstickTodos} from "./yardstick_decision.ts"; @@ -43,17 +46,19 @@ export function projectWorkYardstickSourceRequests(params: JsonObject): JsonObje })}; } export function validateWorkYardstickContinuation(params: JsonObject): JsonObject { - if (params.enabled !== true) throw new EffectRuntimeRequestError("work yardstick capture requires explicit yardstick_enabled configuration"); + if (!evaluateMethodologyActivation({enabled: params.enabled}).active) { + throw new EffectRuntimeRequestError("work yardstick capture requires explicit yardstick_enabled configuration"); + } const binding = requireJsonObject(params.binding, "work binding"); const decision = normalizeWorkYardstickDecision(params.decision, params.evidence_refs); const current = currentWorkYardstickFinding((params.findings ?? []) as JsonObject[], binding); // Only the settlement adapter supplies the original timestamp of an exact // committed replay. Preserve a strictly older missing capture as history; // it cannot displace the current choice or admit a newer stale predecessor. - const replayTime = typeof params.replay_committed_at === "string" - ? parseTodoTimestampMicros(params.replay_committed_at) : null; - const currentTime = current ? parseTodoTimestampMicros(String(current.last_updated_at ?? "")) : null; - const historicalReplay = replayTime !== null && currentTime !== null && replayTime < currentTime; + const historicalReplay = isHistoricalReplayPrecedingCurrent( + params.replay_committed_at, + current?.last_updated_at, + ); if (decision.prior_finding_id !== (current?.finding_id ?? null) && !historicalReplay) { throw new EffectRuntimeRequestError("yardstick basis changed; read the current same-Todo decision before revising it"); } diff --git a/loopx/presentation/explore_results_api.py b/loopx/presentation/explore_results_api.py index 2077f0030c..cbe2a213bd 100644 --- a/loopx/presentation/explore_results_api.py +++ b/loopx/presentation/explore_results_api.py @@ -60,7 +60,7 @@ def _finding_rows(events, todos, *, goal_id, validated: bool = False): "question": nodes.get(finding["node_id"], {}).get("title", ""), "scope": nodes.get(finding["node_id"], {}).get("summary", "")} for finding in projection["findings"] - ], goal_id=goal_id) + ], goal_id=goal_id, findings=projection["findings"]) def _history_rows(runtime_root, goal_id): diff --git a/tests/control_plane/test_quantitative_planning_cli.py b/tests/control_plane/test_quantitative_planning_cli.py new file mode 100644 index 0000000000..63a5deddbc --- /dev/null +++ b/tests/control_plane/test_quantitative_planning_cli.py @@ -0,0 +1,823 @@ +"""End-to-end File + SQLite real-path CLI, Turn-context, and loopback API tests +for the 5-stage methodology operator catalog, scoped multi-point Yardstick +fidelity, budgeted continuation planning, and counterfactual pair evaluation.""" +from __future__ import annotations + +from copy import deepcopy +import json +import os +from pathlib import Path +import subprocess +import sys + +import pytest + +from loopx.capabilities.explore.quantitative_planning import verify_methodology_operator_output +from loopx.capabilities.explore.turn_context import explore_turn_context +from loopx.presentation.explore_results_api import _result_rows +from tests.control_plane.canonical_authority_fixture import isolate_sqlite_runtime +from tests.control_plane.test_quota_settlement_cli import ( + AGENT_ID, + GOAL_ID, + TODO_ID, + _initialize_fixture_authority, + _run_cli, + _write_fixture, +) + + +def _setup_goal(tmp_path: Path, *, enabled: bool = True, provider: str = "sqlite"): + project, runtime, registry_path = _write_fixture(tmp_path) + config = json.loads(registry_path.read_text(encoding="utf-8")) + config["goals"][0]["explore_graph"] = {"enabled": True} + config["goals"][0]["quota"]["allowed_slots"] = 8 + config["goals"][0].setdefault("execution_profile", {})["yardstick_enabled"] = enabled + registry_path.write_text(json.dumps(config), encoding="utf-8") + _initialize_fixture_authority(project, runtime, provider=provider, handoff_mode="soft_claim") + return project, runtime, registry_path + + +def _profiles(): + auth = { + "metric": "accepted_score", + "unit": "points", + "higher_is_better": True, + "comparison_scope": "validation:fixed", + "input_revision": "inputs-v1", + "evaluation_ref": "evaluator:outcome-v1", + } + proxy = { + "metric": "proxy_loss", + "unit": "nats", + "higher_is_better": False, + "comparison_scope": "validation:fixed", + "input_revision": "inputs-v1", + "evaluation_ref": "evaluator:proxy-v1", + } + return auth, proxy + + +@pytest.mark.parametrize("provider", ["file", "sqlite"]) +@pytest.mark.parametrize("calibration", ["frontier_cliff", "two_point_reversal"]) +def test_yardstick_fidelity_cli_turn_context_and_api_across_providers( + tmp_path: Path, provider: str, calibration: str, +) -> None: + project, runtime, registry_path = _setup_goal(tmp_path, enabled=True, provider=provider) + auth, proxy = _profiles() + + decision = { + "schema_version": "work_yardstick_decision_v0", + "target": { + "summary": "Maximize accepted score within fixed validation budget", + "scope": "Fixed benchmark split and compute allowance", + "acceptance_ref": "acceptance:task-v1", + }, + "authoritative": auth, + "proxy": proxy, + "counterevidence_refs": [], + "prior_finding_id": None, + "next_action": { + "method_id": "method:calibrated_search", + "reason": "Track proxy concordance across multiple candidates and audit frontier reversals", + "cost_bound": "Bounded confirmation evaluations within allowed quota", + }, + } + + init_attach = tmp_path / "init_attach.json" + init_attach.write_text( + json.dumps( + { + "schema_version": "explore_result_attachment_v0", + "question": "Does proxy_loss remain concordant near the top frontier?", + "applicability": "Fixed split inputs-v1", + "input_revision": "inputs-v1", + "observation": "Initial baseline candidate c1 evaluated on both metrics", + "interpretation": "Continue multi-point tracking", + "status": "confirmed", + "evidence_refs": ["ev:c1:a", "ev:c1:p"], + "measurements": [ + {**auth, "candidate_ref": "cand:c1", "evidence_ref": "ev:c1:a", "value": 20.0}, + {**proxy, "candidate_ref": "cand:c1", "evidence_ref": "ev:c1:p", "value": 2.40}, + ], + "work_yardstick_decision": decision, + } + ), + encoding="utf-8", + ) + + rc, _ = _run_cli( + registry_path, runtime, + "quota", "should-run", "--codex-app", + "--goal-id", GOAL_ID, "--agent-id", AGENT_ID, "--todo-id", TODO_ID, "--turn-instance-id", "turn-1", + cwd=project, + ) + assert rc == 0 + rc, settled = _run_cli( + registry_path, runtime, + "refresh-state", + "--goal-id", GOAL_ID, "--agent-id", AGENT_ID, "--todo-id", TODO_ID, "--turn-instance-id", "turn-1", + "--classification", "validated_change", + "--delivery-batch-scale", "implementation", + "--delivery-outcome", "outcome_progress", + "--explore-result-json", str(init_attach), + "--no-global-sync", "--suppress-external-sinks", + cwd=project, + ) + assert rc == 0, settled + + # The bottom two agree and the authoritative frontier reverses. Two + # candidates tie at the third-place cutoff and both belong in that frontier. + candidates = [ + ("cand:c2", 35.0, 1.90), + ("cand:c3", 72.0, 0.80), + ("cand:c4", 81.0, 0.92), + ("cand:c5", 93.0, 1.10), + ("cand:c6", 72.0, 0.80), + ] + if calibration == "two_point_reversal": + # Only one observed order: c2 beats c1 on the outcome and loses on the proxy. + candidates = [("cand:c2", 35.0, 3.50)] + for idx, (cand_ref, a_val, p_val) in enumerate(candidates, start=2): + res_path = tmp_path / f"cand_{idx}.json" + res_path.write_text( + json.dumps( + { + "schema_version": "explore_result_attachment_v0", + "question": "Does proxy_loss remain concordant near the top frontier?", + "applicability": "Fixed split inputs-v1", + "input_revision": "inputs-v1", + "observation": f"Evaluated {cand_ref}", + "interpretation": "Recorded paired outcome and proxy", + "status": "confirmed", + "evidence_refs": [f"ev:{cand_ref}:a", f"ev:{cand_ref}:p"], + "measurements": [ + {**auth, "candidate_ref": cand_ref, "evidence_ref": f"ev:{cand_ref}:a", "value": a_val}, + {**proxy, "candidate_ref": cand_ref, "evidence_ref": f"ev:{cand_ref}:p", "value": p_val}, + ], + } + ), + encoding="utf-8", + ) + rc, out = _run_cli( + registry_path, runtime, + "explore", "finding", + "--goal-id", GOAL_ID, "--agent-id", AGENT_ID, + "--todo-id", TODO_ID, "--turn-instance-id", f"turn-{idx}", + "--result-json", str(res_path), + cwd=project, + ) + assert rc == 0, out + + proxy_star_path = tmp_path / "proxy_star.json" + proxy_star_path.write_text( + json.dumps( + { + "schema_version": "explore_result_attachment_v0", + "question": "Does proxy_loss remain concordant near the top frontier?", + "applicability": "Fixed split inputs-v1", + "input_revision": "inputs-v1", + "observation": "Proxy-screened candidate star", + "interpretation": "Awaits authoritative confirmation", + "status": "confirmed", + "evidence_refs": ["ev:star:p"], + "measurements": [ + {**proxy, "candidate_ref": "cand:proxy_star", "evidence_ref": "ev:star:p", "value": 0.65}, + ], + } + ), + encoding="utf-8", + ) + rc, out = _run_cli( + registry_path, runtime, + "explore", "finding", + "--goal-id", GOAL_ID, "--agent-id", AGENT_ID, + "--todo-id", TODO_ID, "--turn-instance-id", "turn-star", + "--result-json", str(proxy_star_path), + cwd=project, + ) + assert rc == 0, out + + # 1. CLI `explore yardstick-fidelity` + rc, fidelity_packet = _run_cli( + registry_path, runtime, + "explore", "yardstick-fidelity", + "--goal-id", GOAL_ID, "--frontier-k", "3", + cwd=project, + ) + assert rc == 0, fidelity_packet + assert fidelity_packet["evaluation_count"] == 1 + eval_row = fidelity_packet["evaluations"][0] + if calibration == "two_point_reversal": + assert eval_row["calibration_tier"] == "two_point" + assert eval_row["counts"]["paired_candidates"] == 2 + assert eval_row["fidelity_status"] == "order_reversed" + assert eval_row["rank_concordance"]["discordant_pairs"] == 1 + + rc, packet = _run_cli( + registry_path, runtime, "explore", "evolutionary-plan", "--goal-id", GOAL_ID, cwd=project, + ) + assert rc == 0, packet + plan, = packet["plans"] + assert [r["candidate_ref"] for r in plan["elites"]] == ["cand:c2", "cand:c1"] + candidate, = plan["proxy_quarantine"] + assert candidate["candidate_ref"] == "cand:proxy_star" + assert candidate["disposition"] == "confirm_before_breeding" + assert plan["grants_execution_authority"] is False + return + + assert eval_row["fidelity_status"] == "frontier_bias_cliff" + assert eval_row["counts"]["paired_candidates"] == 6 + assert eval_row["frontier_neighborhood"]["frontier_k"] == 4 + assert eval_row["frontier_neighborhood"]["frontier_kendall_tau_b"] == -1 + assert eval_row["audit_candidates"]["unconfirmed_proxy_winners"][0]["candidate_ref"] == "cand:proxy_star" + + # 1b. CLI `explore evolutionary-plan` discovers declared Yardstick, selects elites, and quarantines proxy star + rc, evo_packet = _run_cli( + registry_path, runtime, + "explore", "evolutionary-plan", + "--goal-id", GOAL_ID, "--elite-limit", "3", + cwd=project, + ) + assert rc == 0, evo_packet + assert evo_packet["plan_count"] == 1 + evo_row = evo_packet["plans"][0] + assert evo_row["selection_status"] == "mechanism_unresolved" + assert evo_row["counts"]["unknown_mechanism_candidates"] == 6 + assert evo_row["counts"]["mechanism_islands"] == 0 + assert evo_row["crossover_pairs"] == [] + assert evo_row["elites"][0]["candidate_ref"] == "cand:c5" + assert evo_row["proxy_fidelity_status"] == "frontier_bias_cliff" + assert evo_row["counts"]["selected_elites"] == 3 + assert evo_row["proxy_quarantine"][0]["candidate_ref"] == "cand:proxy_star" + assert evo_row["proxy_quarantine"][0]["disposition"] == "quarantined_proxy_cliff" + + # 2. Compact & Node-Detail `explore_turn_context` + compact_ctx = explore_turn_context( + registry_path=registry_path, runtime_root=runtime, + goal_id=GOAL_ID, agent_id=AGENT_ID, todo_id=TODO_ID, + ) + compact_cue = compact_ctx["graph"]["work_yardsticks"][0] + assert compact_cue["fidelity_status"] == "frontier_bias_cliff" + assert compact_cue["paired_candidates"] == 6 + assert "eligible authoritative comparison" in compact_cue["fidelity_guidance"] + + detail_ctx = explore_turn_context( + registry_path=registry_path, runtime_root=runtime, + goal_id=GOAL_ID, agent_id=AGENT_ID, todo_id=TODO_ID, + result_node=compact_cue["node_id"], + ) + detail_cue = detail_ctx["graph"]["work_yardsticks"][0] + assert detail_cue["yardstick_fidelity"]["fidelity_status"] == "frontier_bias_cliff" + + # 3. Loopback API `_result_rows` enriches `work_yardstick` findings with `yardstick_fidelity` + api_rows = _result_rows(runtime, GOAL_ID, registry_path) + yardstick_rows = [r for r in api_rows if "work_yardstick" in r] + assert len(yardstick_rows) == 1 + assert yardstick_rows[0]["yardstick_fidelity"]["fidelity_status"] == "frontier_bias_cliff" + + # 4. Feature-off rollback parity: disabling `yardstick_enabled` removes runtime cues cleanly + config = json.loads(registry_path.read_text(encoding="utf-8")) + config["goals"][0]["execution_profile"]["yardstick_enabled"] = False + registry_path.write_text(json.dumps(config), encoding="utf-8") + disabled_ctx = explore_turn_context( + registry_path=registry_path, runtime_root=runtime, + goal_id=GOAL_ID, agent_id=AGENT_ID, todo_id=TODO_ID, + ) + assert "work_yardsticks" not in disabled_ctx["graph"] + assert "work_yardstick_capture" not in disabled_ctx["graph"] + + +def test_stateless_planning_cli_commands_end_to_end(tmp_path: Path) -> None: + repo = Path(__file__).resolve().parents[2] + env = {**os.environ, "LOOPX_USAGE_PING": "0"} + + def invoke(*args: str, expect_rc: int = 0) -> subprocess.CompletedProcess[str]: + proc = subprocess.run( + [sys.executable, "-m", "loopx.cli", *args], + cwd=repo, + text=True, + capture_output=True, + env=env, + timeout=45, + ) + assert proc.returncode == expect_rc, proc.stdout + proc.stderr + return proc + + # 1. `explore methodology-operators` (JSON + Markdown) + cat_json = json.loads(invoke("explore", "methodology-operators", "--format", "json").stdout) + assert cat_json["ok"] is True + assert cat_json["schema_version"] == "stage_methodology_operator_catalog_v0" + assert len(cat_json["operators"]) == 6 + cat_md = invoke("explore", "methodology-operators", "--stage", "feedback_evolve", "--format", "markdown").stdout + assert "5-Stage Methodology Operator Catalog" in cat_md + assert "budgeted_continuation" in cat_md + assert "evolutionary_population" in cat_md + + # 2. `explore budgeted-plan` + plan_doc = tmp_path / "budgeted_plan.json" + plan_doc.write_text( + json.dumps( + { + "remaining_budget": {"wall_minutes": 120, "token_budget": None, "authoritative_eval_slots": 1}, + "confirmation": {"eval_duration_minutes": 30, "cooldown_minutes": 10}, + "incumbent": {"candidate_ref": "cand:inc", "value": 0.5, "higher_is_better": True, "metric": "acc", "unit": "ratio"}, + "baseline_route": { + "route_id": "route:base", + "summary": "Continue local step", + "probe_minutes": 0, + "implementation_minutes": 20, + "expected_tokens": 10000, + "requires_authoritative_eval": True, + "expected_gain_min": 0.01, + "expected_gain_expected": 0.02, + "expected_gain_max": 0.03, + }, + "candidate_routes": [ + { + "route_id": "route:probe_switch", + "summary": "Probe new kernel", + "kind": "switch_mechanism", + "probe_minutes": 15, + "implementation_minutes": 35, + "expected_tokens": None, + "requires_authoritative_eval": True, + "expected_gain_min": 0.0, + "expected_gain_expected": 0.08, + "expected_gain_max": 0.12, + } + ], + } + ), + encoding="utf-8", + ) + plan_out = json.loads(invoke("explore", "budgeted-plan", "--document", str(plan_doc), "--format", "json").stdout) + assert plan_out["ok"] is True + assert plan_out["candidate_routes"][0]["recommended_disposition"] == "probe_then_decide" + assert plan_out["candidate_routes"][0]["total_expected_tokens"] is None + limited = json.loads(plan_doc.read_text(encoding="utf-8")) + limited["remaining_budget"]["token_budget"] = 100 + limited["baseline_route"]["expected_tokens"] = 150 + limited["candidate_routes"][0]["expected_tokens"] = 20 + plan_doc.write_text(json.dumps(limited), encoding="utf-8") + plan_out = json.loads(invoke("explore", "budgeted-plan", "--document", str(plan_doc), "--format", "json").stdout) + assert plan_out["baseline_route"]["feasibility_status"] == "token_infeasible" + assert plan_out["baseline_route"]["total_expected_tokens"] is None + assert plan_out["effective_safe_gain"] == 0 + assert plan_out["candidate_routes"][0]["recommended_disposition"] in { + "viable_direct_continuation", "probe_then_decide", + } + + limited["confirmation"]["eval_tokens"] = 10 + limited["baseline_route"]["expected_tokens"] = None + candidate = limited["candidate_routes"][0] + candidate.update(probe_minutes=0, expected_gain_expected=0.005, expected_gain_max=0.01) + plan_doc.write_text(json.dumps(limited), encoding="utf-8") + plan_out = json.loads(invoke("explore", "budgeted-plan", "--document", str(plan_doc), "--format", "json").stdout) + assert plan_out["baseline_route"]["feasibility_status"] == "feasible_with_unknown_bounds" + assert plan_out["effective_safe_gain"] == 0 + assert plan_out["candidate_routes"][0]["feasibility_status"] == "feasible" + assert plan_out["candidate_routes"][0]["recommended_disposition"] == "viable_direct_continuation" + + limited["baseline_route"]["expected_tokens"] = 40 + candidate.update(expected_gain_min=None, expected_gain_expected=None, expected_gain_max=None) + plan_doc.write_text(json.dumps(limited), encoding="utf-8") + plan_out = json.loads(invoke("explore", "budgeted-plan", "--document", str(plan_doc), "--format", "json").stdout) + assert plan_out["effective_safe_gain"] == 0.02 + unestimated = plan_out["candidate_routes"][0] + assert unestimated["feasibility_status"] == "feasible" + assert unestimated["expected_gain_expected"] is None + assert unestimated["p_star_at_expected_gain"] is None + assert unestimated["recommended_disposition"] == "viable_direct_continuation" + + limited["incumbent"].update(candidate_ref=None, value=None) + candidate.update(expected_gain_min=-0.03, expected_gain_expected=-0.02, expected_gain_max=-0.01) + plan_doc.write_text(json.dumps(limited), encoding="utf-8") + plan_out = json.loads(invoke("explore", "budgeted-plan", "--document", str(plan_doc), "--format", "json").stdout) + assert plan_out["effective_safe_gain"] is None + assert plan_out["candidate_routes"][0]["recommended_disposition"] == "viable_direct_continuation" + + # 3. `explore counterfactual-eval` (strict failure vs. audit mode) + sha = "d" * 64 + cf_doc = tmp_path / "cf_eval.json" + cf_doc.write_text( + json.dumps( + { + "pairs": [ + { + "pair_id": "pair:clean", + "task_id": "task:1", + "checkpoint": { + "checkpoint_id": "ckpt:1", + "source_state_sha256": sha, + "input_revision": "rev-1", + "evaluation_ref": "eval:v1", + "metric": "score", + "unit": "pts", + "higher_is_better": True, + "comparison_scope": "scope:fixed", + "terminal_boundary_ref": "horizon:paired", + "fork_recorded_at": "2026-01-01T00:00:00Z", + }, + "treatment": { + "arm_id": "arm:treat", + "checkpoint_id": "ckpt:1", + "source_state_sha256": sha, + "input_revision": "rev-1", + "evaluation_ref": "eval:v1", + "workspace_isolation_id": "ws:treat", + "terminal_boundary_ref": "horizon:paired", + "decision_context_latest_at": "2026-01-01T00:00:00Z", + "first_action_signature": "act:a", + "terminal_value": 88.0, + "valid_submission": True, + "costs": {"wall_seconds": 600, "total_tokens": 50000}, + }, + "control": { + "arm_id": "arm:ctrl", + "checkpoint_id": "ckpt:1", + "source_state_sha256": sha, + "input_revision": "rev-1", + "evaluation_ref": "eval:v1", + "workspace_isolation_id": "ws:ctrl", + "terminal_boundary_ref": "horizon:paired", + "decision_context_latest_at": "2026-01-01T00:00:00Z", + "first_action_signature": "act:b", + "terminal_value": 75.0, + "valid_submission": True, + "costs": {"wall_seconds": 580, "total_tokens": 48000}, + }, + }, + { + "pair_id": "pair:shared_ws", + "task_id": "task:2", + "checkpoint": { + "checkpoint_id": "ckpt:2", + "source_state_sha256": sha, + "input_revision": "rev-1", + "evaluation_ref": "eval:v1", + "metric": "score", + "unit": "pts", + "higher_is_better": True, + "comparison_scope": "scope:fixed", + "terminal_boundary_ref": "horizon:paired", + "fork_recorded_at": "2026-01-01T00:00:00Z", + }, + "treatment": { + "arm_id": "arm:treat", + "checkpoint_id": "ckpt:2", + "source_state_sha256": sha, + "input_revision": "rev-1", + "evaluation_ref": "eval:v1", + "workspace_isolation_id": "ws:SHARED", + "terminal_boundary_ref": "horizon:paired", + "decision_context_latest_at": "2026-01-01T00:00:00Z", + "terminal_value": 99.0, + "valid_submission": True, + "costs": {}, + }, + "control": { + "arm_id": "arm:ctrl", + "checkpoint_id": "ckpt:2", + "source_state_sha256": sha, + "input_revision": "rev-1", + "evaluation_ref": "eval:v1", + "workspace_isolation_id": "ws:SHARED", + "terminal_boundary_ref": "horizon:paired", + "decision_context_latest_at": "2026-01-01T00:00:00Z", + "terminal_value": 70.0, + "valid_submission": True, + "costs": {}, + }, + }, + ] + } + ), + encoding="utf-8", + ) + failed = json.loads( + invoke("explore", "counterfactual-eval", "--document", str(cf_doc), "--format", "json", expect_rc=1).stdout + ) + assert failed["ok"] is False + + audited = json.loads( + invoke( + "explore", "counterfactual-eval", + "--document", str(cf_doc), + "--no-strict-identification", + "--format", "json", + ).stdout + ) + assert audited["ok"] is True + assert audited["counts"]["valid_pairs"] == 1 + assert audited["counts"]["contaminated_pairs"] == 1 + assert audited["estimand_summary"]["mean_tau_b"] == 13.0 + assert audited["pairs"][1]["identification_violations"] == ["shared_workspace_interference"] + clean = json.loads(cf_doc.read_text(encoding="utf-8"))["pairs"][0] + + def counterfactual(pairs, *, strict=True, expect_rc=0): + cf_doc.write_text(json.dumps({"pairs": pairs}), encoding="utf-8") + args = () if strict else ("--no-strict-identification",) + return json.loads(invoke( + "explore", "counterfactual-eval", "--document", str(cf_doc), + *args, "--format", "json", expect_rc=expect_rc, + ).stdout) + + def independent_workspace(label): + pair = deepcopy(clean) + pair["pair_id"] = f"pair:{label}" + for side in ("treatment", "control"): + pair[side]["workspace_isolation_id"] = f"ws:{label}:{side}" + return pair + + # Relabeling workspaces must not multiply a single observed execution pair. + replicas = [independent_workspace(f"replica-{i}") for i in range(5)] + inflated = [clean, *replicas] + audited = counterfactual(inflated, strict=False) + assert audited["counts"]["valid_pairs"] == audited["counts"]["wins"] == 0 + assert audited["counts"]["contaminated_pairs"] == 6 + assert all(row["identification_violations"] == ["reused_arm_execution"] + for row in audited["pairs"]) + assert "reused_arm_execution" in json.dumps(counterfactual(inflated, expect_rc=1)) + + # A distinct partner or a side swap does not make a consumed arm independent. + shared_arm = independent_workspace("shared-arm") + shared_arm["treatment"] = deepcopy(clean["treatment"]) + shared_arm["control"]["arm_id"] = "arm:new-control" + shared_arm["control"]["terminal_value"] = 100 + shared_arm["control"]["costs"]["wall_seconds"] = 900 + independent = independent_workspace("independent-pair") + for side in ("treatment", "control"): + independent[side]["arm_id"] = f"arm:independent-{side}" + batch = [clean, shared_arm, independent] + for order in (batch, list(reversed(batch))): + audited = counterfactual(order, strict=False) + assert audited["counts"]["valid_pairs"] == 1 + assert audited["counts"]["contaminated_pairs"] == 2 + assert audited["estimand_summary"]["mean_tau_b"] == 13 + assert audited["complete_marginal_costs"]["observed_delta_sums"]["wall_seconds"] == 20 + accepted_rows = [row for row in audited["pairs"] if row["identification_valid"]] + assert [row["pair_id"] for row in accepted_rows] == [independent["pair_id"]] + assert all("reused_arm_execution" in row["identification_violations"] + for row in audited["pairs"] if row not in accepted_rows) + assert "reused_arm_execution" in json.dumps(counterfactual(batch, expect_rc=1)) + + same_arm = independent_workspace("same-arm-both-sides") + same_arm["control"]["arm_id"] = same_arm["treatment"]["arm_id"] + assert "reused_arm_execution" in json.dumps(counterfactual([same_arm], expect_rc=1)) + audited = counterfactual([same_arm], strict=False) + assert audited["counts"]["scored_valid_pairs"] == 0 + assert audited["pairs"][0]["identification_violations"] == ["reused_arm_execution"] + + duplicate = {**clean, "pair_id": "pair:relabeled"} + for order in ([clean, duplicate], [duplicate, clean]): + audited = counterfactual(order, strict=False) + assert audited["counts"]["valid_pairs"] == audited["counts"]["wins"] == 1 + assert audited["estimand_summary"]["exact_sign_flip_p_value"] == 1 + assert audited["complete_marginal_costs"]["observed_delta_sums"]["wall_seconds"] == 20 + by_id = {row["pair_id"]: row for row in audited["pairs"]} + assert by_id[clean["pair_id"]]["identification_valid"] + assert set(by_id[duplicate["pair_id"]]["identification_violations"]) == { + "reused_run_workspace", "reused_arm_execution", + } + + # Independent declared executions under the same checkpoint can still pool. + for i, pair in enumerate(replicas): + for side in ("treatment", "control"): + pair[side]["arm_id"] = f"arm:{side}-{i}" + accepted = counterfactual([clean, *replicas]) + assert accepted["counts"]["valid_pairs"] == accepted["counts"]["wins"] == 6 + assert accepted["estimand_summary"]["exact_sign_flip_p_value"] == 0.03125 + assert accepted["complete_marginal_costs"]["observed_delta_sums"]["wall_seconds"] == 120 + + other_checkpoint = independent_workspace("another-checkpoint") + for key in ("checkpoint", "treatment", "control"): + other_checkpoint[key]["checkpoint_id"] = "ckpt:independent" + accepted = counterfactual([clean, other_checkpoint]) + assert accepted["counts"]["valid_pairs"] == 2 + + +def test_due_stride_review_core_persisted_decision_passes_methodology_verifier( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, +) -> None: + isolate_sqlite_runtime(tmp_path, monkeypatch) + project, runtime, registry_path = _write_fixture(tmp_path) + config = json.loads(registry_path.read_text(encoding="utf-8")) + config["goals"][0]["quota"]["allowed_slots"] = 12 + registry_path.write_text(json.dumps(config), encoding="utf-8") + _initialize_fixture_authority(project, runtime, provider="sqlite") + + def call(*args: str, expected_code: int = 0) -> dict: + code, result = _run_cli(registry_path, runtime, *args, cwd=project) + assert code == expected_code, json.dumps(result, indent=2) + return result + + call( + "configure-goal", "--goal-id", GOAL_ID, + "--execution-replan-after-turns", "2", + "--execution-stride-strategy", "evidence_conditioned_v1", + "--execute", + ) + for index in range(2): + turn = f"stride-turn-{index}" + guard = call( + "quota", "should-run", "--codex-app", + "--goal-id", GOAL_ID, "--agent-id", AGENT_ID, "--turn-instance-id", turn, + ) + assert guard["selected_todo"]["todo_id"] == TODO_ID + call( + "refresh-state", "--goal-id", GOAL_ID, "--agent-id", AGENT_ID, + "--todo-id", TODO_ID, "--turn-instance-id", turn, + "--classification", "validated_progress", + "--delivery-batch-scale", "implementation", + "--delivery-outcome", "outcome_progress", + "--delivery-boundary", "in_flight_continuation", + "--no-global-sync", "--suppress-external-sinks", + ) + call( + "quota", "spend-slot", "--goal-id", GOAL_ID, "--agent-id", AGENT_ID, + "--todo-id", TODO_ID, "--turn-instance-id", turn, + "--slots", "1", "--source", "heartbeat", "--execute", + ) + + due_guard = call( + "quota", "should-run", "--codex-app", + "--goal-id", GOAL_ID, "--agent-id", AGENT_ID, "--turn-instance-id", "stride-due", + ) + obligation = due_guard["autonomous_replan_obligation"] + successor = call( + "todo", "add", "--goal-id", GOAL_ID, "--role", "agent", + "--claimed-by", AGENT_ID, "--text", "Validate a new independent source", + "--task-class", "advancement_task", "--action-kind", "validate", + "--target-key", "independent-source", "--replan-obligation-id", obligation["obligation_id"], + ) + reread = call( + "quota", "should-run", "--codex-app", + "--goal-id", GOAL_ID, "--agent-id", AGENT_ID, + "--todo-id", successor["todo_id"], "--turn-instance-id", "stride-due", + ) + control = reread["replan_action_packet"]["stride_control"] + assert control["required_on_periodic_review"] is True + + # Core refuses omission of stride_decision when due. + refused = call( + "refresh-state", "--goal-id", GOAL_ID, "--agent-id", AGENT_ID, + "--replan-obligation-id", obligation["obligation_id"], + "--turn-instance-id", "stride-due", + "--classification", "typed_blocker_writeback", + "--delivery-batch-scale", "single_surface", + "--delivery-outcome", "outcome_gap", + "--progress-result-class", "blocked", + "--progress-blocker-id", "fixture:blocked-source", + "--progress-evidence-id", "fixture:blocked-source-evidence", + "--no-global-sync", "--suppress-external-sinks", + expected_code=1, + ) + assert refused.get("appended") is not True + assert "stride_decision" in refused["error"] + + stride_decision = { + "schema_version": "stride_decision_v2", + "strategy_version": "evidence_conditioned_v1", + "reference_context_id": control["reference_context_id"], + "reference_obligation_id": obligation["obligation_id"], + "basis": "uncertain_evidence", + "action": "verify", + "axes": { + "work_slice": { + "plan": "Complete the independent source validation.", + "reason": "One complete check resolves the pending evidence question.", + }, + "route_distance": { + "plan": "Retain the current delivery route while checking its source.", + "reason": "No result yet supports replacing the route.", + }, + "validation_scope": { + "plan": "Check the source result and a contrary case.", + "reason": "Both cases distinguish a usable source from a false lead.", + }, + }, + "next_probe": { + "scope": "One bounded independent source validation.", + "prediction": "The source supports the original acceptance.", + "adoption_criterion": "Adopt only if the checked source meets acceptance.", + "cost_bound": "One source and its contrary case before replanning.", + "minimum_evidence": "A checked result and a contrary-case result.", + }, + } + vision_path = tmp_path / "stride_vision.json" + vision_path.write_text( + json.dumps( + { + "schema_version": "goal_vision_replan_contract_v0", + "state": "vision_patch_proposed", + "vision_patch": { + "acceptance_summary": "The original delivery remains open while the new source is validated.", + }, + "path_delta": { + "schema_version": "goal_path_delta_v0", + "outcome": "replan", + "prior_assumption": "The first source was sufficient.", + "observed_reality": "Need one independent verification before completion.", + "retained": ["Original acceptance boundary"], + "changed": ["Validate the independent source first"], + "evidence_refs": ["evidence:independent-source-review"], + }, + "stride_decision": stride_decision, + "todo_delta": [f"create:{successor['todo_id']}"], + } + ), + encoding="utf-8", + ) + accepted = call( + "refresh-state", "--goal-id", GOAL_ID, "--agent-id", AGENT_ID, + "--replan-obligation-id", obligation["obligation_id"], + "--turn-instance-id", "stride-due", + "--agent-vision-json", str(vision_path), + "--classification", "bounded_replan_progress", + "--delivery-batch-scale", "implementation", + "--delivery-outcome", "outcome_progress", + "--no-global-sync", "--suppress-external-sinks", + ) + assert accepted["autonomous_replan_ack"]["semantic_delta"]["accepted"] is True + persisted = json.loads(Path(accepted["json_path"]).read_text(encoding="utf-8")) + persisted_decision = persisted["agent_vision"]["stride_decision"] + assert "advisory_only" not in persisted_decision + assert "grants_execution_authority" not in persisted_decision + + # 1. Catalog distinguishes stride_review as required_due_decision (advisory_only=False) + catalog = call("explore", "methodology-operators") + by_id = {row["operator_id"]: row for row in catalog["operators"]} + assert by_id["stride_review"]["operator_role"] == "required_due_decision" + assert by_id["stride_review"]["advisory_only"] is False + assert by_id["stride_review"]["grants_execution_authority"] is False + for adv_id in ( + "work_yardstick", + "scoped_yardstick_fidelity", + "budgeted_continuation", + "counterfactual_continuation", + "evolutionary_population", + ): + assert by_id[adv_id]["operator_role"] == "advisory_cue" + assert by_id[adv_id]["advisory_only"] is True + + # 2. Core-persisted stride_decision_v2 passes both RPC and direct TS verifier cleanly + verified = verify_methodology_operator_output( + operator_id="stride_review", + output=persisted_decision, + ) + assert verified["valid"] is True, verified + assert verified["violations"] == [] + + # 3. Any nested/root authority escalation on stride_decision_v2 still fails, and quiet due_obligation stays silent + escalated_root = verify_methodology_operator_output( + operator_id="stride_review", + output={**persisted_decision, "grants_execution_authority": True}, + ) + assert escalated_root["valid"] is False + assert any("grants_execution_authority" in v for v in escalated_root["violations"]) + + escalated_nested = verify_methodology_operator_output( + operator_id="stride_review", + output={ + **persisted_decision, + "axes": { + **persisted_decision["axes"], + "work_slice": { + **persisted_decision["axes"]["work_slice"], + "should_run": True, + "nested": {"admin_bypass": True, "grants_execution_authority": True}, + }, + }, + }, + ) + assert escalated_nested["valid"] is False + assert len(escalated_nested["violations"]) == 3 + + repo_root = Path(__file__).resolve().parents[2] + ts_script = """ + import assert from 'node:assert/strict'; + import { + evaluateMethodologyActivation, + verifyMethodologyOperatorOutput, + } from './loopx/control_plane/capabilities/stage_methodology_operator.ts'; + const decision = JSON.parse(process.argv[1]); + const direct = verifyMethodologyOperatorOutput({operator_id: 'stride_review', output: decision}); + assert.equal(direct.valid, true); + assert.deepEqual(direct.violations, []); + assert.deepEqual( + evaluateMethodologyActivation({enabled: true, quiet_turn_policy: 'due_obligation_only', due_obligation_active: false}), + {active: false, reason: 'quiet_turn'}, + ); + """ + proc = subprocess.run( + [ + "node", "--no-warnings", "--experimental-strip-types", "--input-type=module", + "-e", ts_script, json.dumps(persisted_decision), + ], + cwd=repo_root, + capture_output=True, + text=True, + check=False, + timeout=30, + ) + assert proc.returncode == 0, proc.stdout + proc.stderr diff --git a/tests/control_plane/test_work_yardstick_cli.py b/tests/control_plane/test_work_yardstick_cli.py index 00eb5fe4f2..9d27339d7e 100644 --- a/tests/control_plane/test_work_yardstick_cli.py +++ b/tests/control_plane/test_work_yardstick_cli.py @@ -987,12 +987,14 @@ def test_later_plain_measurement_cannot_replace_cited_decision_evidence(tmp_path refs = ["result:preview-a", "result:preview-b"] plain = attachment() chosen = plain.pop("work_yardstick_decision") - plain["evidence_refs"] = refs + plain["evidence_refs"] = [*refs, "result:preview-uncited"] plain["measurements"] = [ {**chosen["proxy"], "candidate_ref": f"candidate:{candidate}", "evidence_ref": ref, "value": value} for candidate, ref, value in zip(("a", "b"), refs, (8, 6)) ] + plain["measurements"].append({**chosen["proxy"], "candidate_ref": "candidate:c", + "evidence_ref": "result:preview-uncited", "value": 100}) def retain(turn_id, payload): evidence = tmp_path / f"{turn_id}.json" @@ -1006,16 +1008,20 @@ def retain(turn_id, payload): first = retain("previous-proxy", plain) selected = attachment() - selected.update(node_id=first["node_id"], evidence_refs=[*refs, "result:outcome-a", "result:outcome-b"]) + selected.update(node_id=first["node_id"], evidence_refs=[ + *refs, "result:outcome-c", "result:outcome-a", "result:outcome-b", + ]) selected["work_yardstick_decision"]["counterevidence_refs"] = refs selected["measurements"] = [ {**chosen["authoritative"], "candidate_ref": f"candidate:{candidate}", "evidence_ref": f"result:outcome-{candidate}", "value": value} - for candidate, value in (("a", 20), ("b", 14)) + for candidate, value in (("c", 1), ("a", 20), ("b", 14)) ] result = retain("current-decision", selected) before = context(runtime, path)["graph"]["work_yardsticks"][0] assert before["search_comparison"]["proxy_order_reversed"] is False + assert "yardstick_fidelity" not in before, "An uncited sibling cannot turn two authorized pairs into three" + assert "fidelity_guidance" not in before late = {**plain, "node_id": first["node_id"], "evidence_refs": refs[1:], "measurements": [{**plain["measurements"][1], "value": 99}]} diff --git a/tests/control_plane_ts/content_digest_single_owner.test.ts b/tests/control_plane_ts/content_digest_single_owner.test.ts index eccc73e8ae..b6943f223e 100644 --- a/tests/control_plane_ts/content_digest_single_owner.test.ts +++ b/tests/control_plane_ts/content_digest_single_owner.test.ts @@ -105,11 +105,10 @@ const CANONICAL_CONSUMERS = [ "control_plane/capabilities/content_media.ts", "control_plane/capabilities/content_media_generation.ts", "control_plane/capabilities/content_media_image.ts", - "control_plane/capabilities/explore_execution_observation.ts", - "control_plane/capabilities/explore_result_page.ts", "control_plane/capabilities/external_evidence.ts", "control_plane/capabilities/goal_capability_organization.ts", "control_plane/capabilities/native_child_result_inbox.ts", + "control_plane/capabilities/stage_methodology_operator.ts", "control_plane/capabilities/trajectory_assessment.ts", "control_plane/capabilities/trajectory_input_usage.ts", "control_plane/capabilities/trajectory_tool_context.ts", @@ -187,12 +186,10 @@ const CANONICAL_CONSUMERS = [ "control_plane/work_items/replan_history_snapshot.ts", "control_plane/work_items/replan_semantics.ts", "control_plane/work_items/refresh_recommendation.ts", - "control_plane/work_items/stride_policy.ts", "control_plane/work_items/task_lease_acquire.ts", "control_plane/work_items/task_lease_lifecycle.ts", "control_plane/work_items/task_lease_lifecycle_request.ts", "control_plane/work_items/task_lease_workspace.ts", - "control_plane/work_items/yardstick_decision.ts", "zcode_goal_mode/cli.ts", "zcode_goal_mode/runtime.ts", ]; diff --git a/tests/control_plane_ts/explore_history.test.ts b/tests/control_plane_ts/explore_history.test.ts index 9f2e1ac452..1bab70dce4 100644 --- a/tests/control_plane_ts/explore_history.test.ts +++ b/tests/control_plane_ts/explore_history.test.ts @@ -43,6 +43,34 @@ test("parent grouping and 200 nodes do not manufacture supporting evidence or dr assert.equal(layoutExploreHistory(projectExploreHistory(cyclic, goal).nodes, []).length, 2); }); +test("the current Yardstick calibration follows the last version of each finding", () => { + const authoritative = {metric: "score", unit: "points", higher_is_better: true, + comparison_scope: "native", input_revision: "revision-1", evaluation_ref: "eval:native"}; + const proxy = {...authoritative, comparison_scope: "local", evaluation_ref: "eval:local"}; + const measured = [1, 2, 3].map(value => { + const refs = [`ev:outcome:${value}`, `ev:local:${value}`]; + return event({event_id: `e${value + 1}`, event_kind: "finding", result_id: `f${value}`, node_id: "route", + agent_id: "agent-1", evidence_refs: refs, status: "confirmed", + measurements: [{...authoritative, candidate_ref: `cand:${value}`, evidence_ref: refs[0], value}, + {...proxy, candidate_ref: `cand:${value}`, evidence_ref: refs[1], value}], + ...(value === 1 ? {work_yardstick: { + binding: {goal_id: goal, agent_id: "agent-1", todo_id: "todo_test", turn_instance_id: "turn-1"}, + decision: {schema_version: "work_yardstick_decision_v0", + target: {summary: "Improve the accepted score", scope: "Current route", acceptance_ref: "acceptance:score"}, + authoritative, proxy, counterevidence_refs: [], prior_finding_id: null, + next_action: {method_id: "method:compare", reason: "Compare both measures", cost_bound: "One evaluation"}}, + }} : {}), + }); + }); + const rows = [event(), ...measured]; + assert.equal(projectExploreHistory(rows, goal).yardstickFidelityByEvent.size, 0); + assert.equal(projectExploreHistory(rows, goal, true).yardstickFidelityByEvent.get("e2")!.counts.paired_candidates, 3); + const withdrawal = {...measured[2]!, event_id: "e5", status: "refuted", measurements: undefined}; + const revised = projectExploreHistory([...rows, withdrawal], goal, true); + assert.equal(revised.yardstickFidelityByEvent.get("e2")!.counts.paired_candidates, 2); + assert.ok(revised.events.find(row => row.event_id === "e4")?.measurements?.length); +}); + test("explicit links and unattached findings remain source records; missing anchors and times stay unknown", () => { const rows = [event({recorded_at: "unknown"}), event({event_id: "e2", event_kind: "edge", result_id: "relation", from_node: "route", to_node: "unloaded", edge_type: "refutes"}), diff --git a/tests/control_plane_ts/methodology_operator_invariants.test.ts b/tests/control_plane_ts/methodology_operator_invariants.test.ts new file mode 100644 index 0000000000..8f8604a579 --- /dev/null +++ b/tests/control_plane_ts/methodology_operator_invariants.test.ts @@ -0,0 +1,353 @@ +import assert from "node:assert/strict"; +import test from "node:test"; +import { + BUILTIN_STAGE_METHODOLOGY_OPERATORS, + LOOPX_CONTROL_STAGES, + committedInWorkWindow, + evaluateMethodologyActivation, + isHistoricalReplayPrecedingCurrent, + isValidContentSha256, + latestCommittedScopedRecord, + listStageMethodologyOperators, + matchesExactScopeTuple, + matchesWorkIdentity, + normalizeCanonicalSourceSnapshot, + projectBoundedMethodologyItems, + recordedBeforePrefix, + selectCanonicalEvidenceWindow, + verifyMethodologyOperatorOutput, +} from "../../loopx/control_plane/capabilities/stage_methodology_operator.ts"; +import {projectWorkYardstickCue, workYardstickTodos} from "../../loopx/control_plane/work_items/yardstick_decision.ts"; + +test("Bug Class 1: strict feature-off & quiet-turn silence reject truthy non-booleans and untriggered turns", () => { + for (const nonTrue of [false, undefined, null, 0, 1, "true", "false", {}, []]) { + assert.deepEqual(evaluateMethodologyActivation({enabled: nonTrue}), { + active: false, + reason: "disabled", + }); + assert.deepEqual( + workYardstickTodos({ + yardstick_enabled: nonTrue, + agent_id: "worker", + work_todos: [{todo_id: "todo_a", role: "agent", done: false, claimed_by: "worker", status: "open"}], + }), + [], + ); + assert.equal( + projectWorkYardstickCue({enabled: nonTrue, goal_id: "g", agent_id: "worker", todo_id: "todo_a", findings: []}), + null, + ); + } + + assert.deepEqual( + evaluateMethodologyActivation({ + enabled: true, + required_scope: {goal_id: "g", todo_id: " "}, + }), + {active: false, reason: "missing_scope"}, + ); + assert.deepEqual( + evaluateMethodologyActivation({ + enabled: true, + quiet_turn_policy: "silent_when_untriggered", + trigger_active: false, + }), + {active: false, reason: "quiet_turn"}, + ); + assert.deepEqual( + evaluateMethodologyActivation({ + enabled: true, + quiet_turn_policy: "silent_when_untriggered", + trigger_active: true, + required_scope: {goal_id: "g", todo_id: "todo_a"}, + }), + {active: true, reason: "active"}, + ); + assert.deepEqual( + evaluateMethodologyActivation({ + enabled: true, + quiet_turn_policy: "due_obligation_only", + due_obligation_active: false, + }), + {active: false, reason: "quiet_turn"}, + ); + assert.deepEqual( + evaluateMethodologyActivation({ + enabled: true, + quiet_turn_policy: "due_obligation_only", + trigger_active: false, + }), + {active: false, reason: "quiet_turn"}, + ); + assert.deepEqual( + evaluateMethodologyActivation({ + enabled: true, + quiet_turn_policy: "due_obligation_only", + due_obligation_active: true, + }), + {active: true, reason: "active"}, + ); + assert.deepEqual( + evaluateMethodologyActivation({ + enabled: true, + quiet_turn_policy: "due_obligation_only", + }), + {active: true, reason: "active"}, + ); +}); + +test("Bug Class 2: canonical evidence window blocks late replays, tied clocks, and post-prefix uncited injections", () => { + const sha = "a".repeat(64); + const snapshot = normalizeCanonicalSourceSnapshot({event_count: 3, event_ids_sha256: sha}); + assert.equal(snapshot.event_count, 3); + assert.throws(() => normalizeCanonicalSourceSnapshot({event_count: -1, event_ids_sha256: sha}), /event count/); + assert.throws(() => normalizeCanonicalSourceSnapshot({event_count: 1, event_ids_sha256: "short"}), /digest/); + + const binding = {goal_id: "goal-alpha", agent_id: "agent-1", todo_id: "todo_work"}; + const decisionRow = { + finding_id: "f-decision", + node_id: "node-q", + recorded_order: 3, + last_updated_at: "2026-06-01T10:00:00Z", + work_yardstick: {binding, decision: {target: {summary: "T", scope: "S", acceptance_ref: "acc:1"}}}, + }; + // Late delivery of an OLD primary (committed 09:59 < 10:00, appended at order 5 >= 3): must NOT win or enter window! + const delayedOldPrimary = { + finding_id: "f-delayed-old", + node_id: "node-q", + status: "confirmed", + recorded_order: 5, + last_updated_at: "2026-06-01T09:59:00Z", + work_binding: binding, + }; + // Tied timestamp at order 6: tied clock does NOT prove newer observation! + const tiedClockPrimary = { + finding_id: "f-tied", + node_id: "node-q", + status: "confirmed", + recorded_order: 6, + last_updated_at: "2026-06-01T10:00:00Z", + work_binding: binding, + }; + // Genuinely newer confirmed observation in same work window: admitted! + const validNewPrimary = { + finding_id: "f-new", + node_id: "node-q", + status: "confirmed", + recorded_order: 7, + last_updated_at: "2026-06-01T10:01:00Z", + work_binding: binding, + }; + // Explicitly cited pre-prefix finding vs. post-prefix spoofed citation + const citedBeforePrefix = { + finding_id: "f-cited-early", + node_id: "node-q", + agent_id: "agent-1", + recorded_order: 2, + evidence_refs: ["ev:cited"], + }; + const spoofedAfterPrefix = { + finding_id: "f-cited-late-spoof", + node_id: "node-q", + agent_id: "agent-1", + recorded_order: 3, + evidence_refs: ["ev:cited"], + }; + + assert.equal(isValidContentSha256(sha), true); + assert.equal(isValidContentSha256("A".repeat(64)), false); + assert.equal( + isHistoricalReplayPrecedingCurrent("2026-06-01T09:59:00Z", "2026-06-01T10:00:00Z"), + true, + ); + assert.equal( + isHistoricalReplayPrecedingCurrent("2026-06-01T10:00:00Z", "2026-06-01T10:00:00Z"), + false, + ); + assert.equal(recordedBeforePrefix(citedBeforePrefix, 3), true); + assert.equal(recordedBeforePrefix(spoofedAfterPrefix, 3), false); + assert.equal(committedInWorkWindow(delayedOldPrimary, 3, 1780308000000000n), false); + + const latest = latestCommittedScopedRecord([decisionRow, {...delayedOldPrimary, work_yardstick: decisionRow.work_yardstick}], () => true); + assert.equal(latest?.finding_id, "f-decision"); + + const selected = selectCanonicalEvidenceWindow({ + findings: [decisionRow, delayedOldPrimary, tiedClockPrimary, validNewPrimary], + cited_sources: [ + { + source_finding_id: "f-decision", + source_snapshot: snapshot, + findings: [citedBeforePrefix, spoofedAfterPrefix], + }, + ], + source_finding: decisionRow, + binding, + source_snapshot: snapshot, + cited_refs: new Set(["ev:cited"]), + has_decision_attachment: row => row.work_yardstick !== undefined, + matches_decision_target: () => true, + }); + + assert.deepEqual( + selected.scoped_findings.map(r => r.finding_id), + ["f-decision", "f-new", "f-cited-early"], + ); +}); + +test("Bug Class 3 & 4: exact scope-tuple isolation and bounded cold-path projection", () => { + const expected = { + metric: "accuracy", + unit: "pct", + higher_is_better: true, + comparison_scope: "val:fixed", + input_revision: "rev-1", + evaluation_ref: "eval:v1", + }; + const keys = ["metric", "unit", "higher_is_better", "comparison_scope", "input_revision", "evaluation_ref"] as const; + assert.equal(matchesExactScopeTuple({...expected}, expected, keys), true); + assert.equal(matchesExactScopeTuple({...expected, input_revision: "rev-2"}, expected, keys), false); + assert.equal(matchesExactScopeTuple({...expected, higher_is_better: false}, expected, keys), false); + assert.equal( + matchesWorkIdentity( + {work_binding: {goal_id: "g1", agent_id: "a1", todo_id: "todo_1"}}, + {goal_id: "g1", agent_id: "a1", todo_id: "todo_2"}, + ), + false, + ); + + const bounded = projectBoundedMethodologyItems(["a", "b", "c", "d", "e"], 3, ["loopx", "explore", "turn-context"]); + assert.deepEqual(bounded, { + visible: ["a", "b", "c"], + omitted_count: 2, + details_command: ["loopx", "explore", "turn-context"], + }); +}); + +test("Bug Class 5: 5-stage operator catalog completeness and non-escalation output verifier", () => { + const catalog = listStageMethodologyOperators(); + assert.deepEqual(catalog.stages, [...LOOPX_CONTROL_STAGES]); + assert.equal(BUILTIN_STAGE_METHODOLOGY_OPERATORS.length, 6); + for (const op of BUILTIN_STAGE_METHODOLOGY_OPERATORS) { + assert.equal(op.grants_execution_authority, false); + assert.equal(op.rollback_contract, "stateless_disable_preserves_history"); + assert.ok(op.lightweight_guidance.length > 20 && op.lightweight_guidance.length <= 240); + if (op.operator_id === "stride_review") { + assert.equal(op.operator_role, "required_due_decision"); + assert.equal(op.advisory_only, false); + } else { + assert.equal(op.operator_role, "advisory_cue"); + assert.equal(op.advisory_only, true); + } + } + + const executeStageOps = listStageMethodologyOperators({stage: "execute_verify"}); + const ids = (executeStageOps.operators as Array<{operator_id: string}>).map(o => o.operator_id); + assert.ok(ids.includes("work_yardstick")); + assert.ok(ids.includes("scoped_yardstick_fidelity")); + assert.ok(ids.includes("counterfactual_continuation")); + assert.ok(ids.includes("evolutionary_population")); + + const clean = verifyMethodologyOperatorOutput({ + operator_id: "scoped_yardstick_fidelity", + output: {advisory_only: true, grants_execution_authority: false, score: 0.75, unknown_cost: null}, + }); + assert.equal(clean.valid, true); + + const missingAdvisory = verifyMethodologyOperatorOutput({ + operator_id: "scoped_yardstick_fidelity", + output: {score: 0.75}, + }); + assert.equal(missingAdvisory.valid, false); + assert.deepEqual(missingAdvisory.violations, [ + "$.advisory_only: must be explicitly true", + "$.grants_execution_authority: must be explicitly false", + ]); + + const coreStrideDecision = { + schema_version: "stride_decision_v2", + strategy_version: "evidence_conditioned_v1", + reference_context_id: "replan-context-1234", + reference_obligation_id: "replan-abcd1234abcd1234", + basis: "uncertain_evidence", + action: "verify", + axes: { + work_slice: {plan: "Complete check.", reason: "Resolve uncertainty."}, + route_distance: {plan: "Keep current route.", reason: "No counterexample yet."}, + validation_scope: {plan: "Verify positive and negative cases.", reason: "Independent check."}, + }, + next_probe: { + scope: "One bounded check", + prediction: "Distinguish candidates", + adoption_criterion: "Pass acceptance", + cost_bound: "One turn", + minimum_evidence: "Verified run", + }, + }; + const strideClean = verifyMethodologyOperatorOutput({ + operator_id: "stride_review", + output: coreStrideDecision, + }); + assert.equal(strideClean.valid, true); + assert.deepEqual(strideClean.violations, []); + + const strideRootAuthority = verifyMethodologyOperatorOutput({ + operator_id: "stride_review", + output: {...coreStrideDecision, grants_execution_authority: true}, + }); + assert.equal(strideRootAuthority.valid, false); + assert.deepEqual(strideRootAuthority.violations, [ + "$.grants_execution_authority: methodology operator cannot grant execution authority", + ]); + + const strideNestedAuthority = verifyMethodologyOperatorOutput({ + operator_id: "stride_review", + output: { + ...coreStrideDecision, + axes: { + ...coreStrideDecision.axes, + work_slice: { + ...coreStrideDecision.axes.work_slice, + should_run: true, + grants_execution_authority: true, + nested: {admin_bypass: true}, + }, + }, + }, + }); + assert.equal(strideNestedAuthority.valid, false); + assert.deepEqual(strideNestedAuthority.violations, [ + "$.axes.work_slice.should_run: methodology operator cannot emit authority/gate field", + "$.axes.work_slice.grants_execution_authority: methodology operator cannot grant execution authority", + "$.axes.work_slice.nested.admin_bypass: methodology operator cannot emit authority/gate field", + ]); + + const nestedAuthority = verifyMethodologyOperatorOutput({ + operator_id: "scoped_yardstick_fidelity", + output: { + advisory_only: true, + grants_execution_authority: false, + result: {should_run: true, points: [{admin_bypass: false}]}, + }, + }); + assert.equal(nestedAuthority.valid, false); + assert.deepEqual(nestedAuthority.violations, [ + "$.result.should_run: methodology operator cannot emit authority/gate field", + "$.result.points[0].admin_bypass: methodology operator cannot emit authority/gate field", + ]); + + const escalated = verifyMethodologyOperatorOutput({ + operator_id: "rogue_op", + output: { + advisory_only: true, + grants_execution_authority: false, + should_run: true, + nested: {bad: Number.NaN, ctrl: "line1\n\u0007"}, + }, + max_bytes: 32, + }); + assert.equal(escalated.valid, false); + assert.ok(escalated.violations.some(v => v.includes("should_run"))); + assert.ok(escalated.violations.some(v => v.includes("non-finite"))); + assert.ok(escalated.violations.some(v => v.includes("control characters"))); + assert.ok(escalated.violations.some(v => v.includes("exceeds cap"))); +}); diff --git a/tests/control_plane_ts/quantitative_research_operators.test.ts b/tests/control_plane_ts/quantitative_research_operators.test.ts new file mode 100644 index 0000000000..57734a0c52 --- /dev/null +++ b/tests/control_plane_ts/quantitative_research_operators.test.ts @@ -0,0 +1,438 @@ +import assert from "node:assert/strict"; +import test from "node:test"; +import {evaluateScopedYardstickFidelity} from "../../loopx/control_plane/capabilities/yardstick_fidelity.ts"; +import {evaluateBudgetedContinuationPlan} from "../../loopx/control_plane/capabilities/budgeted_continuation.ts"; +import {evaluateCounterfactualContinuations} from "../../loopx/control_plane/capabilities/counterfactual_continuation.ts"; +import {evaluateEvolutionaryPopulation} from "../../loopx/control_plane/capabilities/evolutionary_search.ts"; + +const authMax = { + metric: "heldout_score", + unit: "points", + higher_is_better: true, + comparison_scope: "bench:v1", + input_revision: "rev-1", + evaluation_ref: "eval:authoritative-v1", +}; +const proxyMin = { + metric: "val_loss", + unit: "nats", + higher_is_better: false, + comparison_scope: "bench:v1", + input_revision: "rev-1", + evaluation_ref: "eval:proxy-v1", +}; + +function makeFinding( + id: string, + order: number, + cand: string, + authVal: number | null | undefined, + proxyVal: number | null | undefined, + extra: Record = {}, +) { + const measurements = []; + if (authVal !== undefined) { + measurements.push({...authMax, candidate_ref: cand, evidence_ref: `ev:${id}:a`, value: authVal}); + } + if (proxyVal !== undefined) { + measurements.push({...proxyMin, candidate_ref: cand, evidence_ref: `ev:${id}:p`, value: proxyVal}); + } + return { + finding_id: id, + node_id: "node-opt", + status: "confirmed", + recorded_order: order, + measurements, + ...extra, + }; +} + +test("ScopedYardstickFidelity: detects Goodhart frontier_bias_cliff across opposite-direction metrics and audits unconfirmed proxy winners", () => { + // 6 paired candidates: + // Candidates c1..c3 (bottom half): proxy and authoritative strongly agree. + // Candidates c4..c6 (top-3 authoritative frontier): proxy reverses order (lower val_loss picks c4 over c6, while c6 has highest heldout_score)! + const findings = [ + makeFinding("f1", 1, "cand:c1", 10, 2.50), + makeFinding("f2", 2, "cand:c2", 20, 2.10), + makeFinding("f3", 3, "cand:c3", 30, 1.80), + makeFinding("f4", 4, "cand:c4", 70, 0.85), + makeFinding("f5", 5, "cand:c5", 80, 0.95), + makeFinding("f6", 6, "cand:c6", 92, 1.15), + // Unconfirmed proxy winner (val_loss 0.70 beats best paired 0.85, no authoritative score yet) + makeFinding("f7", 7, "cand:unconfirmed_star", undefined, 0.70), + // Null unknown measurement: counted as unknown, never coerced to 0 + makeFinding("f8", 8, "cand:unknown", null, 1.40), + // Quarantined finding with mismatched execution_observation.input_revision: ignored! + makeFinding("f9", 9, "cand:spoof", 999, 0.10, { + execution_observation: { + planned_method_id: "m:a", + executed_method_id: "m:a", + input_revision: "rev-FOREIGN", + candidate_sha256: "b".repeat(64), + elapsed_seconds: 10, + limit_seconds: 60, + search_iterations: 1, + stop_reason: "completed", + evidence_refs: ["ev:f9:a"], + }, + }), + ]; + + const report = evaluateScopedYardstickFidelity({ + node_id: "node-opt", + authoritative: authMax, + proxy: proxyMin, + frontier_k: 3, + findings, + }); + + assert.equal(report.fidelity_status, "frontier_bias_cliff"); + assert.equal(report.calibration_tier, "multi_point"); + assert.equal(report.counts.paired_candidates, 6); + assert.equal(report.counts.authoritative_unknowns, 1); + assert.equal(report.counts.proxy_only_candidates, 2); + assert.ok((report.rank_concordance?.kendall_tau_b ?? 0) > 0.3, "Global concordance remains positive"); + assert.equal(report.frontier_neighborhood?.frontier_kendall_tau_b, -1); + assert.equal(report.frontier_neighborhood?.bias_cliff_detected, true); + assert.equal(report.frontier_neighborhood?.authoritative_best_candidate_ref, "cand:c6"); + assert.equal(report.frontier_neighborhood?.proxy_best_paired_candidate_ref, "cand:c4"); + assert.deepEqual( + report.audit_candidates.unconfirmed_proxy_winners.map(w => w.candidate_ref), + ["cand:unconfirmed_star"], + ); + assert.equal(report.advisory_only, true); + assert.equal(report.grants_execution_authority, false); +}); + +test("A proxy tie leaves the best candidate unresolved despite positive global concordance", () => { + const result = evaluateScopedYardstickFidelity({ + node_id: "node-opt", authoritative: authMax, proxy: proxyMin, + findings: [ + makeFinding("f1", 1, "cand:low", 10, 2), + makeFinding("f2", 2, "cand:a", 50, 1), + makeFinding("f3", 3, "cand:b", 60, 1), + ], + }); + assert.equal(result.fidelity_status, "aligned_in_tested_scope"); + assert.equal(result.frontier_neighborhood?.authoritative_best_candidate_ref, "cand:b"); + assert.equal(result.frontier_neighborhood?.proxy_best_paired_candidate_ref, null); + assert.equal(result.frontier_neighborhood?.best_candidate_agrees, null); +}); + +test("Tied authoritative ranks are invariant to candidate names and preserve the proxy follow-up", () => { + for (const winner of ["cand:d", "cand:a"]) { + const findings = [ + makeFinding("f1", 1, "cand:b", 100, 2), + makeFinding("f2", 2, "cand:c", 100, 1), + makeFinding("f3", 3, winner, 100, 0), + makeFinding("f4", 4, "cand:z", 90, 3), + makeFinding("f5", 5, "cand:proxy-only", undefined, -1), + ]; + const request = {node_id: "node-opt", authoritative: authMax, proxy: proxyMin, findings}; + const fidelity = evaluateScopedYardstickFidelity(request); + assert.equal(fidelity.rank_concordance?.discordant_pairs, 0); + assert.equal(fidelity.frontier_neighborhood?.authoritative_best_candidate_ref, null); + assert.equal(fidelity.frontier_neighborhood?.proxy_best_paired_candidate_ref, winner); + assert.equal(fidelity.fidelity_status, "aligned_in_tested_scope"); + + const population = evaluateEvolutionaryPopulation(request); + assert.deepEqual( + population.proxy_quarantine.map(row => [row.candidate_ref, row.disposition]), + [["cand:proxy-only", "confirm_before_breeding"]], + ); + } +}); + +test("BudgetedContinuationPlan: back-schedules confirmation deadlines, preserves null bounds, and computes exact p*", () => { + const plan = evaluateBudgetedContinuationPlan({ + remaining_budget: { + wall_minutes: 180, + token_budget: 200000, + authoritative_eval_slots: 2, + }, + confirmation: { + eval_duration_minutes: 45, + cooldown_minutes: 15, + eval_tokens: 20000, + pending_in_flight_minutes: 10, + }, + incumbent: { + candidate_ref: "cand:incumbent", + value: 0.72, + higher_is_better: true, + metric: "score", + unit: "ratio", + }, + baseline_route: { + route_id: "route:continue_tuning", + summary: "Continue local tuning on current architecture", + probe_minutes: 0, + implementation_minutes: 30, + expected_tokens: 50000, + requires_authoritative_eval: true, + expected_gain_min: 0.01, + expected_gain_expected: 0.02, + expected_gain_max: 0.04, + }, + candidate_routes: [ + { + route_id: "route:reversible_probe_switch", + summary: "20m diagnostic probe then switch solver", + kind: "switch_mechanism", + probe_minutes: 20, + implementation_minutes: 40, + expected_tokens: null, + requires_authoritative_eval: true, + expected_gain_min: -0.02, + expected_gain_expected: 0.06, + expected_gain_max: 0.10, + }, + { + route_id: "route:too_late_rewrite", + summary: "Full rewrite missing confirmation deadline", + kind: "switch_mechanism", + probe_minutes: 30, + implementation_minutes: 96, + expected_tokens: 120000, + requires_authoritative_eval: true, + expected_gain_min: 0.0, + expected_gain_expected: 0.12, + expected_gain_max: 0.20, + }, + ], + }); + + // The 15-minute cooldown starts now and overlaps work after the pending result. + assert.equal(plan.confirmation.latest_confirmation_start_minute, 135); + assert.equal(plan.confirmation.remaining_work_window_minutes, 125); + assert.equal(plan.effective_safe_gain, 0.02); + + const probeRoute = plan.candidate_routes.find(r => r.route_id === "route:reversible_probe_switch")!; + assert.equal(probeRoute.feasibility_status, "feasible_with_unknown_bounds"); + assert.equal(probeRoute.total_expected_tokens, null, "Unknown token cost stays null, never 0"); + assert.equal(probeRoute.total_wall_minutes, 115); + assert.equal(probeRoute.reversible_probe_window_feasible, true); + assert.equal(probeRoute.recommended_disposition, "probe_then_decide"); + // Retained-incumbent floor max(0, -0.02) = 0: + // p*_optimistic = 0.02 / 0.10 = 0.2; p*_expected = 0.02 / 0.06 = 0.333333 + assert.equal(probeRoute.p_star_optimistic, 0.2); + assert.equal(probeRoute.p_star_at_expected_gain, 0.333333); + + const lateRoute = plan.candidate_routes.find(r => r.route_id === "route:too_late_rewrite")!; + assert.equal(lateRoute.feasibility_status, "deadline_infeasible"); + assert.equal(lateRoute.recommended_disposition, "reject_infeasible"); + assert.ok(lateRoute.slack_before_deadline_minutes < 0); +}); + +test("CounterfactualContinuations: enforces checkpoint & workspace non-interference and computes direction-normalized tau_B", () => { + const sha = "c".repeat(64); + const makePair = (id: string, task: string, treatLoss: number, ctrlLoss: number, overrides: Record = {}) => ({ + pair_id: id, + task_id: task, + checkpoint: { + checkpoint_id: `ckpt:${id}`, + source_state_sha256: sha, + input_revision: "rev-1", + evaluation_ref: "eval:v1", + metric: "rmse", + unit: "error", + higher_is_better: false, + comparison_scope: "scope:rmse-v1", + terminal_boundary_ref: "horizon:one-hour", + incumbent_at_fork: 1.50, + fork_recorded_at: "2026-07-01T12:00:00Z", + }, + treatment: { + arm_id: "arm:budgeted_yardstick", + checkpoint_id: `ckpt:${id}`, + source_state_sha256: sha, + input_revision: "rev-1", + evaluation_ref: "eval:v1", + workspace_isolation_id: `ws:${id}:treat`, + terminal_boundary_ref: "horizon:one-hour", + decision_context_latest_at: "2026-07-01T12:00:00Z", + first_action_signature: "act:switch_solver", + terminal_value: treatLoss, + valid_submission: true, + costs: {wall_seconds: 3600, total_tokens: 120000, evaluator_runs: 2, control_plane_requests: 6, recovery_events: null}, + ...(overrides.treatment as object ?? {}), + }, + control: { + arm_id: "arm:baseline", + checkpoint_id: `ckpt:${id}`, + source_state_sha256: sha, + input_revision: "rev-1", + evaluation_ref: "eval:v1", + workspace_isolation_id: `ws:${id}:ctrl`, + terminal_boundary_ref: "horizon:one-hour", + decision_context_latest_at: "2026-07-01T11:59:59Z", + first_action_signature: "act:tune_hyperparams", + terminal_value: ctrlLoss, + valid_submission: true, + costs: {wall_seconds: 3500, total_tokens: 110000, evaluator_runs: 3, control_plane_requests: 5, recovery_events: 0}, + ...(overrides.control as object ?? {}), + }, + }); + + // Lower rmse is better -> treatment 0.90 vs control 1.20 => tau_B = +0.30 + const pairs = [ + makePair("p1", "task:a", 0.90, 1.20), + makePair("p2", "task:a", 1.00, 1.25), + makePair("p3", "task:b", 0.80, 1.10), + makePair("p4", "task:b", 0.95, 1.15), + ]; + + const cleanEval = evaluateCounterfactualContinuations({pairs}); + assert.equal(cleanEval.all_pairs_identification_valid, true); + assert.equal(cleanEval.counts.wins, 4); + assert.equal(cleanEval.counts.action_diverged_pairs, 4); + // tau_B values: 0.30, 0.25, 0.30, 0.20 -> mean = 0.2625, exact sign-flip p = 2 / 16 = 0.125 + assert.equal(cleanEval.estimand_summary.mean_tau_b, 0.2625); + assert.equal(cleanEval.estimand_summary.exact_sign_flip_p_value, 0.125); + assert.equal(cleanEval.complete_marginal_costs.missing_pair_counts.recovery_events, 4); + const accuracy = makePair("p5", "task:a", 81, 80, {treatment: {first_action_signature: null}}); + Object.assign(accuracy.checkpoint, {metric: "accuracy", unit: "percentage-points", higher_is_better: true}); + const mixed = evaluateCounterfactualContinuations({pairs: [pairs[0], accuracy]}); + assert.equal(mixed.estimand_summary.comparable_scope_count, 2); + assert.equal(mixed.estimand_summary.mean_tau_b, null); + assert.equal(mixed.estimand_summary.exact_sign_flip_p_value, null); + assert.deepEqual(mixed.by_task.map(row => row.mean_tau_b).sort(), [0.3, 1]); + assert.equal(mixed.pairs[1]?.outcome_classification, "action_unobserved"); + assert.equal(mixed.pairs[1]?.tau_b_direction_normalized, 1); + const missingClock = makePair("p6", "task:a", 0.9, 1.2, {control: {decision_context_latest_at: null}}); + assert.throws(() => evaluateCounterfactualContinuations({pairs: [missingClock]}), /missing_context_timing/); + assert.equal(cleanEval.complete_marginal_costs.observed_delta_sums.evaluator_runs, -4); + + // Shared workspace_isolation_id + post-fork future context leakage: + const contaminated = makePair("p-dirty", "task:c", 0.50, 1.20, { + treatment: { + workspace_isolation_id: "ws:p-dirty:ctrl", + decision_context_latest_at: "2026-07-01T12:05:00Z", + }, + }); + assert.throws( + () => evaluateCounterfactualContinuations({strict_identification: true, pairs: [contaminated]}), + /non-interference|future/, + ); + + const audited = evaluateCounterfactualContinuations({strict_identification: false, pairs: [...pairs, contaminated]}); + assert.equal(audited.all_pairs_identification_valid, false); + assert.equal(audited.counts.contaminated_pairs, 1); + assert.equal(audited.counts.valid_pairs, 4); + assert.equal(audited.estimand_summary.mean_tau_b, 0.2625, "Contaminated pair excluded from estimand"); +}); + +test("evolutionary_population preserves distinct mechanism islands, generates orthogonal crossovers, and quarantines Goodharted proxy winners", () => { + const authoritative = { + metric: "accuracy", + unit: "pct", + higher_is_better: true, + comparison_scope: "bench:test-v1", + input_revision: "rev-1", + evaluation_ref: "eval:full", + }; + const proxy = { + metric: "proxy_score", + unit: "pct", + higher_is_better: true, + comparison_scope: "bench:dev-v1", + input_revision: "rev-1", + evaluation_ref: "eval:fast", + }; + + // Suppose 4 hyperparameter tweaks of mech:alpha score 85.0, 84.8, 84.6, 84.4, + // while orthogonal mech:beta scores 83.5 and mech:gamma scores 82.0. + // Plus a Goodharted proxy frontier reversal on top-3 and an unconfirmed proxy star (99.5)! + const rows: Array<[string, string, number | null, number]> = [ + ["cand:a1", "mech:alpha", 85.0, 86.0], + ["cand:a2", "mech:alpha", 84.8, 87.0], + ["cand:a3", "mech:alpha", 84.6, 88.0], + ["cand:a4", "mech:alpha", 84.4, 70.0], + ["cand:b1", "mech:beta", 83.5, 68.0], + ["cand:g1", "mech:gamma", 82.0, 65.0], + ["cand:proxy_only_star", "mech:alpha", null, 99.5], + ]; + + const findings = rows.map(([cand, , authVal, proxyVal], idx) => ({ + finding_id: `f-${idx}`, + node_id: "node:evo", + status: "confirmed", + recorded_order: idx, + measurements: [ + {...authoritative, value: authVal, candidate_ref: cand, evidence_ref: `ev:auth:${idx}`}, + {...proxy, value: proxyVal, candidate_ref: cand, evidence_ref: `ev:proxy:${idx}`}, + ], + })); + + const plan = evaluateEvolutionaryPopulation({ + node_id: "node:evo", + authoritative, + proxy, + findings, + elite_limit: 3, + crossover_limit: 2, + mutation_limit: 2, + mechanism_by_candidate: Object.fromEntries(rows.map(([cand, mech]) => [cand, mech])), + }); + + assert.equal(plan.selection_status, "multi_island_evolution"); + assert.equal(plan.proxy_fidelity_status, "frontier_bias_cliff"); + assert.equal(plan.counts.confirmed_candidates, 6); + assert.equal(plan.counts.mechanism_islands, 3); + assert.equal(plan.counts.authoritative_unknowns, 1); + // Island elitism: with elite_limit=3 across 3 mechanism islands, champions of mech:alpha, mech:beta, + // and mech:gamma MUST all survive instead of letting cand:a2/cand:a3 crowd out beta & gamma! + assert.deepEqual( + plan.elites.map(e => [e.candidate_ref, e.mechanism_id, e.selection_reason]), + [ + ["cand:a1", "mech:alpha", "island_champion"], + ["cand:b1", "mech:beta", "island_champion"], + ["cand:g1", "mech:gamma", "island_champion"], + ], + ); + // Orthogonal crossover pairs combine distinct mechanisms only: + assert.equal(plan.crossover_pairs.length, 2); + assert.equal(plan.crossover_pairs[0]?.pair_id, "cand:a1+cand:b1"); + assert.equal(plan.crossover_pairs[1]?.pair_id, "cand:a1+cand:g1"); + // Unconfirmed proxy winner is quarantined due to frontier_bias_cliff, never mixed into elites: + assert.equal(plan.proxy_quarantine.length, 1); + assert.equal(plan.proxy_quarantine[0]?.candidate_ref, "cand:proxy_only_star"); + assert.equal(plan.proxy_quarantine[0]?.disposition, "quarantined_proxy_cliff"); + assert.equal(plan.advisory_only, true); + assert.equal(plan.grants_execution_authority, false); +}); + +test("Evolutionary choices are invariant to equivalent latency units", () => { + const candidates = ["cand:a", "cand:b", "cand:z"]; + const mechanisms = ["mech:a", "mech:b", "mech:z"]; + for (const [unit, values] of [["s", [4e-7, 3e-7, 1e-7]], ["ns", [400, 300, 100]]] as const) { + const authoritative = {...authMax, metric: "latency", unit, higher_is_better: false}; + const findings = candidates.map((candidate, i) => makeFinding( + String(i), i + 1, candidate, undefined, undefined, { + measurements: [{ + ...authoritative, candidate_ref: candidate, evidence_ref: "ev:latency:" + i, value: values[i], + }], + }, + )); + const request = {node_id: "node-opt", authoritative, findings}; + const singleIsland = evaluateEvolutionaryPopulation({ + ...request, + elite_limit: 2, + mechanism_by_candidate: Object.fromEntries(candidates.map(candidate => [candidate, "mech:same"])), + }); + assert.deepEqual(singleIsland.elites.map(row => row.candidate_ref), ["cand:z", "cand:b"]); + assert.equal(singleIsland.islands[0]?.champion_candidate_ref, "cand:z"); + + const diverse = evaluateEvolutionaryPopulation({ + ...request, + elite_limit: 3, + mechanism_by_candidate: Object.fromEntries(candidates.map((candidate, i) => [candidate, mechanisms[i]])), + }); + assert.deepEqual(diverse.elites.map(row => row.candidate_ref), ["cand:z", "cand:b", "cand:a"]); + assert.deepEqual(diverse.crossover_pairs.map(row => row.pair_id), ["cand:z+cand:b", "cand:z+cand:a"]); + assert.deepEqual( + diverse.mutation_targets.map(row => row.mutation_kind), + ["island_champion_refinement", "island_champion_refinement"], + ); + } +});