feat(site): score your own pasted predictions against their floor

CHANGELOG.md

@@ -29,7 +29,16 @@ different event from one that moved because it was wrong.
   with `plumbline report` prints the same Dataset section, with the read,
   loaded, and refused counts, as the one written at run time (#58). Artifacts
   written before this still report, without the section.
-
+- The site scores your own predictions (#56). Paste a probability and an
+  outcome per line, as CSV, tabs, or spaces, with or without a header, and the
+  page computes the ECE, the floor for exactly those predictions, and the
+  verdict a report would print, word for word. A row that cannot be read is
+  named by its line, and nothing is scored until every row reads. Nothing
+  pasted leaves the browser: it is not uploaded, stored, or put in the address.
+  The parity check holds the path to `ece_figure` on pasted text in four
+  formats, two of them on and beside a rounding tie, and pins the parser's
+  refusals; the smoke test pastes a bad row and the worked example's rows,
+  which must reproduce the report's line.
 - A browser calculator for the ECE floor (`site/`), a JavaScript port of
   `synthetic_floor` that reproduces numpy's seeded random stream draw for draw.
   `scripts/floor_golden.py` exports golden values from the Python and

scripts/check_floor_parity.mjs

@@ -53,6 +53,58 @@ for (const c of fixture.cases) {
   console.log(`${label.padEnd(36)} mean ${band.mean.toFixed(6)}  p95 ${band.p95.toFixed(6)}  ${elapsed}ms`);
 }
 
+// Pasted predictions: the page parses the text, then scores it as a report
+// does. Each case must parse to the rows the Python scored and print the same
+// line, character for character.
+for (const c of fixture.pasted) {
+  const label = `pasted, ${c.label}`;
+  const parsed = floor.parsePredictions(c.text);
+  if (parsed.errors.length) {
+    failures.push(`${label}: refused: ${JSON.stringify(parsed.errors.slice(0, 2))}`);
+    continue;
+  }
+  if (parsed.rows !== c.n) failures.push(`${label}: read ${parsed.rows} rows, the Python scored ${c.n}`);
+  const measured = floor.ece(parsed.probabilities, parsed.correct, c.n_bins);
+  const band = floor.calibrationFloor(parsed.probabilities, c.n_bins);
+  for (const [key, actual] of [["ece", measured], ["mean", band.mean], ["p95", band.p95]]) {
+    const gap = Math.abs(actual - c[key]);
+    worst = Math.max(worst, gap);
+    if (!(gap <= tolerance)) failures.push(`${label}: ${key} ${actual} against Python ${c[key]} (off by ${gap})`);
+  }
+  const line = floor.statement(measured, band);
+  if (line !== c.statement) failures.push(`${label}: verdict differs\n  js:     ${line}\n  python: ${c.statement}`);
+}
+console.log(`${fixture.pasted.length} pasted texts parsed and scored`);
+
+// What the parser refuses, and the line it names. There is no Python to hold
+// these to: the report never reads pasted text. They pin the page's own rules.
+const refusals = [
+  ["0.5,1\n1.2,0", [2, "the probability 1.2 is outside 0 to 1"]],
+  ["probability,outcome\n0.5,1\nabc,1", [3, 'the probability "abc" is not a number']],
+  ["NaN,1\n0.5,1", [1, 'the probability "NaN" is not a number']],
+  ["0.5,2", [1, 'the outcome "2" is not 0 or 1']],
+  ["0.5\n0.5,1", [1, "expected two values, a probability and an outcome, and found 1"]],
+  ["0.5,1,0.3", [1, "expected two values, a probability and an outcome, and found 3"]],
+  ["", [null, "there are no rows to score. Paste one row per line: a probability, then 0 or 1"]],
+  ["probability,outcome\n\n", [null, "there are no rows to score. Paste one row per line: a probability, then 0 or 1"]],
+];
+for (const [text, [line, message]] of refusals) {
+  const { errors } = floor.parsePredictions(text);
+  const got = errors[0];
+  if (!got || got.line !== line || got.message !== message) {
+    failures.push(`parser: ${JSON.stringify(text)} gave ${JSON.stringify(got)}, expected line ${line}: ${message}`);
+  }
+}
+const capped = floor.parsePredictions("0.5,1\n0.5,0\n0.5,1", { maxRows: 2 }).errors;
+if (!(capped.length === 1 && capped[0].line === null && capped[0].message.startsWith("3 rows is more than this page scores"))) {
+  failures.push(`parser: a paste over the row cap gave ${JSON.stringify(capped)}`);
+}
+const lenient = floor.parsePredictions(' 0.5, 1,\n\n"0.25"\tfalse \n1e-1 TRUE\n');
+if (lenient.errors.length || lenient.probabilities.join() !== "0.5,0.25,0.1" || lenient.correct.join() !== "true,false,true") {
+  failures.push(`parser: lenient input read as ${JSON.stringify(lenient)}`);
+}
+console.log(`${refusals.length + 2} parser cases checked`);
+
 // Every figure on the page is printed through fixed4, which must round as
 // Python's "%.4f" does, including on and beside a tie.
 const formats = Object.entries(fixture.fixed4);

scripts/floor_golden.py

@@ -25,7 +25,8 @@
 
 import numpy as np
 
-from plumbline.metrics.calibration import CalibrationFigure, synthetic_floor
+from plumbline.metrics.calibration import CalibrationFigure, ece_figure, synthetic_floor
+from plumbline.types import ProbabilitySeries
 
 FIXTURE = Path(__file__).resolve().parent.parent / "site" / "floor-golden.json"
 
@@ -93,6 +94,81 @@ def _formatting_cases() -> dict[str, str]:
     return {repr(value): f"{value:.4f}" for value in sorted(values)}
 
 
+def _pasted_cases() -> list[dict[str, Any]]:
+    """Pasted text in the formats the page reads, with what a report prints for it.
+
+    The page parses the text itself, so each case is written the way a person
+    might paste it, and the Python scores the rows the text was written from
+    through ``ece_figure``, the call a report makes. Probabilities are written
+    with ``repr``, which round-trips exactly, so both sides score the same
+    doubles. Two cases put the measured ECE on and beside a rounding tie.
+    """
+    rng = np.random.default_rng(56)
+    confident = [float(p) for p in rng.beta(5.0, 2.0, 300)]
+    confident_hits = [bool(u < p * 0.9) for u, p in zip(rng.random(300), confident, strict=True)]
+    spread = [float(p) for p in rng.uniform(0.0, 1.0, 150)]
+    spread_hits = [bool(u < p) for u, p in zip(rng.random(150), spread, strict=True)]
+    tie = [0.75] * 32
+    tie_hits = [True] * 23 + [False] * 9  # 23/32 against 0.75: ECE is exactly 0.03125
+    near = [0.80125] * 400
+    near_hits = [True] * 320 + [False] * 80  # 0.8 against 0.80125: beside 0.00125
+
+    def rows(ps: list[float], hits: list[bool], line: str) -> list[str]:
+        return [
+            line.format(p=repr(p), y=int(y), word=str(y).lower())
+            for p, y in zip(ps, hits, strict=True)
+        ]
+
+    spread_lines = rows(spread, spread_hits, "{p}\t{word}")
+    spread_lines.insert(40, "")
+    series = [
+        (
+            "comma separated, with a header",
+            10,
+            confident,
+            confident_hits,
+            "probability,outcome\n" + "\n".join(rows(confident, confident_hits, "{p},{y}")) + "\n",
+        ),
+        (
+            "tab separated, CRLF line ends, true and false, a blank line",
+            15,
+            spread,
+            spread_hits,
+            "\r\n".join(spread_lines),
+        ),
+        (
+            "space separated, the ECE exactly on a rounding tie",
+            10,
+            tie,
+            tie_hits,
+            "\n".join(rows(tie, tie_hits, "  {p}   {y}")),
+        ),
+        (
+            "semicolons and quotes, the ECE beside a tie",
+            10,
+            near,
+            near_hits,
+            'p;"correct"\n' + "\n".join(rows(near, near_hits, '"{p}";"{y}"')),
+        ),
+    ]
+    cases = []
+    for label, n_bins, ps, hits, text in series:
+        figure = ece_figure(ProbabilitySeries(tuple(ps), "calibrated_claim"), hits, n_bins=n_bins)
+        cases.append(
+            {
+                "label": label,
+                "n_bins": n_bins,
+                "text": text,
+                "n": figure.n,
+                "ece": figure.value,
+                "mean": figure.floor.mean,
+                "p95": figure.floor.p95,
+                "statement": figure.statement(),
+            }
+        )
+    return cases
+
+
 def _pcg64_state(seed: int) -> dict[str, str]:
     state = np.random.default_rng(seed).bit_generator.state["state"]
     return {"state": str(state["state"]), "inc": str(state["inc"])}
@@ -125,8 +201,9 @@ def build() -> dict[str, Any]:
         )
     return {
         "about": (
-            "Golden values from plumbline.metrics.calibration.synthetic_floor, "
-            "which site/floor.js must reproduce. Written by scripts/floor_golden.py."
+            "Golden values from plumbline.metrics.calibration (synthetic_floor for the "
+            "calculator, ece_figure for pasted rows), which site/floor.js must "
+            "reproduce. Written by scripts/floor_golden.py."
         ),
         "function": "synthetic_floor(n, n_bins, accuracy), all other arguments default",
         "defaults": {"binning": "equal_width", "n_boot": 2000, "concentration": 6.0, "seed": 0},
@@ -135,6 +212,7 @@ def build() -> dict[str, Any]:
         "pcg64_initial_state": {"0": _pcg64_state(0), "1": _pcg64_state(1)},
         "cases": cases,
         "fixed4": _formatting_cases(),
+        "pasted": _pasted_cases(),
     }
 
 
@@ -147,6 +225,16 @@ def _check(fresh: dict[str, Any]) -> list[str]:
         problems.append("numpy's seeded PCG64 states differ from the fixture")
     if committed.get("fixed4") != fresh["fixed4"]:
         problems.append("the :.4f formatting cases differ from the fixture")
+    old_pasted = committed.get("pasted", [])
+    if [c["text"] for c in old_pasted] != [c["text"] for c in fresh["pasted"]]:
+        problems.append("the pasted-text cases differ from the fixture; regenerate it")
+    else:
+        for old, new in zip(old_pasted, fresh["pasted"], strict=True):
+            for key in ("ece", "mean", "p95"):
+                if abs(old[key] - new[key]) > TOLERANCE:
+                    problems.append(f"pasted, {new['label']}: {key} is {new[key]!r} now")
+            if old["statement"] != new["statement"]:
+                problems.append(f"pasted, {new['label']}: verdict wording changed")
     if len(committed["cases"]) != len(fresh["cases"]):
         problems.append("the fixture holds a different set of cases; regenerate it")
         return problems

scripts/smoke_site.mjs

@@ -17,6 +17,8 @@
 //   field invalid;
 // - the calculator runs, and writes its inputs into the address;
 // - a link carrying those inputs runs the calculator as the page opens;
+// - pasted predictions with a bad row name the line and score nothing, and the
+//   worked example's rows, pasted, reproduce the report's line;
 // - search, opened from the header on a doc page, finds a section of that doc.
 
 import { spawn } from "node:child_process";
@@ -313,6 +315,27 @@ await check("a link carrying the inputs runs the calculator as the page opens",
   await violations();
 });
 
+await check("a pasted row that cannot be read names its line and scores nothing", async () => {
+  await until(`!${$("paste-fill")}.hidden`, "the example's rows to be offered");
+  await evaluate(`${$("paste-data")}.value = "probability,outcome\\n0.5,1\\n1.4,0"; ${$("paste-go")}.click();`);
+  const status = await evaluate(`${$("paste-status")}.textContent`);
+  expect(status.startsWith("Line 3: the probability 1.4 is outside 0 to 1."), `status reads ${JSON.stringify(status)}`);
+  expect((await evaluate(`${$("paste-data")}.getAttribute("aria-invalid")`)) === "true", "the box is not marked invalid");
+  expect(await evaluate(`${$("paste-result")}.hidden`), "a result was shown for rows that did not read");
+});
+
+await check("the worked example's rows, pasted, give the report's line", async () => {
+  const before = await evaluate("location.search");
+  await evaluate(`${$("paste-fill")}.click(); ${$("paste-go")}.click();`);
+  await until(`!${$("paste-result")}.hidden && !${$("paste-go")}.disabled`, "a scored paste");
+  const verdict = await evaluate(`${$("paste-verdict")}.querySelector("span").textContent`);
+  const report = await evaluate(`${$("ex-report-line")}.lastElementChild.textContent`);
+  expect(verdict === report, `pasted verdict ${JSON.stringify(verdict.slice(0, 80))} is not the report's ${JSON.stringify(report.slice(0, 80))}`);
+  expect(!(await evaluate(`${$("paste-data")}.hasAttribute("aria-invalid")`)), "the box is still marked invalid");
+  expect((await evaluate("location.search")) === before, "scoring pasted rows changed the address");
+  await violations();
+});
+
 await check("search on a doc page finds a section of it", async () => {
   await open("docs/methodology.html");
   await evaluate("document.querySelector('.search-open').click()");

site/explainer.css

@@ -6,12 +6,16 @@ section { padding: 2rem 0; border-top: 1px solid var(--line); }
 form { display: grid; grid-template-columns: repeat(auto-fit, minmax(9.5rem, 1fr)); gap: 1rem; margin: 1.25rem 0 1rem; }
 label { display: block; font-size: 0.9rem; font-weight: 600; margin-bottom: 0.25rem; }
 .hint { display: block; font-size: 0.8rem; font-weight: 400; color: var(--muted); }
-input { width: 100%; font: 1rem var(--mono); padding: 0.5rem 0.6rem; color: var(--fg); background: var(--bg); border: 1px solid var(--control); border-radius: 4px; }
+input, textarea { width: 100%; font: 1rem var(--mono); padding: 0.5rem 0.6rem; color: var(--fg); background: var(--bg); border: 1px solid var(--control); border-radius: 4px; }
 .actions { grid-column: 1 / -1; display: flex; gap: 1rem; align-items: center; flex-wrap: wrap; }
 button { font: 600 1rem var(--sans); padding: 0.55rem 1.1rem; color: var(--bg); background: var(--fg); border: 0; border-radius: 4px; cursor: pointer; }
 button[disabled] { opacity: 0.55; cursor: progress; }
 progress { flex: 1 1 8rem; height: 0.5rem; }
 .error { color: var(--error); }
+textarea { display: block; min-height: 9rem; resize: vertical; font-size: 0.9rem; line-height: 1.4; }
+form .wide { grid-column: 1 / -1; }
+.note { border-left: 3px solid var(--line); padding: 0.2rem 0 0.2rem 0.8rem; }
+.paste-errors { margin: 0 0 1rem; padding-left: 1.25rem; font-size: 0.9rem; }
 
 .result { border: 1px solid var(--line); border-radius: 6px; padding: 1rem; background: var(--panel); }
 .result[hidden], [hidden] { display: none !important; }
@@ -98,5 +102,5 @@ a.button.secondary, button.secondary { color: var(--fg); background: var(--bg);
 a.button.secondary:hover, button.secondary:hover { border-color: var(--fg); }
 button.small-button { font-size: 0.85rem; font-weight: 400; padding: 0.3rem 0.7rem; }
 .share { display: flex; flex-wrap: wrap; gap: 0.5rem; margin: 0 0 1rem; }
-input[aria-invalid="true"] { border-color: var(--error); outline: 1px solid var(--error); }
+input[aria-invalid="true"], textarea[aria-invalid="true"] { border-color: var(--error); outline: 1px solid var(--error); }
 @media print { .rail, .cta, .share, .actions { display: none !important; } }