diff --git a/AGENTS.md b/AGENTS.md index 735988f0..e474b9de 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -73,6 +73,13 @@ in the same spirit) -- never against real data, per the hard rule above. against a live local stack (`make up`) and self-skip without one -- see [README.md](README.md#local-product-stack-docker-compose). +Period leftover pairs (ADR 0017 / 0018) are computed in +`lineageweave/leftover_pairs.py` from the residual after a real +GRM/GPCM score, never invented. Missing cells stay out of the +Gabriel factorization. Closest and farthest post–criterion pairs +persist to `report_leftover_pair` and sit above the member list so +a click opens that post. + `frontend/` has its own toolchain (Node pinned via `frontend/mise.toml`, pnpm via Corepack -- do not add a second Node package manager or a floating Node version): diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 617b8b95..5166e1e7 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -504,7 +504,9 @@ so PU/team/project thetas stay on one metric. Later periods EAP-score on those same fixed parameters (Kim, 2006 FIPC). After scoring, `information_polytomous` ranks the shared-bank items by Fisher information at the group's mean θ (Lord, 1980 max-info CAT). Rankings -persist to `report_item_information`. Results persist to +persist to `report_item_information`. After those IRT main effects, +residual SVD leftover pairs (Jeon et al., 2021; ADR 0017) persist to +`report_leftover_pair`. Results persist to `report_period_score` / `report_member_score`. `GET /api/reports/{grouping}` lists the trend; `GET /api/reports/{grouping}/{period}` is ABAC-filtered; @@ -515,7 +517,8 @@ post) that already have constructed IRT cells into the same shared bank as the dummy high/low band rows, so comparison-strip click through opens those DAG posts. Report members include the earliest open ticket title, status lookup label, and due date when one exists. The home page renders -the actual mean θ, the FIPC delta, the CAT-selected item, and the +the actual mean θ, the FIPC delta, the CAT-selected item, leftover +closest/farthest pairs above the member list, and the PU / corp / thread comparison -- never a placeholder. TEPP is unchanged. ## Phase 6b: Knowledge Graph as a real Ontology + Semantic Layer diff --git a/CHANGELOG.d/0.71.2-leftover-pairs.md b/CHANGELOG.d/0.71.2-leftover-pairs.md new file mode 100644 index 00000000..0c6b1e1f --- /dev/null +++ b/CHANGELOG.d/0.71.2-leftover-pairs.md @@ -0,0 +1,9 @@ +# 0.71.2 — Leftover post–criterion pairs + +## Added + +- Persist closest and farthest leftover pairs from the residual + interaction map after GRM/GPCM scoring (ADR 0017). +- After `make seed`, period reports show the closest and farthest + leftover pairs above the member list; clicking a pair opens that post + (ADR 0018). A leftover pair for a hidden post is omitted. diff --git a/CHANGELOG.md b/CHANGELOG.md index 0096828a..e7ffaa1f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,16 @@ All notable changes to this project are documented here. Format follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/); versioning follows [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [0.71.2] - 2026-08-17 + +### Added + +- Period reports now persist closest and farthest leftover + post–criterion pairs after the IRT main effects (Jeon leftover + map). After `make seed`, leftover pairs sit above the member + list; clicking a pair opens that post. A leftover pair for a + hidden post is omitted the same way a hidden member is. + ## [0.71.0] - 2026-08-14 ### Added diff --git a/backend/app/main.py b/backend/app/main.py index c69609a9..91ad406b 100644 --- a/backend/app/main.py +++ b/backend/app/main.py @@ -758,7 +758,14 @@ async def read_period_reports( members = [member for member in report["members"] if _can_see_post(account, member)] if not members: continue - visible.append({**report, "members": members, "post_count": len(members)}) + leftover_pairs = [ + pair + for pair in report.get("leftover_pairs", []) + if _can_see_post(account, pair) + ] + visible.append( + {**report, "members": members, "leftover_pairs": leftover_pairs, "post_count": len(members)} + ) return {"grouping_kind": grouping_kind, "period_code": period_code, "reports": visible} diff --git a/backend/app/report_ingestion.py b/backend/app/report_ingestion.py index 07c0bf56..eff621d0 100644 --- a/backend/app/report_ingestion.py +++ b/backend/app/report_ingestion.py @@ -256,7 +256,7 @@ async def persist_period_report( period_code: str, report: PeriodReport, ) -> None: - """Replace the stored report, member scores, and item bank.""" + """Replace the stored report, member scores, leftover pairs, and item bank.""" await conn.execute( """ delete from report_period_score @@ -343,6 +343,24 @@ async def persist_period_report( item.rank, item.information, ) + for pair in report.leftover_pairs: + await conn.execute( + """ + insert into report_leftover_pair ( + grouping_kind, grouping_key, period_code, rubric_version, + pair_kind, post_id, criterion_code, leftover_distance, leftover_residual + ) values ($1,$2,$3,$4,$5,$6,$7,$8,$9) + """, + grouping_kind, + grouping_key, + period_code, + RUBRIC_VERSION, + pair.pair_kind, + pair.post_id, + pair.criterion_code, + pair.leftover_distance, + pair.leftover_residual, + ) def _groups_from_rows( @@ -478,6 +496,22 @@ async def fetch_period_reports( period_code, RUBRIC_VERSION, ) + leftover = await conn.fetch( + """ + select lp.grouping_key, lp.pair_kind, lp.post_id, lp.criterion_code, + lp.leftover_distance, lp.leftover_residual, p.post_title, + p.visibility_code, p.corporate_entity_id + from report_leftover_pair lp + join source_post p on p.post_id = lp.post_id + where lp.grouping_kind = $1 and lp.period_code = $2 and lp.rubric_version = $3 + order by lp.grouping_key, + case lp.pair_kind when 'closest' then 0 else 1 end, + p.post_title + """, + grouping_kind, + period_code, + RUBRIC_VERSION, + ) status_labels = await labels_for_codes( conn, [row["ticket_status_code"] for row in members if row["ticket_status_code"]], @@ -488,6 +522,9 @@ async def fetch_period_reports( selected_by_group: dict[str, list[asyncpg.Record]] = defaultdict(list) for row in selected: selected_by_group[row["grouping_key"]].append(row) + leftover_by_group: dict[str, list[asyncpg.Record]] = defaultdict(list) + for row in leftover: + leftover_by_group[row["grouping_key"]].append(row) payload: list[dict[str, Any]] = [] for header in headers: payload.append( @@ -545,6 +582,19 @@ async def fetch_period_reports( } for row in selected_by_group.get(header["grouping_key"], []) ], + "leftover_pairs": [ + { + "pair_kind": str(row["pair_kind"]), + "post_id": str(row["post_id"]), + "post_title": row["post_title"], + "criterion_code": str(row["criterion_code"]), + "leftover_distance": float(row["leftover_distance"]), + "leftover_residual": float(row["leftover_residual"]), + "visibility_code": row["visibility_code"], + "corporate_entity_id": str(row["corporate_entity_id"]), + } + for row in leftover_by_group.get(header["grouping_key"], []) + ], } ) return payload diff --git a/backend/tests/test_api.py b/backend/tests/test_api.py index 74ab4470..1db483ee 100644 --- a/backend/tests/test_api.py +++ b/backend/tests/test_api.py @@ -2365,6 +2365,10 @@ def test_seed_period_report_surfaces_on_get_reports(client, demo_analyst_token, assert high_report["link_method"] == "fipc" assert high_report["selected_model"] in {"grm", "gpcm"} assert high_report["delta_mean_theta"] is None + leftover_kinds = {pair["pair_kind"] for pair in high_report.get("leftover_pairs", [])} + assert leftover_kinds <= {"closest", "farthest"} + assert all(pair["post_title"] for pair in high_report.get("leftover_pairs", [])) + assert all(pair["leftover_distance"] >= 0 for pair in high_report.get("leftover_pairs", [])) week3 = client.get( "/api/reports/process_unit/2026-W03", diff --git a/docker/postgres-init/Dockerfile b/docker/postgres-init/Dockerfile index 0c6323a9..51ac998c 100644 --- a/docker/postgres-init/Dockerfile +++ b/docker/postgres-init/Dockerfile @@ -18,6 +18,7 @@ COPY migrations/0008_post_summary_result.sql /docker-entrypoint-initdb.d/09-post COPY migrations/0009_shared_metric_bank.sql /docker-entrypoint-initdb.d/10-shared-metric-bank.sql COPY migrations/0010_report_item_information.sql /docker-entrypoint-initdb.d/11-report-item-information.sql COPY migrations/0011_post_chat_result.sql /docker-entrypoint-initdb.d/12-post-chat-result.sql +COPY migrations/0012_report_leftover_pair.sql /docker-entrypoint-initdb.d/13-report-leftover-pair.sql # Official image already drops to this account at runtime; declare it so # the Dockerfile itself satisfies DS-0002 (explicit non-root USER). USER postgres diff --git a/docs/adr/0003-fast-mlsirm-report-integration.md b/docs/adr/0003-fast-mlsirm-report-integration.md index 031bb0ea..bdf234b6 100644 --- a/docs/adr/0003-fast-mlsirm-report-integration.md +++ b/docs/adr/0003-fast-mlsirm-report-integration.md @@ -100,6 +100,10 @@ than one large PR: `information_polytomous` (Lord, 1980 max-info). Persist the ranking (`report_item_information`) and show the rank-1 item on the Period reports panel. Do not reimplement an information function here. +7. **Leftover-pair slice** (shipped in 0.71.2; ADR 0017 / 0018): after + IRT main effects, persist closest and farthest post–criterion pairs + from the residual leftover map. Do not fork LSIRM; do not invent a + leftover-pair API inside `fast-mlsirm` in this slice. **TEPP boundary.** [ARCHITECTURE.md](../../ARCHITECTURE.md) already assigns calibrated temporal/event measurement to diff --git a/docs/adr/0017-persist-lsirm-leftover-pairs.md b/docs/adr/0017-persist-lsirm-leftover-pairs.md new file mode 100644 index 00000000..3ceb7157 --- /dev/null +++ b/docs/adr/0017-persist-lsirm-leftover-pairs.md @@ -0,0 +1,63 @@ +# ADR 0017 — Persist LSIRM leftover post–criterion pairs + +**Decision status:** Accepted +**Date:** 2026-08-17 + +## Context + +Period reports already persist IRT main effects: EAP θ per post, a +shared GRM/GPCM item bank, FIPC linking, and Lord (1980) max-info CAT +ranks. After those main effects, Jeon et al. (2021, eq. 3) leave a +leftover interaction `−γ‖ξ_p − ζ_i‖` on the person–item map. Closest +pairs are the smallest Euclidean leftover-map distances; farthest +pairs are the largest. + +`fast-mlsirm` implements the leftover term inside MLSRM fitting but +exposes no leftover-pair API. LineageWeave must not fork LSIRM or +invent a second IRT fit. Buyers still need a durable, clickable +answer to “which post–criterion pair is unexpectedly aligned, and +which pair is unexpectedly opposed?” + +## Decision + +After a real GRM/GPCM score, compute the residual matrix +`R = Y − E[Y|θ, item]` from the already-fitted category +probabilities. A Gabriel (1971) biplot of the **complete-case** +submatrix of `R` supplies person positions `ξ` and item positions +`ζ`. Missing response cells are excluded from the factorization; +they are never filled with zero. Persist exactly one `closest` +and one `farthest` observed cell per period report in +`report_leftover_pair` (3NF, two-or-more-word `snake_case`). + +The biplot lives in `lineageweave/leftover_pairs.py` so leftover +tests do not import `period_report` or `fast_mlsirm`. + +Cascade the rows with `report_period_score`. A leftover post must +also be a `report_member_score` row, and the leftover criterion +must be a `report_item_information` item on that same report. +Do not store a second theta. Do not invent leftover numbers when +the IRT matrix is unusable. A rank-0 residual still emits a +stable pair so `make seed` is not empty; the stored distance is +then zero, not a fabricated interaction. + +The UI contract is ADR 0018. + +## Consequences + +Rebuild and seed write leftover pairs in the same transaction as +member scores. `GET /api/reports/{grouping}/{period}` returns +`leftover_pairs` with the post title so the buyer can open that post. +Hidden posts stay hidden: leftover pairs join `source_post` and use +the same ABAC gate as members. Migration `0012_report_leftover_pair.sql` +upgrades volumes that already applied `0001`. + +## References + +Gabriel, K. R. (1971). The biplot graphic display of matrices with +application to principal component analysis. *Biometrika, 58*(3), +453–467. https://doi.org/10.1093/biomet/58.3.453 + +Jeon, M., Jin, I. H., Schweinberger, M., & Baugh, S. (2021). Mapping +unobserved item–respondent interactions: A latent space item response +model with interaction map. *Psychometrika, 86*(2), 378–403. +https://doi.org/10.1007/s11336-021-09762-5 diff --git a/docs/adr/0018-leftover-pair-report-ui.md b/docs/adr/0018-leftover-pair-report-ui.md new file mode 100644 index 00000000..2fcbd896 --- /dev/null +++ b/docs/adr/0018-leftover-pair-report-ui.md @@ -0,0 +1,42 @@ +# ADR 0018 — Leftover pairs sit above the report member list + +**Decision status:** Accepted +**Date:** 2026-08-17 + +## Context + +ADR 0017 persists closest and farthest leftover post–criterion pairs. +Those pairs only help if a buyer can see them on the Period reports +panel and open the named post without hunting through the member list. + +The member list is already the click-through to Event Lineage, Keyman, +and evaluation. Leftover pairs must not replace that list or invent a +second navigation surface. + +## Decision + +On each period-report group, render leftover pairs **above** the +member list. Each pair is a button: closest or farthest label, post +title, criterion short label, leftover-map distance, and the next +action (“Open this post to read the criterion it sat closest to / +farthest from after main effects.”). Clicking the button opens that +post with the same handler as a member row. + +After `make seed`, closest and farthest leftover pairs sit above the +member list. Click a pair to open that post. + +Missing leftover rows render nothing — never a placeholder pair. +A hidden post never appears as a leftover pair. + +## Consequences + +The authorized report payload carries `leftover_pairs` next to +`members` and `selected_items`. Screen-reader names are +`Open leftover closest pair: {title}` and +`Open leftover farthest pair: {title}` so the control announces the +next action, not only the distance. + +## Related + +Depends on [ADR 0017](0017-persist-lsirm-leftover-pairs.md) and +[ADR 0003](0003-fast-mlsirm-report-integration.md). diff --git a/frontend/package.json b/frontend/package.json index 9c84795d..49535e11 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -1,7 +1,7 @@ { "name": "frontend", "private": true, - "version": "0.71.0", + "version": "0.71.2", "type": "module", "scripts": { "dev": "vite", diff --git a/frontend/src/App.test.tsx b/frontend/src/App.test.tsx index 415e1419..bc84dd3f 100644 --- a/frontend/src/App.test.tsx +++ b/frontend/src/App.test.tsx @@ -292,6 +292,24 @@ describe("App, authenticated", () => { { item_code: "general_sentiment_positive", rank: 2, information: 0.4 }, { item_code: "general_sentiment_negative", rank: 3, information: 0.2 }, ], + leftover_pairs: [ + { + pair_kind: "closest", + post_id: "post-1", + post_title: "Public post", + criterion_code: "sales_lead_specificity", + leftover_distance: 0.12, + leftover_residual: 0.4, + }, + { + pair_kind: "farthest", + post_id: "post-2", + post_title: "Specification revision requested", + criterion_code: "general_sentiment_negative", + leftover_distance: 1.84, + leftover_residual: -1.1, + }, + ], members: [ { post_id: "post-1", @@ -1265,6 +1283,23 @@ describe("App, authenticated", () => { ); expect(screen.getByRole("button", { name: /open report post: public post/i })).toHaveTextContent("Open"); expect(screen.getByRole("button", { name: /open report post: public post/i })).toHaveTextContent("due 2026-01-12"); + expect(screen.getByLabelText("Leftover pairs")).toBeInTheDocument(); + const closestPair = screen.getByRole("button", { name: /open leftover closest pair: public post/i }); + const farthestPair = screen.getByRole("button", { + name: /open leftover farthest pair: specification revision requested/i, + }); + expect(closestPair).toHaveTextContent("Closest leftover: Public post · sales-lead"); + expect(closestPair).toHaveTextContent( + "Open this post to read the criterion it sat closest to after main effects.", + ); + expect(closestPair).toHaveTextContent("d 0.12"); + expect(farthestPair).toHaveTextContent("Farthest leftover: Specification revision requested · negative"); + expect(farthestPair).toHaveTextContent( + "Open this post to read the criterion it sat farthest from after main effects.", + ); + expect(farthestPair).toHaveTextContent("d 1.84"); + const memberButton = screen.getByRole("button", { name: /open report post: public post/i }); + expect(closestPair.compareDocumentPosition(memberButton) & Node.DOCUMENT_POSITION_FOLLOWING).toBeTruthy(); expect( screen.getByRole("button", { name: /open report post: specification revision requested/i }), ).toHaveTextContent("Send Westfield Power the revised specification"); @@ -1308,6 +1343,16 @@ describe("App, authenticated", () => { expect(periodInput).toHaveValue("2026-W03"); }); + it("opens a leftover pair post from the report panel", async () => { + stubBackend(); + render(); + + await userEvent.click( + await screen.findByRole("button", { name: /open leftover closest pair: public post/i }), + ); + await waitFor(() => expect(screen.getByText("The full body text.")).toBeInTheDocument()); + }); + it("opens Event Lineage, Keyman, and evaluation from a report member click", async () => { stubBackend(); render(); diff --git a/frontend/src/App.tsx b/frontend/src/App.tsx index 1e39a925..ff11eb16 100644 --- a/frontend/src/App.tsx +++ b/frontend/src/App.tsx @@ -1527,6 +1527,37 @@ function ReportsPanel({ {report.selected_items[0].information.toFixed(2)} )} + {report.leftover_pairs && report.leftover_pairs.length > 0 && ( + + )} {report.members.length > 0 && (