diff --git a/.github/workflows/prov-o-contract.yml b/.github/workflows/prov-o-contract.yml new file mode 100644 index 00000000..eae45315 --- /dev/null +++ b/.github/workflows/prov-o-contract.yml @@ -0,0 +1,94 @@ +name: PROV-O contract + +on: + pull_request: + branches: [main] + paths: + - "lineageweave/prov_o.py" + - "tests/test_prov_o.py" + - "tests/test_prov_o_schema.py" + - "migrations/0017_prov_o_standard_relations.sql" + - "docs/ontology/prov-o-support-profile.ttl" + - "pyproject.toml" + - "uv.lock" + - ".github/workflows/prov-o-contract.yml" + push: + branches: [main] + paths: + - "lineageweave/prov_o.py" + - "tests/test_prov_o.py" + - "tests/test_prov_o_schema.py" + - "migrations/0017_prov_o_standard_relations.sql" + - "docs/ontology/prov-o-support-profile.ttl" + - "pyproject.toml" + - "uv.lock" + - ".github/workflows/prov-o-contract.yml" + +permissions: + contents: read + +concurrency: + group: prov-o-contract-${{ github.ref }} + cancel-in-progress: true + +jobs: + standards-contract: + name: Registry, inference, coverage, PostgreSQL + runs-on: ubuntu-latest + services: + postgres: + image: postgres:16-alpine@sha256:57c72fd2a128e416c7fcc499958864df5301e940bca0a56f58fddf30ffc07777 + env: + POSTGRES_PASSWORD: postgres + ports: + - 5432:5432 + options: >- + --health-cmd "pg_isready -U postgres" + --health-interval 10s + --health-timeout 5s + --health-retries 5 + env: + LINEAGEWEAVE_TEST_POSTGRES_ADMIN_DSN: postgresql://postgres:postgres@localhost:5432/postgres + steps: + - name: Checkout repository + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # actions/checkout@v7 + with: + persist-credentials: false + + - name: Set up Python + uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # actions/setup-python@v6 + with: + python-version: "3.12" + + - name: Set up locked Python dependency manager + uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0 + with: + version: "0.11.28" + enable-cache: false + + - name: Install the committed universal lock + run: uv sync --frozen --extra dev + + - name: Verify complete relation behavior and 100 percent coverage + run: | + uv run --frozen python -m coverage run --branch --source=lineageweave.prov_o \ + -m pytest -q tests/test_prov_o.py + uv run --frozen python -m coverage report --fail-under=100 lineageweave/prov_o.py + + - name: Require a reachable PostgreSQL service + run: | + set -euo pipefail + for _ in $(seq 1 30); do + if pg_isready -h localhost -p 5432 -U postgres; then + exit 0 + fi + sleep 2 + done + echo "PostgreSQL service is unreachable; the schema contract would silently skip." >&2 + exit 1 + + - name: Verify normalized PostgreSQL contracts + run: uv run --frozen python -m pytest -q tests/test_prov_o_schema.py + + - name: Compile owned Python surface + run: uv run --frozen python -m compileall -q lineageweave/prov_o.py tests/test_prov_o.py tests/test_prov_o_schema.py diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index cb4c7f95..1cad1f17 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -4,7 +4,6 @@ on: push: branches: [main] pull_request: - branches: [main] permissions: contents: read @@ -17,6 +16,20 @@ jobs: pytest: name: Full test suite runs-on: ubuntu-latest + services: + postgres: + image: postgres:16-alpine@sha256:57c72fd2a128e416c7fcc499958864df5301e940bca0a56f58fddf30ffc07777 + env: + POSTGRES_PASSWORD: postgres + ports: + - 5432:5432 + options: >- + --health-cmd "pg_isready -U postgres" + --health-interval 10s + --health-timeout 5s + --health-retries 5 + env: + LINEAGEWEAVE_TEST_POSTGRES_ADMIN_DSN: postgresql://postgres:postgres@localhost:5432/postgres steps: - name: Checkout repository uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # actions/checkout@v7 @@ -28,19 +41,22 @@ jobs: with: python-version: "3.12" - - name: Install pinned Rust toolchain + - name: Set up locked dependency manager + uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0 + with: + version: "0.11.28" + enable-cache: false + + - name: Select pinned Rust toolchain run: | - # Same pin as backend/Dockerfile: fast-mlsirm's PyO3/maturin - # core has no wheel, so pip install -e ".[backend]" compiles it. - curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | \ - sh -s -- -y --profile minimal --default-toolchain 1.97.1 - echo "$HOME/.cargo/bin" >> "$GITHUB_PATH" + rustup toolchain install 1.97.1 --profile minimal + rustup default 1.97.1 - - name: Install package and test dependencies - run: python -m pip install -e ".[dev,backend]" + - name: Install the committed universal lock + run: uv sync --frozen --extra dev --extra backend - - name: Run full test suite - run: python -m pytest -q + - name: Run full test suite against PostgreSQL + run: uv run --frozen python -m pytest -q frontend: name: Frontend lint, test, build @@ -54,7 +70,6 @@ jobs: - name: Set up Node uses: actions/setup-node@a0853c24544627f65ddf259abe73b1d18a591444 # actions/setup-node@v5 with: - # Matches frontend/mise.toml's pin (setup-node doesn't parse mise.toml). node-version: "24" - name: Enable Corepack @@ -75,3 +90,7 @@ jobs: - name: Build working-directory: frontend run: pnpm run build + + - name: Build Storybook + working-directory: frontend + run: pnpm run build-storybook diff --git a/AGENTS.md b/AGENTS.md index c790995c..e26c146d 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -57,7 +57,14 @@ summary/chat, or invented commitment. A missing signal and a confidently-negative signal are different things. Keyman extraction, entity-relationship classification, post summary, in-popup chat, and commitment derivation go through contextual-orchestrator the same way -adjudication does -- never a raw LLM API. +adjudication does -- never a raw LLM API. Demo TEPP seed goes through +`tepp_client` the same way: a missing transport or an unpublished +envelope is Failed (`tepp_not_available` / `tepp_result_not_persisted`). +A published accepted acknowledgement is Failed / +`tepp_completed_result_unsupported` and may be shown as aggregate +transport evidence (ADR 0035). Never stamp Succeeded from that ack or +from a LineageWeave-local completed envelope, and never invent a theta +or a local psychometric substitute. ## Tests @@ -75,7 +82,7 @@ in the same spirit) -- never against real data, per the hard rule above. against a live local stack (`make up`) and self-skip without one -- see [README.md](README.md#local-product-stack-docker-compose). -Period leftover pairs (ADR 0017 / 0018) are computed in +Period leftover pairs (ADR 0028 / 0029) are computed in `lineageweave/leftover_pairs.py` from the residual after a real GRM/GPCM score, never invented. Missing cells stay out of the Gabriel factorization. Closest and farthest post–criterion pairs @@ -89,11 +96,60 @@ floating Node version): ```bash cd frontend && pnpm install pnpm run lint && pnpm run test && pnpm run build +# Storybook inventory (ADR 0020 tokens): pnpm run build-storybook ``` +A run-bearing analysis-run registry empties only after an unrevoked +`analysis_run_retention_grant` and `GRANT analysis_run_retention_admin` +(ADR 0020 / v0.87.0). The documented phrase is not a secret. The same +call empties reconstruction children when those tables exist +(ADR 0032 / v2.10.4). Do not expose purge on a public HTTP route. + +`POST /api/analysis-runs` records Pending lineage only (ADR 0017 / +v2.7.1). TEPP and period-report kinds 422 before any snapshot write. +`POST /api/analysis-runs/{id}/start` reconstructs a Pending lineage +cutoff bag through `reconstruct()` / `lineage_edge_specs` (ADR 0021 / +v0.88.0). Do not invent a theta. +Opening a cutoff-rewritten title shows **Body this run knew** from +`source_post_revision` beside the live rewrite (ADR 0025 / v2.1.0). +Do not invent the earlier sentence when no revision covers the cutoff. +After `make seed`, the January 12 Demo Corp lineage and TEPP runs list +Demo public post and do not list Late Demo public post (2026-01-13). +The live post list still shows Late Demo (ADR 0016). + +A corporate-entity similarity result has three outcomes: unique, miss, +or tie (ADR 0026). A tie is not a miss. Keep the organization name +unbound and do not create an `AUTO-` catalog row, even when live name +resolution, hierarchy inference, and verification are available. Keyman +must test the raw organization name before any abbreviation rewrite so a +rewrite cannot turn an existing tie into an apparent creation miss. + +R&R chips read the catalog id stored on `post_summary_role` +(ADR 0019 / 0027), including `cataloged_person_id`. Do not rejoin +`corporate_entity` or `cataloged_person` by display name. Historical +backfill leaves a role unbound when two same-named mentions already +exist on the post. + +A listed analysis-run that then 404s must stay generic: do not name the thread or the cutoff, +and do not say the run is not visible (ADR 0014 / ADR 0018). After +that 404, re-read the authorized list so the stale row does not stay +clickable. Announce the next action with `role="alert"` without +moving focus. Remaining visible runs stay clickable. Request remains +the named reconstruction control. + ## CI gates `.github/workflows/tests.yml` runs the full suite on every PR to `main`. Do not weaken, skip, or `continue-on-error` a failing check -- fix the underlying cause or, for a genuine false positive in a third-party scanner, add a narrow, documented suppression referencing the specific finding. +## W3C PROV-O boundary + +- Add standard provenance through `lineageweave.prov_o` and the + normalized `provenance_*` schema, never by inventing another + `edge_type` alias for a W3C property. +- Qualified relations retain their Influence resource and imply the + corresponding unqualified relation. +- Appendix B inverse names normalize to the preferred W3C direction; + do not proliferate private inverse vocabulary. +- Keep `knowledge_graph_edge` an explicit navigation projection. diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index f8a83ceb..5bdebfab 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -64,7 +64,7 @@ flowchart LR | `chunking.py` | Splits a document into meaning-identifiable units (paragraph, sentence, DOM, conversation-turn) plus embedded-image extraction, in document order | | `embedding_client.py` | Pluggable text-embedding channel (`Null` default, `OpenAiCompatible` real impl) + `chunked_max_similarity` | | `adjudication_client.py` | Pluggable LLM-judgment channel (`Null` default, `ContextualOrchestrator` real impl) | -| `image_content.py` | Pluggable vision channel: OCR + object recognition/tagging for embedded images (`Null` default, `OpenAiCompatibleVisionClient` real impl) | +| `image_content.py` | Pluggable vision channel: OCR + object recognition/tagging for embedded images (`Null` default, `OpenAiCompatibleVisionClient` real impl). The product popup (`frontend/src/PostBody.tsx`) and `extract_base64_images` parse with the same HTML rules as `chunk_by_dom` (ADR 0031) so invoice-like `alt` values still show the picture; GET does not call the vision client. | | `tepp_client.py` | TEPP's published `AnalysisRunRequest` wire contract, pluggable transport | | `rankweave_client.py` | Fail-closed RankWeave ranking port (`weighted_reciprocal_rank_fuse` in-process; never invent a fused score or a theta) | | `reconstruct.py` | The pipeline: group → candidate window → score → fuse → thread | @@ -122,7 +122,7 @@ flowchart LR `rankweave_client.py`'s default transport raises `RankWeaveNotAvailable`. `GET /api/rankings` then returns `rankweave_not_available` and an empty ranking list. Hidden posts - are omitted from every channel. See ADR 0024. + are omitted from every channel. See ADR 0030. ## Standards and citations @@ -142,6 +142,8 @@ identities and content) and `migrations/0001_initial_schema.sql` for the `role_permission` / `account_role_assignment`, `abac_policy`, `post` / `post_counterparty_entity`, `person` / `person_affiliation` / `post_person_mention`, `knowledge_graph_edge`, `issue_ticket`, + +Keyman and R&R person mentions are separate replaceable projections (`post_person_mention` and `post_summary_person_mention`). The read-only `combined_post_person_mention` view feeds lineage discovery. Materialized KG edges are unique and carry normalized `knowledge_graph_edge_evidence`; only evidence from an ABAC-visible post participates in RWR. `post_lineage_edge`). Real-database tests: `tests/test_schema.py` (skipped without a reachable PostgreSQL server, same pattern as the real-provider LLM tests). @@ -260,11 +262,14 @@ pattern and then hide the action button so it cannot 503 again. `find_linked_post_ids` first expands to every post sharing a mentioned person before calling `backend/app/knowledge_graph.py::load_visible_subgraph` -- that function -only loads edges among an *already-known* post set (its other caller, -`related_for_person`, pre-resolves the full set itself), it does not -discover new posts on its own; a real bug from calling it with only the -single starting post was caught while building this and is now -regression-tested (`test_post_chat_cites_a_post_linked_only_via_a_shared_keyman`). +only loads edges among an *already-known* post set (its other callers, +`related_for_person` / `related_for_entity` / `related_for_team`, +pre-resolve the full set themselves), it does not discover new posts on +its own; a real bug from calling it with only the single starting post +was caught while building this and is now regression-tested +(`test_post_chat_cites_a_post_linked_only_via_a_shared_keyman`). +Person, team, and organization mention channels load independently +(ADR 0018): a team-only or organization-only post still walks. ### Frontend (`frontend/`) @@ -280,8 +285,9 @@ summary/key-events/R&R, VOC evidence excerpts, an Event Lineage panel affiliate tree (resolved ancestors plus unresolved org roots), Keyman + counterparty panels (a Keyman click loads RWR related nodes; a related corporate-entity node, a resolved Keyman affiliation, -or a classified name that resolves to a cataloged org continues -the same walk via `GET /api/corporate-entities/{id}/related`; +a classified name that resolves to a cataloged org, or an R&R team +continues the same walk via `GET /api/corporate-entities/{id}/related` +or `GET /api/teams/{id}/related`; `post_admin` can extract), and an in-popup chat whose cited sources open a sliding evidence panel (`EvidencePanel`, CSS @@ -326,7 +332,9 @@ is the same never-guess-a-parent rule `corporate_hierarchy_resolution` already applies. Entity levels and Keyman sides are labeled from `common_lookup_value` (`Our side`, `Plant`, `Company`) so the popup never shows raw `our_side` / `plant` -codes when a label exists. +codes when a label exists. Related-node person chips use the same +side lookup label (for example, `Our side` or `Counterparty`) rather +than exposing the generic PROV-O `Person` class as business context. `GET /api/posts` and `GET /api/posts/{post_id}` include `voc_type_label` / `visibility_label` from `common_lookup_value` so @@ -456,6 +464,85 @@ lists the same dated tickets the period-report members already show. Re-seed is idempotent. The empty-state copy is only for accounts that truly have no dated open tickets. +## Phase 6-M2: authorized analysis-run evidence (read projection) + +Issue #79's first buyer-visible Milestone 2 slice is a source-redacting +read of the #89 registry. `GET /api/analysis-runs` and +`GET /api/analysis-runs/{id}` require `post_read` and apply the scope +in SQL: the requester always sees their own run; a corporate-entity or +process-unit scope is visible only to affiliated accounts; a +thread-group scope is visible only when the account can already see a +post in that group; `all_visible` is requester-only. Hidden runs 404. Detail also lists ABAC-visible post titles in the +run's scope whose `created_at` is at or before `knowledge_cutoff` +(ADR 0016) so a buyer can open a post the run was allowed to know +without seeing later live rows or hidden bodies. Detail also returns +revision and configuration digest prefixes. +`POST /api/analysis-runs` records a Pending lineage run on a new +authorized cutoff capture (ADR 0017): snapshot, counts, frozen +membership, run, scope, and the first status in one transaction. TEPP +and period-report kinds are 422. Request a lineage reconstruction from +the home list after affiliated corps load (choose a corp if you walk +more than one), then open the Pending row to confirm the cutoff corpus. +`POST /api/analysis-runs/{id}/start` then commits Running plus a +durable outbox row, wakes Valkey, and delivers ThreadWeave on that +frozen bag (ADR 0021 / ADR 0023) or submits TEPP through +`tepp_client` (ADR 0022). It does not invent a TEPP score. +Request a lineage reconstruction from the home list, open the Pending +row, then start reconstruction. A Pending TEPP row starts a +measurement; a missing transport stays Failed / +`tepp_not_available`. Hover the Result digest +prefix, then confirm the designed A-100 fork before treating the live +Event Lineage panel as that run's tree. +`make seed` also records a TEPP measurement run through +`tepp_client` on that same snapshot; the default transport is +unavailable, so that run is Failed rather than a fabricated score. +A second Demo Corp TEPP run uses an in-process published accepted +acknowledgement and stays Failed / `tepp_completed_result_unsupported` +with aggregate transport evidence (ADR 0035). +The home list is clickable: `GET /api/analysis-runs/{id}` fills a +labeled detail (cutoff, requested date, 12-character digest prefixes +with full digests on hover, counts, status history) +without exposing a DSN or raw record. Opening a cutoff title still +shows the live body and names both clocks when the title was +rewritten after the run. A marked title also shows the body that +run knew (`GET /api/posts/{id}?as_of=`) so the operator can compare +two texts, not two clocks. Status history is detail-only +and uses lookup labels plus occurrence times; a failure event keeps +its machine `failure_code` rather than an invented caption. Failed +TEPP list rows add a next-action line (open the run, then connect the +measurement service) so `tepp_not_available` is not mistaken for a +calibrated negative result. A failed lineage row tells the operator +to retry reconstruction, not to connect TEPP. A failed period-report +row tells the operator to rebuild the report. A pending TEPP row +does not claim a calibrated measurement and does not say +reconstruction. The list button accessible name includes the +next-action sentence; detail repeats it (ADR 0014). A pending lineage row +says reconstruction has not started yet; open it and start +reconstruction. The +payload is lookup labels plus non-negative aggregate counts -- never +source SQL, a DSN, a raw record, or a provider body. After `make seed`, +Demo Analyst and Demo Admin see "Lineage reconstruction · Succeeded · +Demo Corp" with "3 documents" and Pending / Running / Succeeded times, +the designed A-100 fork as clickable reconstructed edges, Claimed +then Delivered outbox times, "TEPP measurement · Failed · Demo +Corp" whose detail history ends in Failed / `tepp_not_available`, +and a second "TEPP measurement · Failed · Demo Corp" whose detail +shows Measurement evidence for the published accepted acknowledgement +(ADR 0035). +Seed also records "Period report · Succeeded · Demo Corp" on that +same snapshot after the calibrated report tables are written +(ADR 0024). Open that row to confirm the cutoff posts; mean θ stays +on the period-report panel. Start stays 422. +A run-bearing registry is emptied only after an unrevoked +`analysis_run_retention_grant` and `GRANT analysis_run_retention_admin`, +then `purge_analysis_run_registry('approved-retention-purge')` +(ADR 0020); a raw `DELETE` and a runtime role that only knows the +public phrase stay rejected. When start reconstruction has persisted +run-scoped edges, that same call empties those children instead of +stopping on an immutable-trigger or foreign-key error (ADR 0032). +Repeated chip and close controls use +`frontend/src/styles/tokens.css` and the Storybook inventory. + ## Phase 6a: fast-mlsirm dependency + Rust toolchain (infra only) First of three staged slices toward the brief's weekly/monthly @@ -511,7 +598,7 @@ on those same fixed parameters (Kim, 2006 FIPC). After scoring, `information_polytomous` ranks the shared-bank items by Fisher information at the group's mean θ (Lord, 1980 max-info CAT). Rankings persist to `report_item_information`. After those IRT main effects, -residual SVD leftover pairs (Jeon et al., 2021; ADR 0017) persist to +residual SVD leftover pairs (Jeon et al., 2021; ADR 0028) persist to `report_leftover_pair`. Results persist to `report_period_score` / `report_member_score`. `GET /api/reports/{grouping}` lists the trend; @@ -691,3 +778,205 @@ checks a well-known public foundation name ("Mozilla Foundation") against a deliberately fabricated one in the same request, asserting the former comes back `verify_corroborated` with a real evidence URL and the latter `verify_uncorroborated` with none. + +## Phase 7: R&R's named actor is a PROV-O Agent, not always a person + +`post_summary.py`'s R&R extraction forced every named actor into a +person slot, but business correspondence routinely names an +organization acting in its own name ("당사" [our company], +"Demo Corp"), not an individual. See +[ADR 0006](docs/adr/0006-role-responsibility-agent-ontology.md). + +Grounded in W3C PROV-O (Lebo, Sahoo, & McGuinness, 2013): +`RoleResponsibility` (renamed field `actor_name`, was `person_name` -- +the field can hold an organization's name now, so "person" in the name +would be wrong) gains `actor_type_code` (`prov_person` / +`prov_organization`, defaulting to person when the model omits it) and +`affiliated_organization_name` (an LLM-inferred affiliation for a +person actor, since a bare name without an employer is hard to place). +The ontology gains `:RoleActorPerson rdfs:subClassOf prov:Person` and +`:RoleActorOrganization rdfs:subClassOf prov:Organization` -- genuine +subclasses of the real external PROV-O classes (imported via the +`prov:` namespace), kept distinct from the ontology's existing `:Person` +(node_type's cataloged Keyman with a stable `person_id`) since an R&R +actor is a free-text name with no cataloged identity of its own. +`migrations/0012_role_responsibility_agent_type.sql` renames the +`post_summary_role` column via `RENAME COLUMN` (preserves existing +rows) rather than a drop/recreate. The popup's R&R list shows a +Person/Organization badge and the inferred affiliation; only a person +actor still links to the Keyman panel. + +## Phase 8: same-name Keymen are not silently merged; titles are captured + +Two different real people can share a name -- `keyman_extraction.py` +never captured a stated job title/position, so nothing distinguished +"Kim Cheolsu, sales manager" from an unrelated "Kim Cheolsu, purchasing +lead" beyond the bare name. `PersonMention` gains `job_title: str | +None`, and the extraction prompt now explicitly asks for one when the +text states it (never left out as a same-name disambiguation signal). + +Persistence, in two places for a reason: `person_affiliation.role_title` +(a schema column that already existed, previously never populated) for +a title tied to a specific organization, and a new +`cataloged_person.last_known_job_title` (`migrations/0013_person_job_title.sql`) +for a title stated without a named organization to attach it to (e.g. +"our legal counsel, Sam Okonkwo" -- `fixtures.ambiguous_keyman_post()`'s +own real example, which has zero affiliated organizations for Sam). +Both feed `_upsert_person`'s disambiguation check +(`backend/app/keyman_ingestion.py`): a same person_name+person_side_code +match is only reused when the new mention's stated title, if any, does +not conflict with a title already on file -- a genuine stated conflict +creates a fresh `cataloged_person` row instead of merging two different +people. A missing title on either side is not treated as a conflict +(titles legitimately change -- a promotion -- and most mentions state no +title at all), so this only splits on an actual stated disagreement, +verified by a real test that two posts naming the same name with +genuinely different stated titles produce two distinct person rows. + +## Phase 9: an R&R actor can be a team, meso-level between person and organization + +Real post text named "설계팀" (design team) -- neither a person nor the +company itself, but a sub-unit of one. See +[ADR 0007](docs/adr/0007-team-actor-type.md). `actor_type_code` gains a +third value, `prov_team`, grounded in the W3C Organization Ontology's +`org:OrganizationalUnit` (Reynolds, 2014) -- a different, complementary +W3C vocabulary from PROV-O (which models "who acted," not "how a +company is internally structured"). The prompt now offers three actor +types and requires `affiliated_organization_name` for a team actor too +(not just a person): a team's own name never answers "which company," +unlike an organization actor's. `migrations/0014_role_responsibility_team_actor_type.sql` +adds the lookup row -- purely additive, no schema change, since +`actor_type_code` already stores an arbitrary FK'd code. + +## Phase 10: an abbreviated organization name is resolved and search-verified, not left opaque + +Real post text names organizations by abbreviation ("AGP" for +"Aurora Grid Power") that character-similarity matching +(`corporate_hierarchy_resolution`) structurally cannot bridge -- an +initialism shares almost no substring with its expansion. See +[ADR 0008](docs/adr/0008-organization-abbreviation-resolution.md). + +New module `lineageweave/organization_name_resolution.py`: an LLM +proposes the full name from context (or declines with `UNKNOWN`), then +the *existing* `relation_verification` Searxng client cross-verifies +the specific raw/resolved pairing (no second web-search integration +built). Only a search-corroborated resolution is ever substituted in +for `resolve_corporate_entity` -- an unresolved or unverified name +still flows through unchanged. Cached in a new +`organization_name_resolution` table +(`migrations/0015_organization_name_resolution.sql`) keyed by the raw +name, so the same abbreviation across many posts is resolved once. +Grounded in SKOS `skos:altLabel`/`skos:prefLabel` (Miles & Bechhofer, +2009). Wired into `backend/app/keyman_ingestion.py`'s affiliation loop +and the offline synthetic-batch script's paced re-implementation of it +(the batch script's own copy was also missing `role_title` persistence +entirely -- fixed alongside this). + +Also fixed while running this against synthetic embedded-image fixtures: +`image_content.py`'s `_parse_description` required an exact single-pass +`TEXT:`/`CAPTION:`/`TAGS:` match, which was rejecting real vision +responses whose formatting was close but not exact (markdown-bolded +labels, reordered labels, a missing TAGS line) -- silently producing +the same "content unavailable" placeholder as a genuinely unconfigured +vision channel. Fields are now recovered independently per label line; +only a response with neither TEXT nor CAPTION is treated as unusable. + +## Phase 11: R&R team/organization actors get a shared cross-post identity + +Extraction runs per-post; a team's or organization's identity did not +survive across posts the way a Keyman's already did via +`cataloged_person`. See +[ADR 0009](docs/adr/0009-cross-post-actor-identity.md). New +`cataloged_team` catalog (`migrations/0016_cross_post_actor_identity.sql`, +identity key `(team_name, affiliated_organization_name)` -- a bare team +name like "설계팀" is not by itself identifying) plus two mention join +tables (`post_team_mention`, `post_organization_mention`); an +organization actor reuses the existing `corporate_entity` catalog, no +new table needed. `lineageweave/knowledge_graph.py`'s +`knowledge_graph_edges_for_post` extended with three new edge kinds +(`edge_mention_team`, `edge_team_affiliation`, `edge_mention_organization`); +`backend/app/post_summary_ingestion.py`'s `persist_post_summary` now +resolves each R&R actor's identity, stores that id on +`post_summary_role` (ADR 0019 — `entity_name` is not unique), and calls +the same `persist_edges_for_post` Keyman ingestion already uses. A person R&R +actor is opportunistically joined to an existing `cataloged_person` row +by name (never originated by R&R itself -- documented gap in the ADR: +`cataloged_person` needs `person_side_code`, which R&R's prompt does +not currently capture). ADR 0019 stores that resolved catalog id on +`post_summary_role` (`cataloged_team_id` / +`cataloged_corporate_entity_id` / `cataloged_person_id`, ADR 0019 / +0027) so a later read does not rejoin `corporate_entity` by +`entity_name`. Fetch returns the person foreign key as +`catalog_node_id` the same way. Historical backfill leaves a role +unbound when two same-named mentions already exist on the post. +Open a post whose R&R names an organization that shares a display name +with another catalog row: the chip keeps the id persist stored. Click +it to walk that organization, not the homonym. Click a person chip to +walk the stored person even when Keyman was not extracted on that post. + +## Phase 12: a real counterparty organization is auto-created, not left permanently unresolved + +`corporate_hierarchy_resolution`'s similarity matching only ever finds +an ALREADY-cataloged entity. Real Milestone 2 data confirmed the actual +gap: 0 of 4,154 person affiliations and 0 of 9,852 R&R organization +mentions ever resolved -- the standing "통합 고객사 계열 tree AI" +requirement was never actually populated. See +[ADR 0010](docs/adr/0010-corporate-hierarchy-auto-creation.md). + +New `lineageweave/corporate_hierarchy_inference.py` proposes a +Group/Company/Plant placement from context; new +`backend/app/corporate_entity_ingestion.py`'s +`get_or_create_corporate_entity` tries similarity matching first, then +creates a real new `corporate_entity` row once the proposal is +Searxng-corroborated (reusing `relation_verification`, no new search +integration), recursing up a bounded parent chain so the whole +hierarchy gets real links. Auto-created rows get a deterministic +`AUTO-`-prefixed code so they can never collide with a real login corp +code. Wired into both `keyman_ingestion.py`'s affiliation loop and +`post_summary_ingestion.py`'s R&R organization-actor loop. +## Standards-complete W3C PROV-O provenance layer + +ADR 0011 separates standards-complete provenance from the compact +buyer-facing navigation graph. `lineageweave/prov_o.py` validates +and materializes all 50 normative PROV-O properties, including +literal-valued times/values and qualified Influence resources. +`migrations/0017_prov_o_standard_relations.sql` stores definitions, +class/property hierarchies, domains, ranges, qualification maps, +inverse names, typed resources, literals, assertions, and inference +premises in third normal form. Existing product nodes cross the +boundary only through `provenance_resource_binding`; projection to +`knowledge_graph_edge` is explicit and reversible. + +See `docs/PROV_O_IMPLEMENTATION.md`, the complete implementation +matrix, and `docs/adr/0011-prov-o-standard-relations.md`. + +## Phase 13: corporate-entity creation is serialized against a real observed deadlock + +Phase 12's creation path made real concurrent writes for the first +time. A synthetic regression corpus batch run under real concurrency surfaced a +genuine `DeadlockDetectedError`: two concurrent transactions each +creating a different new entity, mentioned in opposite order across +two different posts, took row-level locks in opposite order and +deadlocked. See [ADR 0012](docs/adr/0012-corporate-entity-creation-lock.md). + +`get_or_create_corporate_entity` now takes a single named Postgres +advisory transaction lock (`pg_advisory_xact_lock`) immediately before +the write -- never held across the slow LLM inference/Searxng +verification calls that precede it -- and re-checks candidates fresh +under the lock before inserting. The lock key is fixed, not per-name, +so it also covers the multi-entity opposite-order case a per-name lock +would still deadlock on. Every already-cataloged entity still resolves +through the unchanged, lock-free similarity-matching fast path; only +the rare creation branch serializes. + +## Phase 14: customer-group tree plus Searxng abbreviation cross-check + +Operators navigate the authorized Group / Company / Plant catalog +(`GET /api/customer-group-tree`), not only the post-scoped affiliate +tree or the flat `/api/me` corp list. Abbreviations on a post are +cross-checked against that tree through the existing Searxng client +(`abbreviation_tree_corroboration`). A unique corroborated node binds; +a down, empty, or tied search stays unbound and does not invent a +parent or AUTO row. See +[ADR 0033](docs/adr/0033-customer-group-tree-abbreviation-corroboration.md). +This path does not reimplement ADR 0008 or ADR 0010. diff --git a/CHANGELOG.d/0.71.2-leftover-pairs.md b/CHANGELOG.d/0.71.2-leftover-pairs.md index 0c6b1e1f..30a56188 100644 --- a/CHANGELOG.d/0.71.2-leftover-pairs.md +++ b/CHANGELOG.d/0.71.2-leftover-pairs.md @@ -3,7 +3,7 @@ ## Added - Persist closest and farthest leftover pairs from the residual - interaction map after GRM/GPCM scoring (ADR 0017). + interaction map after GRM/GPCM scoring (ADR 0028). - After `make seed`, period reports show the closest and farthest leftover pairs above the member list; clicking a pair opens that post - (ADR 0018). A leftover pair for a hidden post is omitted. + (ADR 0029). A leftover pair for a hidden post is omitted. diff --git a/CHANGELOG.d/0.76.0.md b/CHANGELOG.d/0.76.0.md new file mode 100644 index 00000000..a1620816 --- /dev/null +++ b/CHANGELOG.d/0.76.0.md @@ -0,0 +1,3 @@ +# 0.76.0 — W3C PROV-O standard relations + +This release fragment is the machine-local source for the root changelog entry added by the one-shot documentation workflow. It records complete support for the Recommendation's 30 classes, 50 normative properties, 14 qualification patterns, Appendix B inverse names, normalized PostgreSQL persistence, exact RDF serialization, and 100% owned-module statement/branch coverage. diff --git a/CHANGELOG.d/0.77.0-review-hardening.md b/CHANGELOG.d/0.77.0-review-hardening.md new file mode 100644 index 00000000..873e9345 --- /dev/null +++ b/CHANGELOG.d/0.77.0-review-hardening.md @@ -0,0 +1,21 @@ +# 0.77.0 — Review hardening follow-up + +- Vision response parsing now treats caption and tag values as single-line + fields, strips balanced outer Markdown emphasis, and prevents trailing model + commentary from entering searchable tags or captions. +- The dedicated PROV-O workflow now fails when its PostgreSQL service is + unreachable instead of allowing the database contract module to skip. +- PROV-O database fixtures preserve DSN query options and quote generated + database identifiers through `psycopg2.sql.Identifier`. +- PostgreSQL lexical `xsd:dateTime` validation now rejects offsets outside + the XSD range of `Z` / `±hh:mm` with a maximum of `±14:00`, so + `+14:01` fails closed instead of being accepted as `timestamptz`. +- Corporate-entity inference, external verification, and the short advisory- + lock creation transaction now finish before the atomic post-summary + replacement transaction begins, so network latency never extends summary + write locks while post-owned rows still commit or roll back together. +- Keyman and R&R person mentions now replace separate source-owned projections, + while one canonical graph edge retains normalized per-post evidence. This + prevents a summary refresh from deleting Keyman facts or leaving removed R&R + actors visible in buyer navigation. +- The implementation matrix follows portable Markdown table spacing. diff --git a/CHANGELOG.d/0.79.0-analysis-run-authorized-read.md b/CHANGELOG.d/0.79.0-analysis-run-authorized-read.md new file mode 100644 index 00000000..7233020b --- /dev/null +++ b/CHANGELOG.d/0.79.0-analysis-run-authorized-read.md @@ -0,0 +1,9 @@ +# 0.79.0 — Authorized analysis-run read projection + +## Added + +- `GET /api/analysis-runs` and `GET /api/analysis-runs/{id}` expose + source-redacting registry evidence to `post_read` accounts. +- Home-page Analysis runs panel shows the seeded Demo Corp lineage run + after `make seed`. Hidden scopes 404. No raw source, DSN, or provider + payload is returned. diff --git a/CHANGELOG.d/0.80.0-analysis-run-detail-click.md b/CHANGELOG.d/0.80.0-analysis-run-detail-click.md new file mode 100644 index 00000000..090ebb82 --- /dev/null +++ b/CHANGELOG.d/0.80.0-analysis-run-detail-click.md @@ -0,0 +1,5 @@ +# 0.80.0 analysis-run detail click + +Home Analysis runs rows open `GET /api/analysis-runs/{id}`. The +detail shows labeled aggregates and dates only. Hidden runs stay +404 / "not visible". Synthetic Demo Corp seed only. diff --git a/CHANGELOG.d/0.81.0-analysis-run-status-history.md b/CHANGELOG.d/0.81.0-analysis-run-status-history.md new file mode 100644 index 00000000..9fa4b870 --- /dev/null +++ b/CHANGELOG.d/0.81.0-analysis-run-status-history.md @@ -0,0 +1,5 @@ +# 0.81.0 analysis-run status history + +Detail of `GET /api/analysis-runs/{id}` shows the labeled append-only +lifecycle. The list stays latest-status only. Hidden runs 404. +Synthetic Demo Corp seed only. diff --git a/CHANGELOG.d/0.82.0-analysis-run-post-clickthrough.md b/CHANGELOG.d/0.82.0-analysis-run-post-clickthrough.md new file mode 100644 index 00000000..1802a564 --- /dev/null +++ b/CHANGELOG.d/0.82.0-analysis-run-post-clickthrough.md @@ -0,0 +1,4 @@ +# 0.82.0 analysis-run post click-through + +Detail lists ABAC-visible post titles in the run scope. Hidden +other-corp private posts never appear. Synthetic titles only. diff --git a/CHANGELOG.d/0.83.0-analysis-run-cutoff-and-keyman-upgrade.md b/CHANGELOG.d/0.83.0-analysis-run-cutoff-and-keyman-upgrade.md new file mode 100644 index 00000000..f2de7cfa --- /dev/null +++ b/CHANGELOG.d/0.83.0-analysis-run-cutoff-and-keyman-upgrade.md @@ -0,0 +1,2 @@ +Analysis-run detail applies knowledge_cutoff to visible posts. Migration +0016 no longer deletes overlapping Keyman mention_context. diff --git a/CHANGELOG.d/0.84.0-tepp-analysis-run.md b/CHANGELOG.d/0.84.0-tepp-analysis-run.md new file mode 100644 index 00000000..df127a9d --- /dev/null +++ b/CHANGELOG.d/0.84.0-tepp-analysis-run.md @@ -0,0 +1,8 @@ +# 0.84.0 TEPP analysis-run seed + +Seed writes `analysis_run_tepp` via `tepp_client` on the shared Demo +Corp snapshot. The home list shows Failed and a kind-specific next +action; detail history keeps `tepp_not_available`. Missing transport +is not a fake measurement. A failed lineage row does not mention TEPP. +A failed period-report row rebuilds the report. A pending TEPP row +does not claim a calibrated measurement. diff --git a/CHANGELOG.d/0.84.1-analysis-run-digest-a11y.md b/CHANGELOG.d/0.84.1-analysis-run-digest-a11y.md new file mode 100644 index 00000000..213eb545 --- /dev/null +++ b/CHANGELOG.d/0.84.1-analysis-run-digest-a11y.md @@ -0,0 +1,5 @@ +# 0.84.1 Analysis-run digest a11y and live-body warning + +Detail prefixes stay audible and hoverable. Open a cutoff title only +after reading that the live body may have changed since the run. +The list stays aggregates-only. diff --git a/CHANGELOG.d/0.85.0-analysis-run-create.md b/CHANGELOG.d/0.85.0-analysis-run-create.md new file mode 100644 index 00000000..505320a1 --- /dev/null +++ b/CHANGELOG.d/0.85.0-analysis-run-create.md @@ -0,0 +1,5 @@ +# 0.85.0 authorized analysis-run create + +`POST /api/analysis-runs` records Pending on an authorized cutoff +capture. Request a lineage reconstruction from the home list. This +write does not invent a measurement. diff --git a/CHANGELOG.d/0.86.0-related-nodes-team-org-walk.md b/CHANGELOG.d/0.86.0-related-nodes-team-org-walk.md new file mode 100644 index 00000000..6e4605d5 --- /dev/null +++ b/CHANGELOG.d/0.86.0-related-nodes-team-org-walk.md @@ -0,0 +1,5 @@ +Related-node walks include team and organization mention edges. Click an +R&R team to open sibling posts. Thread-group run lists honor knowledge_cutoff. +Failed period-report rows rebuild the report; a pending TEPP corpus +does not claim a calibrated measurement. A pending lineage row says +reconstruction has not started yet. diff --git a/CHANGELOG.d/0.86.2-role-catalog-identity.md b/CHANGELOG.d/0.86.2-role-catalog-identity.md new file mode 100644 index 00000000..a56b3d47 --- /dev/null +++ b/CHANGELOG.d/0.86.2-role-catalog-identity.md @@ -0,0 +1,3 @@ +R&R organization buttons walk the catalog id stored on the role row. A +shared display name no longer attaches a homonym. Team related matches +person/entity 403/404. diff --git a/CHANGELOG.d/0.87.0-retention-purge-grant-admin.md b/CHANGELOG.d/0.87.0-retention-purge-grant-admin.md new file mode 100644 index 00000000..a2bcb7d8 --- /dev/null +++ b/CHANGELOG.d/0.87.0-retention-purge-grant-admin.md @@ -0,0 +1,13 @@ +# 0.87.0 analysis-run retention purge + +Operators empty a run-bearing analysis-run registry only after an +unrevoked `analysis_run_retention_grant` and +`GRANT analysis_run_retention_admin`. Then +`select purge_analysis_run_registry('approved-retention-purge')`. +Export `analysis_run_retention_event`, delete those rows, then roll +back 0020 and 0018. A raw DELETE, a published token without a grant, +and a runtime role that is not the admin role still fail (ADR 0020). + +Designers can change chip and close-button appearance in +`frontend/src/styles/tokens.css` and preview the next click in Storybook +(`cd frontend && pnpm run storybook`). diff --git a/CHANGELOG.d/0.88.0-analysis-run-start.md b/CHANGELOG.d/0.88.0-analysis-run-start.md new file mode 100644 index 00000000..a94209c2 --- /dev/null +++ b/CHANGELOG.d/0.88.0-analysis-run-start.md @@ -0,0 +1,5 @@ +# 0.88.0 start a pending lineage reconstruction + +`POST /api/analysis-runs/{id}/start` runs ThreadWeave on a Pending +lineage cutoff bag and persists run-scoped edges. Start reconstruction +from the open run. This path does not invent a TEPP measurement. diff --git a/CHANGELOG.d/0.89.0-analysis-run-live-write-clock.md b/CHANGELOG.d/0.89.0-analysis-run-live-write-clock.md new file mode 100644 index 00000000..bb4bb980 --- /dev/null +++ b/CHANGELOG.d/0.89.0-analysis-run-live-write-clock.md @@ -0,0 +1,5 @@ +# 0.89.0 Analysis-run live write clock + +In-cutoff titles now say whether the live row was rewritten after the +run. Open Demo public post as the edited counter-example; Demo private +post still matches the January cutoff. Bodies stay live. diff --git a/CHANGELOG.d/0.90.0-analysis-run-cutoff-body-warning.md b/CHANGELOG.d/0.90.0-analysis-run-cutoff-body-warning.md new file mode 100644 index 00000000..2ec2dfc3 --- /dev/null +++ b/CHANGELOG.d/0.90.0-analysis-run-cutoff-body-warning.md @@ -0,0 +1,5 @@ +# 0.90.0 Analysis-run cutoff body warning + +Opening a title marked updated after cutoff now says the popup body is +live and to compare it with this run. The popup does not invent the +earlier text. diff --git a/CHANGELOG.d/0.91.0-analysis-run-reconstructed-edge-click.md b/CHANGELOG.d/0.91.0-analysis-run-reconstructed-edge-click.md new file mode 100644 index 00000000..ed51dafe --- /dev/null +++ b/CHANGELOG.d/0.91.0-analysis-run-reconstructed-edge-click.md @@ -0,0 +1,4 @@ +# 0.91.0 Analysis-run reconstructed edge click + +Started reconstruction edges are buttons. Open the revised-quote child +or the pricing-follow-up parent. Live-body warning still applies. diff --git a/CHANGELOG.d/0.92.0-analysis-run-tepp-start.md b/CHANGELOG.d/0.92.0-analysis-run-tepp-start.md new file mode 100644 index 00000000..6ea75f0a --- /dev/null +++ b/CHANGELOG.d/0.92.0-analysis-run-tepp-start.md @@ -0,0 +1,4 @@ +# 0.92.0 Analysis-run TEPP start + +Pending TEPP start goes through tepp_client. Missing transport stays +Failed. No invented theta. diff --git a/CHANGELOG.d/0.93.0-seed-analysis-run-reconstruction.md b/CHANGELOG.d/0.93.0-seed-analysis-run-reconstruction.md new file mode 100644 index 00000000..9bd198d8 --- /dev/null +++ b/CHANGELOG.d/0.93.0-seed-analysis-run-reconstruction.md @@ -0,0 +1,4 @@ +# 0.93.0 Seed analysis-run reconstruction + +make seed persists the designed A-100 fork on the Demo Corp Succeeded +lineage run. Open that run and click the revised-quote child. diff --git a/CHANGELOG.d/0.94.0-analysis-run-outbox.md b/CHANGELOG.d/0.94.0-analysis-run-outbox.md new file mode 100644 index 00000000..49143656 --- /dev/null +++ b/CHANGELOG.d/0.94.0-analysis-run-outbox.md @@ -0,0 +1,5 @@ +# 0.94.0 Analysis-run start outbox + +Start commits Running plus a durable outbox row, wakes Valkey, then +delivers ThreadWeave or tepp_client. A crash no longer loses the work +item. No TEPP theta is invented. diff --git a/CHANGELOG.d/0.95.0-analysis-run-outbox-delivery.md b/CHANGELOG.d/0.95.0-analysis-run-outbox-delivery.md new file mode 100644 index 00000000..a92a4149 --- /dev/null +++ b/CHANGELOG.d/0.95.0-analysis-run-outbox-delivery.md @@ -0,0 +1,4 @@ +# 0.95.0 Analysis-run outbox delivery history + +Open a seeded Succeeded lineage run to see Claimed then Delivered. +Stream ids stay off the payload. diff --git a/CHANGELOG.d/0.96.0-seed-period-report-run.md b/CHANGELOG.d/0.96.0-seed-period-report-run.md new file mode 100644 index 00000000..037723ba --- /dev/null +++ b/CHANGELOG.d/0.96.0-seed-period-report-run.md @@ -0,0 +1,4 @@ +# 0.96.0 Seeded period-report analysis run + +After `make seed`, open **Period report · Succeeded · Demo Corp**. +Mean θ stays on the period-report panel. No theta is copied. diff --git a/CHANGELOG.d/0.97.0-analysis-run-open-period-report.md b/CHANGELOG.d/0.97.0-analysis-run-open-period-report.md new file mode 100644 index 00000000..e42dc8be --- /dev/null +++ b/CHANGELOG.d/0.97.0-analysis-run-open-period-report.md @@ -0,0 +1,5 @@ +# 0.97.0 Open the seeded period report from its analysis run + +Open Period report · Succeeded · Demo Corp, then Open period report +2026-W02. The report period field is focused. Failed rows stay +closed. Mean θ stays on the report panel. diff --git a/CHANGELOG.d/0.98.0-analysis-run-open-report-grouping.md b/CHANGELOG.d/0.98.0-analysis-run-open-report-grouping.md new file mode 100644 index 00000000..26f16420 --- /dev/null +++ b/CHANGELOG.d/0.98.0-analysis-run-open-report-grouping.md @@ -0,0 +1,4 @@ +# 0.98.0 Open the corp grouping with the scored week + +Open period report 2026-W02 also switches Report grouping to Corporate +entity and marks Demo Corp current. Mean θ stays on the report panel. diff --git a/CHANGELOG.d/0.98.1-analysis-run-sql-constants.md b/CHANGELOG.d/0.98.1-analysis-run-sql-constants.md new file mode 100644 index 00000000..37e5d425 --- /dev/null +++ b/CHANGELOG.d/0.98.1-analysis-run-sql-constants.md @@ -0,0 +1,4 @@ +# 0.98.1 Analysis-run SQL constants + +List and detail visibility queries are complete parameterized literals. +The $1 / $2 / $3 binds are unchanged. diff --git a/CHANGELOG.d/0.99.0-analysis-run-comparison-strip-landing.md b/CHANGELOG.d/0.99.0-analysis-run-comparison-strip-landing.md new file mode 100644 index 00000000..1b3df70f --- /dev/null +++ b/CHANGELOG.d/0.99.0-analysis-run-comparison-strip-landing.md @@ -0,0 +1,4 @@ +# 0.99.0 Land the comparison strip on the opened Demo Corp row + +Open period report 2026-W02 when already on that week. The grouping +comparison strip lands on Demo Corp. Mean θ stays on the report panel. diff --git a/CHANGELOG.d/1.0.0-analysis-run-opened-report-next-action.md b/CHANGELOG.d/1.0.0-analysis-run-opened-report-next-action.md new file mode 100644 index 00000000..a42a6b53 --- /dev/null +++ b/CHANGELOG.d/1.0.0-analysis-run-opened-report-next-action.md @@ -0,0 +1,5 @@ +# 1.0.0 Name the next action on the landed Demo Corp report + +Open period report 2026-W02 names the opened grouping and puts the +visible Corporate entity caption plus mean θ in the focused chip. +Mean θ stays on the report panel. diff --git a/CHANGELOG.d/1.1.0-analysis-run-opened-report-members.md b/CHANGELOG.d/1.1.0-analysis-run-opened-report-members.md new file mode 100644 index 00000000..d4a9b2c0 --- /dev/null +++ b/CHANGELOG.d/1.1.0-analysis-run-opened-report-members.md @@ -0,0 +1,5 @@ +# 1.1.0 Land Demo Corp members under the opened-report next action + +Open period report 2026-W02 puts Demo Corp mean θ and member posts +immediately under the next action, ahead of Other Corp. Mean θ stays +on the report panel. diff --git a/CHANGELOG.d/1.2.0-analysis-run-opened-member-next-action.md b/CHANGELOG.d/1.2.0-analysis-run-opened-member-next-action.md new file mode 100644 index 00000000..b247d670 --- /dev/null +++ b/CHANGELOG.d/1.2.0-analysis-run-opened-member-next-action.md @@ -0,0 +1,5 @@ +# 1.2.0 Name the next action on the opened Demo Corp member + +Open Public post from the landed Demo Corp report. The panel says that +post is open and to read Event Lineage, Keyman, and evaluation. Mean θ +stays on the report panel. diff --git a/CHANGELOG.d/1.3.0-focus-event-lineage-from-report-member.md b/CHANGELOG.d/1.3.0-focus-event-lineage-from-report-member.md new file mode 100644 index 00000000..33f0436a --- /dev/null +++ b/CHANGELOG.d/1.3.0-focus-event-lineage-from-report-member.md @@ -0,0 +1,4 @@ +# 1.3.0 Focus Event Lineage from a report member + +Open Public post from the landed Demo Corp members and the popup Event +Lineage heading takes focus. Home list opens do not. diff --git a/CHANGELOG.d/1.4.0-current-event-lineage-node.md b/CHANGELOG.d/1.4.0-current-event-lineage-node.md new file mode 100644 index 00000000..a41cb412 --- /dev/null +++ b/CHANGELOG.d/1.4.0-current-event-lineage-node.md @@ -0,0 +1,4 @@ +# 1.4.0 Current Event Lineage node + +Open Public post from the landed Demo Corp members and the popup Event +Lineage DAG marks that post current. The home DAG does not. diff --git a/CHANGELOG.d/1.5.0-event-lineage-keyman-eval-next.md b/CHANGELOG.d/1.5.0-event-lineage-keyman-eval-next.md new file mode 100644 index 00000000..f047f513 --- /dev/null +++ b/CHANGELOG.d/1.5.0-event-lineage-keyman-eval-next.md @@ -0,0 +1,4 @@ +# 1.5.0 Name Keyman and evaluation after the current Event Lineage node + +Open Public post from the landed Demo Corp members and the popup names +Keyman and evaluation after the current Event Lineage node. diff --git a/CHANGELOG.d/1.6.0-land-keyman-eval-under-next-action.md b/CHANGELOG.d/1.6.0-land-keyman-eval-under-next-action.md new file mode 100644 index 00000000..e7115e36 --- /dev/null +++ b/CHANGELOG.d/1.6.0-land-keyman-eval-under-next-action.md @@ -0,0 +1,5 @@ +# 1.6.0 Land Keyman and evaluation under the Event Lineage next action + +Open Public post from the landed Demo Corp members and Keyman plus +evaluation sit under the Event Lineage next action, ahead of Affiliate +tree. Home list opens keep the earlier order. diff --git a/CHANGELOG.d/1.7.0-name-first-keyman-after-eval.md b/CHANGELOG.d/1.7.0-name-first-keyman-after-eval.md new file mode 100644 index 00000000..a6683348 --- /dev/null +++ b/CHANGELOG.d/1.7.0-name-first-keyman-after-eval.md @@ -0,0 +1,4 @@ +# 1.7.0 Name the first Keyman after landed evaluation + +Open Public post from the landed Demo Corp members and the popup names +the first Keyman after landed evaluation. diff --git a/CHANGELOG.d/1.8.0-land-first-keyman-related.md b/CHANGELOG.d/1.8.0-land-first-keyman-related.md new file mode 100644 index 00000000..a3314ec7 --- /dev/null +++ b/CHANGELOG.d/1.8.0-land-first-keyman-related.md @@ -0,0 +1,5 @@ +# 1.8.0 Land first Keyman related nodes + +Open Public post from the landed Demo Corp members and Ada West related +nodes sit under the first-Keyman next action. Home list opens still +wait for a Keyman click. diff --git a/CHANGELOG.d/1.9.0-name-first-related-after-ada.md b/CHANGELOG.d/1.9.0-name-first-related-after-ada.md new file mode 100644 index 00000000..2f36620a --- /dev/null +++ b/CHANGELOG.d/1.9.0-name-first-related-after-ada.md @@ -0,0 +1,4 @@ +# 1.9.0 Name the first related node after Ada West related + +Open Public post from the landed Demo Corp members and the popup names +the first related node after landed Ada West related. diff --git a/CHANGELOG.d/2.0.0-land-first-related-after-priya.md b/CHANGELOG.d/2.0.0-land-first-related-after-priya.md new file mode 100644 index 00000000..2a45ea23 --- /dev/null +++ b/CHANGELOG.d/2.0.0-land-first-related-after-priya.md @@ -0,0 +1,4 @@ +# 2.0.0 Land the first related node after Priya Nair is named + +Open Public post from the landed Demo Corp members and the popup lands +Priya Nair related nodes under the first-related next action. diff --git a/CHANGELOG.d/2.1.0-source-post-revision.md b/CHANGELOG.d/2.1.0-source-post-revision.md new file mode 100644 index 00000000..79b50c76 --- /dev/null +++ b/CHANGELOG.d/2.1.0-source-post-revision.md @@ -0,0 +1,5 @@ +# 2.1.0 Source-post revision at cutoff + +Open a marked Demo public post: the January sentence is **Body this run +knew**; the live body is the later delivery window. Compare those two +texts. Analysis-run detail still has no post body. diff --git a/CHANGELOG.d/2.10.0-leftover-pairs-and-rankings.md b/CHANGELOG.d/2.10.0-leftover-pairs-and-rankings.md new file mode 100644 index 00000000..0b712589 --- /dev/null +++ b/CHANGELOG.d/2.10.0-leftover-pairs-and-rankings.md @@ -0,0 +1,13 @@ +# 2.10.0 — Leftover pairs and fail-closed Rankings + +## Added + +- Home Rankings panel fuses visible posts through `RankWeaveClient` + (ADR 0030). After login with the port disabled or the library + missing, Demo Analyst sees **Rankings · RankWeave not available**. + An accepted hit lists the title; click opens that post. A hidden + post is omitted. Never invent a fused score or a theta. +- Period reports persist closest and farthest leftover post–criterion + pairs after IRT main effects (ADR 0028 / 0029). After `make seed`, + leftover pairs sit above the member list; clicking a pair opens that + post. A leftover pair for a hidden post is omitted. diff --git a/CHANGELOG.d/2.10.1-analysis-run-accessible-next-action.md b/CHANGELOG.d/2.10.1-analysis-run-accessible-next-action.md new file mode 100644 index 00000000..3a0e27a0 --- /dev/null +++ b/CHANGELOG.d/2.10.1-analysis-run-accessible-next-action.md @@ -0,0 +1,5 @@ +# 2.10.1 Include next-action in analysis-run accessible names + +List button names include the kind-specific next-action sentence. +Open a Failed TEPP row and hear connect the measurement service +(ADR 0014). diff --git a/CHANGELOG.d/2.10.2-embedded-image-html-parser.md b/CHANGELOG.d/2.10.2-embedded-image-html-parser.md new file mode 100644 index 00000000..cb3b3f82 --- /dev/null +++ b/CHANGELOG.d/2.10.2-embedded-image-html-parser.md @@ -0,0 +1,5 @@ +# 2.10.2 Parse invoice HTML images with an HTML parser + +Opening a post whose embedded picture uses invoice-like HTML +(`alt="Invoice > 1000"`) shows the picture between the surrounding +sentences. The raw base64 string is gone (ADR 0031). diff --git a/CHANGELOG.d/2.10.3-stale-hidden-run-list.md b/CHANGELOG.d/2.10.3-stale-hidden-run-list.md new file mode 100644 index 00000000..b1d8cda0 --- /dev/null +++ b/CHANGELOG.d/2.10.3-stale-hidden-run-list.md @@ -0,0 +1,7 @@ +# 2.10.3 Drop a stale analysis-run row after its detail 404s + +Opening a listed analysis-run that then 404s drops that stale row +from the home list after an authorized re-read, announces the next +action with a status alert, and leaves Request as the named +reconstruction control. The message still does not name the thread +or the cutoff (ADR 0014 / ADR 0018). diff --git a/CHANGELOG.d/2.10.4-retention-purge-reconstruction-children.md b/CHANGELOG.d/2.10.4-retention-purge-reconstruction-children.md new file mode 100644 index 00000000..1343d758 --- /dev/null +++ b/CHANGELOG.d/2.10.4-retention-purge-reconstruction-children.md @@ -0,0 +1,6 @@ +# 2.10.4 Empty reconstruction children during granted retention purge + +After a Demo Corp lineage reconstruction has started, the same +granted retention purge empties reconstruction edges, the +reconstruction digest, and frozen snapshot members (ADR 0032). +Do not `DISABLE TRIGGER` as superuser. diff --git a/CHANGELOG.d/2.11.0-customer-group-tree-searxng.md b/CHANGELOG.d/2.11.0-customer-group-tree-searxng.md new file mode 100644 index 00000000..6ffd0f55 --- /dev/null +++ b/CHANGELOG.d/2.11.0-customer-group-tree-searxng.md @@ -0,0 +1,7 @@ +# 2.11.0 Customer-group tree and Searxng abbreviation cross-check + +Home shows the authorized Group / Company / Plant forest. A click opens +that Demo Corp node as the corporate-entity report grouping. Post +abbreviations are cross-checked against that tree through the existing +Searxng client. When Searxng is down, empty, or tied, the mention stays +unbound — no invented parent and no AUTO row (ADR 0033). diff --git a/CHANGELOG.d/2.12.0-persistable-tepp-result.md b/CHANGELOG.d/2.12.0-persistable-tepp-result.md new file mode 100644 index 00000000..cba49b6f --- /dev/null +++ b/CHANGELOG.d/2.12.0-persistable-tepp-result.md @@ -0,0 +1,8 @@ +# 2.12.0 Persistable TEPP result is Succeeded + +A live TEPP transport that returns a time / multilevel / +multi-affiliation envelope is stored on the analysis-run and marked +Succeeded. Home list and detail show clocks and affiliation counts. +A screen reader on that Succeeded Demo Corp row hears the next action. +An accepted ack or a missing transport stays Failed. No invented theta +(ADR 0034). diff --git a/CHANGELOG.d/2.12.1-tepp-accepted-transport-evidence.md b/CHANGELOG.d/2.12.1-tepp-accepted-transport-evidence.md new file mode 100644 index 00000000..6619d928 --- /dev/null +++ b/CHANGELOG.d/2.12.1-tepp-accepted-transport-evidence.md @@ -0,0 +1,9 @@ +# 2.12.1 TEPP accepted acknowledgements are transport evidence + +A published TEPP `AnalysisRunAccepted` envelope is stored as +aggregate transport evidence. The run stays Failed / +`tepp_completed_result_unsupported`. A LineageWeave-local +`time_multilevel_multi_affiliation` envelope is not a completed TEPP +measurement and must not stamp Succeeded. Authorized detail shows +**Measurement evidence** with a copyable SHA-256. No invented theta +(ADR 0035). diff --git a/CHANGELOG.d/2.12.2-tepp-accepted-clocks.md b/CHANGELOG.d/2.12.2-tepp-accepted-clocks.md new file mode 100644 index 00000000..56af89e3 --- /dev/null +++ b/CHANGELOG.d/2.12.2-tepp-accepted-clocks.md @@ -0,0 +1,7 @@ +# 2.12.2 TEPP accepted evidence stores distinct receipt and row-write clocks + +Accepted transport evidence persists `received_at` as the +transport-response receipt and `recorded_at` as the row-write +instant. Measurement evidence shows the second clock only when those +instants differ. Digest recomputation is unchanged. No invented +theta (ADR 0035 follow-up). diff --git a/CHANGELOG.d/2.12.3-late-demo-cutoff-post.md b/CHANGELOG.d/2.12.3-late-demo-cutoff-post.md new file mode 100644 index 00000000..d442279e --- /dev/null +++ b/CHANGELOG.d/2.12.3-late-demo-cutoff-post.md @@ -0,0 +1,6 @@ +# 2.12.3 Late Demo public post + +Seed inserts Late Demo public post after the January 12 knowledge +cutoff so the ADR 0016 list filter has a falsifiable own-corp +counter-example. No second cutoff implementation. TEPP honesty +unchanged. diff --git a/CHANGELOG.d/2.2.0-name-ask-after-priya-related.md b/CHANGELOG.d/2.2.0-name-ask-after-priya-related.md new file mode 100644 index 00000000..37a463a4 --- /dev/null +++ b/CHANGELOG.d/2.2.0-name-ask-after-priya-related.md @@ -0,0 +1,4 @@ +# 2.2.0 Name Ask after Priya Nair related nodes are current + +Open Public post from the landed Demo Corp members and the popup names +Ask about this lineage after Priya Nair related nodes are current. diff --git a/CHANGELOG.d/2.3.0-focus-ask-after-priya-related.md b/CHANGELOG.d/2.3.0-focus-ask-after-priya-related.md new file mode 100644 index 00000000..bf2c4f35 --- /dev/null +++ b/CHANGELOG.d/2.3.0-focus-ask-after-priya-related.md @@ -0,0 +1,5 @@ +# 2.3.0 Focus Ask after Priya related + +Open Public post from the landed Demo Corp members and the Ask heading +takes focus after Priya Nair related nodes are current. Home list +opens do not. diff --git a/CHANGELOG.d/2.4.0-land-ask-under-next-action.md b/CHANGELOG.d/2.4.0-land-ask-under-next-action.md new file mode 100644 index 00000000..b1b1f6d7 --- /dev/null +++ b/CHANGELOG.d/2.4.0-land-ask-under-next-action.md @@ -0,0 +1,5 @@ +# 2.4.0 Land Ask under the next action + +Open Public post from the landed Demo Corp members and Ask about this +lineage sits under the Ask next action, ahead of Affiliate tree. Home +list opens keep Ask at the bottom. diff --git a/CHANGELOG.d/2.5.0-name-first-ask-after-landed-chat.md b/CHANGELOG.d/2.5.0-name-first-ask-after-landed-chat.md new file mode 100644 index 00000000..63b074c8 --- /dev/null +++ b/CHANGELOG.d/2.5.0-name-first-ask-after-landed-chat.md @@ -0,0 +1,4 @@ +# 2.5.0 Name the first Ask after landed chat + +Open Public post from the landed Demo Corp members and the first +seeded Ask is named after landed chat. Home list opens do not. diff --git a/CHANGELOG.d/2.6.0-land-first-ask-answer.md b/CHANGELOG.d/2.6.0-land-first-ask-answer.md new file mode 100644 index 00000000..7a3c0c80 --- /dev/null +++ b/CHANGELOG.d/2.6.0-land-first-ask-answer.md @@ -0,0 +1,5 @@ +# 2.6.0 Land the first Ask answer under the named seed next action + +Open Public post from the landed Demo Corp members and the first seeded +Ask answer sits under the Ask seed next action, ahead of the input. +Home list opens keep that answer after the input. diff --git a/CHANGELOG.d/2.7.0-name-first-cited-after-ask-answer.md b/CHANGELOG.d/2.7.0-name-first-cited-after-ask-answer.md new file mode 100644 index 00000000..93d454ce --- /dev/null +++ b/CHANGELOG.d/2.7.0-name-first-cited-after-ask-answer.md @@ -0,0 +1,5 @@ +# 2.7.0 Name the first cited source after the landed Ask answer + +Open Public post from the landed Demo Corp members and the first +cited source on the landed Ask answer is named. Home list opens do +not. diff --git a/CHANGELOG.d/2.7.1-wait-affiliated-corps-lineage-create.md b/CHANGELOG.d/2.7.1-wait-affiliated-corps-lineage-create.md new file mode 100644 index 00000000..a5a7aa04 --- /dev/null +++ b/CHANGELOG.d/2.7.1-wait-affiliated-corps-lineage-create.md @@ -0,0 +1,7 @@ +# 2.7.1 Wait for affiliated corps before lineage create + +`POST /api/analysis-runs` records Pending lineage on an authorized +cutoff capture. TEPP and period-report kinds are 422. Open Analysis +runs and wait until affiliated corps load; choose a corp if you walk +more than one, then click Request a lineage reconstruction. Preview +the picker in Storybook (`Analysis/LineageEntityPicker`). diff --git a/CHANGELOG.d/2.7.2-role-person-catalog-identity.md b/CHANGELOG.d/2.7.2-role-person-catalog-identity.md new file mode 100644 index 00000000..cdf3414e --- /dev/null +++ b/CHANGELOG.d/2.7.2-role-person-catalog-identity.md @@ -0,0 +1,6 @@ +# 2.7.2 Persist R&R person catalog ids + +R&R person chips read the stored catalog id. Historical person backfill +leaves homonym mentions unbound. Open a post whose R&R names a cataloged +person: the chip is a button even when Keyman was not extracted on that +post. diff --git a/CHANGELOG.d/2.8.0-land-first-cited-evidence.md b/CHANGELOG.d/2.8.0-land-first-cited-evidence.md new file mode 100644 index 00000000..6af53e0e --- /dev/null +++ b/CHANGELOG.d/2.8.0-land-first-cited-evidence.md @@ -0,0 +1,5 @@ +# 2.8.0 Land the first cited evidence under the named citation next action + +Open Public post from the landed Demo Corp members and the first +cited evidence sits under the citation next action, ahead of the +input. Home list opens still wait for a citation click. diff --git a/CHANGELOG.d/2.9.0-name-evidence-lineage-after-cited.md b/CHANGELOG.d/2.9.0-name-evidence-lineage-after-cited.md new file mode 100644 index 00000000..c41af2d6 --- /dev/null +++ b/CHANGELOG.d/2.9.0-name-evidence-lineage-after-cited.md @@ -0,0 +1,5 @@ +# 2.9.0 Name Event Lineage after landed cited evidence + +Open Public post from the landed Demo Corp members and the popup names +Event Lineage after Linked post evidence is current. Home list opens +do not. diff --git a/CHANGELOG.d/milestone2-analysis-run-registry.md b/CHANGELOG.d/milestone2-analysis-run-registry.md new file mode 100644 index 00000000..5d7e0288 --- /dev/null +++ b/CHANGELOG.d/milestone2-analysis-run-registry.md @@ -0,0 +1,21 @@ +## Added + +- Added a normalized PostgreSQL registry for immutable source snapshots, + aggregate reconciliation counts, authenticated analysis requests, product + scopes, and append-only lifecycle evidence. +- Added a run-owned knowledge cutoff and snapshot-owned evidence-availability + clock so one capture can support multiple historically valid analyses without + future-information leakage. +- Added canonical account-scoped idempotency, immutable request and scope evidence, + deletion resistance, serialized count/run locking, scope-required request-time- + ordered lifecycle transitions, database-owned record time, and a derived + current-status view. +- Added fail-closed rollback, real-PostgreSQL contract tests, ADR 0013, and APA + 7th standards traceability. + +## Security + +- The registry deliberately excludes source SQL, DSNs, raw records, inline + images, provider payloads, credentials, private source identifiers, and raw + exceptions. Necessary PII remains in purpose-bound authorized product/source + contexts rather than being copied into audit metadata or blanket-masked. diff --git a/CHANGELOG.d/tied-organization-no-auto-create.md b/CHANGELOG.d/tied-organization-no-auto-create.md new file mode 100644 index 00000000..64842f48 --- /dev/null +++ b/CHANGELOG.d/tied-organization-no-auto-create.md @@ -0,0 +1,6 @@ +# Tied organization names do not create catalog rows + +A tied top organization similarity score now stays unbound. Even with live +name resolution, hierarchy inference, and verification, the ingestion path +does not insert an `AUTO-` catalog row. Only a genuine below-threshold miss +may enter the corroborated creation path (ADR 0026). diff --git a/CHANGELOG.md b/CHANGELOG.md index 6bfcaa28..4a9b3f19 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,25 +4,854 @@ All notable changes to this project are documented here. Format follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/); versioning follows [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## [0.75.0] - 2026-08-17 +## [2.12.5] - 2026-08-18 + +### Fixed + +- Migrations 0019 and 0025 (R&R role-catalog identity backfills) both + used `min(uuid_column)` to pick "the" value from a `having count(*) + = 1` group -- Postgres has no built-in `min(uuid)` aggregate, so + both failed outright the first time either was actually run against + a real, non-trivial dataset. Fixed to + `min(uuid_column::text)::uuid`, safe given the query's own + `having count(*) = 1` already guarantees exactly one value per + group. Applying the full migration set 0001-0029 against a real, + long-lived dataset also surfaced that this database's original + bootstrap had left several *earlier* migrations (0001, 0016) + partially applied -- specific tables/indexes/backfills their own + later statements defined were missing even though their initial + `create table` statements had run. All 29 migrations are now + confirmed genuinely, fully applied end to end against a real + 43,814-post dataset; every table/index any migration defines is now + present, verified via direct schema comparison, not assumption. + +### Known issue (not fixed here, flagged for follow-up) + +- `backend/tests/test_api.py::test_start_analysis_run_recovers_the_a100_fork` + and `::test_tepp_start_persists_published_accepted_evidence` + deterministically fail against a real live PostgreSQL/Keycloak/Valkey + stack (this whole test module is `skipif`-guarded and never runs in + CI) with `CheckViolationError` on `analysis_run_status_time_check` + (`occurred_at <= recorded_at`): the row's `occurred_at` + (`datetime.now(timezone.utc)`, captured in `backend/app/analysis_run_start.py`) + reproducibly lands ~15-20ms *after* `recorded_at` + (`clock_timestamp()`, evaluated later, at actual insert time, inside + a `before insert` trigger) -- the wrong direction, given + `recorded_at` is evaluated strictly after `occurred_at` is captured + in every code path. Confirmed via a direct clock-sync measurement + (5 samples, Python vs. Postgres `clock_timestamp()` interleaved) + that there is no measurable systemic clock drift between the test + process and this Postgres instance under normal conditions, and + confirmed the failure is 100% reproducible in isolation (not a + concurrency/load artifact) and entirely pre-existing (verified via + `git diff` that no file in this change touches + `analysis_run_start.py` or the 0018 migration that defines this + constraint). Root cause not yet conclusively identified; deferred + as out of scope for this migration-catchup change (a different + feature area -- analysis-run/TEPP lifecycle, not R&R/summary/ + verification) rather than rushed. 553 other tests unaffected. + +- `get_or_create_corporate_entity`'s post-lock duplicate-create re-check + fuzzy-matched against every cataloged entity, not just an exact + concurrent duplicate of the entity being created. A newly-created + parent whose name is a prefix of the child now being created (e.g. + "Acme" as parent of "Acme Gwangju Plant") scored ~0.7 similarity + against that child under the shared 0.6 threshold, so the child was + silently bound to its own parent's id instead of getting its own + catalog row -- undermining exactly the "통합 고객사 계열 tree AI" + (integrated customer affiliate tree) hierarchy the feature exists + for. The re-check now requires an exact post-normalization match + (`min_similarity=1.0`); real mention resolution against the full + candidate set is unchanged. Caught locally by + `test_first_mention_of_a_new_counterparty_creates_a_real_corporate_entity`, + which requires a live PostgreSQL/Keycloak/Valkey stack and is + therefore skipped in CI (`make up` required) -- confirmed CI's own + "Full test suite" run has never actually executed this assertion. +- `test_start_analysis_run_recovers_the_a100_fork` seeded + `snapshot_sha256`/`configuration_sha256`/`code_revision_sha` with + `"t"`/`"u"`/`"v"`-repeated literals; none are valid hex characters, + so the very first insert failed its own `analysis_source_snapshot` + check constraint every time this test actually ran. Same CI-blind + gap as above -- fixed to valid hex placeholders matching this file's + existing convention. + +## [2.12.3] - 2026-08-18 + +### Added + +- `make seed` inserts Late Demo public post (2026-01-13) so the + January 12 Demo Corp lineage and TEPP runs' knowledge cutoff is + falsifiable: Demo public post still opens; Late Demo does not. + The live post list still shows Late Demo. The cutoff filter + itself already lives on this stack (ADR 0016). TEPP honesty is + unchanged: accepted acks stay Failed transport evidence, not + Succeeded. Never invent a theta. + +## [2.12.2] - 2026-08-18 + +### Fixed + +- Accepted TEPP transport evidence now stores **received** (transport + response) and **recorded** (row write) as distinct clocks when those + instants differ (ADR 0035 follow-up). After `make seed`, Demo Analyst + opens **TEPP measurement · Failed · Demo Corp** Measurement evidence + and sees one Received clock when seed receipt and persist share an + instant. A later start that persists in a later minute shows both + clocks. Digest recomputation is unchanged. Hidden runs stay 404. + Never invent a theta. + +## [2.12.1] - 2026-08-17 + +### Fixed + +- A published TEPP **accepted** acknowledgement is stored as + **aggregate transport evidence** and stays Failed / + `tepp_completed_result_unsupported` (ADR 0035). After `make seed`, + Demo Analyst opens that Failed Demo Corp row to read contract + version, accepted run id, clocks, and a copyable SHA-256. The + section says completed-artifact identity is unavailable until TEPP + publishes a versioned completed-result contract. A + LineageWeave-local `time_multilevel_multi_affiliation` envelope, or + any other unpublished completed shape, stays Failed / + `tepp_result_not_persisted` and must not stamp Succeeded. Missing + `TEPP_TRANSPORT_URL` stays Failed / `tepp_not_available`. Never + invent a theta. + +## [2.12.0] - 2026-08-17 + +### Added + +- A persistable TEPP **time / multilevel / multi-affiliation** result + is stored on the analysis-run and marked Succeeded (ADR 0034). After + `make seed`, Demo Analyst sees **TEPP measurement · Succeeded · Demo + Corp** next to the Failed missing-transport row. Home list and detail + show measured clocks and affiliation counts. A screen reader on that + Succeeded row hears open the run to read those aggregates, not only + the title. An `accepted` ack or an envelope this product cannot store + stays Failed / `tepp_result_not_persisted`. Missing + `TEPP_TRANSPORT_URL` stays Failed / `tepp_not_available`. Never + invent a theta. + +## [2.11.0] - 2026-08-17 + +### Added + +- Home now shows the authorized customer-group tree (Group / Company / + Plant) instead of only a flat corp list. After `make seed`, Demo + Analyst walks Demo Group → Demo Corp → Demo Plant; a click opens that + entity as the corporate-entity report grouping. The post-scoped + affiliate tree is unchanged (ADR 0033). +- Abbreviations on a post are cross-checked against that tree through + the existing Searxng client. A unique corroborated hit binds; a down, + empty, or tied search stays unbound and does not invent a parent or + AUTO row. Seeded `DC` on Demo Corp is synthetic Demo Corp only. + +## [2.10.4] - 2026-08-17 + +### Fixed + +- After a Demo Corp lineage reconstruction has started, the same + granted retention purge empties `analysis_run_lineage_edge`, + `analysis_run_reconstruction`, and `analysis_source_snapshot_member` + when those tables exist, including their delete-reject triggers + (ADR 0032). Follow the same grant + admin + phrase path — do not + `DISABLE TRIGGER` as superuser. + +## [2.10.3] - 2026-08-17 + +### Fixed + +- Opening a listed analysis-run that then 404s drops that stale row + from the home list after an authorized re-read. The next action is + announced as a status alert: open a remaining visible run, or + request a lineage reconstruction. The message still does not name + the thread or the cutoff (ADR 0014 / ADR 0018). + +## [2.10.2] - 2026-08-17 + +### Fixed + +- Opening a post whose embedded picture uses invoice-like HTML + (`alt="Invoice > 1000"`, unquoted `width`, newlines in the base64) + now shows the picture. The raw payload no longer returns when a + remote-only or SVG tag is the whole body. Re-export as PNG or JPEG + if the type is rejected. The popup, `extract_base64_images`, and + `chunk_by_dom` share one raster allowlist (ADR 0031). + +## [2.10.1] - 2026-08-17 + +### Fixed + +- Analysis-run list buttons now include the kind-specific next-action + sentence in the accessible name (WCAG 2.2 SC 4.1.2). Open a Failed + TEPP row: a screen reader hears connect the measurement service, not + only the run title. `aria-label` replaces button contents (ADR 0014). + No TEPP theta is invented. + +## [2.10.0] - 2026-08-17 ### Added - Home Rankings panel fuses visible posts through `RankWeaveClient` - (ADR 0024). After login with the port disabled or the library + (ADR 0030). After login with the port disabled or the library missing, Demo Analyst sees **Rankings · RankWeave not available**. An accepted hit lists the title; click opens that post. A hidden post is omitted. Never invent a fused score or a theta. +- Period reports now persist closest and farthest leftover + post–criterion pairs after the IRT main effects (Jeon leftover + map, ADR 0028 / 0029). After `make seed`, leftover pairs sit above + the member list; clicking a pair opens that post. A leftover pair + for a hidden post is omitted the same way a hidden member is. -## [0.71.2] - 2026-08-17 +## [2.9.0] - 2026-08-17 ### Added -- Period reports now persist closest and farthest leftover - post–criterion pairs after the IRT main effects (Jeon leftover - map). After `make seed`, leftover pairs sit above the member - list; clicking a pair opens that post. A leftover pair for a - hidden post is omitted the same way a hidden member is. +- Opening Public post from the landed Demo Corp members now names the + next action after landed cited evidence: Linked post evidence is + current, then read Event Lineage on that post. Home list opens do + not add that copy. No TEPP theta is invented. No cutoff body is + invented (ADR 0016). + +## [2.8.0] - 2026-08-17 + +### Added + +- Opening Public post from the landed Demo Corp members now puts the + first cited evidence immediately under the named citation next + action, ahead of the chat input. Home list opens still wait for a + citation click. No TEPP theta is invented. No cutoff body is + invented (ADR 0016). + +## [2.7.2] - 2026-08-17 + +### Fixed + +- R&R person chips now read `cataloged_person_id` from + `post_summary_role` (ADR 0027). Open a post whose R&R names a + cataloged person: the chip is a button even when Keyman extraction + was not run on that post. Click it to walk that person, not a later + same-named row. Historical backfill leaves a role unbound when two + same-named mentions already exist. + +## [2.7.1] - 2026-08-17 + +### Fixed + +- `POST /api/analysis-runs` records Pending lineage only (ADR 0017). + TEPP and period-report kinds are 422 so this path cannot invent a + measurement. Open Analysis runs and wait until affiliated corps + load; choose a corp if you walk more than one, then click + **Request a lineage reconstruction**. Preview the picker in + Storybook (`Analysis/LineageEntityPicker`). Failed TEPP stays + terminal on this write. + +## [2.7.0] - 2026-08-17 + +### Added + +- Opening Public post from the landed Demo Corp members now names the + first cited source after the landed first Ask answer: Linked post, + then open that evidence. Home list opens do not add that copy. No + TEPP theta is invented. No cutoff body is invented (ADR 0016). + +## [2.6.0] - 2026-08-17 + +### Added + +- Opening Public post from the landed Demo Corp members now puts the + first Ask answer immediately under the named seed next action, ahead + of the chat input. Home list opens keep that answer after the input. + No TEPP theta is invented. No cutoff body is invented (ADR 0016). + +## [2.5.0] - 2026-08-17 + +### Added + +- Opening Public post from the landed Demo Corp members now names the + first Ask after landed chat: What happened between these events, + then read that answer. Home list opens do not add that copy. No + TEPP theta is invented. No cutoff body is invented (ADR 0016). + +## [2.4.0] - 2026-08-17 + +### Added + +- Opening Public post from the landed Demo Corp members now puts Ask + about this lineage immediately under the Ask next action, ahead of + Affiliate tree. Home list opens keep Ask after Keyman. No TEPP theta + is invented. No cutoff body is invented (ADR 0016). + +## [2.3.0] - 2026-08-17 + +### Added + +- Opening Public post from the landed Demo Corp members now focuses + the Ask heading after Priya Nair related nodes are current. Home + list opens do not steal that focus. No TEPP theta is invented. No + cutoff body is invented (ADR 0016). + +## [2.2.0] - 2026-08-17 + +### Added + +- Opening Public post from the landed Demo Corp members now names the + next action after Priya Nair related nodes are current: **Ask about + this lineage**. Home list opens do not add that copy. No TEPP theta + is invented. No cutoff body is invented (ADR 0016). + +## [2.1.0] - 2026-08-17 + +### Added + +- Opening a title marked **Updated after cutoff** now shows the body + that run knew beside the live rewrite. After `make seed`, open Demo + public post from the Demo Corp lineage run: **Body this run knew** is + the January follow-up; the live body names the later delivery window. + `GET /api/posts/{id}?as_of=` reads `source_post_revision`. Analysis-run + detail stays titles and clocks. A missing revision is omitted — never + a fabricated cutoff sentence or a TEPP theta (ADR 0025). + +## [2.0.0] - 2026-08-17 + +### Added + +- Opening Public post from the landed Demo Corp members now lands + Priya Nair related nodes under the first-related next action, ahead + of Affiliate tree. Home list opens still wait for a related click. + No TEPP theta is invented. No cutoff body is invented (ADR 0016). + +## [1.9.0] - 2026-08-17 + +### Added + +- Opening Public post from the landed Demo Corp members now names the + first related node after landed Ada West related: Priya Nair, then + read that person. Home list opens do not add that copy. No TEPP + theta is invented. No cutoff body is invented (ADR 0016). + +## [1.8.0] - 2026-08-17 + +### Added + +- Opening Public post from the landed Demo Corp members now lands + Ada West related nodes under the first-Keyman next action, ahead + of Affiliate tree. Home list opens still wait for a Keyman click. + No TEPP theta is invented. No cutoff body is invented (ADR 0016). + +## [1.7.0] - 2026-08-17 + +### Added + +- Opening Public post from the landed Demo Corp members now names the + first Keyman after landed evaluation: Ada West, then read that + person. Home list opens do not add that copy. No TEPP theta is + invented. No cutoff body is invented (ADR 0016). + +## [1.6.0] - 2026-08-17 + +### Added + +- Opening Public post from the landed Demo Corp members now puts + Keyman and evaluation immediately under the Event Lineage next + action, ahead of Affiliate tree. Home list opens keep evaluation + above Event Lineage. No TEPP theta is invented. No cutoff body is + invented (ADR 0016). + +## [1.5.0] - 2026-08-17 + +### Added + +- Opening Public post from the landed Demo Corp members names the next + action after the current Event Lineage node: read Keyman and + evaluation. Home list opens do not add that copy. No TEPP theta is + invented. No cutoff body is invented (ADR 0016). + +## [1.4.0] - 2026-08-17 + +### Added + +- Opening Public post from the landed Demo Corp members marks that + post current in the popup Event Lineage DAG, so the focused heading + has a you-are-here node. The home DAG stays unmarked. No TEPP theta + is invented. No cutoff body is invented (ADR 0016). + +## [1.3.0] - 2026-08-17 + +### Added + +- Opening Public post from the landed Demo Corp members focuses the + popup Event Lineage heading, matching the next-action copy. Home + post-list opens do not steal that focus. No TEPP theta is invented. + +## [1.2.0] - 2026-08-17 + +### Added + +- Opening **Public post** from the landed Demo Corp report now names + the next action: that post is open from Demo Corp, so read Event + Lineage, Keyman, and evaluation. The opened member is current. Mean + θ stays on the report panel. No TEPP theta is invented. No cutoff + body is invented (ADR 0016). + +## [1.1.0] - 2026-08-17 + +### Added + +- Opening **Open period report 2026-W02** now puts the Demo Corp + report (mean θ and member posts) immediately under the named next + action, ahead of Other Corp and the week strip. The Public post + member stays clickable. Mean θ stays on the report panel. No TEPP + theta is invented. No cutoff body is invented (ADR 0016). + +## [1.0.0] - 2026-08-17 + +### Added + +- Opening **Open period report 2026-W02** now names the next action on + the landed Demo Corp report: read its mean θ and member posts, then + open a post. The focused comparison chip uses the visible + `Corporate entity: Demo Corp` caption and the persisted mean θ + (WCAG 2.5.3). Changing the week still focuses the report period + field. Mean θ stays on the report panel. No TEPP theta is invented. + +## [0.99.0] - 2026-08-17 + +### Added + +- Opening **Open period report 2026-W02** when the operator is already + on that week lands the grouping comparison strip on Demo Corp. The + Demo Corp chip is current and focused. Changing the week still + focuses the report period field. Mean θ stays on the report panel. + No TEPP theta is invented. + +## [0.98.1] - 2026-08-17 + +### Fixed + +- Authorized analysis-run list and detail queries are now complete + parameterized SQL literals. Semgrep no longer treats the visibility + predicate as string-concatenated user input. The $1 / $2 / $3 binds + are unchanged. No TEPP theta is invented. + +## [0.98.0] - 2026-08-17 + +### Added + +- Opening **Open period report 2026-W02** from a corporate-entity + analysis run also switches Report grouping to Corporate entity and + marks the Demo Corp grouping current. The opened report is named + Demo Corp, not a UUID. Mean θ stays on the report panel. No TEPP + theta is invented. + +## [0.97.0] - 2026-08-17 + +### Added + +- A Succeeded period-report analysis run now opens the scored week. + After `make seed`, open **Period report · Succeeded · Demo Corp** + and click **Open period report 2026-W02**: the report period field + is focused on that week. Failed rows stay closed. Mean θ stays on + the report panel. No TEPP theta is invented. + +## [0.96.0] - 2026-08-17 + +### Added + +- `make seed` now records **Period report · Succeeded · Demo Corp** on + the shared snapshot after the calibrated report tables are written + (ADR 0024). Open that row to confirm the cutoff posts. Mean θ stays + on the period-report panel. Start stays 422. No TEPP theta is + invented. + +## [0.95.0] - 2026-08-17 + +### Added + +- Analysis-run detail now lists labeled outbox delivery: Claimed then + Delivered (ADR 0023). After `make seed`, open the Demo Corp lineage + run to see those times. Stream entry ids stay off the payload. No + TEPP theta is invented. + +## [0.94.0] - 2026-08-17 + +### Added + +- `POST /api/analysis-runs/{id}/start` now commits Running plus one + durable outbox row, wakes Valkey (`analysis-run-outbox`), then + delivers ThreadWeave or `tepp_client` (ADR 0023). A crash after + Start leaves the work item; refresh finishes it. Period-report + stays 422. No TEPP theta is invented. + +## [0.93.0] - 2026-08-17 + +### Added + +- `make seed` now persists the designed A-100 fork on the Demo Corp + Succeeded lineage run. Open that run: the revised quote and delivery + question follow the pricing follow-up and are buttons. Start is + unchanged. No TEPP theta is invented. + +## [0.92.0] - 2026-08-17 + +### Added + +- **Start TEPP measurement** on a Pending TEPP row submits + `AnalysisRunRequest` through `tepp_client` (ADR 0022). A missing + `TEPP_TRANSPORT_URL` or a refused URL is Failed / + `tepp_not_available`. An accepted envelope is Failed / + `tepp_result_not_persisted`. Failed stays terminal: **Request a new + TEPP measurement** records a new Pending run. Period-report start + stays 422. No TEPP theta is invented. + +## [0.91.0] - 2026-08-17 + +### Added + +- After **Start reconstruction**, the titled A-100 edges are buttons. + Click the revised-quote child to open the live post; click the + pricing-follow-up parent to open that post. A child marked + **Updated after cutoff** still shows the live-body warning. The + popup does not invent a cutoff snapshot. No TEPP theta is invented. + +## [0.90.0] - 2026-08-17 + +### Added + +- Opening an analysis-run title marked **Updated after cutoff** now + shows a popup status that the body is live, not a cutoff snapshot + (ADR 0016). After `make seed`, open the Demo Corp lineage run and + click Demo public post: the warning appears above the live body + and tells you to compare it with this run. Demo private post and + the home post list do not. The popup does not invent the earlier + text. No TEPP theta is invented. + +## [0.89.0] - 2026-08-17 + +### Added + +- Analysis-run detail now compares each in-cutoff title's live + `updated_at` with that run's knowledge cutoff. After `make seed`, + open the Demo Corp lineage run: Demo public post is marked + **Updated after cutoff**; Demo private post is not. Opening a + marked title still shows the live body -- cutoff body versioning + stays a later slice (ADR 0016). The list stays aggregates-only. + No TEPP theta is invented. + +## [0.88.0] - 2026-08-16 + +### Added + +- `POST /api/analysis-runs/{id}/start` runs ThreadWeave on a visible + Pending lineage cutoff bag and persists run-scoped parent choices + (ADR 0021). Open the Pending Demo Corp row, then start reconstruction. + The designed A-100 fork (revised quote and delivery question under the + pricing follow-up) is the acceptance tree. TEPP and period-report + start are 422 — this path does not invent a theta. A Succeeded retry + returns the stored digest. A Running restart is 409. Create freezes + authorized post ids so start cannot pick up a later backfill. Live + Event Lineage stays a separate rebuild. + +## [0.87.0] - 2026-08-16 + +### Added + +- Operators can empty a run-bearing analysis-run registry without a + superuser trigger disable. Insert an unrevoked + `analysis_run_retention_grant` for `session_user`, grant + `analysis_run_retention_admin`, then + `select purge_analysis_run_registry('approved-retention-purge')` + (ADR 0020). Export `analysis_run_retention_event`, delete those + rows, then roll back 0020 and 0018. A raw `DELETE`, a published + token without a grant, and a runtime role that is not the admin + role still fail. +- Repeated citation chips and close buttons use named design tokens + in `frontend/src/styles/tokens.css`. Preview them in Storybook + (`cd frontend && pnpm run storybook`). + +## [0.86.2] - 2026-08-16 + +### Fixed + +- An R&R organization button now walks the catalog id stored on that + role row (ADR 0019). Two catalog orgs can share a display name; open + the post, click the name, and you stay on the resolved org — not a + homonym. `GET /api/teams/{id}/related` matches person/entity authz: + another corp's private-only team is 403; an unknown UUID is 404. + +## [0.86.1] - 2026-08-16 + +### Changed + +- Opening a post or its evidence panel now shows each embedded + `data:image` picture in document order, with the surrounding sentences + as text. The raw base64 string is no longer dumped into the popup. + Remote `http(s)` image URLs stay unloaded. After `make seed`, a post + whose body includes a data-URI image shows the picture; Extract Keyman + or Ask still runs OCR on that image for search. + +### Fixed + +- Opening a Pending lineage run repeats that reconstruction has not + started. Pending next-action copy is pinned to the registered run + kinds, so a Pending TEPP row does not say reconstruction. + +## [0.86.0] - 2026-08-16 + +### Added + +- Related-node walks now include team and organization mention edges. + After `make seed` and a summary that names 설계팀 on two posts, open + either post, click the R&R team, and open the sibling post (ADR 0018). + A team-only follow-up is no longer an island. +- `GET /api/teams/{team_id}/related` starts the same RWR walk Keyman + and corporate-entity related already use. Related team chips are + buttons. + +### Fixed + +- Thread-group analysis-run *lists* now require an in-cutoff visible + post. A later public post in that thread group no longer surfaces a + January run the account was not allowed to know. +- Failed period-report rows tell the operator to rebuild the report. + Next-action copy is pinned to the registered run kinds. A pending + TEPP corpus does not claim a calibrated measurement. + +## [0.85.0] - 2026-08-16 + +### Added + +- `POST /api/analysis-runs` records a Pending lineage or TEPP run on an + authorized cutoff capture (ADR 0017). The home panel's **Request a + lineage reconstruction** button writes that row so an operator can + confirm the cutoff corpus immediately. Reconstruction and live TEPP + execution stay later slices — this write never invents a theta. +- Failed lineage rows tell the operator to retry reconstruction; only + Failed TEPP rows mention the measurement service. Pending rows say + reconstruction has not started yet. + +## [0.84.1] - 2026-08-16 + +### Fixed + +- Analysis-run detail keeps 12-character digest prefixes as visible + text (so assistive technology hears `Code` / `Config` values) and + puts the full digest on hover. Open the Demo Corp lineage run, hover + a prefix, and match it to the API payload. The home list still hides + digests even when the list JSON includes them. +- Opening a cutoff title now says the live body may have changed after + that run. Compare the opened post with the cutoff date before you + treat it as reconstructed evidence (ADR 0016). + +## [0.84.0] - 2026-08-16 + +### Added + +- `make seed` records a Demo Corp TEPP measurement run through + `tepp_client` on the same snapshot as the lineage run (ADR 0013). + The default transport is unavailable, so the home list shows + "TEPP measurement · Failed · Demo Corp" and tells the operator to + open the run, then connect the measurement service. Detail history + keeps `tepp_not_available` -- never a fabricated theta. TEPP stays + a wire client, not a local psychometric engine. `make seed` skips + snapshot-count inserts once counts exist so a re-run does not hit + the freeze trigger. A failed lineage row tells the operator to retry + reconstruction; only a failed TEPP row mentions the measurement + service. A failed period-report row tells the operator to rebuild + the report from a current snapshot. A pending TEPP row does not + claim a calibrated measurement. Stacked PRs now run the same + GitHub Checks as PRs to main. + +## [0.83.0] - 2026-08-16 + +### Fixed + +- Analysis-run detail now lists only ABAC-visible posts whose + `created_at` is at or before that run's `knowledge_cutoff`. After + `make seed`, open the Demo Corp lineage run: Demo public post is + there; a later own-corp follow-up is not. The live post list is + unchanged. Click a listed title to inspect what that cutoff + reconstructed (ADR 0016). +- Upgrading through `0016_cross_post_actor_identity.sql` copies R&R + person names into `post_summary_person_mention` and leaves Keyman + `post_person_mention.mention_context` in place. Re-run Keyman only + when you want a new Keyman set -- a later summary no longer erases + the stolen row. + +## [0.82.0] - 2026-08-16 + +### Added + +- Analysis-run detail lists ABAC-visible posts in the run's scope. + After `make seed`, the Demo Corp lineage run opens the Demo public + post. Hidden other-corp private posts never appear. List payloads + stay aggregates-only. + +## [0.81.0] - 2026-08-16 + +### Added + +- Analysis-run detail shows the labeled lifecycle: Pending, Running, + then Succeeded, with occurrence times from `analysis_run_status_event`. + The list stays latest-status only. Hidden runs still 404 and never + leak events. Failure codes stay machine tokens -- no invented label. + Synthetic Demo Corp seed only. + +## [0.80.0] - 2026-08-16 + +### Added + +- Home Analysis runs rows are buttons. Clicking the seeded Demo Corp + lineage run opens `GET /api/analysis-runs/{id}` and shows cutoff, + requested date, and document count. A hidden run is "This analysis + run is not visible." -- never a raw 404 or a DSN. Still synthetic + aggregates only. + +## [0.79.0] - 2026-08-16 + +### Added + +- Authorized analysis-run evidence on the product home page. After + `make seed`, Demo Analyst sees "Lineage reconstruction · Succeeded · + Demo Corp" with the synthetic document count. `GET /api/analysis-runs` + is scoped in SQL: another tenant's run 404s and never appears in the + list. The payload is labels and aggregates -- never source SQL, a DSN, + or a raw record. TEPP stays behind `tepp_client`; Null channels are + unchanged. + +## [0.78.0] - 2026-08-15 + +### Changed + +- Related-node person chips now use the localized `person_side` lookup label + supplied by the authorized API payload. Users see business context such as + `Our side` or `Counterparty`, while ontology class metadata remains available + separately for semantic processing and provenance. +- Related-person buttons now expose that same caption in the accessible name, so + assistive technology hears `Related nodes for Priya Nair (Counterparty)` + instead of the name alone. +- Structured extraction, summarization, commitment, relationship-classification, + and LLM-as-a-Judge consumers now request contextual-orchestrator `auto` mode + so the orchestration plane can meet the quality requirement and then minimize + known execution cost. Explicit checked `verify` paths remain unchanged + (ADR 0015). + +## [0.77.0] - 2026-08-14 + +### Fixed + +- Keyman and R&R person mentions now replace independent source projections. Knowledge Graph edges have one canonical identity plus post-level evidence, so removed actors and concurrent writes cannot leave stale or duplicate buyer-visible relationships. +- Vision-response parsing now strips balanced outer Markdown emphasis from field values + while still accepting emphasized field labels, so OCR such as + ``TEXT: **LT7**`` is not truncated. +- A real live synthetic regression batch run surfaced a genuine + `DeadlockDetectedError` from concurrent corporate-entity creation: + two concurrent transactions each creating a different new entity, + mentioned in opposite order across two different posts, took + row-level locks in opposite order and deadlocked. Entity *creation* + (the rare, first-mention-only branch) now serializes through a + single named Postgres advisory transaction lock, taken only right + before the write and auto-released at commit/rollback -- the + lock-free similarity-matching fast path every already-cataloged + entity resolves through is unaffected. See ADR 0012. + +## [0.76.0] - 2026-08-14 + +### Added + +- Standards-complete W3C PROV-O support: all 30 classes, all 50 + normative properties, both qualification tables, qualified-to- + unqualified implication, property hierarchy, defined inverses, + Appendix B inverse-name normalization, RDF serialization, and a + normalized PostgreSQL assertion store with fail-closed domain, + range, object-kind, and datatype enforcement (ADR 0011). +- A dedicated exact-head PROV-O contract workflow runs the complete + registry/inference suite, real PostgreSQL migration tests, public + docstring checks, and 100% statement/branch coverage for the owned + runtime module. + +### Changed + +- The product navigation graph remains an explicit projection; + literal-valued and qualified provenance is no longer forced into + `knowledge_graph_edge`. + +### Fixed + +- Review hardening verifies complete hierarchy placement, rejects parent + failures and cycles, propagates canonical affiliations, replaces stale + actor projections, enforces atomic team identity, validates timezone-aware + `xsd:dateTime` literals (including the XSD `±14:00` offset bound), and + protects referenced provenance rows. + +## [0.75.0] - 2026-08-14 + +### Added + +- A real counterparty organization mentioned for the first time now + gets auto-created into the corporate hierarchy, not left permanently + unresolved. Synthetic regression fixtures prove the first-mention gap. + An LLM proposes a + Group/Company/Plant placement from context; a real new + `corporate_entity` row is only created once the proposal is + search-corroborated (reusing the existing Searxng verification + client, no new search integration). Auto-created rows get a + deterministic `AUTO-`-prefixed code, kept structurally separate from + the real login corp-code namespace. Wired into both Keyman + affiliation resolution and R&R organization-actor resolution. + +## [0.74.0] - 2026-08-14 + +### Added + +- R&R team and organization actors now get a shared identity across + posts, not just per-post free text -- the same "설계팀" (design + team) named in ten posts resolves to one `cataloged_team` row + (identity key: team name + parent org, since a bare team name is not + by itself identifying), and an organization actor resolves against + the existing `corporate_entity` catalog. Each resolved actor gets a + real Knowledge Graph mention edge (`edge_mention_team`, + `edge_team_affiliation`, `edge_mention_organization`), so extraction + results now genuinely become cross-post lineage clues instead of + per-post islands. A person R&R actor is opportunistically joined to + an existing Keyman-cataloged person by name (documented gap: R&R does + not yet originate new person identities itself -- see ADR 0009). + +## [0.73.0] - 2026-08-14 + +### Fixed + +- Image OCR/caption parsing no longer discards a real vision response + just because its formatting was close but not exact (bolded labels + like `**TEXT:**`, reordered labels, or a missing TAGS line) -- + observed live against synthetic embedded-image fixtures in the synthetic regression batch + in format-variation fixtures. Fields are now recovered independently; only a + response with neither TEXT nor CAPTION content is treated as + unusable. A strict format mismatch was silently producing the same + "[image: content unavailable]" placeholder as a genuinely unavailable + vision channel, discarding real, already-paid-for content. + +## [0.72.0] - 2026-08-14 + +### Added + +- Abbreviated/slang organization names (e.g. "AGP") are now resolved + to their canonical name ("Aurora Grid Power") via LLM context, then + cross-verified against external search before being trusted -- new + `lineageweave/organization_name_resolution.py`, reusing the existing + Searxng verification client rather than a second web-search + integration. Cached in a new `organization_name_resolution` table + keyed by the raw name. +- Wired into Keyman affiliation ingestion (both the API path and the + offline synthetic-batch script): a search-corroborated resolution feeds + `resolve_corporate_entity`, an unverified one leaves the raw name + unchanged. + +### Fixed + +- The offline synthetic-batch script's own re-implementation of Keyman + affiliation persistence was missing `role_title` entirely (a stale + copy that predated that feature) -- fixed alongside this change. ## [0.71.0] - 2026-08-14 @@ -34,6 +863,61 @@ All notable changes to this project are documented here. Format follows above the names. Unassigned excerpts stay in the list; a post with no named organization still says so. +## [0.70.0] - 2026-08-14 + +### Added + +- R&R's `actor_type_code` gains `prov_team`, a meso-level actor type for + a named sub-unit of a company (e.g. "설계팀"/design team) -- distinct + from both a person and the company itself. Grounded in the W3C + Organization Ontology's `org:OrganizationalUnit` (Reynolds, 2014). +- A team actor now requires `affiliated_organization_name` in the same + way a person actor does -- a team's own name never answers "which + company." +- R&R badge shows a distinct "Team" label/color, not the prior binary + Person/Organization ternary (which would have mislabeled a team as + "Organization"). + +## [0.69.0] - 2026-08-14 + +### Added + +- Keyman extraction now captures a stated job title/position + (`PersonMention.job_title`), since two different real people can + share a name and a title is real evidence for telling them apart. + Persisted to `person_affiliation.role_title` (an existing schema + column, previously never populated) and a new + `cataloged_person.last_known_job_title` for a title stated without a + named organization to attach it to. +- `_upsert_person` no longer blindly merges a same-name+side match: a + genuinely conflicting stated title creates a fresh person row instead + of reusing one, verified by a real test with two posts naming the + same name and different titles. +- Keyman panel shows the person's title next to their name. After + `make seed`, Ada West is "Account manager" and Priya Nair is + "Procurement lead" so the title is visible without a live LLM. + +### Fixed + +- `scripts/seed_demo_data.py` no longer embeds the local Keycloak admin + password. `make seed` still supplies the compose default via + `KEYCLOAK_ADMIN_PASSWORD`; a direct script run requires that env var + or `--keycloak-admin-password`. + +## [0.68.0] - 2026-08-14 + +### Changed + +- R&R's named actor is no longer forced into a person slot. A + business post can name an organization acting in its own name + ("당사," "Demo Corp"), not an individual -- + `RoleResponsibility.actor_name` (renamed from `person_name`) now + carries `actor_type_code` (Person / Organization, W3C PROV-O + grounded) and an LLM-inferred `affiliated_organization_name` for + person actors. The popup's R&R list shows a Person/Organization + badge and the inferred affiliation; only a person actor still links + to the Keyman panel. See ADR 0006. + ## [0.67.0] - 2026-08-14 ### Added @@ -1117,7 +2001,7 @@ All notable changes to this project are documented here. Format follows embedding, against the live embedding provider. - `docs/lineage-bi-research-notes.md`: new "Chunking" section with the four units' grounding and an explicit, honest note that this project's - real dataset's only free-text field is too short to need chunking in + unseen dataset's only free-text field is too short to need chunking in practice -- the module exists for richer content sources (e.g. the raw MHTML artifacts that dataset's records were derived from). - `lineageweave/image_content.py`: pluggable vision channel for base64 diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 00000000..08e3682a --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,90 @@ +# CLAUDE.md + +Tool-specific pointer. Policy lives in [AGENTS.md](AGENTS.md) and the +ADRs under `docs/adr/`. Do not fork those rules here. + +## Analysis-run retention (v0.87.0) + +To empty a run-bearing registry, insert an unrevoked +`analysis_run_retention_grant` for `session_user` and +`GRANT analysis_run_retention_admin` (ADR 0020). Then +`select purge_analysis_run_registry('approved-retention-purge')`, +export `analysis_run_retention_event`, delete those rows, and roll +back 0020 then 0018. The same call empties reconstruction children +when ADR 0021 tables exist (ADR 0032). The published phrase is not a +secret. Do not `DISABLE TRIGGER` as superuser. Do not grant the admin +role or a retention grant to the application `DATABASE_URL` login. +ADR 0019 is the R&R catalog-id bind, not this purge. Person catalog +identity on that role row is ADR 0027 (`cataloged_person_id`). + +## Analysis-run seed (v0.96.0) + +`make seed` writes a Demo Corp lineage run, a Failed missing-transport +TEPP run, a Failed accepted-evidence TEPP run, and a Succeeded +period-report run on the same snapshot (ADR 0013 / ADR 0024 / ADR 0035). +The TEPP path goes through `tepp_client`. A missing transport or an +unpublished envelope is Failed (`tepp_not_available` / +`tepp_result_not_persisted`). A published accepted acknowledgement is +Failed (`tepp_completed_result_unsupported`) and is shown as aggregate +transport evidence. Do not stamp Succeeded from that ack. Do not invent +a theta or a local psychometric substitute. Measurement evidence shows +Received, and recorded only when that row-write instant differs. +The home list caption stays `kind · status · entity`; the machine +failure code is detail-only (ADR 0014). Open a Failed TEPP row, then +connect a live TEPP transport or read aggregate transport evidence. +Do not treat that row as a validated multilevel estimate. A failed lineage row retries reconstruction -- it does not +mention TEPP. A failed period-report row rebuilds the report. A +pending TEPP row does not claim a calibrated measurement and does +not say reconstruction. The list button name includes the +next-action sentence. A pending lineage row says reconstruction has +not started yet. +Digest prefixes stay audible; hover a prefix to read the full digest. +Opening a cutoff title shows the live post. Titles marked updated +after cutoff were rewritten after the run; the opened body names +both clocks and shows **Body this run knew** beside the live +rewrite. Compare those two texts before treating the live body as +reconstructed evidence (ADR 0016 / 0025). +The January 12 Demo Corp lineage and TEPP runs list Demo public post +and do not list Late Demo public post (2026-01-13). The live post +list still shows Late Demo. +`POST /api/analysis-runs` records Pending lineage only on an +authorized cutoff capture (ADR 0017). TEPP and period-report kinds +are 422. The Request button waits until affiliated corps load; choose +a corp if the token walks more than one. `POST /api/analysis-runs/{id}/start` +commits Running plus a durable outbox row, then reconstructs that +frozen cutoff bag (ADR 0021 / ADR 0023) or submits TEPP through +`tepp_client` (ADR 0022 / ADR 0035). A missing transport or unpublished +envelope is Failed. A published accepted acknowledgement is Failed +transport evidence, not a completed measurement. Failed TEPP is +terminal — connect a TEPP transport from that Failed row or read the +stored evidence. Create does not invent a Pending +TEPP row. Do not invent a theta. Hover the Result prefix to read +the parent-choice digest. +After `make seed`, open **Period report · Succeeded · Demo Corp**, +then **Open period report 2026-W02**. The home week is already +2026-W02, so the grouping comparison strip lands on Demo Corp. Report +grouping is Corporate entity and Demo Corp is current. The focused +chip name contains `Corporate entity: Demo Corp` and the persisted +mean θ. The period-report panel says Demo Corp is the opened grouping +and to read its mean θ and member posts, then open a post. Those +members land immediately under that next action, ahead of Other Corp +and the week strip. Opening Public post names the next action: read +Event Lineage, Keyman, and evaluation on that post. The popup Event +Lineage DAG marks that post current. After that current node, the +popup names Keyman and evaluation as the next read. After landed +evaluation, the popup names the first Keyman as the next read. After +landed Ada West related, the popup names the first related node as +the next read. After that next action, the popup lands Priya Nair +related nodes. After those related nodes land, the popup names Ask +about this lineage as the next read. After that next action, the +popup lands Ask about this lineage. After landed chat, the popup names +the first Ask. After that next action, the popup lands the first Ask +answer. After landed first Ask answer, the popup names the first +cited source. After that next action, the popup lands the first cited +evidence. Changing the week first still +focuses the report period field. Mean θ stays on the period-report +panel. +A listed analysis-run that then 404s stays generic: do not name the +thread or the cutoff. After that 404, re-read the authorized list so +the stale row does not stay clickable. Announce the next action with +`role="alert"` without moving focus. diff --git a/Makefile b/Makefile index 68ee850e..82aa30fa 100644 --- a/Makefile +++ b/Makefile @@ -23,4 +23,5 @@ smoke: # users' real subject ids, plus Valkey ticket_created events so Activity # is not empty (see scripts/seed_demo_data.py). Run after `up`. seed: + @test -n "$${KEYCLOAK_ADMIN_PASSWORD:-}" || { echo "KEYCLOAK_ADMIN_PASSWORD is required" >&2; exit 1; }; \ python3 scripts/seed_demo_data.py diff --git a/README.md b/README.md index a3626d9f..c143f84f 100644 --- a/README.md +++ b/README.md @@ -167,6 +167,14 @@ FastAPI backend over real `fetch()` with the token Keycloak issued. make up make seed cd frontend && cp .env.example .env.local && pnpm install && pnpm run dev +# Repeated chip/close controls: pnpm run storybook +# (Node 24 via frontend/mise.toml; pnpm only) +# Empty a run-bearing registry: insert analysis_run_retention_grant +# for session_user, GRANT analysis_run_retention_admin, then +# select purge_analysis_run_registry('approved-retention-purge'). +# The published token is not a grant (ADR 0020). After a Succeeded +# start, that same call also empties reconstruction children +# (ADR 0032). # -> http://localhost:5173, click "Log in", redirects through the real # Keycloak login page for demo.analyst / lineageweave-demo-only ``` diff --git a/backend/Dockerfile b/backend/Dockerfile index b0c504cc..eb6b8628 100644 --- a/backend/Dockerfile +++ b/backend/Dockerfile @@ -1,33 +1,37 @@ FROM python:3.12-slim@sha256:229a2c5bfa27522db7815ea81f9bed70af17ccb9de9fc7ad142b1877b5830d36 WORKDIR /app -# rankweave and fast-mlsirm install from a git URL (see pyproject.toml) -- -# neither has a PyPI release yet. fast-mlsirm additionally ships a -# PyO3/maturin Rust core with no fallback wheel (ADR 0003), so this build -# needs a real Rust toolchain, not just Python -- build-essential supplies -# the C linker maturin needs on Linux. Create the runtime user here so the -# later USER instruction is not a no-op against a missing account -# (DS-0002: explicit non-root USER). -RUN apt-get update && apt-get install -y --no-install-recommends git build-essential curl ca-certificates \ +# Hash-pinned bootstrap input is copied before dependency installation so the +# build cannot resolve a different uv artifact for the same version. +COPY backend/uv-bootstrap-requirements.txt /tmp/uv-bootstrap-requirements.txt + +# rankweave and fast-mlsirm install from immutable git commit references. +# fast-mlsirm builds a PyO3/maturin core, so the image needs a C linker and +# the repository-pinned Rust toolchain. The runtime user is created before +# dependencies so no application process runs as root. +RUN apt-get update && apt-get install -y --no-install-recommends \ + git build-essential curl ca-certificates \ && rm -rf /var/lib/apt/lists/* \ && groupadd --gid 1000 appuser \ - && useradd --uid 1000 --gid appuser --create-home appuser + && useradd --uid 1000 --gid appuser --create-home appuser \ + && python -m pip install --no-cache-dir --no-deps \ + --require-hashes --only-binary=:all: \ + -r /tmp/uv-bootstrap-requirements.txt -# Pinned, non-interactive rustup install (minimal profile: no docs/clippy, -# just rustc+cargo) -- same "pinned, reproducible install" discipline this -# project already applies to rankweave/fast-mlsirm's own commit pins. RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | \ sh -s -- -y --profile minimal --default-toolchain 1.97.1 -ENV PATH="/root/.cargo/bin:${PATH}" +ENV PATH="/app/.venv/bin:/root/.cargo/bin:${PATH}" -COPY pyproject.toml ./ +COPY pyproject.toml uv.lock README.md ./ COPY lineageweave ./lineageweave COPY backend ./backend # lineageweave/ontology.py resolves this path relative to itself # (parents[1] = /app) -- ADR 0004. COPY docs/ontology ./docs/ontology -RUN pip install --no-cache-dir ".[backend]" \ +# Install exactly the committed universal lock. --no-editable prevents a +# runtime dependency on source-tree editability while retaining package data. +RUN uv sync --frozen --no-dev --extra backend --no-editable \ && chown -R appuser:appuser /app USER appuser diff --git a/backend/app/abbreviation_tree_corroboration_ingestion.py b/backend/app/abbreviation_tree_corroboration_ingestion.py new file mode 100644 index 00000000..e8e7d6e9 --- /dev/null +++ b/backend/app/abbreviation_tree_corroboration_ingestion.py @@ -0,0 +1,149 @@ +"""Persist Searxng abbreviation matches against the authorized tree.""" + +from __future__ import annotations + +import asyncio +from typing import Any + +import asyncpg + +from lineageweave.abbreviation_tree_corroboration import ( + AbbreviationTreeMatch, + TreeEntityCandidate, + abbreviation_candidates, + corroborate_abbreviation_against_tree, +) +from lineageweave.customer_group_tree import CatalogEntityRow, authorized_catalog_ids +from lineageweave.relation_verification import RelationVerificationClient + + +async def collect_post_organization_names(conn: asyncpg.Connection, post_id: str) -> tuple[str, ...]: + """Organization strings already extracted onto this post. + + Keyman affiliations and classified counterparties are the mentions + operators can see. This path does not invent a name from post text. + """ + rows = await conn.fetch( + """ + select distinct name from ( + select pa.affiliated_organization_name as name + from post_person_mention ppm + join person_affiliation pa on pa.person_id = ppm.person_id + where ppm.post_id = $1 + union + select c.counterparty_entity_name as name + from post_counterparty_entity c + where c.post_id = $1 + ) mentioned + where name is not null and btrim(name) <> '' + order by name + """, + post_id, + ) + return tuple(row["name"] for row in rows) + + +async def load_authorized_tree_candidates( + conn: asyncpg.Connection, + affiliated_entity_ids: list[str], +) -> tuple[TreeEntityCandidate, ...]: + """Catalog nodes the account may corroborate an abbreviation against.""" + entity_rows = await conn.fetch( + """ + select corporate_entity_id, parent_entity_id, entity_name, entity_level_code + from corporate_entity + """ + ) + entities = tuple( + CatalogEntityRow( + entity_id=str(row["corporate_entity_id"]), + parent_entity_id=str(row["parent_entity_id"]) if row["parent_entity_id"] is not None else None, + entity_name=row["entity_name"], + entity_level_code=row["entity_level_code"], + ) + for row in entity_rows + ) + needed = authorized_catalog_ids(entities, affiliated_entity_ids) + return tuple( + TreeEntityCandidate(entity_id=row.entity_id, entity_name=row.entity_name) + for row in entities + if row.entity_id in needed + ) + + +async def persist_abbreviation_tree_match( + conn: asyncpg.Connection, + match: AbbreviationTreeMatch, +) -> None: + """Upsert one raw mention's tree-constrained Searxng outcome.""" + await conn.execute( + """ + insert into abbreviation_tree_corroboration + (raw_organization_name, corporate_entity_id, + verification_status_code, verification_evidence_url) + values ($1, $2, $3, $4) + on conflict (raw_organization_name) do update set + corporate_entity_id = excluded.corporate_entity_id, + verification_status_code = excluded.verification_status_code, + verification_evidence_url = excluded.verification_evidence_url, + corroborated_at = now() + """, + match.raw_organization_name, + match.corporate_entity_id, + match.verification_status_code, + match.verification_evidence_url, + ) + + +async def fetch_post_abbreviation_matches( + conn: asyncpg.Connection, + post_id: str, +) -> list[dict[str, Any]]: + """Cached tree matches for organization names already on this post.""" + names = await collect_post_organization_names(conn, post_id) + if not names: + return [] + rows = await conn.fetch( + """ + select raw_organization_name, corporate_entity_id, + verification_status_code, verification_evidence_url + from abbreviation_tree_corroboration + where raw_organization_name = any($1::text[]) + order by raw_organization_name + """, + list(names), + ) + return [ + { + "raw_organization_name": row["raw_organization_name"], + "corporate_entity_id": ( + str(row["corporate_entity_id"]) if row["corporate_entity_id"] is not None else None + ), + "verification_status_code": row["verification_status_code"], + "verification_evidence_url": row["verification_evidence_url"], + } + for row in rows + ] + + +async def corroborate_post_abbreviations( + conn: asyncpg.Connection, + verification_client: RelationVerificationClient, + post_id: str, + affiliated_entity_ids: list[str], +) -> list[AbbreviationTreeMatch]: + """Run Searxng against the authorized tree for this post's mentions.""" + names = await collect_post_organization_names(conn, post_id) + candidates = await load_authorized_tree_candidates(conn, affiliated_entity_ids) + to_check = abbreviation_candidates(names, candidates) + matches: list[AbbreviationTreeMatch] = [] + for raw_name in to_check: + match = await asyncio.to_thread( + corroborate_abbreviation_against_tree, + raw_name, + candidates, + verification_client, + ) + await persist_abbreviation_tree_match(conn, match) + matches.append(match) + return matches diff --git a/backend/app/analysis_run_ingestion.py b/backend/app/analysis_run_ingestion.py new file mode 100644 index 00000000..9b903ea8 --- /dev/null +++ b/backend/app/analysis_run_ingestion.py @@ -0,0 +1,1001 @@ +"""Authorized, source-redacting reads of the Milestone 2 analysis-run registry. + +The registry itself is issue #89 / migration 0018. This module is the +product projection: an account sees only runs they requested or whose +scope they already have ABAC authority to walk. Aggregate counts and +lookup labels come back; source SQL, DSNs, raw records, and provider +payloads never do. + +``create_pending_analysis_run`` (ADR 0017) writes snapshot, counts, frozen +membership, run, scope, and the first Pending event atomically. It +records lineage only. It does not reconstruct lineage, accept a TEPP +kind, or invent a score. ``enqueue_pending_analysis_run`` then +``deliver_queued_analysis_run`` later reconstruct lineage (ADR 0021 / +ADR 0023) or submit TEPP through ``tepp_client`` (ADR 0022 / ADR 0035). +A published accepted acknowledgement is stored as aggregate transport +evidence; neither path invents a TEPP score or stamps Succeeded from +that ack. +""" + +from __future__ import annotations + +import hashlib +import json +from dataclasses import dataclass +from datetime import datetime, timezone +from typing import Any +from uuid import UUID + +import asyncpg + +from backend.app.knowledge_graph import labels_for_codes +from lineageweave import __version__ as PACKAGE_VERSION +from lineageweave.tepp_result import tepp_accepted_evidence_sha256 + +_LINEAGE_RUN_KIND = "analysis_run_lineage" +_TEPP_RUN_KIND = "analysis_run_tepp" +_REPORT_RUN_KIND = "analysis_run_report" +_CORPORATE_SCOPE = "analysis_scope_corporate_entity" +_CAPTURE_CONTRACT_VERSION = "analysis-run-capture-v1" +_KIND_SCHEMA_VERSION = { + "analysis_run_lineage": "lineage-run-v1", + "analysis_run_tepp": "tepp-run-v1", +} + +_RUN_LIST_SQL = """ + select + run.analysis_run_id, + run.run_kind_code, + run.knowledge_cutoff, + run.requested_at, + run.configuration_schema_version, + run.configuration_sha256, + run.code_revision_sha, + scope.scope_kind_code, + scope.corporate_entity_id, + scope.process_unit_id, + scope.scope_key, + corp.entity_name as scope_entity_name, + status.status_code, + status.failure_code + from analysis_run run + join analysis_run_scope scope on scope.analysis_run_id = run.analysis_run_id + left join analysis_run_current_status status + on status.analysis_run_id = run.analysis_run_id + left join corporate_entity corp + on corp.corporate_entity_id = scope.corporate_entity_id + where + run.requested_by_account_id = $1 + or ( + scope.scope_kind_code = 'analysis_scope_corporate_entity' + and scope.corporate_entity_id = any($2::uuid[]) + ) + or ( + scope.scope_kind_code = 'analysis_scope_process_unit' + and exists ( + select 1 from account_affiliation aff + where aff.user_account_id = $1 + and aff.process_unit_id = scope.process_unit_id + ) + ) + or ( + scope.scope_kind_code = 'analysis_scope_thread_group' + and exists ( + select 1 from source_post p + where p.thread_group_key = scope.scope_key + and p.created_at <= run.knowledge_cutoff + and ( + p.visibility_code = 'public' + or p.corporate_entity_id = any($2::uuid[]) + ) + ) + ) + order by run.requested_at desc +""" + +_RUN_DETAIL_SQL = """ + select + run.analysis_run_id, + run.run_kind_code, + run.knowledge_cutoff, + run.requested_at, + run.configuration_schema_version, + run.configuration_sha256, + run.code_revision_sha, + scope.scope_kind_code, + scope.corporate_entity_id, + scope.process_unit_id, + scope.scope_key, + corp.entity_name as scope_entity_name, + status.status_code, + status.failure_code + from analysis_run run + join analysis_run_scope scope on scope.analysis_run_id = run.analysis_run_id + left join analysis_run_current_status status + on status.analysis_run_id = run.analysis_run_id + left join corporate_entity corp + on corp.corporate_entity_id = scope.corporate_entity_id + where run.analysis_run_id = $3 + and ( + run.requested_by_account_id = $1 + or ( + scope.scope_kind_code = 'analysis_scope_corporate_entity' + and scope.corporate_entity_id = any($2::uuid[]) + ) + or ( + scope.scope_kind_code = 'analysis_scope_process_unit' + and exists ( + select 1 from account_affiliation aff + where aff.user_account_id = $1 + and aff.process_unit_id = scope.process_unit_id + ) + ) + or ( + scope.scope_kind_code = 'analysis_scope_thread_group' + and exists ( + select 1 from source_post p + where p.thread_group_key = scope.scope_key + and p.created_at <= run.knowledge_cutoff + and ( + p.visibility_code = 'public' + or p.corporate_entity_id = any($2::uuid[]) + ) + ) + ) + ) + order by run.requested_at desc +""" + + +def scope_grouping_key(row: Any) -> str | None: + """Persist the reconstruct grouping key for the run's authorized scope. + + A corporate-entity report run stores the week on ``scope_key``. The + grouping that reconstruct and the period-report panel share is the + corporate entity (or process unit / thread group), never that week + label and never a theta. + """ + scope = row["scope_kind_code"] + if scope == "analysis_scope_corporate_entity" and row["corporate_entity_id"]: + return str(row["corporate_entity_id"]) + if scope == "analysis_scope_process_unit" and row["process_unit_id"]: + return str(row["process_unit_id"]) + if scope == "analysis_scope_thread_group" and row["scope_key"]: + return str(row["scope_key"]) + return None + + +def _iso(value: Any) -> str: + """Serialize a timestamptz the same way post payloads do.""" + return value.isoformat() if hasattr(value, "isoformat") else str(value) + + +def _as_utc(value: datetime) -> datetime: + """Treat a naive clock as UTC so cutoff comparison stays timezone-aware.""" + if value.tzinfo is None: + return value.replace(tzinfo=timezone.utc) + return value.astimezone(timezone.utc) + + +def live_write_after_cutoff(updated_at: datetime, knowledge_cutoff: datetime) -> bool: + """True when the live row was rewritten after the run's analysis clock. + + ``created_at <= knowledge_cutoff`` admits the title. ``updated_at`` is + the live write clock (ADR 0016). Equal times stay in-cutoff evidence. + """ + return _as_utc(updated_at) > _as_utc(knowledge_cutoff) + + +async def _tepp_accepted_by_run( + conn: asyncpg.Connection, + run_ids: list[str], +) -> dict[str, asyncpg.Record]: + """Load published TEPP accepted evidence for the given authorized runs. + + Missing ``analysis_run_tepp_accepted`` means migration 0029 is not + applied. Treat that as no stored transport evidence rather than 500. + Hidden runs never appear in ``run_ids``. + """ + if not run_ids: + return {} + try: + rows = await conn.fetch( + """ + select analysis_run_id, contract_version, accepted_run_id, + run_state, idempotency_key, evidence_sha256, + received_at, recorded_at + from analysis_run_tepp_accepted + where analysis_run_id = any($1::uuid[]) + """, + run_ids, + ) + except asyncpg.UndefinedTableError: + return {} + return {str(row["analysis_run_id"]): row for row in rows} + + +def project_tepp_transport_evidence(row: Any) -> dict[str, Any] | None: + """Project accepted evidence only when the stored digest recomputes. + + Counts, theta, topics, and completed-artifact identity stay omitted. + A digest mismatch fails closed so a substituted row is not shown. + """ + expected = tepp_accepted_evidence_sha256( + contract_version=int(row["contract_version"]), + accepted_run_id=str(row["accepted_run_id"]), + run_state=str(row["run_state"]), + idempotency_key=str(row["idempotency_key"]), + ) + stored = str(row["evidence_sha256"]) + if stored != expected: + return None + return { + "tepp_evidence_kind": "aggregate transport evidence", + "tepp_contract_version": int(row["contract_version"]), + "tepp_accepted_run_id": str(row["accepted_run_id"]), + "tepp_run_state": str(row["run_state"]), + "tepp_idempotency_key": str(row["idempotency_key"]), + "tepp_evidence_sha256": stored, + "tepp_received_at": _iso(row["received_at"]), + "tepp_recorded_at": _iso(row["recorded_at"]), + "tepp_completed_artifact_available": False, + } + + +async def _counts_by_run( + conn: asyncpg.Connection, + run_ids: list[str], +) -> dict[str, list[asyncpg.Record]]: + """Load aggregate snapshot counts for the given runs.""" + if not run_ids: + return {} + rows = await conn.fetch( + """ + select run.analysis_run_id, counts.count_type_code, counts.count_value + from analysis_run run + join analysis_source_count counts + on counts.analysis_source_snapshot_id = run.analysis_source_snapshot_id + where run.analysis_run_id = any($1::uuid[]) + order by counts.count_type_code + """, + run_ids, + ) + grouped: dict[str, list[asyncpg.Record]] = {} + for row in rows: + grouped.setdefault(str(row["analysis_run_id"]), []).append(row) + return grouped + + +async def _status_history( + conn: asyncpg.Connection, + analysis_run_id: str, +) -> list[dict[str, Any]]: + """Labeled append-only lifecycle for one already-visible run.""" + rows = await conn.fetch( + """ + select status_ordinal, status_code, occurred_at, failure_code + from analysis_run_status_event + where analysis_run_id = $1::uuid + order by status_ordinal + """, + analysis_run_id, + ) + labels = await labels_for_codes(conn, [row["status_code"] for row in rows]) + history: list[dict[str, Any]] = [] + for row in rows: + item: dict[str, Any] = { + "status_ordinal": int(row["status_ordinal"]), + "status_code": row["status_code"], + "status_label": labels.get(row["status_code"], row["status_code"]), + "occurred_at": _iso(row["occurred_at"]), + } + if row["failure_code"]: + item["failure_code"] = row["failure_code"] + history.append(item) + return history + + +async def fetch_outbox_deliveries( + conn: asyncpg.Connection, + analysis_run_id: str, +) -> list[dict[str, Any]]: + """Labeled claim/delivery events for one already-visible run. + + Missing outbox tables mean migration 0023 is not applied. Stream + entry ids stay off the payload -- they are not buyer evidence. + """ + try: + rows = await conn.fetch( + """ + select delivery_ordinal, delivery_status_code, occurred_at + from analysis_run_outbox_delivery + where analysis_run_id = $1::uuid + order by delivery_ordinal + """, + analysis_run_id, + ) + except asyncpg.UndefinedTableError: + return [] + labels = await labels_for_codes( + conn, + [row["delivery_status_code"] for row in rows], + ) + return [ + { + "delivery_ordinal": int(row["delivery_ordinal"]), + "delivery_status_code": row["delivery_status_code"], + "delivery_status_label": labels.get( + row["delivery_status_code"], + row["delivery_status_code"], + ), + "occurred_at": _iso(row["occurred_at"]), + } + for row in rows + ] + + +async def _serialize_runs( + conn: asyncpg.Connection, + rows: list[asyncpg.Record], +) -> list[dict[str, Any]]: + """Project registry rows into the authorized buyer-facing payload.""" + if not rows: + return [] + run_ids = [str(row["analysis_run_id"]) for row in rows] + count_rows = await _counts_by_run(conn, run_ids) + tepp_rows = await _tepp_accepted_by_run(conn, run_ids) + labels = await labels_for_codes( + conn, + [row["run_kind_code"] for row in rows] + + [row["scope_kind_code"] for row in rows] + + [row["status_code"] for row in rows if row["status_code"]] + + [ + count["count_type_code"] + for counts in count_rows.values() + for count in counts + ], + ) + payload: list[dict[str, Any]] = [] + for row in rows: + run_id = str(row["analysis_run_id"]) + kind = row["run_kind_code"] + scope = row["scope_kind_code"] + status = row["status_code"] + item: dict[str, Any] = { + "analysis_run_id": run_id, + "run_kind_code": kind, + "run_kind_label": labels.get(kind, kind), + "scope_kind_code": scope, + "scope_kind_label": labels.get(scope, scope), + "status_code": status, + "status_label": labels.get(status, status) if status else None, + "knowledge_cutoff": _iso(row["knowledge_cutoff"]), + "requested_at": _iso(row["requested_at"]), + "source_counts": [ + { + "count_type_code": count["count_type_code"], + "count_type_label": labels.get( + count["count_type_code"], count["count_type_code"] + ), + "count_value": int(count["count_value"]), + } + for count in count_rows.get(run_id, []) + ], + } + if row["scope_entity_name"]: + item["scope_entity_name"] = row["scope_entity_name"] + if row["scope_key"]: + item["scope_key"] = row["scope_key"] + grouping_key = scope_grouping_key(row) + if grouping_key: + item["scope_grouping_key"] = grouping_key + tepp = tepp_rows.get(run_id) + if tepp is not None: + projected = project_tepp_transport_evidence(tepp) + if projected is not None: + item.update(projected) + payload.append(item) + return payload + + +async def fetch_visible_analysis_runs( + conn: asyncpg.Connection, + account_id: str, + affiliated_entity_ids: list[str], +) -> list[dict[str, Any]]: + """Runs the account requested or whose scope they may already walk.""" + rows = await conn.fetch( + _RUN_LIST_SQL, + account_id, + affiliated_entity_ids, + ) + return await _serialize_runs(conn, rows) + + +async def fetch_visible_analysis_run( + conn: asyncpg.Connection, + analysis_run_id: str, + account_id: str, + affiliated_entity_ids: list[str], +) -> dict[str, Any] | None: + """One visible run, or None when it is missing or hidden.""" + rows = await conn.fetch( + _RUN_DETAIL_SQL, + account_id, + affiliated_entity_ids, + analysis_run_id, + ) + payload = await _serialize_runs(conn, rows) + if not payload: + return None + detail = payload[0] + row = rows[0] + detail["configuration_schema_version"] = row["configuration_schema_version"] + detail["configuration_sha256"] = row["configuration_sha256"] + detail["code_revision_sha"] = row["code_revision_sha"] + if row["failure_code"]: + detail["failure_code"] = row["failure_code"] + detail["status_history"] = await _status_history(conn, analysis_run_id) + detail["outbox_deliveries"] = await fetch_outbox_deliveries(conn, analysis_run_id) + detail["visible_posts"] = await fetch_visible_scope_posts( + conn, + row["scope_kind_code"], + row["corporate_entity_id"], + row["process_unit_id"], + row["scope_key"], + affiliated_entity_ids, + row["knowledge_cutoff"], + ) + digest, edges = await fetch_reconstructed_edges( + conn, + analysis_run_id, + affiliated_entity_ids, + ) + if digest is not None: + detail["reconstruction_result_sha256"] = digest + detail["reconstructed_edges"] = edges + return detail + + +def reconstructed_edge_is_visible( + *, + parent_visibility_code: str, + parent_corporate_entity_id: Any, + child_visibility_code: str, + child_corporate_entity_id: Any, + affiliated_entity_ids: list[str], +) -> bool: + """Hide an edge when either endpoint is outside the caller's ABAC bag.""" + affiliated = {str(entity_id) for entity_id in affiliated_entity_ids} + parent_visible = ( + parent_visibility_code == "public" + or str(parent_corporate_entity_id) in affiliated + ) + child_visible = ( + child_visibility_code == "public" + or str(child_corporate_entity_id) in affiliated + ) + return parent_visible and child_visible + + +async def fetch_reconstructed_edges( + conn: asyncpg.Connection, + analysis_run_id: str, + affiliated_entity_ids: list[str], +) -> tuple[str | None, list[dict[str, Any]]]: + """Return the persisted digest and titled edges, or ``(None, [])``. + + Missing reconstruction tables mean this database has not applied + migration 0021 yet; treat that as no stored tree rather than 500. + Titles follow the same public-or-affiliated rule as ``visible_posts``. + """ + try: + header = await conn.fetchrow( + """ + select result_sha256 + from analysis_run_reconstruction + where analysis_run_id = $1 + """, + analysis_run_id, + ) + except asyncpg.UndefinedTableError: + return None, [] + if header is None: + return None, [] + rows = await conn.fetch( + """ + select + edge.parent_post_id, + parent_post.post_title as parent_post_title, + parent_post.visibility_code as parent_visibility_code, + parent_post.corporate_entity_id as parent_corporate_entity_id, + edge.child_post_id, + child_post.post_title as child_post_title, + child_post.visibility_code as child_visibility_code, + child_post.corporate_entity_id as child_corporate_entity_id, + edge.fused_score + from analysis_run_lineage_edge edge + join source_post parent_post on parent_post.post_id = edge.parent_post_id + join source_post child_post on child_post.post_id = edge.child_post_id + where edge.analysis_run_id = $1 + order by parent_post.post_title, child_post.post_title + """, + analysis_run_id, + ) + return header["result_sha256"], [ + { + "parent_post_id": str(row["parent_post_id"]), + "parent_post_title": row["parent_post_title"], + "child_post_id": str(row["child_post_id"]), + "child_post_title": row["child_post_title"], + "fused_score": float(row["fused_score"]), + } + for row in rows + if reconstructed_edge_is_visible( + parent_visibility_code=row["parent_visibility_code"], + parent_corporate_entity_id=row["parent_corporate_entity_id"], + child_visibility_code=row["child_visibility_code"], + child_corporate_entity_id=row["child_corporate_entity_id"], + affiliated_entity_ids=affiliated_entity_ids, + ) + ] + + +async def persist_snapshot_members( + conn: asyncpg.Connection, + snapshot_id: Any, + post_ids: list[str], +) -> None: + """Freeze authorized post ids on a new snapshot. Skip a legacy database.""" + if not post_ids: + return + try: + await conn.executemany( + """ + insert into analysis_source_snapshot_member + (analysis_source_snapshot_id, source_post_id) + values ($1, $2) + on conflict do nothing + """, + [(snapshot_id, post_id) for post_id in post_ids], + ) + except asyncpg.UndefinedTableError: + return + + +async def fetch_visible_scope_posts( + conn: asyncpg.Connection, + scope_kind_code: str, + corporate_entity_id: Any, + process_unit_id: Any, + scope_key: str | None, + affiliated_entity_ids: list[str], + knowledge_cutoff: Any, +) -> list[dict[str, Any]]: + """ABAC-visible post titles known at the run cutoff -- never a hidden body. + + ``knowledge_cutoff`` is the analysis clock (W3C Time / ISO 8601-1:2019; + ADR 0013/0016). A later live post must not appear inside an earlier run. + ``updated_at`` is compared separately so the operator can see which + in-cutoff titles were rewritten after that clock. The live body is + still not returned. + """ + columns = ( + "post_id, post_title, visibility_code, corporate_entity_id, updated_at" + ) + if scope_kind_code == "analysis_scope_corporate_entity" and corporate_entity_id: + rows = await conn.fetch( + f"select {columns} " + "from source_post where corporate_entity_id = $1 " + "and created_at <= $2 " + "order by created_at, post_title", + corporate_entity_id, + knowledge_cutoff, + ) + elif scope_kind_code == "analysis_scope_process_unit" and process_unit_id: + rows = await conn.fetch( + f"select {columns} " + "from source_post where process_unit_id = $1 " + "and created_at <= $2 " + "order by created_at, post_title", + process_unit_id, + knowledge_cutoff, + ) + elif scope_kind_code == "analysis_scope_thread_group" and scope_key: + rows = await conn.fetch( + f"select {columns} " + "from source_post where thread_group_key = $1 " + "and created_at <= $2 " + "order by created_at, post_title", + scope_key, + knowledge_cutoff, + ) + elif scope_kind_code == "analysis_scope_all_visible": + rows = await conn.fetch( + f"select {columns} " + "from source_post where created_at <= $1 " + "order by created_at, post_title", + knowledge_cutoff, + ) + else: + return [] + affiliated = {str(entity_id) for entity_id in affiliated_entity_ids} + posts: list[dict[str, Any]] = [] + for row in rows: + visible = row["visibility_code"] == "public" or str(row["corporate_entity_id"]) in affiliated + if not visible: + continue + updated_at = row["updated_at"] + posts.append( + { + "post_id": str(row["post_id"]), + "post_title": row["post_title"], + "updated_at": _iso(updated_at), + "live_after_cutoff": live_write_after_cutoff( + updated_at, knowledge_cutoff + ), + } + ) + return posts + + +class AnalysisRunCreateError(Exception): + """Fail-closed create: HTTP status plus a next-action detail string.""" + + def __init__(self, status_code: int, detail: str) -> None: + super().__init__(detail) + self.status_code = status_code + self.detail = detail + + +def _require_lineage_create_kind(run_kind_code: str) -> None: + """Reject TEPP and report writes so this path cannot fake those products. + + TEPP stays a ``tepp_client`` wire path. Period reports stay on the + Reports panel rebuild. A Pending TEPP row that never called the + transport is a fabricated measurement request. + """ + if run_kind_code == _TEPP_RUN_KIND: + raise AnalysisRunCreateError( + 422, + "Connect a TEPP transport from a Failed TEPP row; this endpoint " + "does not invent a measurement.", + ) + if run_kind_code == _REPORT_RUN_KIND: + raise AnalysisRunCreateError( + 422, + "Rebuild the period report from the Reports panel.", + ) + if run_kind_code != _LINEAGE_RUN_KIND: + raise AnalysisRunCreateError( + 422, + "Only lineage reconstruction can be requested here.", + ) + + +@dataclass(frozen=True) +class AnalysisRunCapture: + """Immutable capture plan for one authorized create (no source rows).""" + + snapshot_sha256: str + maximum_available_time: datetime + document_count: int + thread_count: int + configuration_sha256: str + configuration_schema_version: str + code_revision_sha: str + + +def utc_iso(value: datetime) -> str: + """Normalize a timestamp to UTC ISO-8601 for digest stability.""" + if value.tzinfo is None: + value = value.replace(tzinfo=timezone.utc) + return value.astimezone(timezone.utc).isoformat() + + +def plan_analysis_run_capture( + *, + run_kind_code: str, + scope_kind_code: str, + corporate_entity_id: str, + knowledge_cutoff: datetime, + idempotency_key: str, + post_ids: list[str], + thread_keys: list[str], + latest_post_created_at: datetime | None, + cutoff_explicit: bool = True, +) -> AnalysisRunCapture: + """Hash the authorized cutoff bag. Never stores a post body or DSN. + + An omitted cutoff is hashed as ``unspecified`` so a retry of the same + client key does not 409 just because the clock moved. + """ + cutoff_token = utc_iso(knowledge_cutoff) if cutoff_explicit else "unspecified" + snapshot_material = json.dumps( + { + "scope_kind_code": scope_kind_code, + "corporate_entity_id": corporate_entity_id, + "knowledge_cutoff": cutoff_token, + "post_ids": sorted(post_ids), + }, + separators=(",", ":"), + sort_keys=True, + ) + configuration_material = json.dumps( + { + "run_kind_code": run_kind_code, + "scope_kind_code": scope_kind_code, + "corporate_entity_id": corporate_entity_id, + "knowledge_cutoff": cutoff_token, + "idempotency_key": idempotency_key, + "configuration_schema_version": _KIND_SCHEMA_VERSION[run_kind_code], + }, + separators=(",", ":"), + sort_keys=True, + ) + available = latest_post_created_at if latest_post_created_at is not None else knowledge_cutoff + return AnalysisRunCapture( + snapshot_sha256=hashlib.sha256(snapshot_material.encode()).hexdigest(), + maximum_available_time=available, + document_count=len(post_ids), + thread_count=len(set(thread_keys)), + configuration_sha256=hashlib.sha256(configuration_material.encode()).hexdigest(), + configuration_schema_version=_KIND_SCHEMA_VERSION[run_kind_code], + code_revision_sha=hashlib.sha256(f"lineageweave-{PACKAGE_VERSION}".encode()).hexdigest(), + ) + + +def _canonical_idempotency_key(raw: str) -> str: + """Trim and reject empty or control-bearing client keys.""" + key = raw.strip() + if not key or len(key) > 256 or any(ord(char) < 32 for char in key): + raise AnalysisRunCreateError( + 422, + "Use a 1–256 character idempotency key without control characters, then retry.", + ) + return key + + +def _resolve_corporate_entity_id( + corporate_entity_id: str | None, + affiliated_entity_ids: list[str], +) -> str: + """Return the affiliated corp this run may cover, or a next-action error.""" + affiliated = [entity_id for entity_id in affiliated_entity_ids if entity_id] + if corporate_entity_id: + try: + UUID(corporate_entity_id) + except ValueError as exc: + raise AnalysisRunCreateError( + 404, + "This corporate entity is not visible to this account.", + ) from exc + if corporate_entity_id not in affiliated: + raise AnalysisRunCreateError( + 404, + "This corporate entity is not visible to this account.", + ) + return corporate_entity_id + if len(affiliated) != 1: + raise AnalysisRunCreateError( + 422, + "Choose the corporate entity this run should cover.", + ) + return affiliated[0] + + +async def create_pending_analysis_run( + conn: asyncpg.Connection, + *, + account_id: str, + affiliated_entity_ids: list[str], + run_kind_code: str, + scope_kind_code: str, + corporate_entity_id: str | None, + knowledge_cutoff: datetime | None, + idempotency_key: str, +) -> dict[str, Any]: + """Insert snapshot, counts, frozen members, run, scope, and Pending. + + Lineage only. Does not reconstruct, call TEPP, or invent a theta. + Kind rejection happens before any snapshot or run insert. + Idempotent retries compare ``configuration_sha256``. + """ + _require_lineage_create_kind(run_kind_code) + if scope_kind_code != _CORPORATE_SCOPE: + raise AnalysisRunCreateError( + 422, + "Request a corporate-entity run. Other scopes are not available yet.", + ) + cutoff_explicit = knowledge_cutoff is not None + if knowledge_cutoff is None: + knowledge_cutoff = datetime.now(timezone.utc) + elif knowledge_cutoff.tzinfo is None: + knowledge_cutoff = knowledge_cutoff.replace(tzinfo=timezone.utc) + now = datetime.now(timezone.utc) + if knowledge_cutoff > now: + raise AnalysisRunCreateError( + 422, + "Choose a knowledge cutoff at or before now, then request the run again.", + ) + key = _canonical_idempotency_key(idempotency_key) + corp_id = _resolve_corporate_entity_id(corporate_entity_id, affiliated_entity_ids) + + existing = await conn.fetchrow( + """ + select analysis_run_id, configuration_sha256 + from analysis_run + where requested_by_account_id = $1 and idempotency_key = $2 + """, + account_id, + key, + ) + + rows = await conn.fetch( + """ + select post_id, post_title, thread_group_key, created_at, + visibility_code, corporate_entity_id + from source_post + where corporate_entity_id = $1 and created_at <= $2 + order by created_at, post_title + """, + corp_id, + knowledge_cutoff, + ) + affiliated = {str(entity_id) for entity_id in affiliated_entity_ids} + visible_rows = [ + row + for row in rows + if row["visibility_code"] == "public" or str(row["corporate_entity_id"]) in affiliated + ] + post_ids = [str(row["post_id"]) for row in visible_rows] + thread_keys = [row["thread_group_key"] for row in visible_rows] + latest = max((row["created_at"] for row in visible_rows), default=None) + capture = plan_analysis_run_capture( + run_kind_code=run_kind_code, + scope_kind_code=scope_kind_code, + corporate_entity_id=corp_id, + knowledge_cutoff=knowledge_cutoff, + idempotency_key=key, + post_ids=post_ids, + thread_keys=thread_keys, + latest_post_created_at=latest, + cutoff_explicit=cutoff_explicit, + ) + if existing is not None: + if existing["configuration_sha256"] != capture.configuration_sha256: + raise AnalysisRunCreateError( + 409, + "This request does not match the earlier run with the same key. " + "Open that run, or retry with a new idempotency key.", + ) + replayed = await fetch_visible_analysis_run( + conn, + str(existing["analysis_run_id"]), + account_id, + affiliated_entity_ids, + ) + if replayed is None: + raise AnalysisRunCreateError(404, "This analysis run is not visible.") + return replayed + + snapshot_id = await conn.fetchval( + """ + insert into analysis_source_snapshot + (snapshot_sha256, source_contract_version, + maximum_available_time, captured_at, created_at) + values ($1, $2, $3, $4, $4) + on conflict (snapshot_sha256) do nothing + returning analysis_source_snapshot_id + """, + capture.snapshot_sha256, + _CAPTURE_CONTRACT_VERSION, + capture.maximum_available_time, + now, + ) + if snapshot_id is None: + snapshot_id = await conn.fetchval( + """ + select analysis_source_snapshot_id + from analysis_source_snapshot + where snapshot_sha256 = $1 + for update + """, + capture.snapshot_sha256, + ) + count_exists = await conn.fetchval( + """ + select 1 from analysis_source_count + where analysis_source_snapshot_id = $1 + limit 1 + """, + snapshot_id, + ) + if count_exists is None: + await conn.execute( + """ + insert into analysis_source_count + (analysis_source_snapshot_id, count_type_code, count_value) + values + ($1, 'analysis_count_document', $2), + ($1, 'analysis_count_thread', $3) + """, + snapshot_id, + capture.document_count, + capture.thread_count, + ) + await persist_snapshot_members(conn, snapshot_id, post_ids) + try: + run_id = await conn.fetchval( + """ + insert into analysis_run + (analysis_source_snapshot_id, run_kind_code, idempotency_key, + requested_by_account_id, knowledge_cutoff, + configuration_schema_version, configuration_sha256, + code_revision_sha, requested_at) + values ($1, $2, $3, $4, $5, $6, $7, $8, $9) + returning analysis_run_id + """, + snapshot_id, + run_kind_code, + key, + account_id, + knowledge_cutoff, + capture.configuration_schema_version, + capture.configuration_sha256, + capture.code_revision_sha, + now, + ) + except asyncpg.UniqueViolationError: + raced = await conn.fetchrow( + """ + select analysis_run_id, configuration_sha256 + from analysis_run + where requested_by_account_id = $1 and idempotency_key = $2 + """, + account_id, + key, + ) + if raced is None or raced["configuration_sha256"] != capture.configuration_sha256: + raise AnalysisRunCreateError( + 409, + "This request does not match the earlier run with the same key. " + "Open that run, or retry with a new idempotency key.", + ) from None + replayed = await fetch_visible_analysis_run( + conn, + str(raced["analysis_run_id"]), + account_id, + affiliated_entity_ids, + ) + if replayed is None: + raise AnalysisRunCreateError(404, "This analysis run is not visible.") + return replayed + await conn.execute( + """ + insert into analysis_run_scope + (analysis_run_id, scope_kind_code, corporate_entity_id) + values ($1, $2, $3) + """, + run_id, + scope_kind_code, + corp_id, + ) + await conn.execute( + """ + insert into analysis_run_status_event + (analysis_run_id, status_ordinal, status_code, occurred_at) + values ($1, 1, 'analysis_status_pending', $2) + """, + run_id, + now, + ) + created = await fetch_visible_analysis_run( + conn, + str(run_id), + account_id, + affiliated_entity_ids, + ) + if created is None: + raise AnalysisRunCreateError(404, "This analysis run is not visible.") + return created diff --git a/backend/app/analysis_run_outbox.py b/backend/app/analysis_run_outbox.py new file mode 100644 index 00000000..7abeb2b3 --- /dev/null +++ b/backend/app/analysis_run_outbox.py @@ -0,0 +1,95 @@ +"""Durable start-work outbox. PostgreSQL is truth; Valkey is the wake-up. + +ADR 0023. Start writes Running plus one immutable outbox row, then a +worker claims that row and runs ThreadWeave or ``tepp_client``. A crash +after enqueue leaves the work item; it does not invent a theta. +""" + +from __future__ import annotations + +import hashlib +import json +from datetime import datetime, timezone +from typing import Any + +import redis.asyncio as redis + +OUTBOX_STREAM_KEY = "analysis-run-outbox" +_CLAIMED = "analysis_outbox_claimed" +_DELIVERED = "analysis_outbox_delivered" + + +def outbox_request_digest( + *, + analysis_run_id: str, + work_kind_code: str, + snapshot_sha256: str, + knowledge_cutoff: datetime, +) -> str: + """SHA-256 of the frozen start request. Never hashes a post body.""" + cutoff = knowledge_cutoff + if cutoff.tzinfo is None: + cutoff = cutoff.replace(tzinfo=timezone.utc) + material = json.dumps( + { + "analysis_run_id": str(analysis_run_id), + "knowledge_cutoff": cutoff.astimezone(timezone.utc).strftime( + "%Y-%m-%dT%H:%M:%SZ" + ), + "snapshot_sha256": snapshot_sha256, + "work_kind_code": work_kind_code, + }, + separators=(",", ":"), + sort_keys=True, + ) + return hashlib.sha256(material.encode()).hexdigest() + + +def outbox_stream_fields( + *, + analysis_run_id: str, + work_kind_code: str, + request_sha256: str, +) -> dict[str, str]: + """Valkey XADD fields for one start-work wake-up. No body, no theta.""" + return { + "analysis_run_id": str(analysis_run_id), + "request_sha256": request_sha256, + "work_kind_code": work_kind_code, + } + + +async def publish_outbox_event( + client: redis.Redis | None, + *, + analysis_run_id: str, + work_kind_code: str, + request_sha256: str, +) -> str | None: + """``XADD`` the wake-up. A missing Valkey leaves PostgreSQL durable.""" + if client is None: + return None + try: + entry_id = await client.xadd( + OUTBOX_STREAM_KEY, + outbox_stream_fields( + analysis_run_id=analysis_run_id, + work_kind_code=work_kind_code, + request_sha256=request_sha256, + ), + maxlen=1000, + approximate=True, + ) + except redis.RedisError: + return None + return str(entry_id) + + +def latest_outbox_delivery_is_delivered(status_code: str | None) -> bool: + """True when the newest delivery event already finished the work.""" + return status_code == _DELIVERED + + +def latest_outbox_delivery_is_claimed(status_code: str | None) -> bool: + """True when a worker already claimed the row and may retry.""" + return status_code == _CLAIMED diff --git a/backend/app/analysis_run_start.py b/backend/app/analysis_run_start.py new file mode 100644 index 00000000..6cf3b657 --- /dev/null +++ b/backend/app/analysis_run_start.py @@ -0,0 +1,801 @@ +"""Start a Pending lineage reconstruction or TEPP measurement. + +ADR 0021 reconstructs lineage. ADR 0022 starts TEPP through +``tepp_client`` only. ADR 0023 enqueues that work on a durable outbox +so a crash after Running does not lose the item. ADR 0035 stores a +published TEPP accepted acknowledgement as aggregate transport +evidence and never stamps Succeeded from that ack or from a +LineageWeave-local completed envelope. Accepted evidence stores +transport-response receipt and row-write time as distinct clocks +when those instants differ. Period-report stays another path. +Neither start invents a theta or a calibrated report score. +""" + +from __future__ import annotations + +import hashlib +import json +from datetime import datetime, timezone +from typing import Any +from uuid import UUID + +import asyncpg + +from backend.app.analysis_run_ingestion import ( + AnalysisRunCreateError, + fetch_visible_analysis_run, +) +from backend.app.analysis_run_outbox import ( + latest_outbox_delivery_is_claimed, + latest_outbox_delivery_is_delivered, + outbox_request_digest, +) +from backend.app.lineage_ingestion import records_from_source_posts +from lineageweave.http_client import HttpClientError, post_json +from lineageweave.lineage_persistence import lineage_edge_specs +from lineageweave.models import Edge +from lineageweave.tepp_client import AnalysisRunRequest, TeppClient, TeppNotAvailable +from lineageweave.tepp_result import TeppAcceptedEvidence, parse_tepp_accepted_evidence + +_LINEAGE_KIND = "analysis_run_lineage" +_TEPP_KIND = "analysis_run_tepp" +_REPORT_KIND = "analysis_run_report" +_PENDING = "analysis_status_pending" +_RUNNING = "analysis_status_running" +_SUCCEEDED = "analysis_status_succeeded" +_FAILED = "analysis_status_failed" +_TEPP_MODEL_CONTRACT = "tepp-analysis-run-v1" +_TEPP_OUTPUT_PROFILE = "calibrated_event_measurement" + + +class AnalysisRunStartError(AnalysisRunCreateError): + """Fail-closed start: HTTP status plus a next-action detail string.""" + + +def reconstruction_result_digest(edges: list[Edge]) -> str: + """SHA-256 of the ordered parent choices. Never hashes a post body.""" + material = json.dumps( + [ + { + "child_post_id": edge.child_id, + "fused_score": round(float(edge.fused_score), 6), + "parent_post_id": edge.parent_id, + } + for edge in sorted(edges, key=lambda item: (item.child_id, item.parent_id)) + ], + separators=(",", ":"), + sort_keys=True, + ) + return hashlib.sha256(material.encode()).hexdigest() + + +def start_kind_rejection(run_kind_code: str) -> AnalysisRunStartError | None: + """Return a 422 when start cannot run this kind. + + Lineage reconstructs the frozen bag. TEPP submits through + ``tepp_client`` and never invents a theta. Period-report stays on + its own rebuild path. + """ + if run_kind_code in {_LINEAGE_KIND, _TEPP_KIND}: + return None + if run_kind_code == _REPORT_KIND: + return AnalysisRunStartError( + 422, + "Rebuild the period report from the reports panel. " + "This start path does not invent a measurement.", + ) + return AnalysisRunStartError( + 422, + "Start reconstructs a Pending lineage run or submits TEPP. " + "This start path does not invent a measurement.", + ) + + +def configured_tepp_client(transport_url: str = "") -> TeppClient: + """Build a TEPP client from an optional HTTP transport URL. + + An empty URL keeps the default unavailable transport. A set URL + POSTs TEPP's published wire payload. File URLs and other schemes + stay unavailable -- this is not a local psychometric substitute. + """ + url = transport_url.strip() + if not url: + return TeppClient() + + def transport(payload: dict[str, Any]) -> dict[str, Any]: + try: + return post_json(url, payload, headers={}, timeout=30.0) + except (HttpClientError, ValueError, TypeError) as exc: + raise TeppNotAvailable(str(exc)) from exc + + return TeppClient(transport=transport) + + +def tepp_run_request( + *, + idempotency_key: str, + snapshot_sha256: str, + knowledge_cutoff: datetime, + corporate_entity_id: str, +) -> AnalysisRunRequest: + """Build TEPP's published request from the frozen run, never a theta.""" + cutoff = knowledge_cutoff + if cutoff.tzinfo is None: + cutoff = cutoff.replace(tzinfo=timezone.utc) + return AnalysisRunRequest( + idempotency_key=idempotency_key, + tenant_workspace_id=str(corporate_entity_id), + snapshot_id=snapshot_sha256, + knowledge_cutoff=cutoff.astimezone(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"), + model_contract_version=_TEPP_MODEL_CONTRACT, + output_profile=_TEPP_OUTPUT_PROFILE, + ) + + +def tepp_accepted_clocks( + *, + started_at: datetime, + received_at: datetime, + recorded_at: datetime, +) -> tuple[datetime, datetime]: + """Return receipt then row-write clocks, monotonic versus start. + + ``received_at`` is the transport-response receipt. ``recorded_at`` + is the later row-write instant. A clock that runs backward is + clamped forward so ``started_at <= received_at <= recorded_at``. + Equal instants stay equal; this helper does not invent a later + recorded clock. + """ + receipt = received_at if received_at >= started_at else started_at + recorded = recorded_at if recorded_at >= receipt else receipt + return receipt, recorded + + +def tepp_submit_outcome( + client: TeppClient, + request: AnalysisRunRequest, +) -> tuple[str, str | None, TeppAcceptedEvidence | None]: + """Submit through ``tepp_client``. Never invent or persist a theta. + + A missing transport is ``tepp_not_available``. A published + ``AnalysisRunAccepted`` envelope is Failed / + ``tepp_completed_result_unsupported`` and returned as aggregate + transport evidence. A LineageWeave-local completed envelope or any + other unpublished shape is Failed / ``tepp_result_not_persisted``. + Succeeded is never stamped from an accepted ack. + """ + try: + envelope = client.submit_analysis_run(request) + except TeppNotAvailable: + return _FAILED, "tepp_not_available", None + parsed = parse_tepp_accepted_evidence( + envelope, + expected_idempotency_key=request.idempotency_key, + ) + if parsed is None: + return _FAILED, "tepp_result_not_persisted", None + return _FAILED, "tepp_completed_result_unsupported", parsed + + +def start_write_conflict_error() -> AnalysisRunStartError: + """Next action when a concurrent start already wrote this run.""" + return AnalysisRunStartError( + 409, + "Open this run. Refresh to see the stored tree if start already finished.", + ) + + +def reconstruction_member_ids( + snapshot_member_ids: list[str], + cutoff_post_ids: list[str], +) -> list[str]: + """Prefer create-time membership over a later cutoff re-query. + + An empty member list means this database has not frozen the bag yet + (migration 0022 missing, or a legacy snapshot). Start then uses the + live cutoff query so those rows still reconstruct. + """ + if snapshot_member_ids: + return list(snapshot_member_ids) + return list(cutoff_post_ids) + + +async def _cutoff_source_posts( + conn: asyncpg.Connection, + *, + corporate_entity_id: Any, + knowledge_cutoff: Any, + affiliated_entity_ids: list[str], +) -> list[asyncpg.Record]: + """ABAC-visible cutoff rows with the grouping keys reconstruct needs.""" + rows = await conn.fetch( + """ + select post_id, post_title, created_at, visibility_code, + corporate_entity_id, process_unit_id, + thread_group_key, secondary_grouping_key + from source_post + where corporate_entity_id = $1 and created_at <= $2 + order by created_at, post_title + """, + corporate_entity_id, + knowledge_cutoff, + ) + affiliated = {str(entity_id) for entity_id in affiliated_entity_ids} + return [ + row + for row in rows + if row["visibility_code"] == "public" + or str(row["corporate_entity_id"]) in affiliated + ] + + +async def _snapshot_member_posts( + conn: asyncpg.Connection, + snapshot_id: Any, +) -> list[asyncpg.Record]: + """Load frozen capture rows, or empty when the member table is absent.""" + try: + return list( + await conn.fetch( + """ + select post.post_id, post.post_title, post.created_at, + post.visibility_code, post.corporate_entity_id, + post.process_unit_id, post.thread_group_key, + post.secondary_grouping_key + from analysis_source_snapshot_member member + join source_post post on post.post_id = member.source_post_id + where member.analysis_source_snapshot_id = $1 + order by post.created_at, post.post_title + """, + snapshot_id, + ) + ) + except asyncpg.UndefinedTableError: + return [] + + +async def _append_status( + conn: asyncpg.Connection, + analysis_run_id: str, + status_ordinal: int, + status_code: str, + occurred_at: datetime, + failure_code: str | None = None, +) -> None: + """Append one legal lifecycle event. Failed rows carry a machine code.""" + await conn.execute( + """ + insert into analysis_run_status_event + (analysis_run_id, status_ordinal, status_code, occurred_at, failure_code) + values ($1, $2, $3, $4, $5) + """, + analysis_run_id, + status_ordinal, + status_code, + occurred_at, + failure_code, + ) + + +async def _next_status_ordinal( + conn: asyncpg.Connection, + analysis_run_id: str, +) -> int: + """Return the next contiguous status ordinal for this run.""" + current_max = await conn.fetchval( + """ + select coalesce(max(status_ordinal), 0) + from analysis_run_status_event + where analysis_run_id = $1 + """, + analysis_run_id, + ) + return int(current_max) + 1 + + +async def _latest_outbox_delivery( + conn: asyncpg.Connection, + analysis_run_id: str, +) -> str | None: + """Newest outbox delivery status, or None when the row was never claimed.""" + return await conn.fetchval( + """ + select delivery_status_code + from analysis_run_outbox_delivery + where analysis_run_id = $1 + order by delivery_ordinal desc + limit 1 + """, + analysis_run_id, + ) + + +async def _next_outbox_delivery_ordinal( + conn: asyncpg.Connection, + analysis_run_id: str, +) -> int: + """Return the next contiguous outbox delivery ordinal for this run.""" + current_max = await conn.fetchval( + """ + select coalesce(max(delivery_ordinal), 0) + from analysis_run_outbox_delivery + where analysis_run_id = $1 + """, + analysis_run_id, + ) + return int(current_max) + 1 + + +async def _append_outbox_delivery( + conn: asyncpg.Connection, + analysis_run_id: str, + delivery_ordinal: int, + delivery_status_code: str, + occurred_at: datetime, + valkey_stream_entry_id: str | None = None, +) -> None: + """Append one claim or delivery event. Stream id is optional.""" + await conn.execute( + """ + insert into analysis_run_outbox_delivery + (analysis_run_id, delivery_ordinal, delivery_status_code, + occurred_at, valkey_stream_entry_id) + values ($1, $2, $3, $4, $5) + """, + analysis_run_id, + delivery_ordinal, + delivery_status_code, + occurred_at, + valkey_stream_entry_id, + ) + + +async def _visible_or_404( + conn: asyncpg.Connection, + analysis_run_id: str, + account_id: str, + affiliated_entity_ids: list[str], +) -> dict[str, Any]: + """Reload the authorized projection or hide the run.""" + started = await fetch_visible_analysis_run( + conn, + analysis_run_id, + account_id, + affiliated_entity_ids, + ) + if started is None: + raise AnalysisRunStartError(404, "This analysis run is not visible.") + return started + + +async def _attach_outbox_digest( + conn: asyncpg.Connection, + started: dict[str, Any], +) -> dict[str, Any]: + """Expose the wake-up digest to the start API, never to the client body.""" + digest = await conn.fetchval( + """ + select request_sha256 + from analysis_run_outbox + where analysis_run_id = $1 + """, + started["analysis_run_id"], + ) + if not digest: + return started + attached = dict(started) + attached["outbox_request_sha256"] = str(digest) + return attached + + +async def _lock_start_run( + conn: asyncpg.Connection, + analysis_run_id: str, +) -> asyncpg.Record: + """Lock the run row used by enqueue and delivery.""" + locked = await conn.fetchrow( + """ + select run.analysis_run_id, run.knowledge_cutoff, run.run_kind_code, + run.idempotency_key, run.analysis_source_snapshot_id, + snapshot.snapshot_sha256, scope.corporate_entity_id + from analysis_run run + join analysis_run_scope scope on scope.analysis_run_id = run.analysis_run_id + join analysis_source_snapshot snapshot + on snapshot.analysis_source_snapshot_id = run.analysis_source_snapshot_id + where run.analysis_run_id = $1 + for update of run + """, + analysis_run_id, + ) + if locked is None: + raise AnalysisRunStartError(404, "This analysis run is not visible.") + return locked + + +async def enqueue_pending_analysis_run( + conn: asyncpg.Connection, + *, + analysis_run_id: str, + account_id: str, + affiliated_entity_ids: list[str], +) -> dict[str, Any]: + """Append Running and one outbox row, or resume an undelivered item. + + Period-report is rejected so this path cannot invent a calibrated + score. A Succeeded retry returns the stored reconstruction. A + Running row with an undelivered outbox is a crash resume. A Running + row without pending work is 409. Hidden runs 404. + """ + try: + UUID(analysis_run_id) + except ValueError as exc: + raise AnalysisRunStartError(404, "This analysis run is not visible.") from exc + + current = await fetch_visible_analysis_run( + conn, + analysis_run_id, + account_id, + affiliated_entity_ids, + ) + if current is None: + raise AnalysisRunStartError(404, "This analysis run is not visible.") + kind_error = start_kind_rejection(current["run_kind_code"]) + if kind_error is not None: + raise kind_error + if current["status_code"] == _SUCCEEDED: + return current + + locked = await _lock_start_run(conn, analysis_run_id) + locked_status = await conn.fetchval( + """ + select status_code + from analysis_run_current_status + where analysis_run_id = $1 + """, + analysis_run_id, + ) + if locked_status == _SUCCEEDED: + return await _visible_or_404( + conn, analysis_run_id, account_id, affiliated_entity_ids + ) + if locked_status == _RUNNING: + latest = await _latest_outbox_delivery(conn, analysis_run_id) + if latest_outbox_delivery_is_delivered(latest): + raise AnalysisRunStartError( + 409, + "Open this run. Start is only for a Pending lineage reconstruction " + "or TEPP measurement.", + ) + has_outbox = await conn.fetchval( + """ + select 1 from analysis_run_outbox where analysis_run_id = $1 + """, + analysis_run_id, + ) + if has_outbox is None: + raise AnalysisRunStartError( + 409, + "Open this run. Start is only for a Pending lineage reconstruction " + "or TEPP measurement.", + ) + return await _attach_outbox_digest( + conn, + await _visible_or_404( + conn, analysis_run_id, account_id, affiliated_entity_ids + ), + ) + if locked_status != _PENDING: + raise AnalysisRunStartError( + 409, + "Open this run. Start is only for a Pending lineage reconstruction " + "or TEPP measurement.", + ) + + now = datetime.now(timezone.utc) + digest = outbox_request_digest( + analysis_run_id=str(locked["analysis_run_id"]), + work_kind_code=str(locked["run_kind_code"]), + snapshot_sha256=str(locked["snapshot_sha256"]), + knowledge_cutoff=locked["knowledge_cutoff"], + ) + try: + await _append_status( + conn, + analysis_run_id, + await _next_status_ordinal(conn, analysis_run_id), + _RUNNING, + now, + ) + await conn.execute( + """ + insert into analysis_run_outbox + (analysis_run_id, work_kind_code, request_sha256, enqueued_at) + values ($1, $2, $3, $4) + """, + analysis_run_id, + locked["run_kind_code"], + digest, + now, + ) + except asyncpg.UniqueViolationError as exc: + raise start_write_conflict_error() from exc + return await _attach_outbox_digest( + conn, + await _visible_or_404( + conn, analysis_run_id, account_id, affiliated_entity_ids + ), + ) + + +async def deliver_queued_analysis_run( + conn: asyncpg.Connection, + *, + analysis_run_id: str, + account_id: str, + affiliated_entity_ids: list[str], + tepp_client: TeppClient | None = None, + valkey_stream_entry_id: str | None = None, +) -> dict[str, Any]: + """Claim the outbox row and finish ThreadWeave or TEPP. + + A delivered row replays the stored result. Missing work is 409. + TEPP stays Failed when the transport is missing, the envelope is + not the published accepted acknowledgement, or TEPP has not + published a completed-result contract. A published accepted + envelope is stored as aggregate transport evidence. No theta is + invented and Succeeded is never stamped. + """ + try: + UUID(analysis_run_id) + except ValueError as exc: + raise AnalysisRunStartError(404, "This analysis run is not visible.") from exc + + current = await fetch_visible_analysis_run( + conn, + analysis_run_id, + account_id, + affiliated_entity_ids, + ) + if current is None: + raise AnalysisRunStartError(404, "This analysis run is not visible.") + if current["status_code"] == _SUCCEEDED: + return current + + outbox = await conn.fetchrow( + """ + select outbox.analysis_run_id, outbox.work_kind_code, + run.knowledge_cutoff, run.idempotency_key, + run.analysis_source_snapshot_id, snapshot.snapshot_sha256, + scope.corporate_entity_id + from analysis_run_outbox outbox + join analysis_run run on run.analysis_run_id = outbox.analysis_run_id + join analysis_run_scope scope on scope.analysis_run_id = run.analysis_run_id + join analysis_source_snapshot snapshot + on snapshot.analysis_source_snapshot_id = run.analysis_source_snapshot_id + where outbox.analysis_run_id = $1 + for update of outbox + """, + analysis_run_id, + ) + if outbox is None: + raise AnalysisRunStartError( + 409, + "Open this run. Start is only for a Pending lineage reconstruction " + "or TEPP measurement.", + ) + latest = await _latest_outbox_delivery(conn, analysis_run_id) + if latest_outbox_delivery_is_delivered(latest): + return await _visible_or_404( + conn, analysis_run_id, account_id, affiliated_entity_ids + ) + now = datetime.now(timezone.utc) + try: + if not latest_outbox_delivery_is_claimed(latest): + await _append_outbox_delivery( + conn, + analysis_run_id, + await _next_outbox_delivery_ordinal(conn, analysis_run_id), + "analysis_outbox_claimed", + now, + valkey_stream_entry_id, + ) + if outbox["work_kind_code"] == _TEPP_KIND: + await _deliver_tepp_measurement( + conn, + analysis_run_id=analysis_run_id, + locked=outbox, + tepp_client=tepp_client or TeppClient(), + ) + else: + await _deliver_lineage_reconstruction( + conn, + analysis_run_id=analysis_run_id, + locked=outbox, + affiliated_entity_ids=affiliated_entity_ids, + ) + finished = datetime.now(timezone.utc) + if finished < now: + finished = now + await _append_outbox_delivery( + conn, + analysis_run_id, + await _next_outbox_delivery_ordinal(conn, analysis_run_id), + "analysis_outbox_delivered", + finished, + valkey_stream_entry_id, + ) + except asyncpg.UniqueViolationError as exc: + raise start_write_conflict_error() from exc + return await _visible_or_404( + conn, analysis_run_id, account_id, affiliated_entity_ids + ) + + +async def start_pending_analysis_run( + conn: asyncpg.Connection, + *, + analysis_run_id: str, + account_id: str, + affiliated_entity_ids: list[str], + tepp_client: TeppClient | None = None, + valkey_stream_entry_id: str | None = None, +) -> dict[str, Any]: + """Enqueue then deliver on one connection. + + The HTTP start path commits the outbox before this delivery so a + crash leaves Running plus a durable work item. Callers that wrap + both steps in one transaction keep the older all-or-nothing + behavior. + """ + queued = await enqueue_pending_analysis_run( + conn, + analysis_run_id=analysis_run_id, + account_id=account_id, + affiliated_entity_ids=affiliated_entity_ids, + ) + if queued["status_code"] == _SUCCEEDED: + return queued + return await deliver_queued_analysis_run( + conn, + analysis_run_id=analysis_run_id, + account_id=account_id, + affiliated_entity_ids=affiliated_entity_ids, + tepp_client=tepp_client, + valkey_stream_entry_id=valkey_stream_entry_id, + ) + + +async def _deliver_lineage_reconstruction( + conn: asyncpg.Connection, + *, + analysis_run_id: str, + locked: asyncpg.Record, + affiliated_entity_ids: list[str], +) -> None: + """Persist ThreadWeave parent choices for the frozen bag.""" + now = datetime.now(timezone.utc) + member_rows = await _snapshot_member_posts( + conn, + locked["analysis_source_snapshot_id"], + ) + if member_rows: + rows = member_rows + else: + rows = await _cutoff_source_posts( + conn, + corporate_entity_id=locked["corporate_entity_id"], + knowledge_cutoff=locked["knowledge_cutoff"], + affiliated_entity_ids=affiliated_entity_ids, + ) + edges = lineage_edge_specs(records_from_source_posts(rows)) + digest = reconstruction_result_digest(edges) + finished = datetime.now(timezone.utc) + if finished < now: + finished = now + await conn.execute( + """ + insert into analysis_run_reconstruction + (analysis_run_id, result_sha256, edge_count, reconstructed_at) + values ($1, $2, $3, $4) + """, + analysis_run_id, + digest, + len(edges), + finished, + ) + for edge in edges: + await conn.execute( + """ + insert into analysis_run_lineage_edge + (analysis_run_id, child_post_id, parent_post_id, + fused_score, reconstructed_at) + values ($1, $2, $3, $4, $5) + """, + analysis_run_id, + edge.child_id, + edge.parent_id, + edge.fused_score, + finished, + ) + await _append_status( + conn, + analysis_run_id, + await _next_status_ordinal(conn, analysis_run_id), + _SUCCEEDED, + finished, + ) + + +async def _persist_tepp_accepted( + conn: asyncpg.Connection, + analysis_run_id: str, + evidence: TeppAcceptedEvidence, + received_at: datetime, + recorded_at: datetime, +) -> bool: + """Store published accepted evidence with receipt and row-write clocks. + + Missing table is not success. Callers pass transport-response + receipt as ``received_at`` and the row-write instant as + ``recorded_at``. This function binds those two values as given and + does not invent a later recorded clock when they are equal. + """ + try: + await conn.execute( + """ + insert into analysis_run_tepp_accepted + (analysis_run_id, contract_version, accepted_run_id, run_state, + idempotency_key, evidence_sha256, received_at, recorded_at) + values ($1, $2, $3, $4, $5, $6, $7, $8) + """, + analysis_run_id, + evidence.contract_version, + evidence.accepted_run_id, + evidence.run_state, + evidence.idempotency_key, + evidence.evidence_sha256(), + received_at, + recorded_at, + ) + except asyncpg.UndefinedTableError: + return False + return True + + +async def _deliver_tepp_measurement( + conn: asyncpg.Connection, + *, + analysis_run_id: str, + locked: asyncpg.Record, + tepp_client: TeppClient, +) -> None: + """Submit the frozen snapshot through ``tepp_client``. Never persist a theta.""" + started_at = datetime.now(timezone.utc) + request = tepp_run_request( + idempotency_key=str(locked["idempotency_key"]), + snapshot_sha256=str(locked["snapshot_sha256"]), + knowledge_cutoff=locked["knowledge_cutoff"], + corporate_entity_id=str(locked["corporate_entity_id"]), + ) + status_code, failure_code, accepted = tepp_submit_outcome(tepp_client, request) + received_at = datetime.now(timezone.utc) + recorded_at = datetime.now(timezone.utc) + receipt, recorded = tepp_accepted_clocks( + started_at=started_at, + received_at=received_at, + recorded_at=recorded_at, + ) + if accepted is not None: + stored = await _persist_tepp_accepted( + conn, analysis_run_id, accepted, receipt, recorded + ) + if not stored: + status_code, failure_code = _FAILED, "tepp_result_not_persisted" + await _append_status( + conn, + analysis_run_id, + await _next_status_ordinal(conn, analysis_run_id), + status_code, + recorded, + failure_code, + ) diff --git a/backend/app/config.py b/backend/app/config.py index 68cad343..082f8756 100644 --- a/backend/app/config.py +++ b/backend/app/config.py @@ -48,7 +48,10 @@ class Settings: # means the verification channel is unavailable, same "no fake # channel" discipline as every other pluggable client. searxng_base_url: str - # RankWeave ranking port (ADR 0024). True = fail-closed + # Optional TEPP HTTP transport. Empty keeps TeppClient's default + # unavailable transport. Never a local psychometric substitute. + tepp_transport_url: str + # RankWeave ranking port (ADR 0030). True = fail-closed # RankWeaveNotAvailable -- never invent a fused score. Default false # uses the in-process library already required by reconstruct.py. rankweave_disabled: bool @@ -84,6 +87,7 @@ def load_settings() -> Settings: vision_model=os.environ.get("VISION_MODEL", ""), valkey_url=os.environ.get("VALKEY_URL", "redis://localhost:16379/0"), searxng_base_url=os.environ.get("SEARXNG_BASE_URL", ""), + tepp_transport_url=os.environ.get("TEPP_TRANSPORT_URL", ""), rankweave_disabled=os.environ.get("RANKWEAVE_DISABLED", "") .strip() .lower() diff --git a/backend/app/corporate_entity_ingestion.py b/backend/app/corporate_entity_ingestion.py new file mode 100644 index 00000000..5c42c0b5 --- /dev/null +++ b/backend/app/corporate_entity_ingestion.py @@ -0,0 +1,223 @@ +"""Resolve an organization mention to the corporate hierarchy catalog. + +Existing unique similarity matches are reused. A tied top score stays +unbound and does not create a row (ADR 0026). A previously unseen entity +-- no candidate at or above the similarity threshold -- is created only +after inference proposes its complete hierarchy placement and external +verification corroborates that placement. Parent failure, cycles, and +excessive depth all fail closed. See ADR 0010. + +Creation writes take one named Postgres advisory transaction lock +(``pg_advisory_xact_lock``) after network inference/verification, then +reload catalog candidates before inserting. See ADR 0012. +""" + +from __future__ import annotations + +import asyncio +import hashlib + +import asyncpg + +from lineageweave.corporate_hierarchy_inference import ( + CorporateHierarchyInferenceClient, + HierarchyProposal, +) +from lineageweave.corporate_hierarchy_resolution import ( + RESOLUTION_TIE, + RESOLUTION_UNIQUE, + CorporateEntityCandidate, + score_corporate_entity, +) +from lineageweave.relation_verification import ( + STATUS_CORROBORATED, + RelationVerificationClient, +) + +_AUTO_CODE_PREFIX = "AUTO-" +_MAX_HIERARCHY_DEPTH = 4 +_CREATION_LOCK_KEY = "lineageweave:corporate_entity_creation" + +# The post-lock re-check exists only to catch a genuine concurrent +# duplicate CREATE of this exact name (see get_or_create_corporate_entity's +# docstring) -- never a fuzzy resolution against an unrelated-but-similar +# sibling or parent. score_corporate_entity's default 0.6 threshold is +# deliberately loose for real mention resolution (an abbreviation, a +# trailing legal suffix), but that same looseness is wrong here: a child +# whose name contains its own just-created parent's name as a prefix +# ("Acme" -> "Acme Gwangju Plant") scores ~0.7 against that parent alone, +# so a loose re-check would silently bind the child TO the parent instead +# of creating its own row. 1.0 (post-normalization exact match) is the +# only threshold that means "this really is the same entity." +_EXACT_MATCH_SIMILARITY = 1.0 + + +def _auto_entity_code(organization_name: str) -> str: + """Return a deterministic, namespace-separated code.""" + digest = hashlib.sha256(organization_name.encode("utf-8")).hexdigest()[:16] + return f"{_AUTO_CODE_PREFIX}{digest}" + + +def _hierarchy_verification_label(proposal: HierarchyProposal) -> str: + """Describe every persisted hierarchy field in one claim.""" + parent = proposal.parent_name if proposal.parent_name is not None else "NO_PARENT" + return f"corporate hierarchy level={proposal.level_code}; immediate_parent={parent}" + + +async def _create_entity( + conn: asyncpg.Connection, + organization_name: str, + level_code: str, + parent_entity_id: str | None, +) -> str: + """Insert one entity atomically and return its catalog id.""" + row = await conn.fetchrow( + """ + insert into corporate_entity + (parent_entity_id, corporate_entity_code, entity_name, entity_level_code) + values ($1, $2, $3, $4) + on conflict (corporate_entity_code) do update set + entity_name = excluded.entity_name, + entity_level_code = excluded.entity_level_code, + parent_entity_id = excluded.parent_entity_id + returning corporate_entity_id + """, + parent_entity_id, + _auto_entity_code(organization_name), + organization_name, + level_code, + ) + return str(row["corporate_entity_id"]) + + +async def _reload_candidates(conn: asyncpg.Connection) -> list[CorporateEntityCandidate]: + """Read every cataloged entity after the creation lock is held.""" + rows = await conn.fetch("select corporate_entity_id, entity_name from corporate_entity") + return [ + CorporateEntityCandidate(str(row["corporate_entity_id"]), row["entity_name"]) + for row in rows + ] + + +def _remember_candidate( + candidates: list[CorporateEntityCandidate], + corporate_entity_id: str, + entity_name: str, +) -> None: + """Keep the caller's in-memory snapshot aligned with a resolved id.""" + if any(candidate.corporate_entity_id == corporate_entity_id for candidate in candidates): + return + candidates.append( + CorporateEntityCandidate( + corporate_entity_id=corporate_entity_id, + entity_name=entity_name, + ) + ) + + +async def get_or_create_corporate_entity( + conn: asyncpg.Connection, + organization_name: str, + context_text: str, + inference_client: CorporateHierarchyInferenceClient, + verification_client: RelationVerificationClient, + candidates: list[CorporateEntityCandidate], + *, + _depth: int = 0, + _visited_names: frozenset[str] = frozenset(), +) -> str | None: + """Return a verified catalog id, otherwise ``None``. + + A unique similarity match is reused. A tied top score stays unbound + and does not create a third same-named row (ADR 0026). Only a genuine + miss -- no candidate at or above ``min_similarity`` -- may enter ADR + 0010 inference. A proposed parent must independently corroborate and + resolve before the child can be inserted. Repeated names in the + recursion path are cycles, including multi-node cycles such as + A -> B -> A. + """ + normalized_name = organization_name.strip() + if not normalized_name: + return None + visit_key = normalized_name.casefold() + if visit_key in _visited_names: + return None + + existing = score_corporate_entity(normalized_name, candidates) + if existing.kind == RESOLUTION_UNIQUE and existing.catalog_id is not None: + return existing.catalog_id + if existing.kind == RESOLUTION_TIE: + return None + if _depth >= _MAX_HIERARCHY_DEPTH or not inference_client.available: + return None + + proposal = await asyncio.to_thread( + inference_client.infer, + normalized_name, + context_text, + ) + if proposal is None or not verification_client.available: + return None + + placement_result = await asyncio.to_thread( + verification_client.verify, + normalized_name, + _hierarchy_verification_label(proposal), + ) + if placement_result.status_code != STATUS_CORROBORATED: + return None + + visited_names = _visited_names | {visit_key} + parent_entity_id: str | None = None + if proposal.parent_name is not None: + normalized_parent = proposal.parent_name.strip() + if not normalized_parent or normalized_parent.casefold() in visited_names: + return None + parent_result = await asyncio.to_thread( + verification_client.verify, + normalized_parent, + f"immediate parent of {normalized_name}", + ) + if parent_result.status_code != STATUS_CORROBORATED: + return None + parent_entity_id = await get_or_create_corporate_entity( + conn, + normalized_parent, + context_text, + inference_client, + verification_client, + candidates, + _depth=_depth + 1, + _visited_names=visited_names, + ) + if parent_entity_id is None: + return None + + async with conn.transaction(): + await conn.execute( + "select pg_advisory_xact_lock(hashtext($1))", + _CREATION_LOCK_KEY, + ) + # Exact match only (see _EXACT_MATCH_SIMILARITY): this re-check's + # sole purpose is catching a genuine concurrent duplicate CREATE of + # THIS name, not re-resolving against a merely-similar candidate -- + # the parent this call may have just created above is now in the + # reloaded pool and must not be mistaken for this (distinct) entity. + fresh = score_corporate_entity( + normalized_name, + await _reload_candidates(conn), + min_similarity=_EXACT_MATCH_SIMILARITY, + ) + if fresh.kind == RESOLUTION_UNIQUE and fresh.catalog_id is not None: + _remember_candidate(candidates, fresh.catalog_id, normalized_name) + return fresh.catalog_id + if fresh.kind == RESOLUTION_TIE: + return None + new_id = await _create_entity( + conn, + normalized_name, + proposal.level_code, + parent_entity_id, + ) + _remember_candidate(candidates, new_id, normalized_name) + return new_id diff --git a/backend/app/customer_group_tree_ingestion.py b/backend/app/customer_group_tree_ingestion.py new file mode 100644 index 00000000..e4b79c76 --- /dev/null +++ b/backend/app/customer_group_tree_ingestion.py @@ -0,0 +1,88 @@ +"""Load the authorized customer-group forest from PostgreSQL.""" + +from __future__ import annotations + +from typing import Any + +import asyncpg + +from lineageweave.customer_group_tree import ( + CatalogEntityRow, + TreeAbbreviation, + build_customer_group_forest, +) +from lineageweave.relation_verification import STATUS_CORROBORATED + +from .knowledge_graph import labels_for_codes + + +async def fetch_customer_group_forest( + conn: asyncpg.Connection, + affiliated_entity_ids: list[str], +) -> list[dict[str, Any]]: + """Authorized Group / Company / Plant forest for one account.""" + entity_rows = await conn.fetch( + """ + select corporate_entity_id, parent_entity_id, entity_name, entity_level_code + from corporate_entity + """ + ) + entities = tuple( + CatalogEntityRow( + entity_id=str(row["corporate_entity_id"]), + parent_entity_id=str(row["parent_entity_id"]) if row["parent_entity_id"] is not None else None, + entity_name=row["entity_name"], + entity_level_code=row["entity_level_code"], + ) + for row in entity_rows + ) + alias_rows = await conn.fetch( + """ + select raw_organization_name, corporate_entity_id, + verification_status_code, verification_evidence_url + from abbreviation_tree_corroboration + where verification_status_code = $1 + and corporate_entity_id is not null + """, + STATUS_CORROBORATED, + ) + abbreviations = tuple( + ( + str(row["corporate_entity_id"]), + TreeAbbreviation( + raw_organization_name=row["raw_organization_name"], + verification_status_code=row["verification_status_code"], + verification_evidence_url=row["verification_evidence_url"], + ), + ) + for row in alias_rows + ) + forest = [ + node.to_dict() + for node in build_customer_group_forest(entities, affiliated_entity_ids, abbreviations) + ] + await _attach_lookup_labels(conn, forest) + return forest + + +def _collect_level_codes(nodes: list[dict[str, Any]]) -> list[str]: + """Every entity-level code in the forest.""" + codes: list[str] = [] + for node in nodes: + if node.get("entity_level_code"): + codes.append(node["entity_level_code"]) + codes.extend(_collect_level_codes(node.get("children", []))) + return codes + + +def _apply_lookup_labels(nodes: list[dict[str, Any]], labels: dict[str, str]) -> None: + """Write display labels onto the JSON forest, falling back to the code.""" + for node in nodes: + level = node.get("entity_level_code") + node["entity_level_label"] = labels.get(level, level) if level else None + _apply_lookup_labels(node.get("children", []), labels) + + +async def _attach_lookup_labels(conn: asyncpg.Connection, forest: list[dict[str, Any]]) -> None: + """Hydrate ``entity_level_label`` from lookup rows.""" + _apply_lookup_labels(forest, await labels_for_codes(conn, _collect_level_codes(forest))) diff --git a/backend/app/entity_relationship_ingestion.py b/backend/app/entity_relationship_ingestion.py index 30fd6236..091e58f4 100644 --- a/backend/app/entity_relationship_ingestion.py +++ b/backend/app/entity_relationship_ingestion.py @@ -6,6 +6,7 @@ from __future__ import annotations +import asyncio from collections.abc import Mapping, Sequence from typing import Any @@ -38,7 +39,9 @@ async def ingest_post_entity_relationships( if not organization_names: return [] - relationships = client.classify(post_title, post_body, organization_names) + relationships = await asyncio.to_thread( + client.classify, post_title, post_body, organization_names + ) for relationship in relationships: await conn.execute( diff --git a/backend/app/keyman_ingestion.py b/backend/app/keyman_ingestion.py index e5451a9b..97c81525 100644 --- a/backend/app/keyman_ingestion.py +++ b/backend/app/keyman_ingestion.py @@ -6,23 +6,72 @@ `person_affiliation` (N:N, matched to a real `corporate_entity` via similarity-based resolution -- see `lineageweave.corporate_hierarchy_resolution`, so an abbreviation or -trailing legal suffix still resolves, not just an exact string match), -and `post_person_mention`. Finishes by calling -`knowledge_graph.persist_edges_for_post` so the Knowledge Graph edges are -computed from the same write, not a separate manual step. +trailing legal suffix still resolves, not just an exact string match -- +plus `role_title`, a schema column that already existed and was +previously never populated by this pipeline), and `post_person_mention`. +Finishes by calling `knowledge_graph.persist_edges_for_post` so the +Knowledge Graph edges are computed from the same write, not a separate +manual step. + +Same-name disambiguation: `_upsert_person`'s name+side match is a real, +known simplification (documented above), but a stated job title is real +evidence a same-name match should NOT blindly trust -- when the new +mention names a title that conflicts with a title already on file for +that name+side (both stated, genuinely different), a fresh +`cataloged_person` row is created rather than merging two people who +happen to share a name. A person's title legitimately changes over time +(a promotion), so this only splits on an actual stated conflict, never +on a missing title on either side. + +Abbreviation resolution (ADR 0008): before matching against +`corporate_entity`, each affiliated organization name is run through +`organization_name_resolution_ingestion.resolve_organization_name` -- +character-similarity matching alone cannot bridge an initialism like +"AGP" to its expansion "Aurora Grid Power". Only a search-corroborated +resolution is substituted in; an unresolved or unverified name still +flows through unchanged. + +Hierarchy auto-creation (ADR 0010): a unseen dataset's first mention of +any new counterparty organization has no existing `corporate_entity` +candidate for similarity matching to find at all -- matching alone can +only ever locate an already-cataloged entity. `get_or_create_corporate_entity` +tries similarity matching first, then falls back to an LLM-proposed, +search-corroborated hierarchy placement (level + parent) before +creating a real new row, so the "통합 고객사 계열 tree AI" requirement +is actually populated from real extraction, not left permanently empty. + +Tie boundary (ADR 0026): a raw organization name whose distinct catalog +candidates share the top qualifying similarity score stays unbound before +abbreviation rewriting. Live name resolution therefore cannot turn known +ambiguity into an apparent miss and manufacture a third `AUTO-` row. """ from __future__ import annotations +import asyncio +from dataclasses import replace + import asyncpg +from lineageweave.corporate_hierarchy_inference import ( + CorporateHierarchyInferenceClient, + NullCorporateHierarchyInferenceClient, +) from lineageweave.corporate_hierarchy_resolution import ( + RESOLUTION_TIE, CorporateEntityCandidate, - resolve_corporate_entity, + score_corporate_entity, ) from lineageweave.keyman_extraction import KeymanExtractionClient, PersonMention +from lineageweave.organization_name_resolution import ( + NullOrganizationNameResolutionClient, + OrganizationNameResolutionClient, +) +from lineageweave.relation_verification import NullRelationVerificationClient, RelationVerificationClient +from .corporate_entity_ingestion import get_or_create_corporate_entity from .knowledge_graph import persist_edges_for_post +from .organization_name_resolution_ingestion import resolve_organization_name async def _load_corporate_entity_candidates(conn: asyncpg.Connection) -> list[CorporateEntityCandidate]: @@ -34,60 +83,213 @@ async def _load_corporate_entity_candidates(conn: asyncpg.Connection) -> list[Co async def _upsert_person(conn: asyncpg.Connection, mention: PersonMention) -> str: - """Reuse a same-name, same-side row so re-extraction does not duplicate.""" - row = await conn.fetchrow( - "select person_id from cataloged_person where person_name = $1 and person_side_code = $2", + """Reuse a same-name, same-side row so re-extraction does not duplicate + -- unless the new mention's stated job title conflicts with a title + already on file for that name+side (`last_known_job_title`, checked + even when this mention names no affiliated organization -- a title + is real same-name-disambiguation evidence on its own, see module + docstring), in which case a same name is not trusted as the same + real person. + """ + candidates = await conn.fetch( + "select person_id, last_known_job_title from cataloged_person " + "where person_name = $1 and person_side_code = $2", mention.person_name, mention.person_side_code, ) - if row is not None: - return str(row["person_id"]) + if candidates and mention.job_title: + for candidate in candidates: + on_file = candidate["last_known_job_title"] + if on_file is not None and on_file != mention.job_title: + continue # stated title conflicts -- do not reuse this row + if on_file is None: + await conn.execute( + "update cataloged_person set last_known_job_title = $1 where person_id = $2", + mention.job_title, + candidate["person_id"], + ) + return str(candidate["person_id"]) + elif candidates: + return str(candidates[0]["person_id"]) + row = await conn.fetchrow( - "insert into cataloged_person (person_name, person_side_code) values ($1, $2) returning person_id", + "insert into cataloged_person (person_name, person_side_code, last_known_job_title) " + "values ($1, $2, $3) returning person_id", mention.person_name, mention.person_side_code, + mention.job_title, ) return str(row["person_id"]) +async def _upsert_affiliation( + conn: asyncpg.Connection, + person_id: str, + raw_name: str, + resolved_name: str, + corporate_entity_id: str | None, + role_title: str | None, +) -> None: + """Promote a raw affiliation row into one canonical identity.""" + await conn.execute( + """ + with legacy_affiliation as ( + select affiliated_corporate_entity_id, role_title + from person_affiliation + where person_id = $1 + and affiliated_organization_name = $2 + ), + canonical_affiliation as ( + insert into person_affiliation + (person_id, affiliated_organization_name, + affiliated_corporate_entity_id, role_title) + values ( + $1, + $3, + coalesce($4, (select affiliated_corporate_entity_id from legacy_affiliation)), + coalesce($5, (select role_title from legacy_affiliation)) + ) + on conflict (person_id, affiliated_organization_name) + do update set + affiliated_corporate_entity_id = coalesce( + excluded.affiliated_corporate_entity_id, + person_affiliation.affiliated_corporate_entity_id + ), + role_title = coalesce( + excluded.role_title, + person_affiliation.role_title + ) + returning person_affiliation_id + ) + delete from person_affiliation + where person_id = $1 + and affiliated_organization_name = $2 + and $2 <> $3 + """, + person_id, + raw_name, + resolved_name, + corporate_entity_id, + role_title, + ) + + +async def _resolve_affiliated_organization( + conn: asyncpg.Connection, + organization_name: str, + context_text: str, + resolution_client: OrganizationNameResolutionClient, + verification_client: RelationVerificationClient, + hierarchy_inference_client: CorporateHierarchyInferenceClient, + candidates: list[CorporateEntityCandidate], +) -> tuple[str, str, str | None]: + """Resolve one affiliation without rewriting a known raw-name tie.""" + raw_outcome = score_corporate_entity(organization_name, candidates) + if raw_outcome.kind == RESOLUTION_TIE: + return organization_name, organization_name, None + + resolved_name = await resolve_organization_name( + conn, + resolution_client, + verification_client, + organization_name, + context_text, + ) + corporate_entity_id = await get_or_create_corporate_entity( + conn, + resolved_name, + context_text, + hierarchy_inference_client, + verification_client, + candidates, + ) + return organization_name, resolved_name, corporate_entity_id + + async def ingest_post_keymen( conn: asyncpg.Connection, client: KeymanExtractionClient, post_id: str, post_title: str, post_body: str, + *, + resolution_client: OrganizationNameResolutionClient | None = None, + verification_client: RelationVerificationClient | None = None, + hierarchy_inference_client: CorporateHierarchyInferenceClient | None = None, + persist_graph: bool = True, ) -> list[PersonMention]: """Extracts, persists, and returns the `PersonMention`s found in one post. + `resolution_client`/`verification_client`/`hierarchy_inference_client` + default to the unavailable Null clients -- callers that don't pass + real ones get the exact same behavior as before ADR 0008/0010 (raw + affiliation names, unresolved). + + Organization resolution and hierarchy creation finish before the Keyman + write transaction. Callers must not wrap this function in an outer + transaction: that would turn ``pg_advisory_xact_lock`` into a savepoint + and hold the creation lock across later LLM work. The post's prior + Keyman mention set is replaced atomically after enrichment. + ``persist_graph=False`` lets a larger caller persist edges in its own + short write transaction after this function returns. + Raises whatever `client.extract` raises (e.g. a `NullKeymanExtractionClient` would raise `RuntimeError`) -- callers should check `client.available` first, same discipline as every other pluggable channel in this repo. """ - mentions = client.extract(post_title, post_body) + resolution_client = resolution_client or NullOrganizationNameResolutionClient() + verification_client = verification_client or NullRelationVerificationClient() + hierarchy_inference_client = hierarchy_inference_client or NullCorporateHierarchyInferenceClient() + mentions = await asyncio.to_thread(client.extract, post_title, post_body) candidates = await _load_corporate_entity_candidates(conn) - + resolved_by_mention: list[tuple[PersonMention, list[tuple[str, str, str | None]]]] = [] for mention in mentions: - person_id = await _upsert_person(conn, mention) + resolved_orgs: list[tuple[str, str, str | None]] = [] + for organization_name in mention.affiliated_organization_names: + resolved_orgs.append( + await _resolve_affiliated_organization( + conn, + organization_name, + post_body, + resolution_client, + verification_client, + hierarchy_inference_client, + candidates, + ) + ) + resolved_by_mention.append((mention, resolved_orgs)) + + normalized_mentions: list[PersonMention] = [] + async with conn.transaction(): await conn.execute( - "insert into post_person_mention (post_id, person_id) values ($1, $2) on conflict do nothing", - post_id, - person_id, + "delete from post_person_mention where post_id = $1", post_id ) - for organization_name in mention.affiliated_organization_names: - corporate_entity_id = resolve_corporate_entity(organization_name, candidates) + for mention, resolved_orgs in resolved_by_mention: + person_id = await _upsert_person(conn, mention) await conn.execute( - """ - insert into person_affiliation (person_id, affiliated_organization_name, affiliated_corporate_entity_id) - values ($1, $2, $3) - on conflict (person_id, affiliated_organization_name) - do update set affiliated_corporate_entity_id = excluded.affiliated_corporate_entity_id - """, + "insert into post_person_mention (post_id, person_id) values ($1, $2) on conflict do nothing", + post_id, person_id, - organization_name, - corporate_entity_id, ) + resolved_names: list[str] = [] + for organization_name, resolved_name, corporate_entity_id in resolved_orgs: + await _upsert_affiliation( + conn, + person_id, + organization_name, + resolved_name, + corporate_entity_id, + mention.job_title, + ) + if resolved_name not in resolved_names: + resolved_names.append(resolved_name) + normalized_mentions.append( + replace( + mention, + affiliated_organization_names=tuple(resolved_names), + ) + ) + if persist_graph: + await persist_edges_for_post(conn, post_id) - if mentions: - await persist_edges_for_post(conn, post_id) - - return mentions + return normalized_mentions diff --git a/backend/app/knowledge_graph.py b/backend/app/knowledge_graph.py index bb398d14..ce7289bb 100644 --- a/backend/app/knowledge_graph.py +++ b/backend/app/knowledge_graph.py @@ -18,9 +18,13 @@ EDGE_AFFILIATION, EDGE_CO_MENTION, EDGE_MENTION, + EDGE_MENTION_ORGANIZATION, + EDGE_MENTION_TEAM, + EDGE_TEAM_AFFILIATION, NODE_CORPORATE_ENTITY, NODE_PERSON, NODE_POST, + NODE_TEAM, KnowledgeGraphEdgeSpec, adjacency_from_edges, knowledge_graph_edges_for_post, @@ -31,6 +35,9 @@ ) +_GRAPH_PROJECTION_LOCK_KEY = "lineageweave:knowledge_graph_projection" + + def edge_spec_from_row(row: asyncpg.Record) -> KnowledgeGraphEdgeSpec: """Map one ``knowledge_graph_edge`` row onto the library spec.""" return KnowledgeGraphEdgeSpec( @@ -63,7 +70,7 @@ async def fetch_post_keymen(conn: asyncpg.Connection, post_id: str) -> list[dict """Load mentioned people and their affiliations for one post.""" person_rows = await conn.fetch( """ - select p.person_id, p.person_name, p.person_side_code, ppm.mention_context + select p.person_id, p.person_name, p.person_side_code, p.last_known_job_title, ppm.mention_context from post_person_mention ppm join cataloged_person p on p.person_id = ppm.person_id where ppm.post_id = $1 @@ -105,16 +112,34 @@ async def fetch_post_keymen(conn: asyncpg.Connection, post_id: str) -> list[dict "person_side_code": row["person_side_code"], "person_side_label": side_labels.get(row["person_side_code"], row["person_side_code"]), "mention_context": row["mention_context"], + "last_known_job_title": row["last_known_job_title"], "affiliations": affiliations_by_person.get(str(row["person_id"]), []), } for row in person_rows ] -async def persist_edges_for_post(conn: asyncpg.Connection, post_id: str) -> list[KnowledgeGraphEdgeSpec]: - """Insert mention, affiliation, and co-mention edges for one post.""" +async def persist_edges_for_post( + conn: asyncpg.Connection, post_id: str +) -> list[KnowledgeGraphEdgeSpec]: + """Reconcile one post's evidence-backed navigation projection. + + Callers own the surrounding transaction. A transaction-scoped + advisory lock serializes the small materialized projection so two + writers cannot interleave evidence deletion and orphan pruning. + Keyman and R&R person sources stay distinct in their writable tables; + ``combined_post_person_mention`` is used only to derive graph edges. + """ + await conn.execute( + "select pg_advisory_xact_lock(hashtext($1))", + _GRAPH_PROJECTION_LOCK_KEY, + ) + await conn.execute( + "delete from knowledge_graph_edge_evidence where evidence_post_id = $1", + post_id, + ) mention_rows = await conn.fetch( - "select person_id from post_person_mention where post_id = $1", + "select person_id from combined_post_person_mention where post_id = $1", post_id, ) affiliation_rows = await conn.fetch( @@ -126,6 +151,23 @@ async def persist_edges_for_post(conn: asyncpg.Connection, post_id: str) -> list """, [row["person_id"] for row in mention_rows], ) + team_mention_rows = await conn.fetch( + "select team_id from post_team_mention where post_id = $1", + post_id, + ) + team_affiliation_rows = await conn.fetch( + """ + select team_id, affiliated_corporate_entity_id + from cataloged_team + where team_id = any($1::uuid[]) + and affiliated_corporate_entity_id is not null + """, + [row["team_id"] for row in team_mention_rows], + ) + organization_mention_rows = await conn.fetch( + "select corporate_entity_id from post_organization_mention where post_id = $1", + post_id, + ) edges = knowledge_graph_edges_for_post( post_id, [str(row["person_id"]) for row in mention_rows], @@ -133,24 +175,27 @@ async def persist_edges_for_post(conn: asyncpg.Connection, post_id: str) -> list (str(row["person_id"]), str(row["affiliated_corporate_entity_id"])) for row in affiliation_rows ], + [str(row["team_id"]) for row in team_mention_rows], + [ + (str(row["team_id"]), str(row["affiliated_corporate_entity_id"])) + for row in team_affiliation_rows + ], + [str(row["corporate_entity_id"]) for row in organization_mention_rows], ) for edge in edges: - await conn.execute( + await conn.fetchrow( """ insert into knowledge_graph_edge ( source_node_type_code, source_node_id, target_node_type_code, target_node_id, edge_type_code, edge_weight - ) - select $1, $2::uuid, $3, $4::uuid, $5, $6 - where not exists ( - select 1 from knowledge_graph_edge - where source_node_type_code = $1 - and source_node_id = $2::uuid - and target_node_type_code = $3 - and target_node_id = $4::uuid - and edge_type_code = $5 - ) + ) values ($1, $2::uuid, $3, $4::uuid, $5, $6) + on conflict ( + source_node_type_code, source_node_id, + target_node_type_code, target_node_id, + edge_type_code + ) do update set edge_weight = excluded.edge_weight + returning knowledge_graph_edge_id """, edge.source_node_type_code, edge.source_node_id, @@ -159,9 +204,19 @@ async def persist_edges_for_post(conn: asyncpg.Connection, post_id: str) -> list edge.edge_type_code, edge.edge_weight, ) + await conn.execute( + """ + delete from knowledge_graph_edge edge_row + where not exists ( + select 1 + from knowledge_graph_edge_evidence evidence + where evidence.knowledge_graph_edge_id = + edge_row.knowledge_graph_edge_id + ) + """ + ) return edges - async def person_exists(conn: asyncpg.Connection, person_id: str) -> bool: """True when ``person_id`` is a UUID that exists in ``cataloged_person``.""" try: @@ -184,83 +239,189 @@ async def corporate_entity_exists(conn: asyncpg.Connection, entity_id: str) -> b return row is not None +async def team_exists(conn: asyncpg.Connection, team_id: str) -> bool: + """True when ``team_id`` is a UUID that exists in ``cataloged_team``.""" + try: + UUID(team_id) + except ValueError: + return False + row = await conn.fetchrow("select 1 from cataloged_team where team_id = $1", team_id) + return row is not None + + async def visible_mention_post_ids( conn: asyncpg.Connection, person_id: str, can_see_post, ) -> list[str]: - """Post ids that mention ``person_id`` and pass the caller's ABAC check.""" + """Visible post ids supported by Keyman or R&R person evidence.""" rows = await conn.fetch( """ - select p.post_id, p.visibility_code, p.corporate_entity_id - from post_person_mention ppm - join source_post p on p.post_id = ppm.post_id - where ppm.person_id = $1 + select post.post_id, post.visibility_code, post.corporate_entity_id + from combined_post_person_mention mention + join source_post post on post.post_id = mention.post_id + where mention.person_id = $1 + order by post.created_at, post.post_id """, person_id, ) return [str(row["post_id"]) for row in rows if can_see_post(row)] - async def visible_affiliation_post_ids( conn: asyncpg.Connection, entity_id: str, can_see_post, ) -> list[str]: - """Post ids that mention someone affiliated with ``entity_id`` and pass ABAC.""" + """Visible posts that mention an entity via a person or a direct org mention.""" rows = await conn.fetch( """ - select distinct p.post_id, p.visibility_code, p.corporate_entity_id - from person_affiliation pa - join post_person_mention ppm on ppm.person_id = pa.person_id - join source_post p on p.post_id = ppm.post_id - where pa.affiliated_corporate_entity_id = $1 + select distinct post.post_id, post.visibility_code, + post.corporate_entity_id, post.created_at + from source_post post + where post.post_id in ( + select mention.post_id + from person_affiliation affiliation + join combined_post_person_mention mention + on mention.person_id = affiliation.person_id + where affiliation.affiliated_corporate_entity_id = $1 + union + select org_mention.post_id + from post_organization_mention org_mention + where org_mention.corporate_entity_id = $1 + ) + order by post.created_at, post.post_id """, entity_id, ) return [str(row["post_id"]) for row in rows if can_see_post(row)] +async def visible_team_mention_post_ids( + conn: asyncpg.Connection, + team_id: str, + can_see_post, +) -> list[str]: + """Visible post ids supported by a cataloged team mention.""" + rows = await conn.fetch( + """ + select post.post_id, post.visibility_code, post.corporate_entity_id + from post_team_mention mention + join source_post post on post.post_id = mention.post_id + where mention.team_id = $1 + order by post.created_at, post.post_id + """, + team_id, + ) + return [str(row["post_id"]) for row in rows if can_see_post(row)] + async def load_visible_subgraph( conn: asyncpg.Connection, visible_post_ids: list[str], ) -> list[KnowledgeGraphEdgeSpec]: - """Edges whose endpoints the account can already see via those posts.""" + """Edges supported by at least one post the account may already see. + + Person, team, and organization mention channels are independent. A + team-only or organization-only post must still walk (ADR 0018). + """ if not visible_post_ids: return [] person_rows = await conn.fetch( - "select distinct person_id from post_person_mention where post_id = any($1::uuid[])", + "select distinct person_id from combined_post_person_mention " + "where post_id = any($1::uuid[])", visible_post_ids, ) person_ids = [row["person_id"] for row in person_rows] - if not person_ids: + team_rows = await conn.fetch( + "select distinct team_id from post_team_mention " + "where post_id = any($1::uuid[])", + visible_post_ids, + ) + team_ids = [row["team_id"] for row in team_rows] + organization_rows = await conn.fetch( + "select distinct corporate_entity_id from post_organization_mention " + "where post_id = any($1::uuid[])", + visible_post_ids, + ) + organization_ids = [row["corporate_entity_id"] for row in organization_rows] + if not person_ids and not team_ids and not organization_ids: return [] rows = await conn.fetch( """ - select source_node_type_code, source_node_id, - target_node_type_code, target_node_id, - edge_type_code, edge_weight - from knowledge_graph_edge - where + select distinct edge.source_node_type_code, edge.source_node_id, + edge.target_node_type_code, edge.target_node_id, + edge.edge_type_code, edge.edge_weight + from knowledge_graph_edge edge + join knowledge_graph_edge_evidence evidence + on evidence.knowledge_graph_edge_id = edge.knowledge_graph_edge_id + and evidence.evidence_post_id = any($1::uuid[]) + where ( - edge_type_code = $3 + edge.edge_type_code = $3 and ( - (source_node_type_code = $4 and source_node_id = any($1::uuid[])) - or (target_node_type_code = $4 and target_node_id = any($1::uuid[])) + (edge.source_node_type_code = $4 + and edge.source_node_id = any($1::uuid[])) + or + (edge.target_node_type_code = $4 + and edge.target_node_id = any($1::uuid[])) ) ) or ( - edge_type_code = $5 - and source_node_type_code = $6 - and target_node_type_code = $6 - and source_node_id = any($2::uuid[]) - and target_node_id = any($2::uuid[]) + edge.edge_type_code = $5 + and edge.source_node_type_code = $6 + and edge.target_node_type_code = $6 + and edge.source_node_id = any($2::uuid[]) + and edge.target_node_id = any($2::uuid[]) ) or ( - edge_type_code = $7 + edge.edge_type_code = $7 and ( - (source_node_type_code = $6 and source_node_id = any($2::uuid[])) - or (target_node_type_code = $6 and target_node_id = any($2::uuid[])) + (edge.source_node_type_code = $6 + and edge.source_node_id = any($2::uuid[])) + or + (edge.target_node_type_code = $6 + and edge.target_node_id = any($2::uuid[])) + ) + ) + or ( + edge.edge_type_code = $8 + and ( + (edge.source_node_type_code = $4 + and edge.source_node_id = any($1::uuid[])) + or + (edge.target_node_type_code = $4 + and edge.target_node_id = any($1::uuid[])) + or + (edge.source_node_type_code = $9 + and edge.source_node_id = any($10::uuid[])) + or + (edge.target_node_type_code = $9 + and edge.target_node_id = any($10::uuid[])) + ) + ) + or ( + edge.edge_type_code = $11 + and ( + (edge.source_node_type_code = $9 + and edge.source_node_id = any($10::uuid[])) + or + (edge.target_node_type_code = $9 + and edge.target_node_id = any($10::uuid[])) + ) + ) + or ( + edge.edge_type_code = $12 + and ( + (edge.source_node_type_code = $4 + and edge.source_node_id = any($1::uuid[])) + or + (edge.target_node_type_code = $4 + and edge.target_node_id = any($1::uuid[])) + or + (edge.source_node_type_code = $13 + and edge.source_node_id = any($14::uuid[])) + or + (edge.target_node_type_code = $13 + and edge.target_node_id = any($14::uuid[])) ) ) """, @@ -271,10 +432,16 @@ async def load_visible_subgraph( EDGE_CO_MENTION, NODE_PERSON, EDGE_AFFILIATION, + EDGE_MENTION_TEAM, + NODE_TEAM, + team_ids, + EDGE_TEAM_AFFILIATION, + EDGE_MENTION_ORGANIZATION, + NODE_CORPORATE_ENTITY, + organization_ids, ) return [edge_spec_from_row(row) for row in rows] - async def hydrate_related_nodes( conn: asyncpg.Connection, related: list[tuple[str, float]], @@ -287,6 +454,7 @@ async def hydrate_related_nodes( person_ids: list[str] = [] post_ids: list[str] = [] corp_ids: list[str] = [] + team_ids: list[str] = [] parsed: list[tuple[str, str, float]] = [] for key, score in related: node_type_code, node_id = parse_node_key(key) @@ -297,6 +465,8 @@ async def hydrate_related_nodes( post_ids.append(node_id) elif node_type_code == NODE_CORPORATE_ENTITY: corp_ids.append(node_id) + elif node_type_code == NODE_TEAM: + team_ids.append(node_id) people = { str(row["person_id"]): row @@ -319,6 +489,17 @@ async def hydrate_related_nodes( corp_ids, ) } if corp_ids else {} + teams = { + str(row["team_id"]): row + for row in await conn.fetch( + "select team_id, team_name from cataloged_team where team_id = any($1::uuid[])", + team_ids, + ) + } if team_ids else {} + + side_labels = await labels_for_codes( + conn, [row["person_side_code"] for row in people.values()] + ) payload: list[dict[str, Any]] = [] for node_type_code, node_id, score in parsed: @@ -329,12 +510,16 @@ async def hydrate_related_nodes( **ontology_annotations(node_type_code), } if node_type_code == NODE_PERSON and node_id in people: + side = people[node_id]["person_side_code"] item["label"] = people[node_id]["person_name"] - item["person_side_code"] = people[node_id]["person_side_code"] + item["person_side_code"] = side + item["person_side_label"] = side_labels.get(side, side) elif node_type_code == NODE_POST and node_id in posts: item["label"] = posts[node_id]["post_title"] elif node_type_code == NODE_CORPORATE_ENTITY and node_id in corps: item["label"] = corps[node_id]["entity_name"] + elif node_type_code == NODE_TEAM and node_id in teams: + item["label"] = teams[node_id]["team_name"] else: continue payload.append(item) @@ -371,3 +556,12 @@ async def related_for_entity( ) -> list[dict[str, Any]]: """Run RWR from ``entity_id`` over the account's visible subgraph.""" return await related_for_start(conn, NODE_CORPORATE_ENTITY, entity_id, visible_post_ids) + + +async def related_for_team( + conn: asyncpg.Connection, + team_id: str, + visible_post_ids: list[str], +) -> list[dict[str, Any]]: + """Run RWR from ``team_id`` over the account's visible subgraph.""" + return await related_for_start(conn, NODE_TEAM, team_id, visible_post_ids) diff --git a/backend/app/main.py b/backend/app/main.py index 27f67911..f944f3c8 100644 --- a/backend/app/main.py +++ b/backend/app/main.py @@ -19,8 +19,11 @@ from __future__ import annotations +import asyncio from contextlib import asynccontextmanager +from datetime import datetime from typing import Any +from uuid import UUID import asyncpg import redis.asyncio as redis @@ -37,10 +40,18 @@ NullEntityRelationshipClient, ) from lineageweave.image_content import orchestrator_vision_client +from lineageweave.corporate_hierarchy_inference import ( + ContextualOrchestratorHierarchyInferenceClient, + NullCorporateHierarchyInferenceClient, +) from lineageweave.keyman_extraction import ( ContextualOrchestratorKeymanExtractionClient, NullKeymanExtractionClient, ) +from lineageweave.organization_name_resolution import ( + ContextualOrchestratorOrganizationNameResolutionClient, + NullOrganizationNameResolutionClient, +) from lineageweave.post_chat import ( ContextualOrchestratorPostChatClient, NullPostChatClient, @@ -56,6 +67,20 @@ from lineageweave.relation_verification import NullRelationVerificationClient, SearxngRelationVerificationClient from lineageweave.rankweave_client import build_rankweave_client +from backend.app.analysis_run_ingestion import ( + AnalysisRunCreateError, + create_pending_analysis_run, + fetch_visible_analysis_run, + fetch_visible_analysis_runs, +) +from backend.app.analysis_run_outbox import publish_outbox_event +from backend.app.analysis_run_start import ( + AnalysisRunStartError, + configured_tepp_client, + deliver_queued_analysis_run, + enqueue_pending_analysis_run, +) +from backend.app.source_post_revision import fetch_known_at_revision, parse_as_of_clock from backend.app.activity_stream import ( create_valkey_client, get_valkey, @@ -64,7 +89,12 @@ ticket_created_summary, ticket_status_changed_summary, ) +from backend.app.abbreviation_tree_corroboration_ingestion import ( + corroborate_post_abbreviations, + fetch_post_abbreviation_matches, +) from backend.app.affiliate_tree_ingestion import fetch_affiliate_forest, fetch_voc_evidence +from backend.app.customer_group_tree_ingestion import fetch_customer_group_forest from backend.app.auth import CurrentAccount, get_current_account from backend.app.config import load_settings from backend.app.db import create_pool, get_pool @@ -98,10 +128,14 @@ fetch_post_keymen, labels_for_codes, person_exists, + persist_edges_for_post, related_for_entity, related_for_person, + related_for_team, + team_exists, visible_affiliation_post_ids, visible_mention_post_ids, + visible_team_mention_post_ids, ) from backend.app.lineage_ingestion import rebuild_lineage, visible_lineage_graph from backend.app.post_chat_ingestion import ( @@ -180,6 +214,26 @@ def _relation_verification_client(): return SearxngRelationVerificationClient(base_url=settings.searxng_base_url) +def _organization_name_resolution_client(): + """Live orchestrator client when configured; otherwise the unavailable null.""" + settings = load_settings() + if not (settings.orchestrator_base_url and settings.orchestrator_api_key): + return NullOrganizationNameResolutionClient() + return ContextualOrchestratorOrganizationNameResolutionClient( + base_url=settings.orchestrator_base_url, api_key=settings.orchestrator_api_key + ) + + +def _corporate_hierarchy_inference_client(): + """Live orchestrator client when configured; otherwise the unavailable null.""" + settings = load_settings() + if not (settings.orchestrator_base_url and settings.orchestrator_api_key): + return NullCorporateHierarchyInferenceClient() + return ContextualOrchestratorHierarchyInferenceClient( + base_url=settings.orchestrator_base_url, api_key=settings.orchestrator_api_key + ) + + def _post_summary_client(): """Live orchestrator client when configured; otherwise the unavailable null.""" settings = load_settings() @@ -238,7 +292,7 @@ def _post_evaluation_client(): def _rankweave_client(): - """In-process RankWeave unless RANKWEAVE_DISABLED=1 (ADR 0024).""" + """In-process RankWeave unless RANKWEAVE_DISABLED=1 (ADR 0030).""" return build_rankweave_client(disabled=load_settings().rankweave_disabled) @@ -278,15 +332,60 @@ async def healthz() -> dict[str, str]: @app.get("/api/me") -async def read_me(account: CurrentAccount = Depends(get_current_account)) -> dict[str, Any]: - """Return the provisioned account that the bearer token resolved to.""" +async def read_me( + account: CurrentAccount = Depends(get_current_account), + pool: asyncpg.Pool = Depends(get_pool), +) -> dict[str, Any]: + """Return the provisioned account and the corps this token may walk. + + Multi-affiliation operators need those names to choose which entity + ``POST /api/analysis-runs`` should cover. + """ + entities: list[dict[str, str]] = [] + if account.corporate_entity_ids: + async with pool.acquire() as conn: + rows = await conn.fetch( + """ + select corporate_entity_id, entity_name + from corporate_entity + where corporate_entity_id = any($1::uuid[]) + order by entity_name + """, + list(account.corporate_entity_ids), + ) + entities = [ + { + "corporate_entity_id": str(row["corporate_entity_id"]), + "entity_name": row["entity_name"], + } + for row in rows + ] return { "user_account_id": account.user_account_id, "display_name": account.display_name, "permission_codes": sorted(account.permission_codes), + "corporate_entities": entities, } +@app.get("/api/customer-group-tree") +async def read_customer_group_tree( + account: CurrentAccount = Depends(get_current_account), + pool: asyncpg.Pool = Depends(get_pool), +) -> dict[str, Any]: + """Authorized Group / Company / Plant forest this token may navigate. + + Affiliated corps pull in ancestors and descendants. A catalog row + the account does not touch is omitted -- a missing affiliation is + not a guessed parent. Corroborated abbreviations attach as + alternative labels; Searxng is not called on this read. + """ + _require_post_read(account) + async with pool.acquire() as conn: + trees = await fetch_customer_group_forest(conn, list(account.corporate_entity_ids)) + return {"trees": trees} + + @app.get("/api/lineage") async def read_lineage_graph( account: CurrentAccount = Depends(get_current_account), @@ -334,11 +433,29 @@ async def list_posts( @app.get("/api/posts/{post_id}") async def read_post( post_id: str, + as_of: str | None = None, account: CurrentAccount = Depends(get_current_account), pool: asyncpg.Pool = Depends(get_pool), ) -> dict[str, Any]: - """Return one source_post, or 404 / 403 if it is missing or out of scope.""" + """Return one source_post, or 404 / 403 if it is missing or out of scope. + + ``as_of`` adds ``known_at`` when a ``source_post_revision`` covers that + clock (ADR 0025). The live ``post_body`` stays the live row. A missing + cover is omitted -- never a fabricated cutoff sentence. Next action: + pass the analysis-run cutoff, then compare ``known_at`` with the live + body before treating the live text as reconstructed evidence. + """ _require_post_read(account) + as_of_clock = None + if as_of is not None: + try: + as_of_clock = parse_as_of_clock(as_of) + except ValueError as exc: + raise HTTPException( + status.HTTP_422_UNPROCESSABLE_ENTITY, + "as_of must be an ISO-8601 timestamp. Use the run cutoff, " + "then compare the known body with the live body.", + ) from exc async with pool.acquire() as conn: row = await conn.fetchrow( "select post_id, post_title, post_body, voc_type_code, visibility_code, corporate_entity_id, created_at " @@ -350,7 +467,13 @@ async def read_post( if not _can_see_post(account, row): raise HTTPException(status.HTTP_403_FORBIDDEN, "not authorized to view this post") labels = await _lookup_post_labels(conn, [row]) - return {**_serialize_post(row, labels), "post_body": row["post_body"]} + known_at = None + if as_of_clock is not None: + known_at = await fetch_known_at_revision(conn, post_id, as_of_clock) + payload = {**_serialize_post(row, labels), "post_body": row["post_body"]} + if known_at is not None: + payload["known_at"] = known_at + return payload async def _load_visible_post( @@ -442,6 +565,34 @@ async def read_related_corporate_entity( } +@app.get("/api/teams/{team_id}/related") +async def read_related_team( + team_id: str, + account: CurrentAccount = Depends(get_current_account), + pool: asyncpg.Pool = Depends(get_pool), +) -> dict[str, Any]: + """RWR-ranked related nodes from one cataloged team, hiding unseen posts.""" + _require_post_read(account) + async with pool.acquire() as conn: + if not await team_exists(conn, team_id): + raise HTTPException(status.HTTP_404_NOT_FOUND, "team not found") + visible_post_ids = await visible_team_mention_post_ids( + conn, team_id, lambda row: _can_see_post(account, row) + ) + if not visible_post_ids: + raise HTTPException(status.HTTP_403_FORBIDDEN, "not authorized to view this team") + team = await conn.fetchrow( + "select team_id, team_name from cataloged_team where team_id = $1", + team_id, + ) + related = await related_for_team(conn, team_id, visible_post_ids) + return { + "team_id": str(team["team_id"]), + "team_name": team["team_name"], + "related": related, + } + + @app.get("/api/posts/{post_id}/counterparties") async def read_post_counterparties( post_id: str, @@ -481,6 +632,64 @@ async def read_post_affiliate_tree( return {"post_id": str(post["post_id"]), "trees": trees} +@app.get("/api/posts/{post_id}/abbreviation-tree-matches") +async def read_post_abbreviation_tree_matches( + post_id: str, + account: CurrentAccount = Depends(get_current_account), + pool: asyncpg.Pool = Depends(get_pool), +) -> dict[str, Any]: + """Cached Searxng tree matches for organization names on this post. + + Does not call Searxng. A missing cache row means the mention has + not been cross-checked yet, not that a parent was invented. + """ + post = await _load_visible_post(post_id, account, pool) + async with pool.acquire() as conn: + matches = await fetch_post_abbreviation_matches(conn, post_id) + return {"post_id": str(post["post_id"]), "matches": matches} + + +@app.post("/api/posts/{post_id}/corroborate-abbreviations") +async def corroborate_post_abbreviation_tree( + post_id: str, + account: CurrentAccount = Depends(get_current_account), + pool: asyncpg.Pool = Depends(get_pool), +) -> dict[str, Any]: + """Cross-check this post's abbreviations against the customer-group tree. + + Reuses the existing Searxng client. Fail-closed: unavailable search + is 503, not an invented parent or AUTO row. A tied or empty result + stays unbound. post_admin only -- a real external-search write. + """ + _require_post_admin(account) + post = await _load_visible_post(post_id, account, pool) + client = _relation_verification_client() + if not client.available: + raise HTTPException( + status.HTTP_503_SERVICE_UNAVAILABLE, + "Abbreviation tree corroboration is unavailable: set SEARXNG_BASE_URL", + ) + async with pool.acquire() as conn: + matches = await corroborate_post_abbreviations( + conn, + client, + post_id, + list(account.corporate_entity_ids), + ) + return { + "post_id": str(post["post_id"]), + "matches": [ + { + "raw_organization_name": match.raw_organization_name, + "corporate_entity_id": match.corporate_entity_id, + "verification_status_code": match.verification_status_code, + "verification_evidence_url": match.verification_evidence_url, + } + for match in matches + ], + } + + @app.get("/api/posts/{post_id}/voc-evidence") async def read_post_voc_evidence( post_id: str, @@ -560,17 +769,28 @@ async def extract_post_keymen( # literal text either blows the token budget or is silently # ignored (see lineageweave/post_content_normalization.py). post_body = normalize_post_body(raw_body, vision_client=_vision_client()).text + mentions = await ingest_post_keymen( + conn, + keyman_client, + post_id, + post["post_title"], + post_body, + resolution_client=_organization_name_resolution_client(), + verification_client=_relation_verification_client(), + hierarchy_inference_client=_corporate_hierarchy_inference_client(), + persist_graph=False, + ) + organization_names = sorted( + {name for mention in mentions for name in mention.affiliated_organization_names} + ) + # relationship_client is gated by the same settings check as + # keyman_client above (both read ORCHESTRATOR_BASE_URL/_API_KEY), + # so reaching here means it is available too. + relationships = await ingest_post_entity_relationships( + conn, relationship_client, post_id, post["post_title"], post_body, organization_names + ) async with conn.transaction(): - mentions = await ingest_post_keymen(conn, keyman_client, post_id, post["post_title"], post_body) - organization_names = sorted( - {name for mention in mentions for name in mention.affiliated_organization_names} - ) - # relationship_client is gated by the same settings check as - # keyman_client above (both read ORCHESTRATOR_BASE_URL/_API_KEY), - # so reaching here means it is available too. - relationships = await ingest_post_entity_relationships( - conn, relationship_client, post_id, post["post_title"], post_body, organization_names - ) + await persist_edges_for_post(conn, post_id) return { "post_id": str(post["post_id"]), "extracted_count": len(mentions), @@ -828,8 +1048,17 @@ async def read_post_summary( ) body_row = await conn.fetchrow("select post_body from source_post where post_id = $1", post_id) normalized_body = normalize_post_body(body_row["post_body"], vision_client=_vision_client()).text - summary = client.summarize(post["post_title"], normalized_body) - return await persist_post_summary(conn, post_id, summary) + summary = await asyncio.to_thread( + client.summarize, post["post_title"], normalized_body + ) + return await persist_post_summary( + conn, + post_id, + summary, + post_body=normalized_body, + hierarchy_inference_client=_corporate_hierarchy_inference_client(), + verification_client=_relation_verification_client(), + ) class ChatRequest(BaseModel): @@ -1112,6 +1341,161 @@ async def derive_post_commitment( return {"post_id": str(post["post_id"]), "has_commitment": True, "ticket": ticket} +@app.get("/api/analysis-runs") +async def list_analysis_runs( + account: CurrentAccount = Depends(get_current_account), + pool: asyncpg.Pool = Depends(get_pool), +) -> dict[str, Any]: + """Authorized analysis-run list: aggregates and labels only. + + Hidden scopes 404 at the item path and never appear here. The + payload has no source SQL, DSN, raw record, or provider body. + """ + _require_post_read(account) + async with pool.acquire() as conn: + runs = await fetch_visible_analysis_runs( + conn, + account.user_account_id, + list(account.corporate_entity_ids), + ) + return {"analysis_runs": runs} + + +class CreateAnalysisRunRequest(BaseModel): + """JSON body for ``POST /api/analysis-runs``. + + Omitting ``corporate_entity_id`` uses the account's sole affiliation. + Only ``analysis_run_lineage`` is accepted. Reconstruction and TEPP + execution stay later slices; this write records Pending lineage only. + """ + + run_kind_code: str = "analysis_run_lineage" + scope_kind_code: str = "analysis_scope_corporate_entity" + corporate_entity_id: str | None = None + knowledge_cutoff: datetime | None = None + idempotency_key: str + + +@app.post("/api/analysis-runs", status_code=status.HTTP_201_CREATED) +async def create_analysis_run( + request: CreateAnalysisRunRequest, + account: CurrentAccount = Depends(get_current_account), + pool: asyncpg.Pool = Depends(get_pool), +) -> dict[str, Any]: + """Record a Pending lineage run on an authorized cutoff capture. + + post_read is enough: the caller requests a run of a corp they + already walk. TEPP and period-report kinds are 422 so this path + cannot invent a measurement. Hidden scopes 404. A matching + idempotent retry returns the same run. + """ + _require_post_read(account) + async with pool.acquire() as conn: + async with conn.transaction(): + try: + created = await create_pending_analysis_run( + conn, + account_id=account.user_account_id, + affiliated_entity_ids=list(account.corporate_entity_ids), + run_kind_code=request.run_kind_code, + scope_kind_code=request.scope_kind_code, + corporate_entity_id=request.corporate_entity_id, + knowledge_cutoff=request.knowledge_cutoff, + idempotency_key=request.idempotency_key, + ) + except AnalysisRunCreateError as exc: + raise HTTPException(exc.status_code, exc.detail) from exc + return created + + +@app.post("/api/analysis-runs/{analysis_run_id}/start") +async def start_analysis_run( + analysis_run_id: str, + account: CurrentAccount = Depends(get_current_account), + pool: asyncpg.Pool = Depends(get_pool), + valkey: redis.Redis = Depends(get_valkey), +) -> dict[str, Any]: + """Enqueue start work, then deliver ThreadWeave or TEPP. + + post_read is enough. Hidden runs 404. Period-report is 422 so this + path cannot invent a calibrated score. TEPP goes through + ``tepp_client`` and stays Failed when the transport is missing, the + envelope is unpublished, or TEPP has not published a completed-result + contract. A published accepted acknowledgement is stored as + aggregate transport evidence. Succeeded is never stamped from that + ack. A Succeeded lineage retry returns + the stored tree. A Running restart with an undelivered outbox + finishes that work. A Running restart without pending work is 409. + The outbox commits before reconstruct/TEPP so a crash leaves a + durable work item (ADR 0023). + """ + _require_post_read(account) + settings = load_settings() + async with pool.acquire() as conn: + async with conn.transaction(): + try: + queued = await enqueue_pending_analysis_run( + conn, + analysis_run_id=analysis_run_id, + account_id=account.user_account_id, + affiliated_entity_ids=list(account.corporate_entity_ids), + ) + except AnalysisRunStartError as exc: + raise HTTPException(exc.status_code, exc.detail) from exc + if queued.get("status_code") == "analysis_status_succeeded": + return queued + request_digest = queued.pop("outbox_request_sha256", None) + stream_id = None + if request_digest: + stream_id = await publish_outbox_event( + valkey, + analysis_run_id=analysis_run_id, + work_kind_code=str(queued.get("run_kind_code") or ""), + request_sha256=request_digest, + ) + async with pool.acquire() as conn: + async with conn.transaction(): + try: + started = await deliver_queued_analysis_run( + conn, + analysis_run_id=analysis_run_id, + account_id=account.user_account_id, + affiliated_entity_ids=list(account.corporate_entity_ids), + tepp_client=configured_tepp_client(settings.tepp_transport_url), + valkey_stream_entry_id=stream_id, + ) + except AnalysisRunStartError as exc: + raise HTTPException(exc.status_code, exc.detail) from exc + return started + + +@app.get("/api/analysis-runs/{analysis_run_id}") +async def read_analysis_run( + analysis_run_id: str, + account: CurrentAccount = Depends(get_current_account), + pool: asyncpg.Pool = Depends(get_pool), +) -> dict[str, Any]: + """One authorized analysis-run projection, or 404 when hidden. + + Detail adds the labeled status history. Hidden runs never leak events. + """ + _require_post_read(account) + try: + UUID(analysis_run_id) + except ValueError: + raise HTTPException(status.HTTP_404_NOT_FOUND, "analysis run not found") from None + async with pool.acquire() as conn: + run = await fetch_visible_analysis_run( + conn, + analysis_run_id, + account.user_account_id, + list(account.corporate_entity_ids), + ) + if run is None: + raise HTTPException(status.HTTP_404_NOT_FOUND, "analysis run not found") + return run + + @app.get("/api/calendar") async def read_calendar( account: CurrentAccount = Depends(get_current_account), @@ -1136,7 +1520,7 @@ async def read_rankings( account: CurrentAccount = Depends(get_current_account), pool: asyncpg.Pool = Depends(get_pool), ) -> dict[str, Any]: - """RankWeave fusion of ABAC-visible posts (ADR 0024). + """RankWeave fusion of ABAC-visible posts (ADR 0030). Hidden posts are omitted from every channel. Never invents a fused score or a theta. Fail-closed when RankWeave is disabled or the diff --git a/backend/app/organization_name_resolution_ingestion.py b/backend/app/organization_name_resolution_ingestion.py new file mode 100644 index 00000000..9300586c --- /dev/null +++ b/backend/app/organization_name_resolution_ingestion.py @@ -0,0 +1,73 @@ + +"""Cache and persist verified organization-name normalization.""" + +from __future__ import annotations + +import asyncio + +import asyncpg + +from lineageweave.organization_name_resolution import ( + OrganizationNameResolutionClient, + resolve_and_verify_organization_name, +) +from lineageweave.relation_verification import ( + STATUS_CORROBORATED, + RelationVerificationClient, +) + + +async def resolve_organization_name( + conn: asyncpg.Connection, + resolution_client: OrganizationNameResolutionClient, + verification_client: RelationVerificationClient, + raw_name: str, + context_text: str, +) -> str: + """Return the corroborated canonical name, otherwise ``raw_name``. + + Synchronous network adapters run in a worker thread so this async + ingestion path does not block unrelated requests. + """ + cached = await conn.fetchrow( + "select resolved_organization_name, verification_status_code " + "from organization_name_resolution where raw_organization_name = $1", + raw_name, + ) + if cached is not None: + if cached["verification_status_code"] == STATUS_CORROBORATED: + return cached["resolved_organization_name"] + return raw_name + if not resolution_client.available: + return raw_name + + resolution = await asyncio.to_thread( + resolve_and_verify_organization_name, + raw_name, + context_text, + resolution_client, + verification_client, + ) + if resolution is None: + return raw_name + + await conn.execute( + """ + insert into organization_name_resolution + (raw_organization_name, resolved_organization_name, + verification_status_code, verification_evidence_url) + values ($1, $2, $3, $4) + on conflict (raw_organization_name) do update set + resolved_organization_name = excluded.resolved_organization_name, + verification_status_code = excluded.verification_status_code, + verification_evidence_url = excluded.verification_evidence_url, + resolved_at = now() + """, + resolution.raw_organization_name, + resolution.resolved_organization_name, + resolution.verification_status_code, + resolution.verification_evidence_url, + ) + if resolution.verification_status_code == STATUS_CORROBORATED: + return resolution.resolved_organization_name + return raw_name diff --git a/backend/app/post_chat_ingestion.py b/backend/app/post_chat_ingestion.py index 2f9fe77f..794faa10 100644 --- a/backend/app/post_chat_ingestion.py +++ b/backend/app/post_chat_ingestion.py @@ -66,13 +66,14 @@ async def find_linked_post_ids(conn: asyncpg.Connection, post_id: str) -> Linked # Discover them here first: every post that mentions any person this # post itself mentions, then load the subgraph over that expanded set. person_rows = await conn.fetch( - "select distinct person_id from post_person_mention where post_id = $1", post_id + "select distinct person_id from combined_post_person_mention where post_id = $1", post_id ) person_ids = [row["person_id"] for row in person_rows] sibling_post_ids = [post_id] if person_ids: sibling_rows = await conn.fetch( - "select distinct post_id from post_person_mention where person_id = any($1::uuid[])", + "select distinct post_id from combined_post_person_mention " + "where person_id = any($1::uuid[])", person_ids, ) sibling_post_ids = list({str(row["post_id"]) for row in sibling_rows} | {post_id}) diff --git a/backend/app/post_summary_ingestion.py b/backend/app/post_summary_ingestion.py index 7e7569d6..03426b0e 100644 --- a/backend/app/post_summary_ingestion.py +++ b/backend/app/post_summary_ingestion.py @@ -1,4 +1,30 @@ -"""Persist and load the popup's Korean summary / key events / R&R.""" +"""Persist and load the popup's Korean summary / key events / R&R. + +ADR 0009 / 0019 / 0027: an R&R actor is not just per-post free text -- +when it is a team, organization, or already-cataloged person, it is +resolved to a shared catalog identity (``cataloged_team`` / +``corporate_entity`` / ``cataloged_person``) stored on the role row and +a Knowledge Graph mention edge is written, so the same "설계팀" or +organization named across two posts becomes one linkable node. Fetch +never reconstructs that id by ``entity_name`` or ``person_name``; those +columns are not unique. A person actor is opportunistically joined to +an *existing* ``cataloged_person`` row by name when Keyman extraction +has already cataloged that name. The resolved ``cataloged_person_id`` +is stored on the role so a later read does not rejoin by display name. +The R&R evidence is written to ``post_summary_person_mention`` rather +than Keyman's ``post_person_mention`` so either extractor can replace +its own result without leaving or deleting the other's evidence. + +ADR 0010: an organization actor's name is resolved via +``get_or_create_corporate_entity`` -- similarity matching first, then +an LLM-proposed, search-corroborated hierarchy placement before +creating a real new row, so a real dataset's first mention of a +counterparty organization actually populates the corporate hierarchy +tree instead of staying permanently unresolved. Inference, +verification, and the short advisory-lock creation transaction finish +before the summary-replacement transaction begins; slow external work +therefore cannot extend the lock or the atomic replacement window. +""" from __future__ import annotations @@ -6,12 +32,44 @@ import asyncpg +from lineageweave.corporate_hierarchy_inference import ( + CorporateHierarchyInferenceClient, + NullCorporateHierarchyInferenceClient, +) from lineageweave.fixtures import fixture_thread_cast -from lineageweave.post_summary import PostSummary, RoleResponsibility +from lineageweave.knowledge_graph import ( + NODE_CORPORATE_ENTITY, + NODE_PERSON, + NODE_TEAM, +) +from lineageweave.ontology import ontology_annotations +from lineageweave.post_summary import ( + ACTOR_TYPE_ORGANIZATION, + ACTOR_TYPE_PERSON, + ACTOR_TYPE_TEAM, + PostSummary, + RoleResponsibility, +) +from lineageweave.relation_verification import ( + NullRelationVerificationClient, + RelationVerificationClient, +) + +from .corporate_entity_ingestion import get_or_create_corporate_entity +from .keyman_ingestion import _load_corporate_entity_candidates +from .knowledge_graph import persist_edges_for_post +from .team_ingestion import upsert_team -async def fetch_persisted_summary(conn: asyncpg.Connection, post_id: str) -> dict[str, Any] | None: - """Return the stored summary payload, or None when none has been written.""" +async def fetch_persisted_summary( + conn: asyncpg.Connection, post_id: str +) -> dict[str, Any] | None: + """Return the stored summary payload, or None when none has been written. + + ``catalog_node_id`` comes from the role row's catalog foreign keys + (ADR 0019 / 0027). This function does not join ``corporate_entity`` + by ``entity_name``. Person chips read ``cataloged_person_id``. + """ header = await conn.fetchrow( "select korean_summary from post_summary_result where post_id = $1", post_id, @@ -23,23 +81,151 @@ async def fetch_persisted_summary(conn: asyncpg.Connection, post_id: str) -> dic post_id, ) roles = await conn.fetch( - "select person_name, responsibility from post_summary_role " - "where post_id = $1 order by person_name", + """ + select role.actor_name, role.responsibility, role.actor_type_code, + role.affiliated_organization_name, + role.cataloged_team_id, + role.cataloged_corporate_entity_id, + role.cataloged_person_id + from post_summary_role role + where role.post_id = $1 + order by role.actor_name + """, post_id, ) + payload_roles: list[dict[str, Any]] = [] + for row in roles: + catalog_node_id = None + catalog_node_type_code = None + if row["cataloged_team_id"] is not None: + catalog_node_id = str(row["cataloged_team_id"]) + catalog_node_type_code = NODE_TEAM + elif row["cataloged_corporate_entity_id"] is not None: + catalog_node_id = str(row["cataloged_corporate_entity_id"]) + catalog_node_type_code = NODE_CORPORATE_ENTITY + elif row["cataloged_person_id"] is not None: + catalog_node_id = str(row["cataloged_person_id"]) + catalog_node_type_code = NODE_PERSON + payload_roles.append( + { + "actor_name": row["actor_name"], + "responsibility": row["responsibility"], + "actor_type_code": row["actor_type_code"], + "affiliated_organization_name": row["affiliated_organization_name"], + "catalog_node_id": catalog_node_id, + "catalog_node_type_code": catalog_node_type_code, + **ontology_annotations(row["actor_type_code"]), + } + ) return { "post_id": post_id, "korean_summary": header["korean_summary"], "key_events": [row["event_text"] for row in events], - "roles_and_responsibilities": [ - {"person_name": row["person_name"], "responsibility": row["responsibility"]} - for row in roles - ], + "roles_and_responsibilities": payload_roles, } -async def persist_post_summary(conn: asyncpg.Connection, post_id: str, summary: PostSummary) -> dict[str, Any]: - """Replace the stored summary for ``post_id`` and return the public payload.""" +async def persist_post_summary( + conn: asyncpg.Connection, + post_id: str, + summary: PostSummary, + *, + post_body: str | None = None, + hierarchy_inference_client: CorporateHierarchyInferenceClient | None = None, + verification_client: RelationVerificationClient | None = None, +) -> dict[str, Any]: + """Replace the stored summary for ``post_id`` and return the public payload. + + ``post_body`` is the context an organization-actor hierarchy proposal + is inferred from (ADR 0010); it falls back to the summary's own Korean + text when not given. The pluggable clients default to unavailable Null + clients, so an organization actor then only resolves against an existing + ``corporate_entity``. + + Organization inference, verification, and any lock-protected catalog + creation complete before the atomic summary transaction. The catalog is + an idempotent shared identity registry; keeping that enrichment separate + prevents network latency and ``pg_advisory_xact_lock`` from extending the + summary replacement transaction while all post-owned rows still commit or + roll back together. + """ + hierarchy_inference_client = ( + hierarchy_inference_client or NullCorporateHierarchyInferenceClient() + ) + verification_client = verification_client or NullRelationVerificationClient() + + context_text = post_body if post_body is not None else summary.korean_summary + candidates = ( + await _load_corporate_entity_candidates(conn) + if summary.roles_and_responsibilities + else [] + ) + resolved_organization_ids: dict[int, str] = {} + for role_index, role in enumerate(summary.roles_and_responsibilities): + if role.actor_type_code != ACTOR_TYPE_ORGANIZATION: + continue + corporate_entity_id = await get_or_create_corporate_entity( + conn, + role.actor_name, + context_text, + hierarchy_inference_client, + verification_client, + candidates, + ) + if corporate_entity_id is not None: + resolved_organization_ids[role_index] = corporate_entity_id + + async with conn.transaction(): + await _replace_summary_projection( + conn, + post_id, + summary, + candidates, + resolved_organization_ids, + ) + + payload = await fetch_persisted_summary(conn, post_id) + if payload is None: + raise RuntimeError("persist_post_summary wrote no row") + return payload + + +async def _resolve_existing_cataloged_person_id( + conn: asyncpg.Connection, person_name: str +) -> str | None: + """Return the earliest existing catalog person id for ``person_name``. + + Lookup orders by ``created_at``, then ``person_id``. This function + does not insert a ``cataloged_person`` row (ADR 0009). A missing + catalog row stays unbound rather than inventing a person. + """ + person_row = await conn.fetchrow( + "select person_id from cataloged_person " + "where person_name = $1 " + "order by created_at, person_id limit 1", + person_name, + ) + if person_row is None: + return None + return str(person_row["person_id"]) + + +async def _replace_summary_projection( + conn: asyncpg.Connection, + post_id: str, + summary: PostSummary, + candidates: list[Any], + resolved_organization_ids: dict[int, str], +) -> None: + """Write one atomic replacement using pre-resolved shared identities.""" + # Summary replacement owns only R&R projections. Keyman mentions remain + # independent and are combined only by the graph read/derivation view. + await conn.execute( + "delete from post_summary_person_mention where post_id = $1", + post_id, + ) + await conn.execute("delete from post_team_mention where post_id = $1", post_id) + await conn.execute("delete from post_organization_mention where post_id = $1", post_id) await conn.execute("delete from post_summary_result where post_id = $1", post_id) await conn.execute( "insert into post_summary_result (post_id, korean_summary) values ($1, $2)", @@ -48,22 +234,72 @@ async def persist_post_summary(conn: asyncpg.Connection, post_id: str, summary: ) for ordinal, event_text in enumerate(summary.key_events): await conn.execute( - "insert into post_summary_event (post_id, event_ordinal, event_text) values ($1, $2, $3)", + "insert into post_summary_event (post_id, event_ordinal, event_text) " + "values ($1, $2, $3)", post_id, ordinal, event_text, ) - for role in summary.roles_and_responsibilities: + # ADR 0009 / 0019 / 0027: resolve catalog identity before writing + # the role row so fetch never reconstructs it by a non-unique name. + for role_index, role in enumerate(summary.roles_and_responsibilities): + cataloged_team_id = None + cataloged_corporate_entity_id = None + cataloged_person_id = None + if role.actor_type_code == ACTOR_TYPE_TEAM: + cataloged_team_id = await upsert_team( + conn, + role.actor_name, + role.affiliated_organization_name, + candidates, + ) + elif role.actor_type_code == ACTOR_TYPE_ORGANIZATION: + cataloged_corporate_entity_id = resolved_organization_ids.get( + role_index + ) + elif role.actor_type_code == ACTOR_TYPE_PERSON: + cataloged_person_id = await _resolve_existing_cataloged_person_id( + conn, + role.actor_name, + ) await conn.execute( - "insert into post_summary_role (post_id, person_name, responsibility) values ($1, $2, $3)", + "insert into post_summary_role " + "(post_id, actor_name, responsibility, actor_type_code, " + "affiliated_organization_name, cataloged_team_id, " + "cataloged_corporate_entity_id, cataloged_person_id) values " + "($1, $2, $3, $4, $5, $6, $7, $8)", post_id, - role.person_name, + role.actor_name, role.responsibility, + role.actor_type_code, + role.affiliated_organization_name, + cataloged_team_id, + cataloged_corporate_entity_id, + cataloged_person_id, ) - payload = await fetch_persisted_summary(conn, post_id) - if payload is None: - raise RuntimeError("persist_post_summary wrote no row") - return payload + if cataloged_team_id is not None: + await conn.execute( + "insert into post_team_mention (post_id, team_id) values ($1, $2) " + "on conflict do nothing", + post_id, + cataloged_team_id, + ) + elif cataloged_corporate_entity_id is not None: + await conn.execute( + "insert into post_organization_mention " + "(post_id, corporate_entity_id) values ($1, $2) " + "on conflict do nothing", + post_id, + cataloged_corporate_entity_id, + ) + elif cataloged_person_id is not None: + await conn.execute( + "insert into post_summary_person_mention (post_id, person_id) " + "values ($1, $2) on conflict do nothing", + post_id, + cataloged_person_id, + ) + await persist_edges_for_post(conn, post_id) def seeded_demo_summary() -> PostSummary: @@ -75,8 +311,21 @@ def seeded_demo_summary() -> PostSummary: ), key_events=("출하 지연 후속 연락",), roles_and_responsibilities=( - RoleResponsibility(person_name="Ada West", responsibility="일정 확인 후속"), - RoleResponsibility(person_name="Priya Nair", responsibility="고객 측 수신"), + RoleResponsibility( + actor_name="Ada West", + responsibility="일정 확인 후속", + affiliated_organization_name="Demo Corp", + ), + RoleResponsibility( + actor_name="Priya Nair", + responsibility="고객 측 수신", + affiliated_organization_name="Northridge Grid", + ), + RoleResponsibility( + actor_name="당사", + responsibility="출하 일정 확정", + actor_type_code=ACTOR_TYPE_ORGANIZATION, + ), ), ) @@ -108,7 +357,11 @@ def _roles_for_fixture(post_title: str) -> tuple[RoleResponsibility, ...]: if cast is None or not cast.person_names: return () return tuple( - RoleResponsibility(person_name=name, responsibility=responsibility) + RoleResponsibility( + actor_name=name, + responsibility=responsibility, + affiliated_organization_name=_FIXTURE_ROLE_AFFILIATION.get(name), + ) for name in cast.person_names if (responsibility := _FIXTURE_ROLE_RESPONSIBILITY.get(name)) ) @@ -120,8 +373,15 @@ def _roles_for_fixture(post_title: str) -> tuple[RoleResponsibility, ...]: "Jordan Hale": "사양 검토", } +_FIXTURE_ROLE_AFFILIATION = { + "Ada West": "Demo Corp", + "Priya Nair": "Northridge Grid", + "Jordan Hale": "Westfield Power", +} + def _summary(korean: str, *events: str) -> PostSummary: + """Create one compact synthetic fixture summary.""" return PostSummary(korean_summary=korean, key_events=events) diff --git a/backend/app/report_ingestion.py b/backend/app/report_ingestion.py index eff621d0..9e204ef1 100644 --- a/backend/app/report_ingestion.py +++ b/backend/app/report_ingestion.py @@ -527,10 +527,14 @@ async def fetch_period_reports( leftover_by_group[row["grouping_key"]].append(row) payload: list[dict[str, Any]] = [] for header in headers: + grouping_key = header["grouping_key"] payload.append( { "grouping_kind": header["grouping_kind"], - "grouping_key": header["grouping_key"], + "grouping_key": grouping_key, + "grouping_label": await resolve_grouping_label( + conn, header["grouping_kind"], grouping_key + ), "period_code": header["period_code"], "rubric_version": header["rubric_version"], "selected_model": header["selected_model"], diff --git a/backend/app/source_post_revision.py b/backend/app/source_post_revision.py new file mode 100644 index 00000000..489f6f48 --- /dev/null +++ b/backend/app/source_post_revision.py @@ -0,0 +1,92 @@ +"""Source-post valid-time revisions for cutoff-known bodies (ADR 0025). + +The analysis-run registry stays aggregates-only. Callers that need the +sentence a run knew must read ``source_post_revision`` through an +authorized post fetch with ``as_of``. A missing cover is omitted -- +never a fabricated cutoff body or a TEPP theta. +""" + +from __future__ import annotations + +from datetime import datetime, timezone +from typing import TYPE_CHECKING, Any + +if TYPE_CHECKING: + import asyncpg + + +def _as_utc(value: datetime) -> datetime: + """Treat a naive clock as UTC so interval tests stay timezone-aware.""" + if value.tzinfo is None: + return value.replace(tzinfo=timezone.utc) + return value.astimezone(timezone.utc) + + +def parse_as_of_clock(value: str) -> datetime: + """Parse an ISO-8601 as-of clock. + + Next action: pass the analysis-run cutoff, then compare ``known_at`` + with the live body. Empty or unparseable values raise ``ValueError``. + """ + text = value.strip() + if not text: + raise ValueError("as_of is empty") + if text.endswith("Z"): + text = text[:-1] + "+00:00" + parsed = datetime.fromisoformat(text) + return _as_utc(parsed) + + +def revision_covers_clock( + written_at: datetime, + superseded_at: datetime | None, + as_of: datetime, +) -> bool: + """True when this revision was current at ``as_of``. + + The interval is half-open: ``written_at <= as_of < superseded_at``. + A null ``superseded_at`` means the revision is still current. + """ + start = _as_utc(written_at) + clock = _as_utc(as_of) + if start > clock: + return False + if superseded_at is None: + return True + return _as_utc(superseded_at) > clock + + +def _iso(value: Any) -> str: + """Serialize a timestamptz the same way post detail already does.""" + return value.isoformat() if hasattr(value, "isoformat") else str(value) + + +async def fetch_known_at_revision( + conn: "asyncpg.Connection", + post_id: str, + as_of: datetime, +) -> dict[str, str] | None: + """Return the title/body current at ``as_of``, or None when none exists. + + Does not invent a sentence. Does not return a live body under a + cutoff label when no revision covers the clock. + """ + row = await conn.fetchrow( + "select post_title, post_body, written_at " + "from source_post_revision " + "where post_id = $1 " + "and written_at <= $2 " + "and (superseded_at is null or superseded_at > $2) " + "order by written_at desc " + "limit 1", + post_id, + as_of, + ) + if row is None: + return None + return { + "post_title": row["post_title"], + "post_body": row["post_body"], + "written_at": _iso(row["written_at"]), + "as_of": _iso(as_of), + } diff --git a/backend/app/team_ingestion.py b/backend/app/team_ingestion.py new file mode 100644 index 00000000..2d0c8787 --- /dev/null +++ b/backend/app/team_ingestion.py @@ -0,0 +1,48 @@ + +"""Resolve an R&R team actor to one shared cross-post identity.""" + +from __future__ import annotations + +import asyncpg + +from lineageweave.corporate_hierarchy_resolution import ( + CorporateEntityCandidate, + resolve_corporate_entity, +) + + +async def upsert_team( + conn: asyncpg.Connection, + team_name: str, + affiliated_organization_name: str | None, + candidates: list[CorporateEntityCandidate], +) -> str: + """Atomically return the unique team identity for the pair. + + ``UNIQUE NULLS NOT DISTINCT`` makes NULL affiliations participate + in the same conflict rule. One upsert removes the prior + read-then-insert race. + """ + corporate_entity_id = ( + resolve_corporate_entity(affiliated_organization_name, candidates) + if affiliated_organization_name + else None + ) + row = await conn.fetchrow( + """ + insert into cataloged_team + (team_name, affiliated_organization_name, + affiliated_corporate_entity_id) + values ($1, $2, $3) + on conflict (team_name, affiliated_organization_name) do update set + affiliated_corporate_entity_id = coalesce( + excluded.affiliated_corporate_entity_id, + cataloged_team.affiliated_corporate_entity_id + ) + returning team_id + """, + team_name, + affiliated_organization_name, + corporate_entity_id, + ) + return str(row["team_id"]) diff --git a/backend/tests/test_api.py b/backend/tests/test_api.py index 1db483ee..910a7848 100644 --- a/backend/tests/test_api.py +++ b/backend/tests/test_api.py @@ -22,6 +22,12 @@ import redis from lineageweave.http_client import HttpClientError, get_json, post_form +from lineageweave.knowledge_graph import knowledge_graph_edges_for_post +from lineageweave.tepp_client import TeppClient +from lineageweave.tepp_result import ( + accepted_tepp_seed_envelope, + tepp_accepted_evidence_sha256, +) _POSTGRES_ADMIN_DSN = os.environ.get( "LINEAGEWEAVE_TEST_POSTGRES_ADMIN_DSN", "postgresql://lineageweave:lineageweave_dev_only@localhost:15432/lineageweave" @@ -30,6 +36,26 @@ _VALKEY_URL = os.environ.get("LINEAGEWEAVE_TEST_VALKEY_URL", "redis://localhost:16379/0") _REALM = "lineageweave-demo" _MIGRATION_PATH = Path(__file__).resolve().parents[2] / "migrations" / "0001_initial_schema.sql" +_REGISTRY_MIGRATION = Path(__file__).resolve().parents[2] / "migrations" / "0018_analysis_run_registry.sql" +_RETENTION_MIGRATION = Path(__file__).resolve().parents[2] / "migrations" / "0020_analysis_run_retention_purge.sql" +_RECONSTRUCTION_MIGRATION = ( + Path(__file__).resolve().parents[2] / "migrations" / "0021_analysis_run_reconstruction.sql" +) +_SNAPSHOT_MEMBER_MIGRATION = ( + Path(__file__).resolve().parents[2] / "migrations" / "0022_analysis_source_snapshot_member.sql" +) +_OUTBOX_MIGRATION = ( + Path(__file__).resolve().parents[2] / "migrations" / "0023_analysis_run_outbox.sql" +) +_REVISION_MIGRATION = ( + Path(__file__).resolve().parents[2] / "migrations" / "0024_source_post_revision.sql" +) +_TEPP_RESULT_MIGRATION = ( + Path(__file__).resolve().parents[2] / "migrations" / "0028_analysis_run_tepp_result.sql" +) +_TEPP_ACCEPTED_MIGRATION = ( + Path(__file__).resolve().parents[2] / "migrations" / "0029_analysis_run_tepp_accepted.sql" +) def _postgres_available() -> bool: @@ -112,10 +138,19 @@ def seeded_db(demo_analyst_token): try: with conn.cursor() as cur: cur.execute(_MIGRATION_PATH.read_text()) + cur.execute(_REGISTRY_MIGRATION.read_text()) + cur.execute(_RETENTION_MIGRATION.read_text()) + cur.execute(_RECONSTRUCTION_MIGRATION.read_text()) + cur.execute(_SNAPSHOT_MEMBER_MIGRATION.read_text()) + cur.execute(_OUTBOX_MIGRATION.read_text()) + cur.execute(_REVISION_MIGRATION.read_text()) + cur.execute(_TEPP_RESULT_MIGRATION.read_text()) + cur.execute(_TEPP_ACCEPTED_MIGRATION.read_text()) cur.execute( "insert into common_lookup_value (lookup_category, lookup_code, lookup_label) values " "('corporate_entity_level', 'group', 'Group'), " "('corporate_entity_level', 'company', 'Company'), " + "('corporate_entity_level', 'plant', 'Plant'), " "('post_visibility', 'public', 'Public'), " "('post_visibility', 'private', 'Private'), " "('voc_type', 'voc', 'Voice of Customer'), " @@ -125,9 +160,13 @@ def seeded_db(demo_analyst_token): "('node_type', 'node_person', 'Person'), " "('node_type', 'node_corporate_entity', 'Corporate entity'), " "('node_type', 'node_post', 'Post'), " + "('node_type', 'node_team', 'Team'), " "('edge_type', 'edge_mention', 'Mentioned in'), " "('edge_type', 'edge_affiliation', 'Affiliated with'), " "('edge_type', 'edge_co_mention', 'Co-mentioned'), " + "('edge_type', 'edge_mention_team', 'Team mentioned in'), " + "('edge_type', 'edge_team_affiliation', 'Team affiliated with'), " + "('edge_type', 'edge_mention_organization', 'Organization mentioned in'), " "('entity_relationship_type', 'rel_voc', 'Voice of Customer'), " "('entity_relationship_type', 'rel_vom', 'Voice of Market'), " "('entity_relationship_type', 'rel_vop', 'Voice of Partner'), " @@ -142,7 +181,10 @@ def seeded_db(demo_analyst_token): "('relation_verification_status', 'verify_uncorroborated', 'No corroborating evidence found'), " "('evaluation_criterion', 'general_sentiment_positive', 'Constructive stance'), " "('evaluation_criterion', 'general_sentiment_negative', 'Negative stance'), " - "('evaluation_criterion', 'sales_lead_specificity', 'Sales-lead specificity')" + "('evaluation_criterion', 'sales_lead_specificity', 'Sales-lead specificity'), " + "('prov_agent_type', 'prov_person', 'Person'), " + "('prov_agent_type', 'prov_organization', 'Organization'), " + "('prov_agent_type', 'prov_team', 'Team')" ) cur.execute( "insert into corporate_entity (corporate_entity_code, entity_name, entity_level_code) " @@ -155,6 +197,12 @@ def seeded_db(demo_analyst_token): (own_group_id,), ) own_corp_id = cur.fetchone()[0] + cur.execute( + "insert into corporate_entity (parent_entity_id, corporate_entity_code, entity_name, entity_level_code) " + "values (%s, 'TEST-PLANT', 'Test Plant', 'plant') returning corporate_entity_id", + (own_corp_id,), + ) + own_plant_id = cur.fetchone()[0] cur.execute( "insert into corporate_entity (corporate_entity_code, entity_name, entity_level_code) " "values ('OTHER-CORP', 'Other Corp', 'group') returning corporate_entity_id" @@ -179,16 +227,134 @@ def seeded_db(demo_analyst_token): "insert into role_permission (access_role_id, permission_code) values (%s, 'post_read')", (role_id,), ) + + def _seed_analysis_run( + digest: str, + idempotency_key: str, + requester_id, + scope_kind: str, + corp_id=None, + ) -> str: + cur.execute( + """ + insert into analysis_source_snapshot + (snapshot_sha256, source_contract_version, + maximum_available_time, captured_at) + values (%s, 'source-contract-v1', + '2026-01-12T00:00:00Z', '2026-01-12T00:05:00Z') + returning analysis_source_snapshot_id + """, + (digest,), + ) + snapshot_id = cur.fetchone()[0] + cur.execute( + """ + insert into analysis_source_count + (analysis_source_snapshot_id, count_type_code, count_value) + values (%s, 'analysis_count_document', 3) + """, + (snapshot_id,), + ) + cur.execute( + """ + insert into analysis_run + (analysis_source_snapshot_id, run_kind_code, idempotency_key, + requested_by_account_id, knowledge_cutoff, + configuration_schema_version, configuration_sha256, + code_revision_sha, requested_at) + values (%s, 'analysis_run_lineage', %s, %s, + '2026-01-12T12:00:00Z', 'lineage-run-v1', %s, %s, + '2026-01-12T12:30:00Z') + returning analysis_run_id + """, + (snapshot_id, idempotency_key, requester_id, "b" * 64, "c" * 40), + ) + run_id = str(cur.fetchone()[0]) + if scope_kind == "analysis_scope_corporate_entity": + cur.execute( + """ + insert into analysis_run_scope + (analysis_run_id, scope_kind_code, corporate_entity_id) + values (%s, %s, %s) + """, + (run_id, scope_kind, corp_id), + ) + else: + cur.execute( + """ + insert into analysis_run_scope + (analysis_run_id, scope_kind_code) + values (%s, %s) + """, + (run_id, scope_kind), + ) + for ordinal, status, occurred in ( + (1, "analysis_status_pending", "2026-01-12T12:31:00Z"), + (2, "analysis_status_running", "2026-01-12T12:32:00Z"), + (3, "analysis_status_succeeded", "2026-01-12T12:33:00Z"), + ): + cur.execute( + """ + insert into analysis_run_status_event + (analysis_run_id, status_ordinal, status_code, occurred_at) + values (%s, %s, %s, %s) + """, + (run_id, ordinal, status, occurred), + ) + return run_id + + cur.execute( + "insert into user_account (external_subject_id, display_name, email_address) " + "values (%s, 'Other Analyst', 'other.analyst@example.test') returning user_account_id", + (f"other-{uuid.uuid4()}",), + ) + other_account_id = cur.fetchone()[0] + visible_run_id = _seed_analysis_run( + "a" * 64, + "visible-own-corp", + account_id, + "analysis_scope_corporate_entity", + own_corp_id, + ) + hidden_run_id = _seed_analysis_run( + "d" * 64, + "hidden-other-corp", + other_account_id, + "analysis_scope_corporate_entity", + other_corp_id, + ) + hidden_all_visible_id = _seed_analysis_run( + "e" * 64, + "hidden-all-visible", + other_account_id, + "analysis_scope_all_visible", + ) cur.execute( "insert into account_role_assignment (user_account_id, access_role_id) values (%s, %s)", (account_id, role_id), ) - def _insert_post(title: str, corporate_entity_id, visibility_code: str, body: str = "body") -> str: + def _insert_post( + title: str, + corporate_entity_id, + visibility_code: str, + body: str = "body", + created_at: str = "2026-01-10T12:00:00Z", + updated_at: str | None = None, + ) -> str: + written_at = updated_at if updated_at is not None else created_at cur.execute( - "insert into source_post (author_account_id, corporate_entity_id, post_title, post_body, voc_type_code, visibility_code) " - "values (%s, %s, %s, %s, 'voc', %s) returning post_id", - (account_id, corporate_entity_id, title, body, visibility_code), + "insert into source_post (author_account_id, corporate_entity_id, post_title, post_body, voc_type_code, visibility_code, created_at, updated_at) " + "values (%s, %s, %s, %s, 'voc', %s, %s, %s) returning post_id", + ( + account_id, + corporate_entity_id, + title, + body, + visibility_code, + created_at, + written_at, + ), ) return str(cur.fetchone()[0]) @@ -201,6 +367,29 @@ def _insert_post(title: str, corporate_entity_id, visibility_code: str, body: st "The weather in Gwangju was irrelevant.", ) other_private_post_id = _insert_post("Other-corp private post", other_corp_id, "private") + late_own_private_post_id = _insert_post( + "Late own-corp private post", + own_corp_id, + "private", + "A follow-up written after the January 2026 run cutoff.", + created_at="2026-01-20T12:00:00Z", + ) + edited_own_post_id = _insert_post( + "Edited own-corp private post", + own_corp_id, + "private", + "A January post before the rewrite.", + created_at="2026-01-10T12:00:00Z", + updated_at="2026-01-10T12:00:00Z", + ) + cur.execute( + "update source_post set post_body = %s, updated_at = %s where post_id = %s", + ( + "A January post rewritten after the run cutoff.", + "2026-01-13T09:00:00Z", + edited_own_post_id, + ), + ) cur.execute( "insert into cataloged_person (person_name, person_side_code) values " @@ -284,12 +473,18 @@ def _insert_post(title: str, corporate_entity_id, visibility_code: str, body: st "public_post_id": public_post_id, "own_group_id": str(own_group_id), "own_corp_id": str(own_corp_id), + "own_plant_id": str(own_plant_id), "other_corp_id": str(other_corp_id), "own_private_post_id": own_private_post_id, + "late_own_private_post_id": late_own_private_post_id, + "edited_own_post_id": edited_own_post_id, "other_private_post_id": other_private_post_id, "our_person_id": our_person_id, "counterpart_person_id": counterpart_person_id, "hidden_person_id": hidden_person_id, + "visible_run_id": visible_run_id, + "hidden_run_id": hidden_run_id, + "hidden_all_visible_id": hidden_all_visible_id, } finally: conn.close() @@ -312,295 +507,819 @@ def client(seeded_db): yield test_client -def test_me_reflects_the_authenticated_account(client, demo_analyst_token) -> None: - response = client.get("/api/me", headers={"Authorization": f"Bearer {demo_analyst_token}"}) - assert response.status_code == 200 - body = response.json() - assert body["display_name"] == "Test Analyst" - assert "post_read" in body["permission_codes"] +def test_analysis_runs_are_labeled_aggregates_and_hide_other_scopes( + client, demo_analyst_token, seeded_db +) -> None: + """Demo analyst sees the Test Corp run, never the Other Corp or outsider run.""" + listed = client.get("/api/analysis-runs", headers={"Authorization": f"Bearer {demo_analyst_token}"}) + assert listed.status_code == 200 + runs = listed.json()["analysis_runs"] + ids = {run["analysis_run_id"] for run in runs} + assert seeded_db["visible_run_id"] in ids + assert seeded_db["hidden_run_id"] not in ids + assert seeded_db["hidden_all_visible_id"] not in ids + visible = next(run for run in runs if run["analysis_run_id"] == seeded_db["visible_run_id"]) + assert visible["run_kind_label"] == "Lineage reconstruction" + assert visible["status_label"] == "Succeeded" + assert visible["scope_kind_label"] == "Corporate entity" + assert visible["scope_entity_name"] == "Test Corp" + assert visible["source_counts"] == [ + { + "count_type_code": "analysis_count_document", + "count_type_label": "Documents", + "count_value": 3, + } + ] + dumped = str(visible) + assert "postgresql://" not in dumped + assert "select " not in dumped.lower() + assert "status_history" not in visible + detail = client.get( + f"/api/analysis-runs/{seeded_db['visible_run_id']}", + headers={"Authorization": f"Bearer {demo_analyst_token}"}, + ) + assert detail.status_code == 200 + body = detail.json() + assert body["configuration_schema_version"] == "lineage-run-v1" + assert "snapshot_sha256" not in body + history = body["status_history"] + assert [event["status_label"] for event in history] == [ + "Pending", + "Running", + "Succeeded", + ] + assert [event["occurred_at"][:16] for event in history] == [ + "2026-01-12T12:31", + "2026-01-12T12:32", + "2026-01-12T12:33", + ] + assert all("failure_code" not in event for event in history) + titles = {post["post_title"] for post in body["visible_posts"]} + assert "Own-corp private post" in titles + assert "Edited own-corp private post" in titles + assert "Late own-corp private post" not in titles + assert "Other-corp private post" not in titles + posts_by_title = {post["post_title"]: post for post in body["visible_posts"]} + assert posts_by_title["Own-corp private post"]["live_after_cutoff"] is False + assert posts_by_title["Edited own-corp private post"]["live_after_cutoff"] is True + assert posts_by_title["Edited own-corp private post"]["updated_at"].startswith("2026-01-13") + assert "post_body" not in posts_by_title["Edited own-corp private post"] + assert "postgresql://" not in str(body) + assert "visible_posts" not in visible + + hidden = client.get( + f"/api/analysis-runs/{seeded_db['hidden_run_id']}", + headers={"Authorization": f"Bearer {demo_analyst_token}"}, + ) + assert hidden.status_code == 404 + assert "tepp_evidence_sha256" not in hidden.text + assert "accepted_run_id" not in hidden.text + assert "aggregate transport evidence" not in hidden.text -def test_post_list_includes_public_and_own_corp_but_excludes_other_corp(client, demo_analyst_token, seeded_db) -> None: - response = client.get("/api/posts", headers={"Authorization": f"Bearer {demo_analyst_token}"}) - assert response.status_code == 200 - titles = {post["post_title"] for post in response.json()} - assert titles == {"Public post", "Own-corp private post"} - public = next(post for post in response.json() if post["post_title"] == "Public post") - assert public["voc_type_label"] == "Voice of Customer" - assert public["visibility_label"] == "Public" + unauthenticated = client.get("/api/analysis-runs") + assert unauthenticated.status_code == 401 -def test_post_detail_uses_lookup_labels_not_raw_codes(client, demo_analyst_token, seeded_db) -> None: - response = client.get( - f"/api/posts/{seeded_db['public_post_id']}", +def test_create_analysis_run_records_pending_without_inventing_a_score( + client, demo_analyst_token, seeded_db +) -> None: + """POST /api/analysis-runs writes Pending on the authorized cutoff bag.""" + created = client.post( + "/api/analysis-runs", headers={"Authorization": f"Bearer {demo_analyst_token}"}, + json={ + "run_kind_code": "analysis_run_lineage", + "corporate_entity_id": seeded_db["own_corp_id"], + "idempotency_key": "buyer-create-2026-w02", + }, ) - assert response.status_code == 200 - body = response.json() - assert body["voc_type_code"] == "voc" - assert body["voc_type_label"] == "Voice of Customer" - assert body["visibility_code"] == "public" - assert body["visibility_label"] == "Public" + assert created.status_code == 201 + body = created.json() + assert body["run_kind_label"] == "Lineage reconstruction" + assert body["status_label"] == "Pending" + assert body["status_history"][0]["status_label"] == "Pending" + assert all(event["status_label"] != "Succeeded" for event in body["status_history"]) + titles = {post["post_title"] for post in body["visible_posts"]} + assert "Own-corp private post" in titles + assert "Other-corp private post" not in titles + assert "theta" not in str(body).lower() + assert "postgresql://" not in str(body) + + replay = client.post( + "/api/analysis-runs", + headers={"Authorization": f"Bearer {demo_analyst_token}"}, + json={ + "run_kind_code": "analysis_run_lineage", + "corporate_entity_id": seeded_db["own_corp_id"], + "idempotency_key": "buyer-create-2026-w02", + }, + ) + assert replay.status_code == 201 + assert replay.json()["analysis_run_id"] == body["analysis_run_id"] + tepp = client.post( + "/api/analysis-runs", + headers={"Authorization": f"Bearer {demo_analyst_token}"}, + json={ + "run_kind_code": "analysis_run_tepp", + "corporate_entity_id": seeded_db["own_corp_id"], + "idempotency_key": "buyer-create-tepp", + }, + ) + assert tepp.status_code == 422 + assert "invent a measurement" in tepp.json()["detail"] + assert "theta" not in tepp.json()["detail"].lower() + + report = client.post( + "/api/analysis-runs", + headers={"Authorization": f"Bearer {demo_analyst_token}"}, + json={ + "run_kind_code": "analysis_run_report", + "corporate_entity_id": seeded_db["own_corp_id"], + "idempotency_key": "buyer-create-report", + }, + ) + assert report.status_code == 422 + assert "Reports panel" in report.json()["detail"] + + conflict = client.post( + "/api/analysis-runs", + headers={"Authorization": f"Bearer {demo_analyst_token}"}, + json={ + "run_kind_code": "analysis_run_lineage", + "corporate_entity_id": seeded_db["own_corp_id"], + "knowledge_cutoff": "2026-01-01T00:00:00Z", + "idempotency_key": "buyer-create-2026-w02", + }, + ) + assert conflict.status_code == 409 + + hidden = client.post( + "/api/analysis-runs", + headers={"Authorization": f"Bearer {demo_analyst_token}"}, + json={ + "run_kind_code": "analysis_run_lineage", + "corporate_entity_id": seeded_db["other_corp_id"], + "idempotency_key": "buyer-create-hidden-corp", + }, + ) + assert hidden.status_code == 404 + + unauthenticated = client.post( + "/api/analysis-runs", + json={"idempotency_key": "buyer-create-unauthenticated"}, + ) + assert unauthenticated.status_code == 401 + + +def test_start_analysis_run_recovers_the_a100_fork( + client, demo_analyst_token, seeded_db +) -> None: + """Starting a Pending lineage run persists the designed fixture tree.""" + from scripts.seed_demo_data import insert_fixture_source_posts -def test_persisted_summary_is_returned_without_an_llm(client, demo_analyst_token, seeded_db) -> None: - """GET /api/posts/{id}/summary must serve a stored row even when the - orchestrator is off -- otherwise a seeded demo popup stays empty. - """ - os.environ.pop("ORCHESTRATOR_BASE_URL", None) - os.environ.pop("ORCHESTRATOR_API_KEY", None) admin_conn = psycopg2.connect(seeded_db["dsn"]) admin_conn.autocommit = True try: with admin_conn.cursor() as cur: cur.execute( - "insert into post_summary_result (post_id, korean_summary) values (%s, %s)", - (seeded_db["public_post_id"], "저장된 한국어 요약입니다."), + "insert into common_lookup_value (lookup_category, lookup_code, lookup_label) " + "values ('voc_type', 'vom', 'Voice of Market') " + "on conflict (lookup_code) do nothing" ) cur.execute( - "insert into post_summary_event (post_id, event_ordinal, event_text) " - "values (%s, 0, '저장된 이벤트')", - (seeded_db["public_post_id"],), + "insert into process_unit (corporate_entity_id, process_unit_code, process_unit_name) " + "select corporate_entity_id, 'TEST-PU-START', 'Start reconstruction' " + "from source_post where post_id = %s returning process_unit_id", + (seeded_db["own_private_post_id"],), ) + process_unit_id = cur.fetchone()[0] cur.execute( - "insert into post_summary_role (post_id, person_name, responsibility) " - "values (%s, 'Ada West', '후속 연락')", - (seeded_db["public_post_id"],), + "select author_account_id, corporate_entity_id from source_post where post_id = %s", + (seeded_db["own_private_post_id"],), ) + author_id, corp_id = cur.fetchone() + insert_fixture_source_posts(cur, author_id, corp_id, process_unit_id) finally: admin_conn.close() - response = client.get( - f"/api/posts/{seeded_db['public_post_id']}/summary", + created = client.post( + "/api/analysis-runs", headers={"Authorization": f"Bearer {demo_analyst_token}"}, + json={ + "run_kind_code": "analysis_run_lineage", + "corporate_entity_id": seeded_db["own_corp_id"], + "knowledge_cutoff": "2026-02-15T00:00:00Z", + "idempotency_key": "buyer-start-2026-w07", + }, ) - assert response.status_code == 200 - body = response.json() - assert body["korean_summary"] == "저장된 한국어 요약입니다." - assert body["key_events"] == ["저장된 이벤트"] - assert body["roles_and_responsibilities"] == [ - {"person_name": "Ada West", "responsibility": "후속 연락"} - ] - + assert created.status_code == 201, created.text + run_id = created.json()["analysis_run_id"] + assert created.json()["status_label"] == "Pending" -def test_seed_demo_summary_surfaces_on_get_summary(client, demo_analyst_token, seeded_db) -> None: - """The same helper `make seed` calls must produce a row GET summary - returns -- even with the orchestrator unset. - """ - os.environ.pop("ORCHESTRATOR_BASE_URL", None) - os.environ.pop("ORCHESTRATOR_API_KEY", None) - from scripts.seed_demo_data import _seed_demo_public_summary + started = client.post( + f"/api/analysis-runs/{run_id}/start", + headers={"Authorization": f"Bearer {demo_analyst_token}"}, + ) + assert started.status_code == 200, started.text + body = started.json() + assert body["status_label"] == "Succeeded" + assert all(event["status_label"] != "Failed" for event in body["status_history"]) + assert body["reconstruction_result_sha256"] + children = { + edge["child_post_title"] + for edge in body["reconstructed_edges"] + if edge["parent_post_title"] == "Pricing renegotiation follow-up" + } + assert "Pricing renegotiation: revised quote sent" in children + assert "Delivery schedule question raised" in children + assert "theta" not in str(body).lower() + assert "outbox_request_sha256" not in body admin_conn = psycopg2.connect(seeded_db["dsn"]) admin_conn.autocommit = True try: with admin_conn.cursor() as cur: - _seed_demo_public_summary(cur, seeded_db["public_post_id"]) + cur.execute( + """ + select outbox.work_kind_code, delivery.delivery_status_code + from analysis_run_outbox outbox + join analysis_run_outbox_delivery delivery + on delivery.analysis_run_id = outbox.analysis_run_id + where outbox.analysis_run_id = %s + order by delivery.delivery_ordinal desc + limit 1 + """, + (run_id,), + ) + outbox_row = cur.fetchone() + assert outbox_row == ("analysis_run_lineage", "analysis_outbox_delivered") finally: admin_conn.close() + valkey = redis.from_url(_VALKEY_URL, decode_responses=True) + try: + entries = valkey.xrevrange("analysis-run-outbox", count=50) + assert any( + fields.get("analysis_run_id") == run_id + and fields.get("work_kind_code") == "analysis_run_lineage" + and "theta" not in str(fields).casefold() + for _entry_id, fields in entries + ) + finally: + valkey.close() - response = client.get( - f"/api/posts/{seeded_db['public_post_id']}/summary", + replay = client.post( + f"/api/analysis-runs/{run_id}/start", headers={"Authorization": f"Bearer {demo_analyst_token}"}, ) - assert response.status_code == 200, response.text - body = response.json() - assert "에이다" in body["korean_summary"] - assert body["key_events"] - assert any(role["person_name"] == "Ada West" for role in body["roles_and_responsibilities"]) - + assert replay.status_code == 200 + assert replay.json()["reconstruction_result_sha256"] == body["reconstruction_result_sha256"] -def test_seed_fixture_summaries_surface_on_get_summary(client, demo_analyst_token, seeded_db) -> None: - """The A-100 fork and calendar commitment `make seed` writes must - answer GET /api/posts/{id}/summary without a live orchestrator. - """ - from scripts.seed_demo_data import ( - _seed_demo_calendar_commitment, - _seed_fixture_summaries, - insert_fixture_source_posts, + tepp_create = client.post( + "/api/analysis-runs", + headers={"Authorization": f"Bearer {demo_analyst_token}"}, + json={ + "run_kind_code": "analysis_run_tepp", + "corporate_entity_id": seeded_db["own_corp_id"], + "knowledge_cutoff": "2026-02-15T00:00:00Z", + "idempotency_key": "buyer-start-tepp-2026-w07", + }, ) + assert tepp_create.status_code == 422 + assert "invent a measurement" in tepp_create.json()["detail"] - os.environ.pop("ORCHESTRATOR_BASE_URL", None) - os.environ.pop("ORCHESTRATOR_API_KEY", None) admin_conn = psycopg2.connect(seeded_db["dsn"]) admin_conn.autocommit = True try: with admin_conn.cursor() as cur: cur.execute( - "insert into common_lookup_value (lookup_category, lookup_code, lookup_label) " - "values ('voc_type', 'vom', 'Voice of Market') " - "on conflict (lookup_code) do nothing" + "select requested_by_account_id from analysis_run where analysis_run_id = %s", + (run_id,), ) + requester_id = cur.fetchone()[0] cur.execute( - "insert into process_unit (corporate_entity_id, process_unit_code, process_unit_name) " - "select corporate_entity_id, 'TEST-PU-SUMMARY', 'Summary thread' " - "from source_post where post_id = %s returning process_unit_id", - (seeded_db["own_private_post_id"],), + """ + insert into analysis_source_snapshot + (snapshot_sha256, source_contract_version, + maximum_available_time, captured_at) + values (%s, 'source-contract-v1', + '2026-02-15T00:00:00Z', '2026-02-15T00:05:00Z') + returning analysis_source_snapshot_id + """, + ("b" * 64,), ) - process_unit_id = cur.fetchone()[0] + tepp_snapshot_id = cur.fetchone()[0] cur.execute( - "select author_account_id, corporate_entity_id from source_post where post_id = %s", - (seeded_db["own_private_post_id"],), + """ + insert into analysis_run + (analysis_source_snapshot_id, run_kind_code, idempotency_key, + requested_by_account_id, knowledge_cutoff, + configuration_schema_version, configuration_sha256, + code_revision_sha, requested_at) + values (%s, 'analysis_run_tepp', 'buyer-start-tepp-seeded', + %s, '2026-02-15T00:00:00Z', 'tepp-run-v1', %s, %s, + '2026-02-15T12:30:00Z') + returning analysis_run_id + """, + (tepp_snapshot_id, requester_id, "c" * 64, "d" * 40), ) - author_id, corp_id = cur.fetchone() - insert_fixture_source_posts(cur, author_id, corp_id, process_unit_id) - _seed_demo_calendar_commitment(cur, author_id, corp_id, process_unit_id) - _seed_fixture_summaries(cur) + tepp_run_id = str(cur.fetchone()[0]) cur.execute( - "select post_id from source_post where post_title = %s", - ("Pricing renegotiation follow-up",), + """ + insert into analysis_run_scope + (analysis_run_id, scope_kind_code, corporate_entity_id) + values (%s, 'analysis_scope_corporate_entity', %s) + """, + (tepp_run_id, seeded_db["own_corp_id"]), ) - fork_id = str(cur.fetchone()[0]) cur.execute( - "select post_id from source_post where post_title = %s", - ("Follow-up on the Riverbend order confirmation",), + """ + insert into analysis_run_status_event + (analysis_run_id, status_ordinal, status_code, occurred_at) + values (%s, 1, 'analysis_status_pending', '2026-02-15T12:31:00Z') + """, + (tepp_run_id,), ) - calendar_id = str(cur.fetchone()[0]) finally: admin_conn.close() - fork = client.get( - f"/api/posts/{fork_id}/summary", - headers={"Authorization": f"Bearer {demo_analyst_token}"}, - ) - assert fork.status_code == 200, fork.text - assert "재협상" in fork.json()["korean_summary"] - assert fork.json()["key_events"] - fork_roles = {role["person_name"] for role in fork.json()["roles_and_responsibilities"]} - assert fork_roles == {"Ada West", "Priya Nair"} - - calendar = client.get( - f"/api/posts/{calendar_id}/summary", + measured = client.post( + f"/api/analysis-runs/{tepp_run_id}/start", headers={"Authorization": f"Bearer {demo_analyst_token}"}, ) - assert calendar.status_code == 200, calendar.text - assert "리버벤드" in calendar.json()["korean_summary"] - assert calendar.json()["roles_and_responsibilities"] == [] - - missing = client.get( - f"/api/posts/{seeded_db['own_private_post_id']}/summary", - headers={"Authorization": f"Bearer {demo_analyst_token}"}, + assert measured.status_code == 200, measured.text + tepp_body = measured.json() + assert tepp_body["status_label"] == "Failed" + assert tepp_body["failure_code"] == "tepp_not_available" + assert any( + event.get("failure_code") == "tepp_not_available" + for event in tepp_body["status_history"] ) - assert missing.status_code == 503 + assert "theta" not in str(tepp_body).lower() - -def test_persisted_chat_is_returned_without_an_llm(client, demo_analyst_token, seeded_db) -> None: - """POST /api/posts/{id}/chat must serve a stored row even when the - orchestrator is off -- otherwise a seeded demo Ask stays empty. - """ - os.environ.pop("ORCHESTRATOR_BASE_URL", None) - os.environ.pop("ORCHESTRATOR_API_KEY", None) admin_conn = psycopg2.connect(seeded_db["dsn"]) admin_conn.autocommit = True try: with admin_conn.cursor() as cur: cur.execute( - "insert into post_chat_result (post_id, question_norm, question_text, answer_text) " - "values (%s, 'what happened between these events', " - "'What happened between these events?', 'Stored follow-up after the site visit.')", - (seeded_db["public_post_id"],), + """ + insert into analysis_source_snapshot + (snapshot_sha256, source_contract_version, + maximum_available_time, captured_at) + values (%s, 'source-contract-v1', + '2026-01-12T00:00:00Z', '2026-01-12T00:05:00Z') + returning analysis_source_snapshot_id + """, + ("9" * 64,), ) + report_snapshot_id = cur.fetchone()[0] cur.execute( - "insert into post_chat_citation " - "(post_id, question_norm, citation_ordinal, cited_post_id) " - "values (%s, 'what happened between these events', 0, %s)", - (seeded_db["public_post_id"], seeded_db["public_post_id"]), + "select requested_by_account_id from analysis_run where analysis_run_id = %s", + (run_id,), + ) + requester_id = cur.fetchone()[0] + cur.execute( + """ + insert into analysis_run + (analysis_source_snapshot_id, run_kind_code, idempotency_key, + requested_by_account_id, knowledge_cutoff, + configuration_schema_version, configuration_sha256, + code_revision_sha, requested_at) + values (%s, 'analysis_run_report', 'buyer-start-report', + %s, '2026-01-12T12:00:00Z', 'lineage-run-v1', %s, %s, + '2026-01-12T12:30:00Z') + returning analysis_run_id + """, + (report_snapshot_id, requester_id, "8" * 64, "7" * 40), + ) + report_run_id = str(cur.fetchone()[0]) + cur.execute( + """ + insert into analysis_run_scope + (analysis_run_id, scope_kind_code, corporate_entity_id) + values (%s, 'analysis_scope_corporate_entity', %s) + """, + (report_run_id, seeded_db["own_corp_id"]), + ) + cur.execute( + """ + insert into analysis_run_status_event + (analysis_run_id, status_ordinal, status_code, occurred_at) + values (%s, 1, 'analysis_status_pending', '2026-01-12T12:31:00Z') + """, + (report_run_id,), + ) + cur.execute( + """ + insert into analysis_source_snapshot + (snapshot_sha256, source_contract_version, + maximum_available_time, captured_at) + values (%s, 'source-contract-v1', + '2026-01-12T00:00:00Z', '2026-01-12T00:05:00Z') + returning analysis_source_snapshot_id + """, + ("6" * 64,), + ) + running_snapshot_id = cur.fetchone()[0] + cur.execute( + """ + insert into analysis_run + (analysis_source_snapshot_id, run_kind_code, idempotency_key, + requested_by_account_id, knowledge_cutoff, + configuration_schema_version, configuration_sha256, + code_revision_sha, requested_at) + values (%s, 'analysis_run_lineage', 'buyer-start-running', + %s, '2026-01-12T12:00:00Z', 'lineage-run-v1', %s, %s, + '2026-01-12T12:30:00Z') + returning analysis_run_id + """, + (running_snapshot_id, requester_id, "5" * 64, "4" * 40), + ) + running_run_id = str(cur.fetchone()[0]) + cur.execute( + """ + insert into analysis_run_scope + (analysis_run_id, scope_kind_code, corporate_entity_id) + values (%s, 'analysis_scope_corporate_entity', %s) + """, + (running_run_id, seeded_db["own_corp_id"]), ) + for ordinal, status, occurred in ( + (1, "analysis_status_pending", "2026-01-12T12:31:00Z"), + (2, "analysis_status_running", "2026-01-12T12:32:00Z"), + ): + cur.execute( + """ + insert into analysis_run_status_event + (analysis_run_id, status_ordinal, status_code, occurred_at) + values (%s, %s, %s, %s) + """, + (running_run_id, ordinal, status, occurred), + ) finally: admin_conn.close() - asked = client.post( - f"/api/posts/{seeded_db['public_post_id']}/chat", - json={"question": "What happened?"}, + report_refused = client.post( + f"/api/analysis-runs/{report_run_id}/start", headers={"Authorization": f"Bearer {demo_analyst_token}"}, ) - assert asked.status_code == 200, asked.text - body = asked.json() - assert body["answer_text"] == "Stored follow-up after the site visit." - assert body["cited_post_ids"] == [seeded_db["public_post_id"]] - assert body["cited_posts"] == [ - {"post_id": seeded_db["public_post_id"], "post_title": "Public post"} - ] + assert report_refused.status_code == 422 + assert "invent a measurement" in report_refused.json()["detail"] - history = client.get( - f"/api/posts/{seeded_db['public_post_id']}/chat", + running = client.post( + f"/api/analysis-runs/{running_run_id}/start", headers={"Authorization": f"Bearer {demo_analyst_token}"}, ) - assert history.status_code == 200, history.text - assert history.json()["exchanges"][0]["answer_text"] == "Stored follow-up after the site visit." + assert running.status_code == 409 + hidden = client.post( + f"/api/analysis-runs/{seeded_db['hidden_run_id']}/start", + headers={"Authorization": f"Bearer {demo_analyst_token}"}, + ) + assert hidden.status_code == 404 -def test_seed_demo_chat_surfaces_on_get_and_post_chat(client, demo_analyst_token, seeded_db) -> None: - """The same helper `make seed` calls must produce a row GET/POST chat - return -- even with the orchestrator unset. - """ - os.environ.pop("ORCHESTRATOR_BASE_URL", None) - os.environ.pop("ORCHESTRATOR_API_KEY", None) - from scripts.seed_demo_data import _seed_demo_public_chat + admin_conn = psycopg2.connect(seeded_db["dsn"]) + admin_conn.autocommit = True + try: + with admin_conn.cursor() as cur: + cur.execute( + """ + insert into analysis_source_snapshot + (snapshot_sha256, source_contract_version, + maximum_available_time, captured_at) + values (%s, 'source-contract-v1', + '2026-01-12T00:00:00Z', '2026-01-12T00:05:00Z') + returning analysis_source_snapshot_id + """, + ("3" * 64,), + ) + crash_snapshot_id = cur.fetchone()[0] + cur.execute( + """ + insert into analysis_run + (analysis_source_snapshot_id, run_kind_code, idempotency_key, + requested_by_account_id, knowledge_cutoff, + configuration_schema_version, configuration_sha256, + code_revision_sha, requested_at) + values (%s, 'analysis_run_lineage', 'buyer-start-outbox-resume', + %s, '2026-02-15T00:00:00Z', 'lineage-run-v1', %s, %s, + '2026-02-15T12:30:00Z') + returning analysis_run_id + """, + (crash_snapshot_id, requester_id, "2" * 64, "1" * 40), + ) + crash_run_id = str(cur.fetchone()[0]) + cur.execute( + """ + insert into analysis_run_scope + (analysis_run_id, scope_kind_code, corporate_entity_id) + values (%s, 'analysis_scope_corporate_entity', %s) + """, + (crash_run_id, seeded_db["own_corp_id"]), + ) + for ordinal, status, occurred in ( + (1, "analysis_status_pending", "2026-02-15T12:31:00Z"), + (2, "analysis_status_running", "2026-02-15T12:32:00Z"), + ): + cur.execute( + """ + insert into analysis_run_status_event + (analysis_run_id, status_ordinal, status_code, occurred_at) + values (%s, %s, %s, %s) + """, + (crash_run_id, ordinal, status, occurred), + ) + cur.execute( + """ + insert into analysis_run_outbox + (analysis_run_id, work_kind_code, request_sha256, enqueued_at) + values (%s, 'analysis_run_lineage', %s, '2026-02-15T12:32:00Z') + """, + (crash_run_id, "a" * 64), + ) + finally: + admin_conn.close() + + resumed = client.post( + f"/api/analysis-runs/{crash_run_id}/start", + headers={"Authorization": f"Bearer {demo_analyst_token}"}, + ) + assert resumed.status_code == 200, resumed.text + resumed_body = resumed.json() + assert resumed_body["status_label"] == "Succeeded" + assert "theta" not in str(resumed_body).lower() + children = { + edge["child_post_title"] + for edge in resumed_body["reconstructed_edges"] + if edge["parent_post_title"] == "Pricing renegotiation follow-up" + } + assert "Pricing renegotiation: revised quote sent" in children + +def test_tepp_start_persists_published_accepted_evidence( + client, demo_analyst_token, seeded_db, monkeypatch +) -> None: + """A published accepted ack is stored as transport evidence, never Succeeded.""" + idempotency_key = "buyer-start-tepp-accepted" + monkeypatch.setattr( + "backend.app.main.configured_tepp_client", + lambda _url="": TeppClient( + transport=lambda _payload: accepted_tepp_seed_envelope( + idempotency_key=idempotency_key + ) + ), + ) admin_conn = psycopg2.connect(seeded_db["dsn"]) admin_conn.autocommit = True try: with admin_conn.cursor() as cur: cur.execute( - "update source_post set post_title = 'Demo public post' where post_id = %s", - (seeded_db["public_post_id"],), + "select requested_by_account_id from analysis_run " + "where analysis_run_id = %s", + (seeded_db["visible_run_id"],), + ) + requester_id = cur.fetchone()[0] + cur.execute( + """ + insert into analysis_source_snapshot + (snapshot_sha256, source_contract_version, + maximum_available_time, captured_at) + values (%s, 'source-contract-v1', + '2026-02-15T00:00:00Z', '2026-02-15T00:05:00Z') + returning analysis_source_snapshot_id + """, + ("8" * 64,), + ) + snapshot_id = cur.fetchone()[0] + cur.execute( + """ + insert into analysis_run + (analysis_source_snapshot_id, run_kind_code, idempotency_key, + requested_by_account_id, knowledge_cutoff, + configuration_schema_version, configuration_sha256, + code_revision_sha, requested_at) + values (%s, 'analysis_run_tepp', 'buyer-start-tepp-accepted', + %s, '2026-02-15T00:00:00Z', 'tepp-run-v1', %s, %s, + '2026-02-15T12:30:00Z') + returning analysis_run_id + """, + (snapshot_id, requester_id, "7" * 64, "6" * 40), + ) + tepp_run_id = str(cur.fetchone()[0]) + cur.execute( + """ + insert into analysis_run_scope + (analysis_run_id, scope_kind_code, corporate_entity_id) + values (%s, 'analysis_scope_corporate_entity', %s) + """, + (tepp_run_id, seeded_db["own_corp_id"]), + ) + cur.execute( + """ + insert into analysis_run_status_event + (analysis_run_id, status_ordinal, status_code, occurred_at) + values (%s, 1, 'analysis_status_pending', '2026-02-15T12:31:00Z') + """, + (tepp_run_id,), ) - _seed_demo_public_chat(cur, seeded_db["public_post_id"]) finally: admin_conn.close() - history = client.get( - f"/api/posts/{seeded_db['public_post_id']}/chat", + measured = client.post( + f"/api/analysis-runs/{tepp_run_id}/start", headers={"Authorization": f"Bearer {demo_analyst_token}"}, ) - assert history.status_code == 200, history.text - questions = [row["question_text"] for row in history.json()["exchanges"]] - assert questions == [ - "What happened between these events?", - "Who is involved?", - "What is the next commitment?", - ] - assert "Northridge Grid" in history.json()["exchanges"][0]["answer_text"] - assert "Ada West" in history.json()["exchanges"][1]["answer_text"] - assert "Priya Nair" in history.json()["exchanges"][1]["answer_text"] - assert "Send Northridge Grid the revised quote" in history.json()["exchanges"][2]["answer_text"] - assert "2026-01-12" in history.json()["exchanges"][2]["answer_text"] + assert measured.status_code == 200, measured.text + body = measured.json() + assert body["status_label"] == "Failed" + assert body["failure_code"] == "tepp_completed_result_unsupported" + assert body["tepp_evidence_kind"] == "aggregate transport evidence" + assert body["tepp_run_state"] == "accepted" + assert body["tepp_accepted_run_id"] == "demo-tepp-accepted-opaque" + assert body["tepp_completed_artifact_available"] is False + expected = tepp_accepted_evidence_sha256( + contract_version=1, + accepted_run_id="demo-tepp-accepted-opaque", + run_state="accepted", + idempotency_key=idempotency_key, + ) + assert body["tepp_evidence_sha256"] == expected + assert "tepp_affiliation_count" not in body + assert "theta" not in str(body).lower() - asked = client.post( - f"/api/posts/{seeded_db['public_post_id']}/chat", - json={"question": "What happened between these events?"}, + listed = client.get( + "/api/analysis-runs", headers={"Authorization": f"Bearer {demo_analyst_token}"}, ) - assert asked.status_code == 200, asked.text - assert "Northridge Grid" in asked.json()["answer_text"] + assert listed.status_code == 200 + listed_run = next( + run for run in listed.json()["analysis_runs"] if run["analysis_run_id"] == tepp_run_id + ) + assert listed_run["status_label"] == "Failed" + assert listed_run["tepp_evidence_sha256"] == expected + assert "tepp_affiliation_count" not in listed_run - involved = client.post( - f"/api/posts/{seeded_db['public_post_id']}/chat", - json={"question": "Who's involved?"}, + +def test_me_reflects_the_authenticated_account(client, demo_analyst_token) -> None: + response = client.get("/api/me", headers={"Authorization": f"Bearer {demo_analyst_token}"}) + assert response.status_code == 200 + body = response.json() + assert body["display_name"] == "Test Analyst" + assert "post_read" in body["permission_codes"] + assert any( + entity["entity_name"] == "Test Corp" for entity in body["corporate_entities"] + ) + + +def test_post_list_includes_public_and_own_corp_but_excludes_other_corp(client, demo_analyst_token, seeded_db) -> None: + response = client.get("/api/posts", headers={"Authorization": f"Bearer {demo_analyst_token}"}) + assert response.status_code == 200 + titles = {post["post_title"] for post in response.json()} + assert titles == { + "Public post", + "Own-corp private post", + "Late own-corp private post", + "Edited own-corp private post", + } + public = next(post for post in response.json() if post["post_title"] == "Public post") + assert public["voc_type_label"] == "Voice of Customer" + assert public["visibility_label"] == "Public" + + +def test_post_detail_uses_lookup_labels_not_raw_codes(client, demo_analyst_token, seeded_db) -> None: + response = client.get( + f"/api/posts/{seeded_db['public_post_id']}", headers={"Authorization": f"Bearer {demo_analyst_token}"}, ) - assert involved.status_code == 200, involved.text - assert "Ada West" in involved.json()["answer_text"] - assert "Priya Nair" in involved.json()["answer_text"] + assert response.status_code == 200 + body = response.json() + assert body["voc_type_code"] == "voc" + assert body["voc_type_label"] == "Voice of Customer" + assert body["visibility_code"] == "public" + assert body["visibility_label"] == "Public" - commitment = client.post( - f"/api/posts/{seeded_db['public_post_id']}/chat", - json={"question": "What's the next commitment?"}, + +def test_post_detail_as_of_returns_the_cutoff_known_body( + client, demo_analyst_token, seeded_db +) -> None: + """Opened marked titles compare two real sentences, not two clocks.""" + headers = {"Authorization": f"Bearer {demo_analyst_token}"} + live = client.get(f"/api/posts/{seeded_db['edited_own_post_id']}", headers=headers) + assert live.status_code == 200 + assert live.json()["post_body"] == "A January post rewritten after the run cutoff." + assert "known_at" not in live.json() + + known = client.get( + f"/api/posts/{seeded_db['edited_own_post_id']}", + params={"as_of": "2026-01-12T12:00:00Z"}, + headers=headers, + ) + assert known.status_code == 200 + body = known.json() + assert body["post_body"] == "A January post rewritten after the run cutoff." + assert body["known_at"]["post_body"] == "A January post before the rewrite." + assert body["known_at"]["written_at"].startswith("2026-01-10") + assert "postgresql://" not in str(body) + + missing = client.get( + f"/api/posts/{seeded_db['edited_own_post_id']}", + params={"as_of": "2026-01-01T00:00:00Z"}, + headers=headers, + ) + assert missing.status_code == 200 + assert "known_at" not in missing.json() + + invalid = client.get( + f"/api/posts/{seeded_db['edited_own_post_id']}", + params={"as_of": "not-a-clock"}, + headers=headers, + ) + assert invalid.status_code == 422 + + +def test_persisted_summary_is_returned_without_an_llm(client, demo_analyst_token, seeded_db) -> None: + """GET /api/posts/{id}/summary must serve a stored row even when the + orchestrator is off -- otherwise a seeded demo popup stays empty. + """ + os.environ.pop("ORCHESTRATOR_BASE_URL", None) + os.environ.pop("ORCHESTRATOR_API_KEY", None) + admin_conn = psycopg2.connect(seeded_db["dsn"]) + admin_conn.autocommit = True + try: + with admin_conn.cursor() as cur: + cur.execute( + "insert into post_summary_result (post_id, korean_summary) values (%s, %s)", + (seeded_db["public_post_id"], "저장된 한국어 요약입니다."), + ) + cur.execute( + "insert into post_summary_event (post_id, event_ordinal, event_text) " + "values (%s, 0, '저장된 이벤트')", + (seeded_db["public_post_id"],), + ) + cur.execute( + "insert into post_summary_role " + "(post_id, actor_name, responsibility, actor_type_code, affiliated_organization_name) " + "values (%s, 'Ada West', '후속 연락', 'prov_person', 'Demo Corp')", + (seeded_db["public_post_id"],), + ) + finally: + admin_conn.close() + + response = client.get( + f"/api/posts/{seeded_db['public_post_id']}/summary", headers={"Authorization": f"Bearer {demo_analyst_token}"}, ) - assert commitment.status_code == 200, commitment.text - assert "Send Northridge Grid the revised quote" in commitment.json()["answer_text"] - assert "2026-01-12" in commitment.json()["answer_text"] + assert response.status_code == 200 + body = response.json() + assert body["korean_summary"] == "저장된 한국어 요약입니다." + assert body["key_events"] == ["저장된 이벤트"] + assert len(body["roles_and_responsibilities"]) == 1 + role = body["roles_and_responsibilities"][0] + assert role["actor_name"] == "Ada West" + assert role["responsibility"] == "후속 연락" + assert role["actor_type_code"] == "prov_person" + assert role["affiliated_organization_name"] == "Demo Corp" + assert role["ontology_label"] == "Role actor (person)" -def test_seed_fixture_chats_surface_on_post_chat(client, demo_analyst_token, seeded_db) -> None: +def test_seed_demo_summary_surfaces_on_get_summary(client, demo_analyst_token, seeded_db) -> None: + """The same helper `make seed` calls must produce a row GET summary + returns -- even with the orchestrator unset. + """ + os.environ.pop("ORCHESTRATOR_BASE_URL", None) + os.environ.pop("ORCHESTRATOR_API_KEY", None) + from scripts.seed_demo_data import _seed_demo_public_summary + + admin_conn = psycopg2.connect(seeded_db["dsn"]) + admin_conn.autocommit = True + try: + with admin_conn.cursor() as cur: + _seed_demo_public_summary(cur, seeded_db["public_post_id"]) + finally: + admin_conn.close() + + response = client.get( + f"/api/posts/{seeded_db['public_post_id']}/summary", + headers={"Authorization": f"Bearer {demo_analyst_token}"}, + ) + assert response.status_code == 200, response.text + body = response.json() + assert "에이다" in body["korean_summary"] + assert body["key_events"] + roles = {role["actor_name"]: role for role in body["roles_and_responsibilities"]} + assert roles["Ada West"]["actor_type_code"] == "prov_person" + assert roles["당사"]["actor_type_code"] == "prov_organization" + assert roles["당사"]["ontology_label"] == "Role actor (organization)" + + +def test_seed_fixture_summaries_surface_on_get_summary(client, demo_analyst_token, seeded_db) -> None: """The A-100 fork and calendar commitment `make seed` writes must - answer POST /api/posts/{id}/chat without a live orchestrator. + answer GET /api/posts/{id}/summary without a live orchestrator. """ from scripts.seed_demo_data import ( _seed_demo_calendar_commitment, - _seed_fixture_chats, + _seed_fixture_summaries, insert_fixture_source_posts, ) @@ -617,7 +1336,7 @@ def test_seed_fixture_chats_surface_on_post_chat(client, demo_analyst_token, see ) cur.execute( "insert into process_unit (corporate_entity_id, process_unit_code, process_unit_name) " - "select corporate_entity_id, 'TEST-PU-CHAT', 'Chat thread' " + "select corporate_entity_id, 'TEST-PU-SUMMARY', 'Summary thread' " "from source_post where post_id = %s returning process_unit_id", (seeded_db["own_private_post_id"],), ) @@ -629,7 +1348,7 @@ def test_seed_fixture_chats_surface_on_post_chat(client, demo_analyst_token, see author_id, corp_id = cur.fetchone() insert_fixture_source_posts(cur, author_id, corp_id, process_unit_id) _seed_demo_calendar_commitment(cur, author_id, corp_id, process_unit_id) - _seed_fixture_chats(cur) + _seed_fixture_summaries(cur) cur.execute( "select post_id from source_post where post_title = %s", ("Pricing renegotiation follow-up",), @@ -643,58 +1362,242 @@ def test_seed_fixture_chats_surface_on_post_chat(client, demo_analyst_token, see finally: admin_conn.close() - fork = client.post( - f"/api/posts/{fork_id}/chat", - json={"question": "What happened between these events?"}, + fork = client.get( + f"/api/posts/{fork_id}/summary", headers={"Authorization": f"Bearer {demo_analyst_token}"}, ) assert fork.status_code == 200, fork.text - assert "pricing renegotiation" in fork.json()["answer_text"].lower() - assert fork.json()["cited_posts"] - - fork_involved = client.post( - f"/api/posts/{fork_id}/chat", - json={"question": "Who is involved?"}, - headers={"Authorization": f"Bearer {demo_analyst_token}"}, - ) - assert fork_involved.status_code == 200, fork_involved.text - assert "Ada West" in fork_involved.json()["answer_text"] - assert "Priya Nair" in fork_involved.json()["answer_text"] + assert "재협상" in fork.json()["korean_summary"] + assert fork.json()["key_events"] + fork_roles = {role["actor_name"] for role in fork.json()["roles_and_responsibilities"]} + assert fork_roles == {"Ada West", "Priya Nair"} - fork_history = client.get( - f"/api/posts/{fork_id}/chat", + calendar = client.get( + f"/api/posts/{calendar_id}/summary", headers={"Authorization": f"Bearer {demo_analyst_token}"}, ) - assert fork_history.status_code == 200, fork_history.text - assert [row["question_text"] for row in fork_history.json()["exchanges"]] == [ - "What happened between these events?", - "Who is involved?", - "What is the next commitment?", - ] + assert calendar.status_code == 200, calendar.text + assert "리버벤드" in calendar.json()["korean_summary"] + assert calendar.json()["roles_and_responsibilities"] == [] - fork_commitment = client.post( - f"/api/posts/{fork_id}/chat", - json={"question": "What is the next commitment?"}, + missing = client.get( + f"/api/posts/{seeded_db['own_private_post_id']}/summary", headers={"Authorization": f"Bearer {demo_analyst_token}"}, ) - assert fork_commitment.status_code == 200, fork_commitment.text - assert "Send Northridge Grid the revised quote" in fork_commitment.json()["answer_text"] - assert "2026-01-12" in fork_commitment.json()["answer_text"] + assert missing.status_code == 503 - calendar = client.post( - f"/api/posts/{calendar_id}/chat", - json={"question": "What happened?"}, - headers={"Authorization": f"Bearer {demo_analyst_token}"}, - ) - assert calendar.status_code == 200, calendar.text - assert "Riverbend" in calendar.json()["answer_text"] - calendar_involved = client.post( - f"/api/posts/{calendar_id}/chat", - json={"question": "Who is involved?"}, - headers={"Authorization": f"Bearer {demo_analyst_token}"}, - ) - assert calendar_involved.status_code == 200, calendar_involved.text +def test_persisted_chat_is_returned_without_an_llm(client, demo_analyst_token, seeded_db) -> None: + """POST /api/posts/{id}/chat must serve a stored row even when the + orchestrator is off -- otherwise a seeded demo Ask stays empty. + """ + os.environ.pop("ORCHESTRATOR_BASE_URL", None) + os.environ.pop("ORCHESTRATOR_API_KEY", None) + admin_conn = psycopg2.connect(seeded_db["dsn"]) + admin_conn.autocommit = True + try: + with admin_conn.cursor() as cur: + cur.execute( + "insert into post_chat_result (post_id, question_norm, question_text, answer_text) " + "values (%s, 'what happened between these events', " + "'What happened between these events?', 'Stored follow-up after the site visit.')", + (seeded_db["public_post_id"],), + ) + cur.execute( + "insert into post_chat_citation " + "(post_id, question_norm, citation_ordinal, cited_post_id) " + "values (%s, 'what happened between these events', 0, %s)", + (seeded_db["public_post_id"], seeded_db["public_post_id"]), + ) + finally: + admin_conn.close() + + asked = client.post( + f"/api/posts/{seeded_db['public_post_id']}/chat", + json={"question": "What happened?"}, + headers={"Authorization": f"Bearer {demo_analyst_token}"}, + ) + assert asked.status_code == 200, asked.text + body = asked.json() + assert body["answer_text"] == "Stored follow-up after the site visit." + assert body["cited_post_ids"] == [seeded_db["public_post_id"]] + assert body["cited_posts"] == [ + {"post_id": seeded_db["public_post_id"], "post_title": "Public post"} + ] + + history = client.get( + f"/api/posts/{seeded_db['public_post_id']}/chat", + headers={"Authorization": f"Bearer {demo_analyst_token}"}, + ) + assert history.status_code == 200, history.text + assert history.json()["exchanges"][0]["answer_text"] == "Stored follow-up after the site visit." + + +def test_seed_demo_chat_surfaces_on_get_and_post_chat(client, demo_analyst_token, seeded_db) -> None: + """The same helper `make seed` calls must produce a row GET/POST chat + return -- even with the orchestrator unset. + """ + os.environ.pop("ORCHESTRATOR_BASE_URL", None) + os.environ.pop("ORCHESTRATOR_API_KEY", None) + from scripts.seed_demo_data import _seed_demo_public_chat + + admin_conn = psycopg2.connect(seeded_db["dsn"]) + admin_conn.autocommit = True + try: + with admin_conn.cursor() as cur: + cur.execute( + "update source_post set post_title = 'Demo public post' where post_id = %s", + (seeded_db["public_post_id"],), + ) + _seed_demo_public_chat(cur, seeded_db["public_post_id"]) + finally: + admin_conn.close() + + history = client.get( + f"/api/posts/{seeded_db['public_post_id']}/chat", + headers={"Authorization": f"Bearer {demo_analyst_token}"}, + ) + assert history.status_code == 200, history.text + questions = [row["question_text"] for row in history.json()["exchanges"]] + assert questions == [ + "What happened between these events?", + "Who is involved?", + "What is the next commitment?", + ] + assert "Northridge Grid" in history.json()["exchanges"][0]["answer_text"] + assert "Ada West" in history.json()["exchanges"][1]["answer_text"] + assert "Priya Nair" in history.json()["exchanges"][1]["answer_text"] + assert "Send Northridge Grid the revised quote" in history.json()["exchanges"][2]["answer_text"] + assert "2026-01-12" in history.json()["exchanges"][2]["answer_text"] + + asked = client.post( + f"/api/posts/{seeded_db['public_post_id']}/chat", + json={"question": "What happened between these events?"}, + headers={"Authorization": f"Bearer {demo_analyst_token}"}, + ) + assert asked.status_code == 200, asked.text + assert "Northridge Grid" in asked.json()["answer_text"] + + involved = client.post( + f"/api/posts/{seeded_db['public_post_id']}/chat", + json={"question": "Who's involved?"}, + headers={"Authorization": f"Bearer {demo_analyst_token}"}, + ) + assert involved.status_code == 200, involved.text + assert "Ada West" in involved.json()["answer_text"] + assert "Priya Nair" in involved.json()["answer_text"] + + commitment = client.post( + f"/api/posts/{seeded_db['public_post_id']}/chat", + json={"question": "What's the next commitment?"}, + headers={"Authorization": f"Bearer {demo_analyst_token}"}, + ) + assert commitment.status_code == 200, commitment.text + assert "Send Northridge Grid the revised quote" in commitment.json()["answer_text"] + assert "2026-01-12" in commitment.json()["answer_text"] + + +def test_seed_fixture_chats_surface_on_post_chat(client, demo_analyst_token, seeded_db) -> None: + """The A-100 fork and calendar commitment `make seed` writes must + answer POST /api/posts/{id}/chat without a live orchestrator. + """ + from scripts.seed_demo_data import ( + _seed_demo_calendar_commitment, + _seed_fixture_chats, + insert_fixture_source_posts, + ) + + os.environ.pop("ORCHESTRATOR_BASE_URL", None) + os.environ.pop("ORCHESTRATOR_API_KEY", None) + admin_conn = psycopg2.connect(seeded_db["dsn"]) + admin_conn.autocommit = True + try: + with admin_conn.cursor() as cur: + cur.execute( + "insert into common_lookup_value (lookup_category, lookup_code, lookup_label) " + "values ('voc_type', 'vom', 'Voice of Market') " + "on conflict (lookup_code) do nothing" + ) + cur.execute( + "insert into process_unit (corporate_entity_id, process_unit_code, process_unit_name) " + "select corporate_entity_id, 'TEST-PU-CHAT', 'Chat thread' " + "from source_post where post_id = %s returning process_unit_id", + (seeded_db["own_private_post_id"],), + ) + process_unit_id = cur.fetchone()[0] + cur.execute( + "select author_account_id, corporate_entity_id from source_post where post_id = %s", + (seeded_db["own_private_post_id"],), + ) + author_id, corp_id = cur.fetchone() + insert_fixture_source_posts(cur, author_id, corp_id, process_unit_id) + _seed_demo_calendar_commitment(cur, author_id, corp_id, process_unit_id) + _seed_fixture_chats(cur) + cur.execute( + "select post_id from source_post where post_title = %s", + ("Pricing renegotiation follow-up",), + ) + fork_id = str(cur.fetchone()[0]) + cur.execute( + "select post_id from source_post where post_title = %s", + ("Follow-up on the Riverbend order confirmation",), + ) + calendar_id = str(cur.fetchone()[0]) + finally: + admin_conn.close() + + fork = client.post( + f"/api/posts/{fork_id}/chat", + json={"question": "What happened between these events?"}, + headers={"Authorization": f"Bearer {demo_analyst_token}"}, + ) + assert fork.status_code == 200, fork.text + assert "pricing renegotiation" in fork.json()["answer_text"].lower() + assert fork.json()["cited_posts"] + + fork_involved = client.post( + f"/api/posts/{fork_id}/chat", + json={"question": "Who is involved?"}, + headers={"Authorization": f"Bearer {demo_analyst_token}"}, + ) + assert fork_involved.status_code == 200, fork_involved.text + assert "Ada West" in fork_involved.json()["answer_text"] + assert "Priya Nair" in fork_involved.json()["answer_text"] + + fork_history = client.get( + f"/api/posts/{fork_id}/chat", + headers={"Authorization": f"Bearer {demo_analyst_token}"}, + ) + assert fork_history.status_code == 200, fork_history.text + assert [row["question_text"] for row in fork_history.json()["exchanges"]] == [ + "What happened between these events?", + "Who is involved?", + "What is the next commitment?", + ] + + fork_commitment = client.post( + f"/api/posts/{fork_id}/chat", + json={"question": "What is the next commitment?"}, + headers={"Authorization": f"Bearer {demo_analyst_token}"}, + ) + assert fork_commitment.status_code == 200, fork_commitment.text + assert "Send Northridge Grid the revised quote" in fork_commitment.json()["answer_text"] + assert "2026-01-12" in fork_commitment.json()["answer_text"] + + calendar = client.post( + f"/api/posts/{calendar_id}/chat", + json={"question": "What happened?"}, + headers={"Authorization": f"Bearer {demo_analyst_token}"}, + ) + assert calendar.status_code == 200, calendar.text + assert "Riverbend" in calendar.json()["answer_text"] + + calendar_involved = client.post( + f"/api/posts/{calendar_id}/chat", + json={"question": "Who is involved?"}, + headers={"Authorization": f"Bearer {demo_analyst_token}"}, + ) + assert calendar_involved.status_code == 200, calendar_involved.text assert "does not name a Keyman" in calendar_involved.json()["answer_text"] calendar_commitment = client.post( @@ -803,40 +1706,156 @@ def test_affiliate_tree_walks_ancestors_and_keeps_unresolved_orgs(client, demo_a assert all(person["person_name"] == "Priya Nair" for node in unresolved for person in node["people"]) -def test_other_corp_private_affiliate_tree_is_forbidden(client, demo_analyst_token, seeded_db) -> None: +def test_customer_group_tree_walks_authorized_ancestors_and_descendants( + client, demo_analyst_token, seeded_db +) -> None: + """Operators navigate Group → Company → Plant, not a flat corp list.""" response = client.get( - f"/api/posts/{seeded_db['other_private_post_id']}/affiliate-tree", + "/api/customer-group-tree", + headers={"Authorization": f"Bearer {demo_analyst_token}"}, + ) + assert response.status_code == 200, response.text + trees = response.json()["trees"] + assert [node["entity_name"] for node in trees] == ["Test Group"] + assert trees[0]["entity_id"] == seeded_db["own_group_id"] + assert trees[0]["entity_level_label"] == "Group" + children = trees[0]["children"] + assert [child["entity_name"] for child in children] == ["Test Corp"] + assert children[0]["entity_id"] == seeded_db["own_corp_id"] + assert children[0]["entity_level_label"] == "Company" + plants = children[0]["children"] + assert [plant["entity_name"] for plant in plants] == ["Test Plant"] + assert plants[0]["entity_id"] == seeded_db["own_plant_id"] + assert plants[0]["entity_level_label"] == "Plant" + assert "Other Corp" not in str(trees) + + +def test_corroborate_abbreviations_requires_post_admin( + client, demo_analyst_token, seeded_db +) -> None: + response = client.post( + f"/api/posts/{seeded_db['own_private_post_id']}/corroborate-abbreviations", headers={"Authorization": f"Bearer {demo_analyst_token}"}, ) assert response.status_code == 403 -def test_voc_evidence_quotes_the_sentence_that_names_the_org(client, demo_analyst_token, seeded_db) -> None: - response = client.get( - f"/api/posts/{seeded_db['own_private_post_id']}/voc-evidence", +def test_corroborate_abbreviations_is_unavailable_without_searxng( + client, demo_analyst_token, seeded_db +) -> None: + _grant_post_admin(seeded_db["dsn"]) + response = client.post( + f"/api/posts/{seeded_db['own_private_post_id']}/corroborate-abbreviations", headers={"Authorization": f"Bearer {demo_analyst_token}"}, ) - assert response.status_code == 200 - body = response.json() - assert body["voc_type_code"] == "voc" - assert body["voc_type_label"] == "Voice of Customer" - assert body["excerpts"] == [ - "Ada West at Test Corp followed up with Priya Nair at Northridge Grid about the delayed shipment." - ] - assert "weather" not in " ".join(body["excerpts"]).lower() - assert body["counterparties"] == [] + assert response.status_code == 503 + assert "SEARXNG_BASE_URL" in response.json()["detail"] -def test_voc_evidence_includes_verification_status(client, demo_analyst_token, seeded_db) -> None: - """GET /voc-evidence must carry the counterparty verification badge - fields -- the VOC panel is not a second unverified claim list. - """ +def test_corroborate_abbreviations_binds_a_unique_tree_node( + client, demo_analyst_token, seeded_db, monkeypatch +) -> None: + """Searxng must corroborate TC against Test Corp; a miss invents nothing.""" + from lineageweave.relation_verification import STATUS_CORROBORATED, RelationVerificationResult + + _grant_post_admin(seeded_db["dsn"]) admin_conn = psycopg2.connect(seeded_db["dsn"]) admin_conn.autocommit = True try: with admin_conn.cursor() as cur: cur.execute( - "insert into post_counterparty_entity " + "insert into person_affiliation (person_id, affiliated_organization_name) " + "values (%s, 'TC')", + (seeded_db["our_person_id"],), + ) + finally: + admin_conn.close() + + class _FakeVerificationClient: + available = True + + def verify(self, organization_name: str, relationship_label: str) -> RelationVerificationResult: + if organization_name == "Test Corp" and relationship_label == "TC": + return RelationVerificationResult( + STATUS_CORROBORATED, "https://example.test/test-corp-tc" + ) + return RelationVerificationResult("verify_uncorroborated", None) + + monkeypatch.setattr("backend.app.main._relation_verification_client", lambda: _FakeVerificationClient()) + response = client.post( + f"/api/posts/{seeded_db['own_private_post_id']}/corroborate-abbreviations", + headers={"Authorization": f"Bearer {demo_analyst_token}"}, + ) + assert response.status_code == 200, response.text + matches = {row["raw_organization_name"]: row for row in response.json()["matches"]} + assert matches["TC"]["corporate_entity_id"] == seeded_db["own_corp_id"] + assert matches["TC"]["verification_status_code"] == "verify_corroborated" + assert "Test Corp" not in matches + for unresolved_name in ("Northridge Grid", "Northridge Holdings"): + if unresolved_name in matches: + assert matches[unresolved_name]["corporate_entity_id"] is None + assert matches[unresolved_name]["verification_status_code"] == "verify_uncorroborated" + + cached = client.get( + f"/api/posts/{seeded_db['own_private_post_id']}/abbreviation-tree-matches", + headers={"Authorization": f"Bearer {demo_analyst_token}"}, + ) + assert cached.status_code == 200 + cached_matches = {row["raw_organization_name"]: row for row in cached.json()["matches"]} + assert cached_matches["TC"]["corporate_entity_id"] == seeded_db["own_corp_id"] + + tree = client.get( + "/api/customer-group-tree", + headers={"Authorization": f"Bearer {demo_analyst_token}"}, + ) + company = tree.json()["trees"][0]["children"][0] + assert [alias["raw_organization_name"] for alias in company["abbreviations"]] == ["TC"] + + +def test_other_corp_private_abbreviation_cross_check_is_forbidden( + client, demo_analyst_token, seeded_db +) -> None: + listed = client.get( + f"/api/posts/{seeded_db['other_private_post_id']}/abbreviation-tree-matches", + headers={"Authorization": f"Bearer {demo_analyst_token}"}, + ) + assert listed.status_code == 403 + + +def test_other_corp_private_affiliate_tree_is_forbidden(client, demo_analyst_token, seeded_db) -> None: + response = client.get( + f"/api/posts/{seeded_db['other_private_post_id']}/affiliate-tree", + headers={"Authorization": f"Bearer {demo_analyst_token}"}, + ) + assert response.status_code == 403 + + +def test_voc_evidence_quotes_the_sentence_that_names_the_org(client, demo_analyst_token, seeded_db) -> None: + response = client.get( + f"/api/posts/{seeded_db['own_private_post_id']}/voc-evidence", + headers={"Authorization": f"Bearer {demo_analyst_token}"}, + ) + assert response.status_code == 200 + body = response.json() + assert body["voc_type_code"] == "voc" + assert body["voc_type_label"] == "Voice of Customer" + assert body["excerpts"] == [ + "Ada West at Test Corp followed up with Priya Nair at Northridge Grid about the delayed shipment." + ] + assert "weather" not in " ".join(body["excerpts"]).lower() + assert body["counterparties"] == [] + + +def test_voc_evidence_includes_verification_status(client, demo_analyst_token, seeded_db) -> None: + """GET /voc-evidence must carry the counterparty verification badge + fields -- the VOC panel is not a second unverified claim list. + """ + admin_conn = psycopg2.connect(seeded_db["dsn"]) + admin_conn.autocommit = True + try: + with admin_conn.cursor() as cur: + cur.execute( + "insert into post_counterparty_entity " "(post_id, counterparty_entity_name, relationship_type_code, " " verification_status_code, verification_evidence_url) " "values (%s, 'Northridge Grid', 'rel_voc', 'verify_pending', null)", @@ -883,6 +1902,8 @@ def test_related_keymen_use_rwr_and_hide_invisible_posts(client, demo_analyst_to counterpart = by_id[seeded_db["counterpart_person_id"]] assert counterpart["ontology_label"] == "Person" assert counterpart["ontology_iri"].endswith("#Person") + assert counterpart["person_side_code"] == "counterparty" + assert counterpart["person_side_label"] == "Counterparty" own_post = by_id[seeded_db["own_private_post_id"]] assert own_post["ontology_label"] == "Post" @@ -902,6 +1923,9 @@ def test_related_corporate_entity_uses_rwr_and_hides_invisible_posts( assert body["entity_name"] == "Test Corp" related_ids = {node["node_id"] for node in body["related"]} assert seeded_db["our_person_id"] in related_ids + our_person = next(node for node in body["related"] if node["node_id"] == seeded_db["our_person_id"]) + assert our_person["person_side_code"] == "our_side" + assert our_person["person_side_label"] == "Our side" assert seeded_db["other_private_post_id"] not in related_ids assert seeded_db["hidden_person_id"] not in related_ids @@ -927,6 +1951,44 @@ def test_unknown_corporate_entity_related_is_not_found( assert response.status_code == 404 +def test_team_only_on_other_corp_private_post_is_forbidden( + client, demo_analyst_token, seeded_db +) -> None: + """A team mentioned only on another corp's private post must 403.""" + + admin_conn = psycopg2.connect(seeded_db["dsn"]) + admin_conn.autocommit = True + try: + with admin_conn.cursor() as cur: + cur.execute( + "insert into cataloged_team (team_name, affiliated_organization_name) " + "values ('비공개 설계팀', 'Other Corp') returning team_id" + ) + team_id = str(cur.fetchone()[0]) + cur.execute( + "insert into post_team_mention (post_id, team_id) values (%s, %s)", + (seeded_db["other_private_post_id"], team_id), + ) + finally: + admin_conn.close() + + response = client.get( + f"/api/teams/{team_id}/related", + headers={"Authorization": f"Bearer {demo_analyst_token}"}, + ) + assert response.status_code == 403 + + +def test_unknown_team_related_is_not_found(client, demo_analyst_token) -> None: + """An unknown team UUID must 404, matching person and entity related.""" + + response = client.get( + f"/api/teams/{uuid.uuid4()}/related", + headers={"Authorization": f"Bearer {demo_analyst_token}"}, + ) + assert response.status_code == 404 + + def test_keyman_only_on_other_corp_private_post_is_forbidden(client, demo_analyst_token, seeded_db) -> None: response = client.get( f"/api/keymen/{seeded_db['hidden_person_id']}/related", @@ -947,6 +2009,557 @@ def test_extract_keymen_requires_post_admin(client, demo_analyst_token, seeded_d _ORCHESTRATOR_API_KEY = os.environ.get("LINEAGEWEAVE_TEST_ORCHESTRATOR_API_KEY") +def test_extract_keymen_does_not_merge_same_name_people_with_conflicting_titles( + client, demo_analyst_token, seeded_db, monkeypatch +) -> None: + """Two different real people can share a name -- extracting a second + post that names the same person_name+side but a genuinely different + stated job_title must NOT reuse the first post's cataloged_person row. + A deterministic fake client (not a real orchestrator call) so this + is CI-stable: the point under test is `_upsert_person`'s own SQL + logic, not LLM extraction quality. + """ + from lineageweave.keyman_extraction import COUNTERPARTY, PersonMention + + _grant_post_admin(seeded_db["dsn"]) + + class _FakeClient: + available = True + + def __init__(self, job_title: str) -> None: + self._job_title = job_title + + def extract(self, post_title: str, post_body: str) -> list[PersonMention]: + return [PersonMention(person_name="Kim Cheolsu", person_side_code=COUNTERPARTY, job_title=self._job_title)] + + admin_conn = psycopg2.connect(seeded_db["dsn"]) + admin_conn.autocommit = True + try: + with admin_conn.cursor() as cur: + post_ids = [] + for title in ("Sales follow-up", "Purchasing follow-up"): + cur.execute( + "insert into source_post (author_account_id, corporate_entity_id, post_title, post_body, voc_type_code, visibility_code) " + "select author_account_id, corporate_entity_id, %s, %s, 'voc', 'public' " + "from source_post where post_id = %s " + "returning post_id", + (title, "placeholder body", seeded_db["own_private_post_id"]), + ) + post_ids.append(str(cur.fetchone()[0])) + finally: + admin_conn.close() + + monkeypatch.setattr("backend.app.main._entity_relationship_client", lambda: _FakeClient("unused")) + + monkeypatch.setattr("backend.app.main._keyman_extraction_client", lambda: _FakeClient("Sales Manager")) + response_a = client.post( + f"/api/posts/{post_ids[0]}/extract-keymen", + headers={"Authorization": f"Bearer {demo_analyst_token}"}, + ) + assert response_a.status_code == 200, response_a.text + + monkeypatch.setattr("backend.app.main._keyman_extraction_client", lambda: _FakeClient("Purchasing Lead")) + response_b = client.post( + f"/api/posts/{post_ids[1]}/extract-keymen", + headers={"Authorization": f"Bearer {demo_analyst_token}"}, + ) + assert response_b.status_code == 200, response_b.text + + admin_conn = psycopg2.connect(seeded_db["dsn"]) + admin_conn.autocommit = True + try: + with admin_conn.cursor() as cur: + cur.execute( + "select count(distinct person_id) from cataloged_person where person_name = 'Kim Cheolsu'" + ) + distinct_people = cur.fetchone()[0] + finally: + admin_conn.close() + + assert distinct_people == 2, "conflicting stated job titles for the same name must not be merged into one person" + + +def test_extract_keymen_resolves_and_caches_an_abbreviated_organization_name( + client, demo_analyst_token, seeded_db, monkeypatch +) -> None: + """ADR 0008: an affiliated organization named by abbreviation + ("AGP") must be resolved to its canonical name + ("Aurora Grid Power") and cross-verified before that name is trusted -- + deterministic fake resolution/verification clients (not a real LLM + or Searxng call) so this is CI-stable; the point under test is the + resolve-then-persist wiring, not model/search quality. + """ + from lineageweave.keyman_extraction import COUNTERPARTY, PersonMention + from lineageweave.relation_verification import STATUS_CORROBORATED, RelationVerificationResult + + _grant_post_admin(seeded_db["dsn"]) + + class _FakeKeymanClient: + available = True + + def extract(self, post_title: str, post_body: str) -> list[PersonMention]: + return [ + PersonMention( + person_name="Kim Cheolsu", + person_side_code=COUNTERPARTY, + affiliated_organization_names=("AGP",), + ) + ] + + class _FakeRelationshipClient: + available = True + + def classify(self, post_title: str, post_body: str, organization_names: list[str]): + return [] + + class _FakeResolutionClient: + available = True + + def resolve(self, raw_name: str, context_text: str) -> str | None: + assert raw_name == "AGP" + return "Aurora Grid Power" + + class _FakeVerificationClient: + available = True + + def verify(self, organization_name: str, relationship_label: str) -> RelationVerificationResult: + assert organization_name == "Aurora Grid Power" + assert relationship_label == "AGP" + return RelationVerificationResult( + status_code=STATUS_CORROBORATED, evidence_url="https://example.org/khnp" + ) + + monkeypatch.setattr("backend.app.main._keyman_extraction_client", lambda: _FakeKeymanClient()) + monkeypatch.setattr("backend.app.main._entity_relationship_client", lambda: _FakeRelationshipClient()) + monkeypatch.setattr( + "backend.app.main._organization_name_resolution_client", lambda: _FakeResolutionClient() + ) + monkeypatch.setattr("backend.app.main._relation_verification_client", lambda: _FakeVerificationClient()) + + response = client.post( + f"/api/posts/{seeded_db['own_private_post_id']}/extract-keymen", + headers={"Authorization": f"Bearer {demo_analyst_token}"}, + ) + assert response.status_code == 200, response.text + + admin_conn = psycopg2.connect(seeded_db["dsn"]) + admin_conn.autocommit = True + try: + with admin_conn.cursor() as cur: + cur.execute( + "select resolved_organization_name, verification_status_code, verification_evidence_url " + "from organization_name_resolution where raw_organization_name = 'AGP'" + ) + cached = cur.fetchone() + cur.execute( + "select pa.affiliated_organization_name from person_affiliation pa " + "join cataloged_person cp on cp.person_id = pa.person_id " + "where cp.person_name = 'Kim Cheolsu'" + ) + affiliation_name = cur.fetchone()[0] + finally: + admin_conn.close() + + assert cached == ("Aurora Grid Power", STATUS_CORROBORATED, "https://example.org/khnp") + assert affiliation_name == "Aurora Grid Power", "a corroborated resolution must be the stored affiliation name" + + +def test_same_team_named_in_two_posts_resolves_to_one_cataloged_team( + client, demo_analyst_token, seeded_db, monkeypatch +) -> None: + """ADR 0009: extraction runs per-post, but "설계팀" (design team) at + the same company named in two different posts must resolve to the + same cataloged_team row -- otherwise every extraction is an island + and can never become a cross-post Knowledge Graph clue. A + deterministic fake summary client (not a real LLM call) so this is + CI-stable; the point under test is the upsert-then-dedupe wiring. + """ + from lineageweave.post_summary import ACTOR_TYPE_TEAM, PostSummary, RoleResponsibility + + class _FakeSummaryClient: + available = True + + def summarize(self, post_title: str, post_body: str) -> PostSummary: + return PostSummary( + korean_summary="설계팀이 도면을 검토했다.", + roles_and_responsibilities=( + RoleResponsibility( + actor_name="설계팀", + responsibility="도면 검토", + actor_type_code=ACTOR_TYPE_TEAM, + affiliated_organization_name="Demo Corp", + ), + ), + ) + + monkeypatch.setattr("backend.app.main._post_summary_client", lambda: _FakeSummaryClient()) + + admin_conn = psycopg2.connect(seeded_db["dsn"]) + admin_conn.autocommit = True + try: + with admin_conn.cursor() as cur: + post_ids = [] + for title in ("설계 검토 회의 1", "설계 검토 회의 2"): + cur.execute( + "insert into source_post (author_account_id, corporate_entity_id, post_title, post_body, voc_type_code, visibility_code) " + "select author_account_id, corporate_entity_id, %s, %s, 'voc', 'public' " + "from source_post where post_id = %s returning post_id", + (title, "placeholder body", seeded_db["own_private_post_id"]), + ) + post_ids.append(str(cur.fetchone()[0])) + finally: + admin_conn.close() + + headers = {"Authorization": f"Bearer {demo_analyst_token}"} + for post_id in post_ids: + response = client.get(f"/api/posts/{post_id}/summary", headers=headers) + assert response.status_code == 200, response.text + + admin_conn = psycopg2.connect(seeded_db["dsn"]) + admin_conn.autocommit = True + try: + with admin_conn.cursor() as cur: + cur.execute("select count(*), count(distinct team_id) from cataloged_team where team_name = '설계팀'") + team_row_count, distinct_team_count = cur.fetchone() + cur.execute( + "select count(distinct pt.post_id) from post_team_mention pt " + "join cataloged_team ct on ct.team_id = pt.team_id " + "where ct.team_name = '설계팀'" + ) + mentioning_post_count = cur.fetchone()[0] + cur.execute( + "select count(*) from knowledge_graph_edge " + "where source_node_type_code = 'node_team' and edge_type_code = 'edge_mention_team'" + ) + team_mention_edge_count = cur.fetchone()[0] + finally: + admin_conn.close() + + assert (team_row_count, distinct_team_count) == (1, 1), "the same team+org pair must dedupe to one row" + assert mentioning_post_count == 2, "both posts must link to the single cataloged team" + assert team_mention_edge_count == 2, "each post's mention must become a real KG edge" + + admin_conn = psycopg2.connect(seeded_db["dsn"]) + try: + with admin_conn.cursor() as cur: + cur.execute("select team_id from cataloged_team where team_name = '설계팀'") + team_id = str(cur.fetchone()[0]) + finally: + admin_conn.close() + + related = client.get( + f"/api/teams/{team_id}/related", + headers=headers, + ) + assert related.status_code == 200, related.text + related_ids = {node["node_id"] for node in related.json()["related"]} + assert set(post_ids) <= related_ids + summaries = [ + client.get(f"/api/posts/{post_id}/summary", headers=headers).json() + for post_id in post_ids + ] + for body in summaries: + role = body["roles_and_responsibilities"][0] + assert role["catalog_node_id"] == team_id + assert role["catalog_node_type_code"] == "node_team" + + +def test_organization_mention_only_posts_appear_in_entity_related( + client, demo_analyst_token, seeded_db +) -> None: + """An org mentioned with no affiliated person must still start a related walk.""" + + admin_conn = psycopg2.connect(seeded_db["dsn"]) + admin_conn.autocommit = True + try: + with admin_conn.cursor() as cur: + cur.execute( + "insert into source_post (author_account_id, corporate_entity_id, post_title, post_body, voc_type_code, visibility_code) " + "select author_account_id, corporate_entity_id, %s, %s, 'voc', 'public' " + "from source_post where post_id = %s returning post_id", + ("Org-only mention", "Test Corp was named without a person.", seeded_db["own_private_post_id"]), + ) + org_only_post_id = str(cur.fetchone()[0]) + cur.execute( + "insert into post_organization_mention (post_id, corporate_entity_id) values (%s, %s)", + (org_only_post_id, seeded_db["own_corp_id"]), + ) + for edge in knowledge_graph_edges_for_post( + org_only_post_id, + [], + organization_corporate_entity_ids=[seeded_db["own_corp_id"]], + ): + cur.execute( + "insert into knowledge_graph_edge (" + "source_node_type_code, source_node_id, target_node_type_code, " + "target_node_id, edge_type_code, edge_weight" + ") values (%s, %s, %s, %s, %s, %s) " + "on conflict do nothing", + ( + edge.source_node_type_code, + edge.source_node_id, + edge.target_node_type_code, + edge.target_node_id, + edge.edge_type_code, + edge.edge_weight, + ), + ) + finally: + admin_conn.close() + + response = client.get( + f"/api/corporate-entities/{seeded_db['own_corp_id']}/related", + headers={"Authorization": f"Bearer {demo_analyst_token}"}, + ) + assert response.status_code == 200, response.text + related_ids = {node["node_id"] for node in response.json()["related"]} + assert org_only_post_id in related_ids + + +def test_private_other_corp_organization_mention_does_not_leak( + client, demo_analyst_token, seeded_db +) -> None: + """The org-mention UNION must still apply ABAC per post.""" + + admin_conn = psycopg2.connect(seeded_db["dsn"]) + admin_conn.autocommit = True + try: + with admin_conn.cursor() as cur: + cur.execute( + "insert into post_organization_mention (post_id, corporate_entity_id) " + "values (%s, %s) on conflict do nothing", + (seeded_db["other_private_post_id"], seeded_db["own_corp_id"]), + ) + cur.execute( + "insert into post_organization_mention (post_id, corporate_entity_id) " + "values (%s, %s) on conflict do nothing", + (seeded_db["other_private_post_id"], seeded_db["other_corp_id"]), + ) + for entity_id in (seeded_db["own_corp_id"], seeded_db["other_corp_id"]): + for edge in knowledge_graph_edges_for_post( + seeded_db["other_private_post_id"], + [], + organization_corporate_entity_ids=[entity_id], + ): + cur.execute( + "insert into knowledge_graph_edge (" + "source_node_type_code, source_node_id, target_node_type_code, " + "target_node_id, edge_type_code, edge_weight" + ") values (%s, %s, %s, %s, %s, %s) " + "on conflict do nothing", + ( + edge.source_node_type_code, + edge.source_node_id, + edge.target_node_type_code, + edge.target_node_id, + edge.edge_type_code, + edge.edge_weight, + ), + ) + finally: + admin_conn.close() + + headers = {"Authorization": f"Bearer {demo_analyst_token}"} + own = client.get( + f"/api/corporate-entities/{seeded_db['own_corp_id']}/related", + headers=headers, + ) + assert own.status_code == 200, own.text + own_ids = {node["node_id"] for node in own.json()["related"]} + assert seeded_db["other_private_post_id"] not in own_ids + + hidden = client.get( + f"/api/corporate-entities/{seeded_db['other_corp_id']}/related", + headers=headers, + ) + assert hidden.status_code == 403 + + +def test_thread_group_run_list_honors_knowledge_cutoff( + client, demo_analyst_token, seeded_db +) -> None: + """A later public post must not surface a previously hidden thread-group run.""" + + admin_conn = psycopg2.connect(seeded_db["dsn"]) + admin_conn.autocommit = True + try: + with admin_conn.cursor() as cur: + cur.execute( + "insert into source_post (author_account_id, corporate_entity_id, post_title, post_body, voc_type_code, visibility_code, thread_group_key, created_at) " + "select author_account_id, corporate_entity_id, %s, %s, 'voc', 'public', %s, %s " + "from source_post where post_id = %s", + ( + "Late thread-group post", + "Written after the January cutoff.", + "late-thread-group", + "2026-01-20T12:00:00Z", + seeded_db["own_private_post_id"], + ), + ) + cur.execute( + """ + insert into analysis_source_snapshot + (snapshot_sha256, source_contract_version, + maximum_available_time, captured_at) + values (%s, 'source-contract-v1', + '2026-01-12T00:00:00Z', '2026-01-12T00:05:00Z') + returning analysis_source_snapshot_id + """, + ("f" * 64,), + ) + snapshot_id = cur.fetchone()[0] + cur.execute( + """ + insert into analysis_run + (analysis_source_snapshot_id, run_kind_code, idempotency_key, + requested_by_account_id, knowledge_cutoff, + configuration_schema_version, configuration_sha256, + code_revision_sha, requested_at) + values (%s, 'analysis_run_lineage', %s, + (select user_account_id from user_account + where email_address = 'other.analyst@example.test'), + '2026-01-12T12:00:00Z', 'lineage-run-v1', %s, %s, + '2026-01-12T12:30:00Z') + returning analysis_run_id + """, + (snapshot_id, "hidden-late-thread", "b" * 64, "c" * 40), + ) + run_id = str(cur.fetchone()[0]) + cur.execute( + """ + insert into analysis_run_scope + (analysis_run_id, scope_kind_code, scope_key) + values (%s, 'analysis_scope_thread_group', 'late-thread-group') + """, + (run_id,), + ) + for ordinal, status, occurred in ( + (1, "analysis_status_pending", "2026-01-12T12:31:00Z"), + (2, "analysis_status_running", "2026-01-12T12:32:00Z"), + (3, "analysis_status_succeeded", "2026-01-12T12:33:00Z"), + ): + cur.execute( + """ + insert into analysis_run_status_event + (analysis_run_id, status_ordinal, status_code, occurred_at) + values (%s, %s, %s, %s) + """, + (run_id, ordinal, status, occurred), + ) + finally: + admin_conn.close() + + listed = client.get( + "/api/analysis-runs", + headers={"Authorization": f"Bearer {demo_analyst_token}"}, + ) + assert listed.status_code == 200 + ids = {run["analysis_run_id"] for run in listed.json()["analysis_runs"]} + assert run_id not in ids + assert seeded_db["visible_run_id"] in ids + + +def test_first_mention_of_a_new_counterparty_creates_a_real_corporate_entity( + client, demo_analyst_token, seeded_db, monkeypatch +) -> None: + """ADR 0010: a person's affiliation to an organization with no + existing corporate_entity candidate must not stay permanently + unresolved -- an LLM-proposed, search-corroborated hierarchy + placement creates a real new row, closing the "통합 고객사 계열 + tree AI" gap synthetic regression corpus data confirmed (0 of thousands of + real affiliations ever resolved before this). Deterministic fake + clients, CI-stable -- the point under test is the create-then-link + wiring, not model/search quality. + """ + from lineageweave.corporate_hierarchy_inference import HierarchyProposal + from lineageweave.keyman_extraction import COUNTERPARTY, PersonMention + from lineageweave.relation_verification import STATUS_CORROBORATED, RelationVerificationResult + + _grant_post_admin(seeded_db["dsn"]) + + class _FakeKeymanClient: + available = True + + def extract(self, post_title: str, post_body: str) -> list[PersonMention]: + return [ + PersonMention( + person_name="Priya Sharma", + person_side_code=COUNTERPARTY, + affiliated_organization_names=("Northwind Turbines Gwangju Plant",), + ) + ] + + class _FakeRelationshipClient: + available = True + + def classify(self, post_title: str, post_body: str, organization_names: list[str]): + return [] + + class _FakeHierarchyInferenceClient: + available = True + + def infer(self, organization_name: str, context_text: str) -> HierarchyProposal | None: + if organization_name == "Northwind Turbines Gwangju Plant": + return HierarchyProposal(level_code="plant", parent_name="Northwind Turbines") + if organization_name == "Northwind Turbines": + return HierarchyProposal(level_code="company", parent_name=None) + return None + + class _FakeVerificationClient: + available = True + + def verify(self, organization_name: str, relationship_label: str) -> RelationVerificationResult: + return RelationVerificationResult( + status_code=STATUS_CORROBORATED, evidence_url=f"https://example.org/{organization_name}" + ) + + monkeypatch.setattr("backend.app.main._keyman_extraction_client", lambda: _FakeKeymanClient()) + monkeypatch.setattr("backend.app.main._entity_relationship_client", lambda: _FakeRelationshipClient()) + monkeypatch.setattr( + "backend.app.main._corporate_hierarchy_inference_client", lambda: _FakeHierarchyInferenceClient() + ) + monkeypatch.setattr("backend.app.main._relation_verification_client", lambda: _FakeVerificationClient()) + + response = client.post( + f"/api/posts/{seeded_db['own_private_post_id']}/extract-keymen", + headers={"Authorization": f"Bearer {demo_analyst_token}"}, + ) + assert response.status_code == 200, response.text + + admin_conn = psycopg2.connect(seeded_db["dsn"]) + admin_conn.autocommit = True + try: + with admin_conn.cursor() as cur: + cur.execute( + "select corporate_entity_id, entity_level_code, parent_entity_id, corporate_entity_code " + "from corporate_entity where entity_name = 'Northwind Turbines Gwangju Plant'" + ) + plant_row = cur.fetchone() + cur.execute( + "select corporate_entity_id, entity_level_code " + "from corporate_entity where entity_name = 'Northwind Turbines'" + ) + company_row = cur.fetchone() + cur.execute( + "select pa.affiliated_corporate_entity_id from person_affiliation pa " + "join cataloged_person cp on cp.person_id = pa.person_id " + "where cp.person_name = 'Priya Sharma'" + ) + affiliation_entity_id = cur.fetchone()[0] + finally: + admin_conn.close() + + assert plant_row is not None, "the plant-level entity must be created" + plant_entity_id, plant_level_code, plant_parent_id, plant_code = plant_row + assert plant_level_code == "plant" + assert plant_code.startswith("AUTO-"), "an auto-created code must never collide with a real login corp code" + assert company_row is not None, "the inferred parent company must also be created, not left dangling" + company_entity_id, company_level_code = company_row + assert company_level_code == "company" + assert str(plant_parent_id) == str(company_entity_id), "the plant's parent must be the real created company" + assert str(affiliation_entity_id) == str(plant_entity_id), "the affiliation must link to the real created plant" + + @pytest.mark.skipif( not (_ORCHESTRATOR_BASE_URL and _ORCHESTRATOR_API_KEY), reason="set LINEAGEWEAVE_TEST_ORCHESTRATOR_BASE_URL and LINEAGEWEAVE_TEST_ORCHESTRATOR_API_KEY to run", diff --git a/backend/tests/test_config.py b/backend/tests/test_config.py index c2f3994d..ba5dc688 100644 --- a/backend/tests/test_config.py +++ b/backend/tests/test_config.py @@ -23,6 +23,14 @@ def test_frontend_origins_drop_blank_entries(monkeypatch) -> None: assert load_settings().frontend_origins == ["http://localhost:5173"] +def test_tepp_transport_url_defaults_empty_and_is_not_a_score(monkeypatch) -> None: + """Missing TEPP_TRANSPORT_URL keeps the channel dropped.""" + monkeypatch.delenv("TEPP_TRANSPORT_URL", raising=False) + assert load_settings().tepp_transport_url == "" + monkeypatch.setenv("TEPP_TRANSPORT_URL", "https://tepp.example/v1/analysis-runs") + assert load_settings().tepp_transport_url == "https://tepp.example/v1/analysis-runs" + + def test_rankweave_disabled_defaults_off(monkeypatch) -> None: monkeypatch.delenv("RANKWEAVE_DISABLED", raising=False) assert load_settings().rankweave_disabled is False diff --git a/backend/uv-bootstrap-requirements.txt b/backend/uv-bootstrap-requirements.txt new file mode 100644 index 00000000..c981ffe9 --- /dev/null +++ b/backend/uv-bootstrap-requirements.txt @@ -0,0 +1,14 @@ +# uv bootstrap dependency for backend/Dockerfile. +# +# All hashes are the PyPI trusted-published uv 0.11.28 binary wheels for the +# Linux architectures supported by the pinned python:3.12-slim base manifest. +# --only-binary=:all: in the Dockerfile makes unsupported platforms fail closed +# instead of falling back to an unhashed source build. +uv==0.11.28 \ + --hash=sha256:49fe42df9f42056037473f3876adec1615709b57d3470ed39178ff420f3afb9f \ + --hash=sha256:041e4b80bebc58d7142ac9394370cacd73185fd8d066d6675d14707d83408f6d \ + --hash=sha256:185416a5316df8c5442b47178349f1f27fc1034468670ac1fb499eae3b25bd68 \ + --hash=sha256:a4a9fe246cb2882532277f5d5e5bd8a59462981462a2f98426f35ecfca82460e \ + --hash=sha256:6f7ce6f6015a3e857bc6a663514afa62856b669ee5c1bd120e4c58ac2ef5513d \ + --hash=sha256:6b3d0ea11e83b373a2166b82dd0864f5677fbadf98db64541ab2e59c42968905 \ + --hash=sha256:8c60294e3be4fa203a04015fc02ac8a31d936e86fde06dcb43c7f8f22661dfff diff --git a/docker/postgres-init/Dockerfile b/docker/postgres-init/Dockerfile index 51ac998c..ce2cf84d 100644 --- a/docker/postgres-init/Dockerfile +++ b/docker/postgres-init/Dockerfile @@ -3,10 +3,9 @@ FROM postgres:16-alpine@sha256:57c72fd2a128e416c7fcc499958864df5301e940bca0a56f5 # Keycloak-database bootstrap and the product schema can be copied from # their single sources of truth. COPY docker/postgres-init/01-create-keycloak-db.sql /docker-entrypoint-initdb.d/01-create-keycloak-db.sql -# The exact same migration file tests/test_schema.py applies -- single -# source of truth, no re-typed copy. Runs against POSTGRES_DB (the "app" -# database) because docker-entrypoint-initdb.d executes each *.sql file -# with that database already selected. +# The exact same migration files the PostgreSQL contract tests apply -- single +# source of truth, no re-typed copy. docker-entrypoint-initdb.d executes each +# file against POSTGRES_DB in lexical order. COPY migrations/0001_initial_schema.sql /docker-entrypoint-initdb.d/02-app-schema.sql COPY migrations/0002_thread_grouping_keys.sql /docker-entrypoint-initdb.d/03-thread-grouping-keys.sql COPY migrations/0003_ticket_commitment_calendar.sql /docker-entrypoint-initdb.d/04-ticket-commitment-calendar.sql @@ -18,7 +17,24 @@ COPY migrations/0008_post_summary_result.sql /docker-entrypoint-initdb.d/09-post COPY migrations/0009_shared_metric_bank.sql /docker-entrypoint-initdb.d/10-shared-metric-bank.sql COPY migrations/0010_report_item_information.sql /docker-entrypoint-initdb.d/11-report-item-information.sql COPY migrations/0011_post_chat_result.sql /docker-entrypoint-initdb.d/12-post-chat-result.sql -COPY migrations/0012_report_leftover_pair.sql /docker-entrypoint-initdb.d/13-report-leftover-pair.sql +COPY migrations/0012_role_responsibility_agent_type.sql /docker-entrypoint-initdb.d/13-role-responsibility-agent-type.sql +COPY migrations/0013_person_job_title.sql /docker-entrypoint-initdb.d/14-person-job-title.sql +COPY migrations/0014_role_responsibility_team_actor_type.sql /docker-entrypoint-initdb.d/15-role-responsibility-team-actor-type.sql +COPY migrations/0015_organization_name_resolution.sql /docker-entrypoint-initdb.d/16-organization-name-resolution.sql +COPY migrations/0016_cross_post_actor_identity.sql /docker-entrypoint-initdb.d/17-cross-post-actor-identity.sql +COPY migrations/0017_prov_o_standard_relations.sql /docker-entrypoint-initdb.d/18-prov-o-standard-relations.sql +COPY migrations/0018_analysis_run_registry.sql /docker-entrypoint-initdb.d/19-analysis-run-registry.sql +COPY migrations/0019_role_catalog_identity.sql /docker-entrypoint-initdb.d/20-role-catalog-identity.sql +COPY migrations/0020_analysis_run_retention_purge.sql /docker-entrypoint-initdb.d/21-analysis-run-retention-purge.sql +COPY migrations/0021_analysis_run_reconstruction.sql /docker-entrypoint-initdb.d/22-analysis-run-reconstruction.sql +COPY migrations/0022_analysis_source_snapshot_member.sql /docker-entrypoint-initdb.d/23-analysis-source-snapshot-member.sql +COPY migrations/0023_analysis_run_outbox.sql /docker-entrypoint-initdb.d/24-analysis-run-outbox.sql +COPY migrations/0024_source_post_revision.sql /docker-entrypoint-initdb.d/25-source-post-revision.sql +COPY migrations/0025_role_person_catalog_identity.sql /docker-entrypoint-initdb.d/26-role-person-catalog-identity.sql +COPY migrations/0026_report_leftover_pair.sql /docker-entrypoint-initdb.d/27-report-leftover-pair.sql +COPY migrations/0027_abbreviation_tree_corroboration.sql /docker-entrypoint-initdb.d/28-abbreviation-tree-corroboration.sql +COPY migrations/0028_analysis_run_tepp_result.sql /docker-entrypoint-initdb.d/29-analysis-run-tepp-result.sql +COPY migrations/0029_analysis_run_tepp_accepted.sql /docker-entrypoint-initdb.d/30-analysis-run-tepp-accepted.sql # Official image already drops to this account at runtime; declare it so # the Dockerfile itself satisfies DS-0002 (explicit non-root USER). USER postgres diff --git a/docs/PROV_O_IMPLEMENTATION.md b/docs/PROV_O_IMPLEMENTATION.md new file mode 100644 index 00000000..96b471e9 --- /dev/null +++ b/docs/PROV_O_IMPLEMENTATION.md @@ -0,0 +1,102 @@ +# W3C PROV-O implementation + +## Requirement + +LineageWeave must accept, validate, persist, infer, and serialize every normative relation in *PROV-O: The PROV Ontology* without flattening qualified influences or literal-valued properties into the existing navigation graph. + +## Runtime architecture + +```mermaid +flowchart LR + A[Product records and external RDF] --> B[PROV-O canonicalizer] + B --> C[Domain/range and literal validator] + C --> D[Explicit provenance assertions] + D --> E[Deterministic materializer] + E --> F[Qualified-to-unqualified implications] + E --> G[Property hierarchy and inverse closure] + E --> H[Qualified event-time shortcuts] + D --> I[(Normalized PostgreSQL provenance store)] + E --> J[RDF/Turtle/JSON-LD via rdflib] + I --> K[Explicit projection] + K --> L[(knowledge_graph_edge navigation graph)] +``` + +## Supported standard surface + +The machine-verifiable inventory is in [`PROV_O_IMPLEMENTATION_MATRIX.md`](PROV_O_IMPLEMENTATION_MATRIX.md): + +- all 30 PROV-O classes; +- all 50 normative properties; +- exact object/datatype distinction; +- direct domains, resource ranges, and `xsd:dateTime` ranges; +- class and property hierarchies; +- both normative qualification tables; +- every Appendix B recommended inverse name. + +## Canonicalization contract + +Inputs may use a local name, `prov:` compact name, full W3C IRI, or a reserved Appendix B inverse name. A canonical property name always retains its standard direction. A reserved inverse name that is not itself one of the 50 normative properties reverses subject and object into the preferred relation. + +```text +source prov:hadDerivation derived + ↓ canonicalize and reverse +derived prov:wasDerivedFrom source +``` + +No inverse alias is accepted for datatype properties because reversing a literal cannot produce a valid RDF subject. + +## Qualification contract + +For each normative mapping: + +```text +influenced --qualifiedRelation--> influence +influence --influencerProperty--> influencer +``` + +LineageWeave materializes: + +```text +influenced --unqualifiedRelation--> influencer +``` + +This applies to Generation, Derivation, Attribution, Usage, Communication, Association, Delegation, generic Influence, PrimarySource, Quotation, Revision, Invalidation, Start, and End. + +## Persistence contract + +`migrations/0017_prov_o_standard_relations.sql` creates a third-normal-form catalog and assertion store. A PostgreSQL trigger recursively checks subject domains and resource ranges through the class hierarchy and checks datatype-property literals before insertion. One assertion has exactly one resource or literal object. Inference provenance is represented by the many-to-many `provenance_assertion_derivation` table. + +## Security and tenancy boundary + +- External IRIs and lexical values are data, never executable instructions. +- RDF serialization performs no external fetch. +- The support profile uses `owl:imports` as metadata; runtime code does not dereference it. +- Assertions are rejected if resources are undeclared or incorrectly typed. +- The migration does not weaken existing row-level access decisions. API exposure must apply the same authenticated product boundary before binding product nodes to provenance resources. + +## Operability + +- Definitions are idempotently seeded. +- Exact W3C IRIs are stable; relational codes are multiword snake case. +- Standard definitions and runtime data are separate, so ontology upgrades can be reviewed without rewriting assertions. +- `provenance_resource_binding` is the only bridge to LineageWeave node identifiers; projections remain reproducible and removable. + +## Acceptance evidence + +```bash +pytest -q tests/test_prov_o.py +coverage run --branch -m pytest -q tests/test_prov_o.py +coverage report -m lineageweave/prov_o.py +python -m compileall -q lineageweave tests +``` + +Expected focused result: all tests pass and `lineageweave/prov_o.py` reports 100% statements and branches. + +## OWL 2 RL compatibility domains are not universal permissions + +Appendix A also publishes broad `prov:Influence` domains for +`prov:hadActivity` and `prov:hadRole` as OWL 2 RL compatibility aids. +The Recommendation explicitly warns that these broad domains must not be +read as permission to use either property on every Influence. Runtime and +database validation therefore enforce the normative union members rather +than weakening the contract. diff --git a/docs/PROV_O_IMPLEMENTATION_MATRIX.md b/docs/PROV_O_IMPLEMENTATION_MATRIX.md new file mode 100644 index 00000000..8a3c2861 --- /dev/null +++ b/docs/PROV_O_IMPLEMENTATION_MATRIX.md @@ -0,0 +1,67 @@ +# PROV-O implementation matrix + +LineageWeave implements the W3C PROV-O Recommendation as a separate standards-complete provenance layer. The product-specific `knowledge_graph_edge` remains a compact navigation projection; it is not used to flatten literal-valued or qualified PROV-O assertions. + +## Coverage contract + +- 30 normative classes. +- 50 normative properties: 44 object properties and 6 datatype properties. +- 14 qualified influence mappings from Tables 2 and 3. +- Qualified forms imply their unqualified forms. +- Transitive subproperty closure, defined inverses, `alternateOf` symmetry, and qualified event-time shortcuts are materialized deterministically. +- All 44 Appendix B inverse names are cataloged; non-canonical reserved names are accepted by reversing into the preferred PROV-O direction. + +## Property matrix + +| PROV-O property | Kind | Domain | Range / datatype | Superproperty | Qualification | Appendix B inverse | +|---|---|---|---|---|---|---| +| `prov:wasGeneratedBy` | object | Entity | Activity | wasInfluencedBy | qualifiedGeneration → Generation.activity | `prov:generated` | +| `prov:wasDerivedFrom` | object | Entity | Entity | wasInfluencedBy | qualifiedDerivation → Derivation.entity | `prov:hadDerivation` | +| `prov:wasAttributedTo` | object | Entity | Agent | wasInfluencedBy | qualifiedAttribution → Attribution.agent | `prov:contributed` | +| `prov:startedAtTime` | datatype | Activity | http://www.w3.org/2001/XMLSchema#dateTime | — | — | — | +| `prov:used` | object | Activity | Entity | wasInfluencedBy | qualifiedUsage → Usage.entity | `prov:wasUsedBy` | +| `prov:wasInformedBy` | object | Activity | Activity | wasInfluencedBy | qualifiedCommunication → Communication.activity | `prov:informed` | +| `prov:endedAtTime` | datatype | Activity | http://www.w3.org/2001/XMLSchema#dateTime | — | — | — | +| `prov:wasAssociatedWith` | object | Activity | Agent | wasInfluencedBy | qualifiedAssociation → Association.agent | `prov:wasAssociateFor` | +| `prov:actedOnBehalfOf` | object | Agent | Agent | wasInfluencedBy | qualifiedDelegation → Delegation.agent | `prov:hadDelegate` | +| `prov:alternateOf` | object | Entity | Entity | — | — | `prov:alternateOf` | +| `prov:specializationOf` | object | Entity | Entity | alternateOf | — | `prov:generalizationOf` | +| `prov:generatedAtTime` | datatype | Entity | http://www.w3.org/2001/XMLSchema#dateTime | — | — | — | +| `prov:hadPrimarySource` | object | Entity | Entity | wasDerivedFrom | qualifiedPrimarySource → PrimarySource.entity | `prov:wasPrimarySourceOf` | +| `prov:value` | datatype | Entity | RDF literal | — | — | — | +| `prov:wasQuotedFrom` | object | Entity | Entity | wasDerivedFrom | qualifiedQuotation → Quotation.entity | `prov:quotedAs` | +| `prov:wasRevisionOf` | object | Entity | Entity | wasDerivedFrom | qualifiedRevision → Revision.entity | `prov:hadRevision` | +| `prov:invalidatedAtTime` | datatype | Entity | http://www.w3.org/2001/XMLSchema#dateTime | — | — | — | +| `prov:wasInvalidatedBy` | object | Entity | Activity | wasInfluencedBy | qualifiedInvalidation → Invalidation.activity | `prov:invalidated` | +| `prov:hadMember` | object | Collection | Entity | wasInfluencedBy | — | `prov:wasMemberOf` | +| `prov:wasStartedBy` | object | Activity | Entity | wasInfluencedBy | qualifiedStart → Start.entity | `prov:started` | +| `prov:wasEndedBy` | object | Activity | Entity | wasInfluencedBy | qualifiedEnd → End.entity | `prov:ended` | +| `prov:invalidated` | object | Activity | Entity | influenced | — | `prov:wasInvalidatedBy` | +| `prov:influenced` | object | Entity / Activity / Agent | Entity / Activity / Agent | — | — | `prov:wasInfluencedBy` | +| `prov:atLocation` | object | Activity / Agent / Entity / InstantaneousEvent | Location | — | — | `prov:locationOf` | +| `prov:generated` | object | Activity | Entity | influenced | — | `prov:wasGeneratedBy` | +| `prov:wasInfluencedBy` | object | Entity / Activity / Agent | Entity / Activity / Agent | — | qualifiedInfluence → Influence.influencer | `prov:influenced` | +| `prov:qualifiedInfluence` | object | Entity / Activity / Agent | Influence | — | — | `prov:qualifiedInfluenceOf` | +| `prov:qualifiedGeneration` | object | Entity | Generation | qualifiedInfluence | — | `prov:qualifiedGenerationOf` | +| `prov:qualifiedDerivation` | object | Entity | Derivation | qualifiedInfluence | — | `prov:qualifiedDerivationOf` | +| `prov:qualifiedPrimarySource` | object | Entity | PrimarySource | qualifiedInfluence | — | `prov:qualifiedSourceOf` | +| `prov:qualifiedQuotation` | object | Entity | Quotation | qualifiedInfluence | — | `prov:qualifiedQuotationOf` | +| `prov:qualifiedRevision` | object | Entity | Revision | qualifiedInfluence | — | `prov:revisedEntity` | +| `prov:qualifiedAttribution` | object | Entity | Attribution | qualifiedInfluence | — | `prov:qualifiedAttributionOf` | +| `prov:qualifiedInvalidation` | object | Entity | Invalidation | qualifiedInfluence | — | `prov:qualifiedInvalidationOf` | +| `prov:qualifiedStart` | object | Activity | Start | qualifiedInfluence | — | `prov:qualifiedStartOf` | +| `prov:qualifiedUsage` | object | Activity | Usage | qualifiedInfluence | — | `prov:qualifiedUsingActivity` | +| `prov:qualifiedCommunication` | object | Activity | Communication | qualifiedInfluence | — | `prov:qualifiedCommunicationOf` | +| `prov:qualifiedAssociation` | object | Activity | Association | qualifiedInfluence | — | `prov:qualifiedAssociationOf` | +| `prov:qualifiedEnd` | object | Activity | End | qualifiedInfluence | — | `prov:qualifiedEndOf` | +| `prov:qualifiedDelegation` | object | Agent | Delegation | qualifiedInfluence | — | `prov:qualifiedDelegationOf` | +| `prov:influencer` | object | Influence | Entity / Activity / Agent | — | — | `prov:hadInfluence` | +| `prov:entity` | object | EntityInfluence | Entity | influencer | — | `prov:entityOfInfluence` | +| `prov:hadUsage` | object | Derivation | Usage | — | — | `prov:wasUsedInDerivation` | +| `prov:hadGeneration` | object | Derivation | Generation | — | — | `prov:generatedAsDerivation` | +| `prov:activity` | object | ActivityInfluence | Activity | influencer | — | `prov:activityOfInfluence` | +| `prov:agent` | object | AgentInfluence | Agent | influencer | — | `prov:agentOfInfluence` | +| `prov:hadPlan` | object | Association | Plan | — | — | `prov:wasPlanOf` | +| `prov:hadActivity` | object | Delegation / Derivation / End / Start | Activity | — | — | `prov:wasActivityOfInfluence` | +| `prov:atTime` | datatype | InstantaneousEvent | http://www.w3.org/2001/XMLSchema#dateTime | — | — | — | +| `prov:hadRole` | object | Association / InstantaneousEvent | Role | — | — | `prov:wasRoleIn` | diff --git a/docs/adr/0003-fast-mlsirm-report-integration.md b/docs/adr/0003-fast-mlsirm-report-integration.md index bdf234b6..e31436b8 100644 --- a/docs/adr/0003-fast-mlsirm-report-integration.md +++ b/docs/adr/0003-fast-mlsirm-report-integration.md @@ -100,7 +100,7 @@ than one large PR: `information_polytomous` (Lord, 1980 max-info). Persist the ranking (`report_item_information`) and show the rank-1 item on the Period reports panel. Do not reimplement an information function here. -7. **Leftover-pair slice** (shipped in 0.71.2; ADR 0017 / 0018): after +7. **Leftover-pair slice** (shipped in 0.71.2; ADR 0028 / 0029): after IRT main effects, persist closest and farthest post–criterion pairs from the residual leftover map. Do not fork LSIRM; do not invent a leftover-pair API inside `fast-mlsirm` in this slice. diff --git a/docs/adr/0006-role-responsibility-agent-ontology.md b/docs/adr/0006-role-responsibility-agent-ontology.md new file mode 100644 index 00000000..8ded02b8 --- /dev/null +++ b/docs/adr/0006-role-responsibility-agent-ontology.md @@ -0,0 +1,107 @@ +# ADR 0006 — R&R's named actor is a PROV-O Agent, not always a person + +**Decision status:** Accepted +**Date:** 2026-08-14 + +## Context + +`post_summary.py`'s R&R (roles & responsibilities) extraction treats +the acting party a post's text names as if it were always a person. +Business correspondence routinely names an organization acting in its +own name -- "당사" (our company), "Demo Corp" -- not a named +individual. The product requirement is to handle that with a general +standard ontology, and to infer a person actor's affiliation so a +bare name is not left unplaced. + +Before this change, `RoleResponsibility.person_name` had no way to +express "this actor is an organization" -- every entry was forced into +a person slot, and an organization actor's name would sit +indistinguishable from an unresolved person. + +## Decision + +Ground the distinction in W3C PROV-O (Lebo, Sahoo, & McGuinness, 2013): +`prov:Agent` is the general acting-party class, with `prov:Person` and +`prov:Organization` as its two recognized subclasses -- an existing, +widely-adopted standard for exactly this "who/what acted" provenance +question, not a bespoke local invention. + +`RoleResponsibility` (`lineageweave/post_summary.py`) gains: +- `actor_name` (renamed from `person_name` -- the field can now hold an + organization's name too, so "person" in the field name would be + actively wrong). +- `actor_type_code`: `prov_person` / `prov_organization` + (`common_lookup_value` category `prov_agent_type`), defaulting to + `prov_person` when the LLM's response omits the field, matching this + repo's existing degrade-gracefully-not-fail discipline. +- `affiliated_organization_name`: for a person actor, the organization + the text names or clearly implies they work for, inferred by the same + LLM call rather than left for a human to cross-reference against the + Keyman panel separately. `None` when the text gives nothing to infer, + or when the actor is itself an organization (its own name already + answers "which organization"). + +The LLM prompt now explicitly instructs the model to decide +person-vs-organization per actor rather than defaulting every named +actor to a person, and to give an affiliation when the text supports +one. + +Ontology (`docs/ontology/lineageweave-kg.ttl`, extending +[ADR 0004](0004-knowledge-graph-ontology.md)'s vocabulary): +`:RoleActorPerson rdfs:subClassOf prov:Person` and +`:RoleActorOrganization rdfs:subClassOf prov:Organization`, each +carrying the `:lookupCode` annotation linking it to the matching +`common_lookup_value` row -- these are genuinely subclasses of the real +external PROV-O classes (imported via the `prov:` prefix), not +same-named local terms that merely resemble the standard. Kept distinct +from the ontology's existing `:Person` (node_type's `node_person`, +i.e. a cataloged Keyman with a stable `person_id`): an R&R actor is a +free-text name with no cataloged identity of its own, and may not even +resolve to a Keyman row. + +Persistence: `post_summary_role` gains `actor_type_code` (FK to +`common_lookup_value`, default `prov_person`) and +`affiliated_organization_name`; `person_name` is renamed to +`actor_name` via `migrations/0012_role_responsibility_agent_type.sql`'s +`ALTER TABLE ... RENAME COLUMN` (preserves every existing row's data, +unlike a drop/recreate) plus the two new `ADD COLUMN IF NOT EXISTS` +statements, with `migrations/0001_initial_schema.sql` updated directly +for a fresh install, matching this repo's established pattern (e.g. +ADR 0005's `verification_status_code` additions). + +UI: the popup's R&R list (`frontend/src/App.tsx`) shows a +Person/Organization badge per actor and the inferred affiliation in +parentheses; only a person actor is still linked to the Keyman panel +(an organization actor has no `person_id` to link to). + +## Consequences + +- `RoleResponsibility.person_name` is a breaking rename to `actor_name` + across the JSON wire contract (`GET /api/posts/{id}/summary`), the + DB column, and every call site. Accepted because the field's old name + was actively misleading once an organization actor is an intended + value, not a hypothetical edge case. +- `prov_agent_type` is a `common_lookup_value` category seeded by its + own migration file (0012), not literally embedded in + `scripts/seed_demo_data.py`'s SQL string the way ADR 0004's original + five covered categories are -- `tests/test_ontology.py`'s round-trip + check reads 0012's file content alongside the seed script's own text + so this still closes the loop, rather than being silently excluded + the way `evaluation_criterion` / `relation_verification_status` + currently are. +- The affiliation inference is opportunistic, not authoritative: it is + a same-request LLM guess from the post's own text, not resolved + against `corporate_entity` the way Keyman affiliations are (see + `lineageweave/corporate_hierarchy_resolution.py`). A future slice + could route it through the same resolver if real usage shows the + free-text name needs matching back to a cataloged organization. + +## Related + +Extends [ADR 0004](0004-knowledge-graph-ontology.md)'s Ontology/ +Semantic-Layer vocabulary and reuses its round-trip enforcement +mechanism (`tests/test_ontology.py`). + +## References (APA 7th) + +Lebo, T., Sahoo, S., & McGuinness, D. (Eds.). (2013). *PROV-O: The PROV ontology*. World Wide Web Consortium. https://www.w3.org/TR/prov-o/ diff --git a/docs/adr/0007-team-actor-type.md b/docs/adr/0007-team-actor-type.md new file mode 100644 index 00000000..1da18883 --- /dev/null +++ b/docs/adr/0007-team-actor-type.md @@ -0,0 +1,90 @@ +# ADR 0007 — R&R's named actor can be a team, a meso-level unit, not just a person/organization + +**Decision status:** Accepted +**Date:** 2026-08-14 + +## Context + +ADR 0006 gave R&R's `actor_type_code` two values: `prov_person` and +`prov_organization`, grounded in W3C PROV-O's `prov:Agent` subclasses. +Real post text surfaced a third, distinct case those two do not cover: +a named sub-unit of a company -- e.g. "설계팀" (design team) -- acting +in the text. A team is not a person, and forcing it into +`prov_organization` is wrong for the same reason ADR 0006 rejected +forcing an organization into a person slot: it collapses a real, +useful distinction. A team is meso-level -- part of a company, not the +company itself, and not an individual either. + +PROV-O has no sub-organization concept to reuse here; `prov:Agent`'s +two subclasses are exhaustive for PROV-O's own purposes (an +organization's internal structure is out of PROV-O's scope). + +## Decision + +Ground the team case in the W3C Organization Ontology (Reynolds, 2014): +`org:OrganizationalUnit`, defined for exactly this -- representing the +division of an organization into sub-organizational units, linked to +its parent via `org:unitOf`/`org:subOrganizationOf`. This is a +different, complementary W3C vocabulary from PROV-O, not a conflicting +one: PROV-O models "who/what acted," ORG models "how an organization is +structured internally" -- a team acting in a post's text needs both a +`prov:Agent`-shaped role (it does something) and an +`org:OrganizationalUnit`-shaped identity (it belongs to a company). +`:RoleActorTeam` is declared `rdfs:subClassOf org:OrganizationalUnit` +for that reason, parallel to how `:RoleActorPerson`/ +`:RoleActorOrganization` subclass PROV-O's classes. + +`post_summary.py` gains `ACTOR_TYPE_TEAM = "prov_team"` +(`common_lookup_value` category `prov_agent_type`, extending ADR +0006's two existing values). The LLM prompt now offers three actor +types (person / organization / team) and explicitly requires a team +actor to also carry `affiliated_organization_name` -- unlike an +organization actor (whose own name already answers "which +organization"), a team's name alone does not identify a company, so +the field is not optional in the same "opportunistic" sense ADR 0006 +described for a person actor; a team is always someone's team, and the +prompt asks the model to infer the parent company from context when +the text supports it. + +No new `RoleResponsibility` field is needed: +`affiliated_organization_name` already exists (ADR 0006) and applies +unchanged to this actor type -- only its *meaning* extends from +"the person's employer" to "the person's or team's parent +organization," which the dataclass docstring now says explicitly. + +Persistence: `migrations/0014_role_responsibility_team_actor_type.sql` +inserts the `prov_team` lookup row -- purely additive +(`insert ... on conflict (lookup_code) do nothing`), no column or +constraint change, since `actor_type_code` already stores an arbitrary +FK'd lookup code and needs no schema change to accept a third value. + +## Consequences + +- `_VALID_ACTOR_TYPE_CODES` in `post_summary.py` now has three members; + any code elsewhere that pattern-matches strictly on the first two + (rather than treating an unrecognized/future code as "not this one") + needs review. Found and fixed one: the frontend badge's CSS class name + (`actor-type-${code}`) was already generic, but its *label text* was a + binary person/organization ternary that would have mislabeled a team + actor as "Organization" -- now a three-way check. +- A team actor is never linked to the Keyman panel (same as an + organization actor in ADR 0006) -- it has no `person_id`. +- Distinguishing "설계팀" (a team) from "Design Corp" (an organization) + is a real LLM judgment call with no hard syntactic rule; the prompt + gives the model the concept and an example, matching this repo's + existing degrade-gracefully discipline for judgment-call extraction + fields (a wrong guess is a labeling error on one row, not lost data -- + the raw `actor_name` string is preserved regardless of which type it + is filed under). + +## Related + +Extends [ADR 0006](0006-role-responsibility-agent-ontology.md), which +itself extends [ADR 0004](0004-knowledge-graph-ontology.md)'s Ontology/ +Semantic-Layer vocabulary. + +## References (APA 7th) + +Reynolds, D. (Ed.). (2014). *The organization ontology* (W3C Recommendation). World Wide Web Consortium. https://www.w3.org/TR/vocab-org/ + +Lebo, T., Sahoo, S., & McGuinness, D. (Eds.). (2013). *PROV-O: The PROV ontology*. World Wide Web Consortium. https://www.w3.org/TR/prov-o/ diff --git a/docs/adr/0008-organization-abbreviation-resolution.md b/docs/adr/0008-organization-abbreviation-resolution.md new file mode 100644 index 00000000..72b12125 --- /dev/null +++ b/docs/adr/0008-organization-abbreviation-resolution.md @@ -0,0 +1,120 @@ +# ADR 0008 — Abbreviated organization names are resolved and search-verified, not left opaque + +**Decision status:** Accepted +**Date:** 2026-08-14 + +## Context + +Real post text names organizations by abbreviated or slang forms a +human reader immediately recognizes but a string-matching pipeline +cannot -- e.g. "AGP," a synthetic contraction of "Aurora Grid Power". +`lineageweave.corporate_hierarchy_resolution` +already resolves near-matches (a trailing legal suffix, a minor +abbreviation) via character-sequence similarity +(`difflib.SequenceMatcher`, grounded in Bhattacharya & Getoor, 2007's +candidate-generation stage), but an initialism/contraction like "AGP" +shares almost no character substring with its expansion -- no +similarity threshold recovers it, because the two strings are not +similar, they are *related by real-world knowledge* the text or an +external source has to supply. + +Left unresolved, every mention of the same real organization under its +abbreviated name creates its own unmatched, un-linkable free-text +string in `person_affiliation`/R&R -- the same organization looks like +N different unknown entities across N posts, each failing to link into +the corporate hierarchy a human reader would recognize instantly. + +## Decision + +A two-stage pipeline, reusing infrastructure this repo already has for +a structurally identical problem (ADR: `lineageweave.relation_verification`, +FEVER-style claim verification) rather than building a second web-search +integration: + +1. **LLM context resolution** + (`lineageweave.organization_name_resolution.ContextualOrchestratorOrganizationNameResolutionClient`): + given the raw abbreviated name and the post's own text as context, + ask the model for the organization's full real-world name, or + `UNKNOWN` when the text gives no real basis to determine one -- + never inventing an expansion from the abbreviation's letters alone. +2. **External search cross-verification** + (reusing `lineageweave.relation_verification.RelationVerificationClient` + as-is, not a new client class): the proposed full name plus the raw + abbreviation together become the search query (e.g. "Aurora Grid Power + AGP") -- a real page mentioning both together is strong + corroboration the specific pairing is correct, not just that the + full name exists as *some* organization. + +Grounded in SKOS (Miles & Bechhofer, 2009): `skos:prefLabel` (a +resource's one preferred/canonical label) and `skos:altLabel` (an +alternative label -- exactly the abbreviation/synonym relationship) is +the standard vocabulary for a raw-name/canonical-name pair. This is a +different, complementary standard from ADR 0006/0007's PROV-O/ORG +classes: SKOS here labels the *string identity* relationship between +two names for the same thing, not the *type* of the named actor. + +Only a search-corroborated resolution is ever substituted in for +downstream entity matching (`resolve_corporate_entity`) -- an +LLM-proposed name with no corroboration, or with verification itself +unavailable, leaves the raw name flowing unchanged. This is the same +never-trust-an-unverified-guess discipline `relation_verification` +itself already established: a wrong resolution corrupts every +downstream Knowledge Graph link through it, so "did not resolve" must +stay a real, distinguishable outcome from "resolved to X." + +Persistence: a new `organization_name_resolution` cache table +(`migrations/0015_organization_name_resolution.sql`), keyed by +`raw_organization_name` -- the same abbreviation is resolved once, not +re-queried on every one of its (potentially many) mentions across +posts. `verification_status_code` reuses the existing +`relation_verification_status` lookup category rather than a +near-duplicate one: a resolved name is corroborated/uncorroborated the +exact same way a classified relationship already is. + +Wired into `backend/app/keyman_ingestion.py`'s affiliation loop (the +concrete case real data surfaced): each affiliated organization name is +resolved before corporate-entity matching and creation. A corroborated +canonical name is returned to the caller as part of the normalized +`PersonMention`, so the same request's relationship classifier uses the +canonical form too rather than reintroducing the raw abbreviation. + +## Consequences + +- The raw abbreviated form is not duplicated onto every row that + mentions it (e.g. `person_affiliation.affiliated_organization_name` + stores the resolved canonical name once corroborated) -- it remains + fully recoverable via a join against `organization_name_resolution`, + which is the authoritative raw-form/canonical-form/evidence record. + This is 3NF-motivated, not a loss: repeating the raw-to-canonical + mapping per affiliation row would be the actual redundancy. +- Resolution availability can improve between extraction runs. When a + prior raw affiliation later resolves, `_upsert_affiliation` promotes + it transactionally: the canonical row is inserted or updated while + preserving any previously resolved corporate-entity link and role + title, and the obsolete raw-name row is deleted when the two names + differ. This avoids leaving duplicate raw and canonical identities for + the same person. +- `ingest_post_keymen` returns the normalized mentions it persisted. + `extract_post_keymen` therefore passes canonical organization names to + entity-relationship classification and returns those same names on + the API response; affiliation persistence, relationship classification, + and the caller-visible payload agree within one transaction. +- Every channel here follows the existing pluggable-client discipline: + `NullOrganizationNameResolutionClient`/an unavailable verification + client degrade to "use the raw name," never a fabricated resolution. + +## Related + +Complements [ADR 0006](0006-role-responsibility-agent-ontology.md) and +[ADR 0007](0007-team-actor-type.md) (actor *type*), and reuses +`lineageweave.relation_verification` (ADR-less, predates this file, see +its own module docstring for FEVER grounding) for the verification +stage rather than duplicating it. + +## References (APA 7th) + +Miles, A., & Bechhofer, S. (Eds.). (2009). *SKOS simple knowledge organization system reference*. World Wide Web Consortium. https://www.w3.org/TR/skos-reference/ + +Bhattacharya, I., & Getoor, L. (2007). Collective entity resolution in relational data. *ACM Transactions on Knowledge Discovery from Data*, 1(1), Article 5. https://doi.org/10.1145/1217299.1217304 + +Thorne, J., Vlachos, A., Christodoulopoulos, C., & Mittal, A. (2018). FEVER: A large-scale dataset for fact extraction and VERification. In *Proceedings of the 2018 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies* (pp. 809–819). Association for Computational Linguistics. https://doi.org/10.18653/v1/N18-1074 diff --git a/docs/adr/0009-cross-post-actor-identity.md b/docs/adr/0009-cross-post-actor-identity.md new file mode 100644 index 00000000..1a970c0f --- /dev/null +++ b/docs/adr/0009-cross-post-actor-identity.md @@ -0,0 +1,133 @@ +# ADR 0009 — R&R team/organization actors get a shared cross-post identity, not just per-post text + +**Decision status:** Accepted +**Date:** 2026-08-14 + +## Context + +Extraction (Keyman, R&R) runs per-post. For a person actor, this was +already not a dead end: Keyman extraction upserts into +`cataloged_person`, so the same name across posts (mostly) resolves to +one row with a stable `person_id` the Knowledge Graph can link through. +R&R's team actor (ADR 0007, `prov_team`) and organization actor +(ADR 0006, `prov_organization`) had no equivalent -- each post's +extraction produced a bare `actor_name` string in `post_summary_role` +with no catalog entry and no Knowledge Graph mention edge. The same +"설계팀" (design team) named in ten different posts was ten unrelated +strings, not one entity a Keyman/team panel could click through to see +every post it appears in -- exactly the "extraction results must +themselves become cross-post lineage clues, not just per-post +artifacts" requirement this product exists to satisfy for people, but +was not yet satisfying for teams or organizations. + +## Decision + +**Team**: a new `cataloged_team` catalog table +(`migrations/0016_cross_post_actor_identity.sql`), the same +catalog-then-mention shape `cataloged_person`/`post_person_mention` +already establishes. Identity key is `(team_name, +affiliated_organization_name)`, not the bare name alone -- "설계팀" +exists at many real companies, so the pair is what is actually +identifying (`backend/app/team_ingestion.py`'s `upsert_team`, mirroring +`keyman_ingestion.py`'s `_upsert_person`). The team's own parent +organization is resolved to a real `corporate_entity` via the *same* +`resolve_corporate_entity` collective-entity-resolution matching +(Bhattacharya & Getoor, 2007) Keyman affiliations already use -- not a +second matching algorithm. + +**Organization**: an R&R organization actor's name is run through the +same `resolve_corporate_entity` matching; a resolved match writes a +`post_organization_mention` row (no new catalog needed -- `corporate_entity` +already is the shared, cross-post organization catalog every VOC +counterparty and Keyman affiliation already resolves against). + +**Person** (an R&R actor, not a Keyman): opportunistically joined to an +*existing* `cataloged_person` row by exact name match, when Keyman +extraction has already cataloged that name on this or another post. +R&R does not create a new person identity itself -- `cataloged_person` +requires `person_side_code` (our-side vs. counterparty), which R&R's +prompt does not currently ask for and Keyman's does; inventing one here +risked a wrong side assignment. Documented as a real, deliberate scope +boundary below, not silently half-done. +Person evidence sources remain separate: Keyman extraction replaces +`post_person_mention`; R&R replacement writes +`post_summary_person_mention`. `combined_post_person_mention` is a +read-only union used for lineage and KG derivation. This prevents a new +summary from deleting Keyman evidence and prevents removed R&R actors +from surviving as stale Keymen. Migration 0016 copies matching R&R +actor names into `post_summary_person_mention` and must not delete +overlapping Keyman rows -- `mention_context` has no R&R column, and a +later summary replacement would otherwise erase the only remaining +person evidence. + + +Each resolved actor gets a real Knowledge Graph mention edge (new +`edge_mention_team` / `edge_team_affiliation` / `edge_mention_organization` +lookup codes, `lineageweave/knowledge_graph.py`'s +`knowledge_graph_edges_for_post` extended, not a second edge-writing +path), reusing the same `persist_edges_for_post` entry point Keyman +ingestion already calls -- one function computes a post's whole edge +set regardless of which extraction step triggered it. + +`knowledge_graph_edge` is a deduplicated materialized registry. +`knowledge_graph_edge_evidence` records every post that currently supports an +edge; readers require support from an ABAC-visible post. Writers reconcile one +post under a transaction-scoped advisory lock, and unsupported registry rows +are pruned. Edge identity therefore cannot duplicate under concurrency, and a +replacement cannot leave a buyer-visible orphan edge. + +Ontology (`docs/ontology/lineageweave-kg.ttl`): `:Team a owl:Class ; +rdfs:subClassOf org:OrganizationalUnit` (same W3C ORG grounding as +ADR 0007's `:RoleActorTeam`, but a distinct term -- `:Team` is a +`cataloged_team` row with a stable identity, `:RoleActorTeam` is the +per-row `actor_type_code` classification, the same +`:Person`/`:RoleActorPerson` split ADR 0006 already established). +`:mentionsTeam` / `:teamAffiliatedWith` / `:mentionsOrganization` are +new, distinct object properties rather than widening `:mentions`'s +domain/range -- stating `rdfs:domain :mentions` twice (once `:Person`, +once `:Team`) would let RDFS entail every `:mentions` subject is BOTH, +which is false. + +## Consequences + +- Team/organization mention persistence only runs when + `summary.roles_and_responsibilities` is non-empty (a real, cheap + guard, not a correctness gap) -- a post with no R&R never touches + `cataloged_team`/`post_organization_mention` at all. +- **Documented, deliberate gap**: R&R never *creates* a new + `cataloged_person` row, only joins to an existing one by exact name. + Two failure modes follow from this, both accepted for now: (1) a + person named only in R&R (never by Keyman on any post) gets no + catalog identity at all until/unless Keyman also names them; (2) an + exact-name join has the same same-name-collision risk + `keyman_ingestion._upsert_person`'s job-title disambiguation exists + to catch, but R&R's join here does not run that check (R&R's own + prompt does not currently capture a job title). A future slice could + extend the R&R prompt to also ask for `person_side_code` (and + optionally a title) so R&R could safely originate new person + identities the same way Keyman does, closing this gap properly rather + than working around it with a guess. +- `cataloged_team` uses PostgreSQL's + `UNIQUE NULLS NOT DISTINCT (team_name, affiliated_organization_name)`. + NULL affiliation therefore participates in the identity key: two + bare-team rows with the same name conflict and the atomic upsert + returns one shared `team_id`. This database constraint, not a + read-before-insert application check, closes the concurrent duplicate + race for both affiliated and unplaced teams. + +## Related + +Depends on [ADR 0006](0006-role-responsibility-agent-ontology.md) and +[ADR 0007](0007-team-actor-type.md) (actor *type*) and +`lineageweave.corporate_hierarchy_resolution` (Bhattacharya & Getoor, +2007, cited there) for the organization-matching this ADR reuses rather +than re-deriving. [ADR 0019](0019-role-catalog-identity.md) stores the +resolved catalog id on `post_summary_role` so fetch does not rejoin by +display name. Person identity on that row is +[ADR 0027](0027-role-person-catalog-identity.md). + +## References (APA 7th) + +Bhattacharya, I., & Getoor, L. (2007). Collective entity resolution in relational data. *ACM Transactions on Knowledge Discovery from Data*, 1(1), Article 5. https://doi.org/10.1145/1217299.1217304 + +Reynolds, D. (Ed.). (2014). *The organization ontology* (W3C Recommendation). World Wide Web Consortium. https://www.w3.org/TR/vocab-org/ diff --git a/docs/adr/0010-corporate-hierarchy-auto-creation.md b/docs/adr/0010-corporate-hierarchy-auto-creation.md new file mode 100644 index 00000000..d034fe9b --- /dev/null +++ b/docs/adr/0010-corporate-hierarchy-auto-creation.md @@ -0,0 +1,100 @@ +# ADR 0010 — a real counterparty organization is auto-created into the corporate hierarchy, not left permanently unresolved + +**Decision status:** Accepted +**Date:** 2026-08-14 + +## Context + +`lineageweave.corporate_hierarchy_resolution`'s similarity-based +matching only ever locates an *already-cataloged* `corporate_entity` +row -- it has no path to create one. This was fine while the only +`corporate_entity` catalog was synthetic demo fixtures with a handful +of names extraction would naturally already know. A synthetic batch +where the catalog holds only the employer's own two-row hierarchy +exposes the same gap: every counterparty named in a post is, by +definition, outside that catalog. Similarity matching then resolves +**0 affiliation rows and 0 R&R organization-actor mentions** -- the +standing integrated customer-affiliate tree requirement (Harbor Group +-> Harbor Devices Korea -> ... in the synthetic brief) stays empty +until a verified creation path exists. + +## Decision + +`get_or_create_corporate_entity` (`backend/app/corporate_entity_ingestion.py`) +extends the existing resolution pipeline with a creation fallback, not +a competing algorithm: + +1. Try `resolve_corporate_entity` (similarity matching, unchanged, + Bhattacharya & Getoor, 2007) first -- an already-cataloged entity + still resolves exactly as before. +2. On a miss, ask an LLM + (`lineageweave.corporate_hierarchy_inference.CorporateHierarchyInferenceClient`) + to propose this organization's place in the Group -> Company -> + Plant hierarchy (`corporate_entity_level`, ADR 0004's existing SKOS + `skos:broader`/`skos:narrower` structure) from the post's own text -- + never inventing a hierarchy the text gives no evidence for; the + model may decline with `UNKNOWN`. +3. The proposal is only trusted after + `lineageweave.relation_verification`'s existing Searxng + corroboration (the same reused verification client + `organization_name_resolution`/ADR 0008 already established this + pattern for) -- an uncorroborated or unavailable-channel proposal + creates nothing, same never-trust-an-unverified-guess discipline as + every other channel here. +4. Only then is a real new `corporate_entity` row inserted. A proposed + parent organization is itself resolved-or-created first (bounded to + 4 levels of recursion, so a misbehaving response chain cannot spin + into unbounded row creation), so the whole chain gets real + `parent_entity_id` links, not an orphaned single-level row. + +**Auto-created code namespace**: `corporate_entity_code` is also the +real login "corp code" attribute Keycloak issues via the `corp_code` +token claim (`docker/keycloak/realm-export.json`) -- an auto-created +counterparty row must never collide with that namespace. Every +auto-created code is prefixed `AUTO-` followed by a deterministic hash +of the entity name (same name -> same code, so a genuine concurrent +duplicate-creation race collides on the real SQL `unique` constraint +and self-resolves via `on conflict`, rather than creating two rows for +one organization under two different codes). + +Wired into both existing organization-resolution call sites -- +`keyman_ingestion.py`'s person-affiliation loop and +`post_summary_ingestion.py`'s R&R organization-actor loop -- rather +than a third, separate code path, so both routes to `corporate_entity` +share one creation policy. + +## Consequences + +- A wrong hierarchy placement (level or parent) is a real risk this + design accepts, bounded by the same LLM-judgment-call discipline + ADR 0007's team-vs-organization classification already accepts: the + raw name is never lost regardless (it is the `entity_name` itself), + so a wrong placement is a correctable graph-structure error, not lost + data. +- The `AUTO-` code namespace is a real, deliberate simplification: an + operator wanting a genuinely curated corp-code scheme for these + entities later would need to re-code them, not just re-run + extraction -- accepted because the alternative (leaving every + counterparty unresolved forever) is strictly worse for this + product's actual purpose. +- Every LLM/search call in this path already existed for a different + purpose (`organization_name_resolution`'s resolution call shape, + `relation_verification`'s verification client) -- no new provider + integration was built, keeping this consistent with the project's + standing discipline of reusing an existing channel over adding a new + one wherever the shape already fits. + +## Related + +Extends [ADR 0008](0008-organization-abbreviation-resolution.md)'s +reuse-the-verification-client pattern and +[ADR 0009](0009-cross-post-actor-identity.md)'s cross-post identity +work -- an R&R organization actor's identity is now genuinely resolved +to a real corporate hierarchy node, not left as free text even when no +prior mention of it existed anywhere in the dataset. + +## References (APA 7th) + +Bhattacharya, I., & Getoor, L. (2007). Collective entity resolution in relational data. *ACM Transactions on Knowledge Discovery from Data*, 1(1), Article 5. https://doi.org/10.1145/1217299.1217304 + +Miles, A., & Bechhofer, S. (Eds.). (2009). *SKOS simple knowledge organization system reference*. World Wide Web Consortium. https://www.w3.org/TR/skos-reference/ diff --git a/docs/adr/0011-prov-o-standard-relations.md b/docs/adr/0011-prov-o-standard-relations.md new file mode 100644 index 00000000..8d214a22 --- /dev/null +++ b/docs/adr/0011-prov-o-standard-relations.md @@ -0,0 +1,79 @@ +# ADR 0011: Preserve W3C PROV-O as a standards-complete provenance layer + +- **Status:** Accepted +- **Date:** 2026-08-14 +- **Decision owners:** ContextualWisdomLab / LineageWeave +- **Standard:** W3C PROV-O Recommendation, 30 April 2013 + +## Context + +PR #74 introduced PROV-O-grounded actor categories, but LineageWeave's existing `knowledge_graph_edge` table only represents a compact binary navigation graph. It cannot faithfully represent PROV-O datatype properties such as `prov:startedAtTime` and `prov:value`, nor the intermediate `prov:Influence` resources required by qualified relations. Adding every standard property as another product edge code would therefore flatten the standard and lose the very provenance details it is intended to preserve. + +The Recommendation defines 30 classes and 50 normative properties grouped into Starting Point, Expanded, and Qualified terms. Tables 2 and 3 define 14 qualification patterns, and consuming applications should treat each qualified form as implying the corresponding unqualified form. Appendix B reserves interoperable inverse names while intentionally preferring the standard property direction. + +## Decision + +1. Add a separate `lineageweave.prov_o` runtime containing the complete normative class/property registry. +2. Validate object-versus-datatype shape, domain, range, subclass membership, timezone-aware `xsd:dateTime`, and Appendix B inverse aliases before accepting an assertion. +3. Deterministically materialize: + - transitive property hierarchy; + - defined inverse properties; + - `prov:alternateOf` symmetry; + - all 14 qualified-to-unqualified implications; + - qualified Generation/Invalidation/Start/End `prov:atTime` shortcuts. +4. Serialize with the exact `http://www.w3.org/ns/prov#` namespace through rdflib. +5. Store standards-complete provenance in normalized `provenance_*` tables. Keep `knowledge_graph_edge` as a buyer-facing navigation projection and bridge existing nodes through `provenance_resource_binding` rather than conflating the two models. +6. Catalog every Appendix B inverse name. Names that are not normative properties are accepted only as import aliases and rewritten by reversing endpoints into the preferred PROV-O relation. +7. Map LineageWeave `Post`, `Person`, `CorporateEntity`, and `Team` classes to PROV-O in a separate support profile that imports rather than redefines the W3C ontology. + +## Relational model + +```mermaid +erDiagram + provenance_class_definition ||--o{ provenance_class_hierarchy : child + provenance_class_definition ||--o{ provenance_class_hierarchy : parent + provenance_relation_definition ||--o{ provenance_relation_domain : has + provenance_class_definition ||--o{ provenance_relation_domain : constrains + provenance_relation_definition ||--o{ provenance_relation_resource_range : has + provenance_class_definition ||--o{ provenance_relation_resource_range : constrains + provenance_relation_definition ||--o{ provenance_relation_hierarchy : child + provenance_relation_definition ||--o{ provenance_relation_hierarchy : parent + provenance_relation_definition ||--|| provenance_inverse_definition : documents + provenance_relation_definition ||--o| provenance_qualification_definition : qualifies + provenance_resource ||--o{ provenance_resource_type : typed_as + provenance_class_definition ||--o{ provenance_resource_type : classifies + provenance_resource ||--o{ provenance_assertion : subject + provenance_relation_definition ||--o{ provenance_assertion : predicate + provenance_resource ||--o{ provenance_assertion : resource_object + provenance_literal_value ||--o{ provenance_assertion : literal_object + provenance_assertion ||--o{ provenance_assertion_derivation : derived + provenance_assertion ||--o{ provenance_assertion_derivation : premise +``` + +## Consequences + +### Positive + +- Complete PROV-O interchange without lossy custom edge codes. +- Qualified provenance retains role, plan, activity, usage, generation, time, and location detail. +- Existing LineageWeave navigation and RWR behavior remains stable. +- Database and runtime share stable multiword snake-case codes while preserving exact W3C IRIs. +- Assertions fail closed in both Python and PostgreSQL. + +### Costs + +- The product now has a standards graph and a navigation projection; projection logic must remain explicit. +- Full OWL reasoning is not embedded. The runtime intentionally materializes only the Recommendation rules needed for deterministic product behavior. +- Bundle serialization remains RDF-technology-specific; the relational layer stores the bundle resource without prescribing TriG. + +## Rejected alternatives + +- **Add 50 `edge_type` lookup rows:** rejected because literal and qualified relations cannot be represented. +- **Store arbitrary RDF triples only:** rejected because domain/range and relational integrity would be deferred to callers. +- **Define every inverse as another preferred property:** rejected because Appendix B explicitly warns that unconstrained inverse proliferation reduces interoperability. + +## Verification + +- Exact registry tests for 30 classes, 50 properties, 6 datatype properties, 14 qualification mappings, and all 44 object-property inverse names. +- Behavior-sensitive tests for validation, subclass domains, every qualification implication, superproperty closure, inverse/symmetry, direct time inference, RDF serialization, SQL seed completeness, support-profile mapping, and public docstrings. +- Owned production module statement and branch coverage: 100%. diff --git a/docs/adr/0012-corporate-entity-creation-lock.md b/docs/adr/0012-corporate-entity-creation-lock.md new file mode 100644 index 00000000..f5a01d85 --- /dev/null +++ b/docs/adr/0012-corporate-entity-creation-lock.md @@ -0,0 +1,45 @@ +# ADR 0012 — corporate-entity creation is serialized with a Postgres advisory transaction lock, not split into separate read/write databases + +**Decision status:** Accepted +**Date:** 2026-08-14 + +## Context + +ADR 0010's `get_or_create_corporate_entity` introduced concurrent writes on the organization-creation path. A deterministic synthetic regression fixture reproduced a `DeadlockDetectedError`: two transactions created different `corporate_entity` rows in opposite order, so each transaction waited for a row-level lock held by the other. + +This repository records the reproducible concurrency shape rather than customer, organization, batch, or production-log details. The architectural defect is independent of any particular dataset: multi-worker extraction can encounter child and parent organizations in different orders. + +## Decision + +Serialize only the *creation* write path with one named Postgres advisory transaction lock: + +```sql +pg_advisory_xact_lock( + hashtext('lineageweave:corporate_entity_creation') +) +``` + +The transaction-scoped lock is acquired immediately before persistence and is released automatically by the enclosing transaction's commit or rollback (PostgreSQL Global Development Group, 2024). + +1. The lock is acquired only after inference and verification complete. Holding it across network I/O would unnecessarily serialize unrelated workers. +2. Under the lock, candidates are reloaded and similarity matching is repeated. Another transaction may have committed the same entity after the caller's original snapshot was read. +3. The key is one fixed creation-path key rather than a per-name key. Per-name locking still permits the opposite-order multi-entity deadlock shape `A: [X, Y]` versus `B: [Y, X]`. +4. The already-cataloged resolution path remains lock-free. + +Splitting the system into separate read and write databases was considered and rejected. Replica separation does not resolve a write-write lock-ordering defect, while a transaction-scoped advisory lock directly enforces one global creation order. + +## Consequences + +- New entity creation is serialized cluster-wide. This is accepted because creation is the uncommon branch and correctness dominates throughput for this path. +- Resolution of existing entities remains concurrent and does not acquire the advisory lock. +- The lock is re-entrant within the same PostgreSQL session and transaction, so bounded parent-chain recursion does not self-deadlock. +- No schema split or additional service is required. +- The regression suite must retain concurrent opposite-order creation coverage. + +## Related + +This decision extends [ADR 0010](0010-corporate-hierarchy-auto-creation.md) with an explicit concurrency-safety property. + +## References — APA 7th + +PostgreSQL Global Development Group. (2024). *PostgreSQL 17 documentation: Advisory lock functions*. https://www.postgresql.org/docs/current/functions-admin.html#FUNCTIONS-ADVISORY-LOCKS diff --git a/docs/adr/0013-normalized-analysis-run-registry.md b/docs/adr/0013-normalized-analysis-run-registry.md new file mode 100644 index 00000000..1c356766 --- /dev/null +++ b/docs/adr/0013-normalized-analysis-run-registry.md @@ -0,0 +1,285 @@ +# ADR 0013 — Milestone 2 uses a normalized, additive analysis-run registry + +**Decision status:** Accepted on this active PR; not protected-main truth until merge +**Date:** 2026-08-15 +**Depends on:** ADR 0011 standards-complete provenance separation and ADR 0012 corporate-entity creation locking + +## Context + +LineageWeave has a reviewed React/FastAPI/PostgreSQL product, compact lineage +navigation, normalized actor identity, report persistence, and a separate +standards-complete PROV-O layer. Milestone 2 must analyze operator-authorized +PostgreSQL evidence without replacing that product, duplicating cross-service +databases, or committing private source identity and content to a public +repository. + +A retained experiment proved that direct PostgreSQL analysis is feasible, but +its parallel application and denormalized run record cannot become product +truth. The product needs a small durable root that answers: + +- which immutable capture was used; +- which evidence was available by the run's knowledge cutoff; +- which authenticated account requested the work; +- which product scope and reproducibility digests governed the run; +- which aggregate counts reconcile the capture; +- which legal lifecycle transitions occurred. + +The registry does not store source SQL, DSNs, raw posts, inline images, provider +payloads, credentials, raw exceptions, or another service's application rows. + +## Decision + +Migration `0018_analysis_run_registry.sql` introduces five normalized relations +and one read projection. + +```mermaid +erDiagram + ANALYSIS_SOURCE_SNAPSHOT ||--o{ ANALYSIS_SOURCE_COUNT : reconciles + ANALYSIS_SOURCE_SNAPSHOT ||--o{ ANALYSIS_RUN : anchors + USER_ACCOUNT ||--o{ ANALYSIS_RUN : requests + ANALYSIS_RUN ||--o| ANALYSIS_RUN_SCOPE : limits + CORPORATE_ENTITY |o--o{ ANALYSIS_RUN_SCOPE : scopes + PROCESS_UNIT |o--o{ ANALYSIS_RUN_SCOPE : scopes + ANALYSIS_RUN ||--o{ ANALYSIS_RUN_STATUS_EVENT : records + + ANALYSIS_SOURCE_SNAPSHOT { + uuid analysis_source_snapshot_id PK + text snapshot_sha256 UK + text source_contract_version + timestamptz maximum_available_time + timestamptz captured_at + } + ANALYSIS_SOURCE_COUNT { + uuid analysis_source_snapshot_id PK,FK + text count_type_code PK,FK + bigint count_value + } + ANALYSIS_RUN { + uuid analysis_run_id PK + uuid analysis_source_snapshot_id FK + uuid requested_by_account_id FK + text idempotency_key UK + timestamptz knowledge_cutoff + text configuration_sha256 + text model_contract_sha256 + text prompt_bundle_sha256 + text code_revision_sha + } + ANALYSIS_RUN_SCOPE { + uuid analysis_run_id PK,FK + text scope_kind_code FK + uuid corporate_entity_id FK + uuid process_unit_id FK + text scope_key + } + ANALYSIS_RUN_STATUS_EVENT { + uuid analysis_run_id PK,FK + int status_ordinal PK + text status_code FK + timestamptz occurred_at + timestamptz recorded_at + text failure_code + boolean retryable + } +``` + +### Temporal ownership + +`analysis_source_snapshot.maximum_available_time` is an evidence fact: the +latest time at which any admitted fact became available. `analysis_run.knowledge_cutoff` +is an analysis fact: the latest information that this particular run may use. +A reusable capture therefore does **not** own one knowledge cutoff. + +Run creation locks the snapshot and requires: + +```text +maximum_available_time <= knowledge_cutoff <= requested_at +captured_at <= requested_at +``` + +This aggregate guard complements TEPP's finer event, assertion, document, +system, availability, and cutoff clocks. It does not replace TEPP temporal or +psychometric computation. + +### Identity and idempotency + +Every run references a real `user_account`. `requested_by_account_id` is not +nullable. Idempotency keys are trimmed, control-free canonical values and are +unique per authenticated account rather than +globally, because independent callers may legitimately choose the same opaque +client key. A later repository must compare request digests on retry and return +a conflict when the same account/key names different evidence or configuration. + +### Immutability and concurrency + +Snapshot identity and availability reject updates. Aggregate count values reject +updates. Count insert/delete and first run creation acquire the same snapshot-row +lock before checking whether a run exists. This shared lock order closes the +race in which a count set and first derivation could otherwise both commit. +After the first run, the complete count set is frozen. + +The analysis request and its authorization scope reject updates and deletes. +Lifecycle changes are represented only by append-only status events, so a cascade +cannot erase the derivation root or its access boundary. + +### Lifecycle state machine + +The parent run row serializes status appends. Events require contiguous +ordinals, monotonic occurrence time, and these transitions: + +```text +pending -> running | cancelled +running -> succeeded | failed | cancelled +succeeded | failed | cancelled -> terminal +``` + +The first event must be `pending`, requires an immutable scope, and cannot predate +the run request. Failed events require a lowercase machine-code identifier; raw +exception text is prohibited. `recorded_at` is overwritten with database system +time on every insert and cannot precede `occurred_at`. +`analysis_run_current_status` is a view, not a second mutable state authority. + +### Authorization scope + +`analysis_run_scope` stores one immutable all-visible, corporate-entity, +process-unit, or thread-group scope. Its shape is database constrained and the +first lifecycle event is rejected until it exists. The next repository/API slice +must insert run, scope, and first status in one +transaction and apply the existing RBAC/ABAC contract when listing or reading +runs. This migration does not claim that an API or UI exists. + +### Service boundaries + +- **LineageWeave** owns product run identity, authorized scope, lifecycle, + aggregate reconciliation, and product-visible derivation references. +- **TEPP** owns exact evidence spans, temporal/event measurement, + multilevel/multiple-membership psychometrics, calibration, and semantic-span + budgeting through a versioned import or REST contract. +- **contextual-orchestrator** owns provider-neutral model routing and bounded + single-model versus multi-agent test-time compute allocation through its + reviewed API. +- **fast-mlsirm** owns Rust psychometric arithmetic and calibration interfaces. +- **Valkey** remains the event queue. Durable registry truth remains in + PostgreSQL; the start outbox (ADR 0023) bridges the two. + +No component reads another service's private application tables. + +## Alternatives considered + +### Merge the parallel experiment unchanged + +Rejected. It replaces reviewed product history, duplicates web and identity +surfaces, and creates a second database authority. + +### Store one JSON document per run + +Rejected. Signed external manifests may be JSON artifacts, but relational +identity, scope, counts, clocks, and lifecycle need independent constraints, +authorization, and query plans. + +### Put the registry only in Valkey + +Rejected. Queue state is transient and replayable. Audit identity, +idempotency, temporal eligibility, and retention evidence require PostgreSQL. + +### Store knowledge cutoff on the snapshot + +Rejected. One immutable capture can support multiple analysis requests with +different historical cutoffs. Putting the cutoff on the snapshot violates the +functional dependency and forces duplicate snapshots. + +## Security, privacy, and compliance consequences + +- Necessary PII remains in its authorized source/product tables rather than + being blanket-masked into operational uselessness. +- This registry stores opaque UUIDs, digests, bounded machine codes, aggregate + counts, and clocks only. +- Logs and public acceptance evidence must not include SQL, DSNs, raw source + text, images, secrets, provider payloads, or private source identifiers. +- Artifact bodies remain in access-controlled deployment storage and are linked + later by content digest and policy-bound reference. +- The design supports SOC 2 and CSAP evidence collection through explicit actor, + configuration, status, retention, and rollback contracts; it does not claim + certification. +- Database RLS is deferred because the current API uses one pooled service + identity and application-level RBAC/ABAC. Adopting actor-bound RLS requires a + separate ADR and transaction-scoped identity propagation. + +## Failure and rollback + +Migration replay is idempotent and rejects lookup-category collisions. The +rollback refuses to remove non-empty registry relations. Evidence must first be +exported, then emptied with `purge_analysis_run_registry` after an unrevoked +`analysis_run_retention_grant` and `analysis_run_retention_admin` membership +(ADR 0020). An empty rollback removes the view, tables, functions, and lookup +rows and is itself replayable. + +## Verification + +Acceptance requires: + +- real-PostgreSQL migration and replay; +- valid snapshot, aggregate, scope, and lifecycle persistence; +- distinct cutoffs over one snapshot; +- rejection of future-information leakage; +- account-scoped idempotency; +- snapshot, count, run, and authorization-scope immutability; +- deletion resistance for request and scope audit evidence; +- scope-required lifecycle, request-time ordering, and database-owned record time; +- canonical idempotency and bounded machine-code failure identifiers; +- count/run concurrency serialization; +- pending-first, contiguous, monotonic, legal status transitions; +- append-only status evidence; +- fail-closed rollback; +- two-or-more-word `snake_case` database-object names; +- complete repository, security, SAST, documentation, and public-content gates + on the exact merge head. + +## Follow-up sequence + +1. Add a transaction repository that creates snapshot, counts, run, scope, and + first status atomically and compares request digests on idempotent retries. + `POST /api/analysis-runs` now records that Pending write (ADR 0017). + `POST /api/analysis-runs/{id}/start` now reconstructs a Pending + lineage cutoff bag in-process from frozen snapshot membership + (ADR 0021). Start now commits a durable outbox row and wakes Valkey + before reconstruct / TEPP (ADR 0023). Live TEPP start submits + through `tepp_client` (ADR 0022). +2. Add RBAC/ABAC-protected run list/detail endpoints and the DB-grounded + read-only administrator surface. +3. Add a normalized PostgreSQL outbox and Valkey delivery worker + (ADR 0023). +4. Add TEPP and contextual-orchestrator adapters only after their versioned + contracts are present on reviewed main branches. Seed and + `POST /api/analysis-runs/{id}/start` now record Failed TEPP through + `tepp_client` on the frozen snapshot when the transport is missing + or the envelope is unpublished. A published accepted acknowledgement + is stored as aggregate transport evidence and stays Failed / + `tepp_completed_result_unsupported` (ADR 0035). A missing or + unpublished TEPP envelope must stay Failed (`tepp_not_available` / + `tepp_result_not_persisted`) and must not write a local psychometric + substitute or stamp Succeeded. Seed also records a + Succeeded `analysis_run_report` on that snapshot after the + period-report tables are written (ADR 0024); the registry row does + not copy a theta. +5. Execute private actual-data analysis and store only signed aggregate and + reproducibility manifests outside public source control. +6. Run browser E2E through real OIDC, product navigation, and evidence drill-down. + +## References — APA 7th + +International Organization for Standardization. (2019). *ISO 8601-1:2019: Date +and time—Representations for information interchange—Part 1: Basic rules* +(confirmed 2024; Amendment 1:2022). + +Moreau, L., & Missier, P. (Eds.). (2013). *PROV-DM: The PROV data model*. +World Wide Web Consortium. https://www.w3.org/TR/prov-dm/ + +PostgreSQL Global Development Group. (2026). *PostgreSQL 18 documentation: +5.5. Constraints*. https://www.postgresql.org/docs/current/ddl-constraints.html + +World Wide Web Consortium. (2013). *PROV-O: The PROV ontology* (W3C +Recommendation). https://www.w3.org/TR/prov-o/ + +World Wide Web Consortium. (2022). *Time ontology in OWL* (W3C Recommendation). +https://www.w3.org/TR/owl-time/ diff --git a/docs/adr/0014-authorized-analysis-run-read.md b/docs/adr/0014-authorized-analysis-run-read.md new file mode 100644 index 00000000..b8df6eef --- /dev/null +++ b/docs/adr/0014-authorized-analysis-run-read.md @@ -0,0 +1,92 @@ +# ADR 0014 — Analysis-run evidence is an authorized, source-redacting read + +**Decision status:** Accepted on this active PR; not protected-main truth until merge +**Date:** 2026-08-16 +**Depends on:** ADR 0013 normalized analysis-run registry +**Refs:** Issue #79 (Milestone 2 parent); closed PR #77 is read-only evidence + +## Context + +PR #89 persists analysis-run identity, aggregate reconciliation, scope, +and lifecycle without exposing a product API. Buyers still cannot see +whether a lineage reconstruction ran, succeeded, or reconciled how many +documents. Closed PR #77 exposed analysis records through a parallel +application that also stored raw metadata payloads -- that shape cannot +become protected product truth. + +## Decision + +LineageWeave owns a fail-closed read projection of the #89 registry: + +- `GET /api/analysis-runs` and `GET /api/analysis-runs/{id}` require + `post_read`. +- Visibility is evaluated in SQL. A run is visible when the caller + requested it, or the scope is a corporate entity / process unit / + thread group the caller may already walk. `all_visible` stays + requester-only so it cannot broaden another tenant's evidence. +- Hidden runs return 404, not 403, and never appear in the list. +- The payload carries lookup labels and non-negative aggregate counts. + It does not carry source SQL, DSNs, raw records, image bytes, provider + payloads, credentials, or another service's table names. +- `GET /api/analysis-runs/{id}` also returns the append-only labeled + `status_history`. The list does not. A failed event may include the + stored machine `failure_code`; this slice does not invent a label. +- TEPP remains a versioned `AnalysisRunRequest` consumer + (`lineageweave.tepp_client`). This slice does not fork TEPP arithmetic. +- contextual-orchestrator remains the only LLM path. This slice does not + call a raw model API. + +## Consequences + +`make seed` writes one synthetic Demo Corp lineage run, one Failed +missing-transport TEPP run, one Failed accepted-evidence TEPP run, and +one Succeeded period-report run on the same snapshot so the existing +React home page can show all three kinds without a second application +(ADR 0024 / ADR 0035). The missing-transport TEPP run is +`tepp_not_available` when the default transport is missing -- the list +keeps that machine code off the caption (this decision) and instead +tells the operator to open the TEPP run, then connect the measurement +service. The accepted-evidence TEPP row tells the operator to read +aggregate transport evidence and that completed measurement identity +is unavailable. A failed lineage row tells the operator to retry +reconstruction, not to connect TEPP. A failed period-report row +tells the operator to rebuild the report from a current snapshot. +A pending or running TEPP row must not claim a calibrated +measurement and must not say reconstruction. The list +button accessible name is `Open analysis run: {caption}. {nextAction}` +when a next action exists (WCAG 2.2 SC 4.1.2); otherwise the caption +alone. `aria-label` replaces button contents (W3C Accessible Name and +Description Computation 1.1), so the next-action sentence must live +in that name. Detail repeats that sentence. A pending +lineage row says reconstruction has not started yet. The detail now shows the legal +lifecycle the registry already stored. `POST /api/analysis-runs` now +records a Pending lineage run on an authorized cutoff capture +(ADR 0017). TEPP and period-report kinds are 422. Reconstruction and +TEPP accepted transport evidence are ADR 0021 / ADR 0035. A fuller Analysis +Run Console remains a later slice. A 404 on a hidden run (including a thread-group row that +still lacks an in-cutoff visible post, ADR 0018) must stay generic: +do not name the thread or the cutoff, and do not say the run is not +visible. Tell the operator to open a visible run from the home list, +or request a lineage reconstruction for a corporation they already +walk. After that 404, re-read `GET /api/analysis-runs` so the stale +list row does not stay clickable, and announce the status with +`role="alert"` (WCAG 2.2 SC 4.1.3) without moving focus. + +## References + +American Educational Research Association, American Psychological +Association, & National Council on Measurement in Education. (2014). +*Standards for educational and psychological testing*. American +Educational Research Association. + +Lebo, T., Sahoo, S., & McGuinness, D. (Eds.). (2013). *PROV-O: The PROV +ontology* (W3C Recommendation). World Wide Web Consortium. +https://www.w3.org/TR/2013/REC-prov-o-20130430/ + +World Wide Web Consortium. (2018). *Accessible name and description +computation 1.1* (W3C Recommendation). +https://www.w3.org/TR/accname-1.1/ + +World Wide Web Consortium. (2023). *Web content accessibility +guidelines (WCAG) 2.2* (W3C Recommendation). +https://www.w3.org/TR/WCAG22/ diff --git a/docs/adr/0013-adaptive-contextual-orchestrator-default.md b/docs/adr/0015-adaptive-contextual-orchestrator-default.md similarity index 96% rename from docs/adr/0013-adaptive-contextual-orchestrator-default.md rename to docs/adr/0015-adaptive-contextual-orchestrator-default.md index ee040207..433432fb 100644 --- a/docs/adr/0013-adaptive-contextual-orchestrator-default.md +++ b/docs/adr/0015-adaptive-contextual-orchestrator-default.md @@ -1,4 +1,4 @@ -# ADR-0013: Adaptive contextual-orchestrator mode is the default +# ADR-0015: Adaptive contextual-orchestrator mode is the default - Status: Accepted - Date: 2026-08-16 diff --git a/docs/adr/0016-analysis-run-knowledge-cutoff-posts.md b/docs/adr/0016-analysis-run-knowledge-cutoff-posts.md new file mode 100644 index 00000000..553d549b --- /dev/null +++ b/docs/adr/0016-analysis-run-knowledge-cutoff-posts.md @@ -0,0 +1,84 @@ +# ADR 0016 — Analysis-run visible posts honor the run knowledge cutoff + +**Decision status:** Accepted +**Date:** 2026-08-16 + +## Context + +ADR 0013 stores `analysis_run.knowledge_cutoff` as the analysis clock: +what that run was allowed to know. The registry trigger already refuses +a cutoff earlier than `analysis_source_snapshot.maximum_available_time`. +The home-page detail, however, listed every ABAC-visible title in the +run's scope from live `source_post` rows. Fixture and seed posts that +defaulted to `created_at = now()` therefore appeared inside a January +2026 run, including a later own-corp follow-up the buyer would treat as +part of that reconstruction. + +W3C Time Ontology in OWL (Hobbs & Pan, 2017) and ISO 8601-1:2019 keep +distinct clocks from collapsing. A knowledge cutoff is not "posts the +account can see today." + +## Decision + +`fetch_visible_scope_posts` filters `created_at <= knowledge_cutoff` on +every scope branch (corporate entity, process unit, thread group, and +all-visible). ABAC visibility is applied after that temporal gate. +Click-through still opens the live post body. Detail compares the live +`updated_at` write clock with `knowledge_cutoff` and marks titles +rewritten after the run. Opening a marked title shows the stored +cutoff-known body (`GET /api/posts/{id}?as_of=`, ADR 0025) beside the +live rewrite. A missing revision is omitted -- never an invented +earlier sentence. The next action is specific: only those marked +titles need a cutoff comparison before treating the live body as +reconstructed evidence. + +Reproducibility digests on the same detail use a labeled group whose +accessible name does not replace the visible prefixes (W3C Accessible +Name and Description Computation 1.1). Full digests stay on `title` +for hover verification and on the API payload; the home list stays +aggregates-only. + +Seed and API fixtures backdate in-cutoff posts. A late own-corp private +post remains on the live post list and stays out of the January 2026 +run. + +## Consequences + +- After `make seed`, the Demo Corp lineage run lists Demo public post + and other in-cutoff Demo Corp titles. Late Demo public post + (2026-01-13) and the later fixture account-review post (2026-02-10) + do not appear. The live post list still shows Late Demo. +- Open the run: Demo public post is marked updated after cutoff + (`updated_at` 2026-01-13). Demo private post is not. +- Open a marked title: the popup shows **Body this run knew** from + `source_post_revision` and the live rewrite. Compare those two texts + before treating the live body as reconstructed evidence (ADR 0025). +- Hover a digest prefix to read the full code or configuration digest + when you need to match the API payload. +- Migration 0024 (ADR 0025) stores each rewrite on + `source_post_revision` so the opened post can show the cutoff-known + body without putting that body on the analysis-run payload. The write + clock remains a projection on `source_post.updated_at`. +- Thread-group *run list* visibility now uses the same cutoff + (ADR 0018). A later public post cannot surface a previously hidden + thread-group run. + +## Follow-up — v2.12.3 Late Demo own-corp counter-example + +v2.12.3 seeds Late Demo public post on 2026-01-13 so the January 12 +Demo Corp lineage and TEPP runs can prove the existing +`created_at <= knowledge_cutoff` filter. This is not a second cutoff +and does not change TEPP honesty. + +## References + +International Organization for Standardization. (2019). *ISO 8601-1:2019: +Date and time—Representations for information interchange—Part 1: Basic +rules* (confirmed 2024; Amendment 1:2022). + +World Wide Web Consortium. (2022). *Time ontology in OWL* (W3C +Recommendation). https://www.w3.org/TR/owl-time/ + +World Wide Web Consortium. (2018). *Accessible name and description +computation 1.1* (W3C Recommendation). +https://www.w3.org/TR/accname-1.1/ diff --git a/docs/adr/0017-authorized-analysis-run-create.md b/docs/adr/0017-authorized-analysis-run-create.md new file mode 100644 index 00000000..81841c46 --- /dev/null +++ b/docs/adr/0017-authorized-analysis-run-create.md @@ -0,0 +1,103 @@ +# ADR 0017 — Operators request a pending lineage run on an authorized capture + +**Decision status:** Accepted on this active PR; not protected-main truth until merge +**Date:** 2026-08-16 +**Depends on:** ADR 0013 normalized analysis-run registry; ADR 0014 authorized +analysis-run read; ADR 0016 knowledge-cutoff posts +**Refs:** Issue #79 (Milestone 2 parent); ADR 0013 follow-up 1 + +## Context + +Home Analysis runs could list seeded lineage and TEPP rows, but a buyer +could not request a new run. Seed-only evidence is a demo, not a product. +ADR 0013 already required a transaction that creates snapshot, counts, +run, scope, and the first status atomically. Follow-up 3 (outbox / worker) +still owns reconstruction and live TEPP execution. + +`#125` landed that write and also accepted a TEPP kind. A Pending TEPP +row that never called `tepp_client` is a fabricated measurement request. +This decision keeps the live cutoff capture and closes that hole. + +## Decision + +`POST /api/analysis-runs` is the authorized write: + +- `post_read` is enough. The caller may only cover a corporate entity + they already walk. An unaffiliated corp is 404, not 403. +- Only `analysis_run_lineage` is accepted. TEPP stays a `tepp_client` + wire path (`tepp_not_available` / `tepp_result_not_persisted`). Period + reports stay on the Reports panel rebuild. +- The capture digest hashes scope, entity, cutoff, and authorized post + ids — never a post body, DSN, source SQL, or a theta. +- The write inserts snapshot, aggregate counts, frozen + `analysis_source_snapshot_member` ids, `analysis_run`, + `analysis_run_scope`, and `analysis_status_pending` in one transaction. +- The first status is Pending. This slice does not reconstruct lineage + and does not call TEPP. +- Account-scoped idempotency compares `configuration_sha256`. An omitted + cutoff is hashed as `unspecified` so a retry of the same client key + does not conflict because the clock moved. +- `GET /api/me` returns the affiliated `corporate_entities` so a + multi-affiliation operator can choose which entity to reconstruct. +- The response is the same authorized detail as `GET /api/analysis-runs/{id}`. + +```mermaid +sequenceDiagram + participant Operator + participant API + participant Registry + Operator->>API: POST /api/analysis-runs + alt TEPP, report, or unknown kind + API-->>Operator: 422 next-action (no registry write) + else same account+key+digest + API->>Registry: compare configuration digest + Registry-->>API: existing run + API-->>Operator: 201 replay + else same key, different digest + API-->>Operator: 409 conflict + else lineage kind, new key + API->>Registry: capture authorized cutoff bag + Registry->>Registry: snapshot + counts + run + scope + pending + API-->>Operator: 201 Pending row + end +``` + +The home panel's **Request a lineage reconstruction** button stays +disabled until `GET /api/me` returns affiliated corps, then records +that Pending row for the chosen entity. Only a failed TEPP row +mentions the measurement service. Failed TEPP is terminal on this +write: create does not invent a Pending TEPP row. + +## Consequences + +- Demo Analyst can request a new Pending Demo Corp lineage run after + `make seed` without inventing a measurement. +- A multi-affiliation account sees the corp picker before the Request + button enables, then chooses the corp before clicking. +- `POST /api/analysis-runs/{id}/start` then reconstructs that frozen bag + (ADR 0021). TEPP start now goes through `tepp_client` (ADR 0022). The + outbox worker is ADR 0023. +- Do not stamp Succeeded or invent a theta from this write. + +## References — APA 7th + +International Organization for Standardization. (2019). *ISO 8601-1:2019: +Date and time—Representations for information interchange—Part 1: Basic +rules* (confirmed 2024; Amendment 1:2022). + +Jensen, C. S., & Snodgrass, R. T. (1999). Temporal data management. +*IEEE Transactions on Knowledge and Data Engineering, 11*(1), 36–44. +https://doi.org/10.1109/69.755613 + +Kent, K., & Souppaya, M. (2006). *Guide to computer security log +management* (NIST Special Publication 800-92). National Institute of +Standards and Technology. https://doi.org/10.6028/NIST.SP.800-92 + +Moreau, L., & Missier, P. (Eds.). (2013). *PROV-DM: The PROV data model*. +World Wide Web Consortium. https://www.w3.org/TR/prov-dm/ + +OpenAPI Initiative. (2025). *OpenAPI specification, version 3.2.0*. +https://spec.openapis.org/oas/v3.2.0.html + +World Wide Web Consortium. (2022). *Time ontology in OWL* (W3C +Recommendation). https://www.w3.org/TR/owl-time/ diff --git a/docs/adr/0018-related-nodes-team-org-walk.md b/docs/adr/0018-related-nodes-team-org-walk.md new file mode 100644 index 00000000..c7f020eb --- /dev/null +++ b/docs/adr/0018-related-nodes-team-org-walk.md @@ -0,0 +1,71 @@ +# ADR 0018 — Related-node walks include team and organization mention edges + +**Decision status:** Accepted +**Date:** 2026-08-16 + +## Context + +ADR 0009 persists `edge_mention_team`, `edge_team_affiliation`, and +`edge_mention_organization` so a cataloged team or organization can +become a cross-post Knowledge Graph clue. The buyer-visible related-node +walk (`load_visible_subgraph` + Tong et al., 2006 random walk with +restart) still loaded only person mention, co-mention, and affiliation +edges, and returned an empty graph when a visible post had no people. +A team-only follow-up therefore never appeared as a related node, and +clicking an R&R team name had no catalog id to start a walk. + +The same temporal honesty ADR 0016 applied to run *detail* posts was +still missing from thread-group *run list* visibility: a later public +post in that thread group could surface a run the account was not +allowed to know at `knowledge_cutoff`. + +ADR 0017 already records an authorized Pending analysis-run write. +This decision is the related-node walk, not that create path. + +## Decision + +`load_visible_subgraph` loads person, team, and organization mention +channels independently. Empty person evidence is not a reason to drop +team or organization edges. `hydrate_related_nodes` labels +`cataloged_team` rows. `GET /api/teams/{team_id}/related` starts the +same RWR walk Keyman and corporate-entity related already use. +`visible_affiliation_post_ids` unions direct `post_organization_mention` +rows with person-affiliation posts so an org-only mention can start a +walk. + +The summary payload exposes `catalog_node_id` / `catalog_node_type_code` +from the catalog foreign keys stored on `post_summary_role` (ADR 0019). +The popup turns that name into a related-node button. Do not reconstruct +the id by `corporate_entity.entity_name`. + +Thread-group run list visibility requires at least one ABAC-visible +`source_post` whose `created_at` is at or before `knowledge_cutoff`. + +## Consequences + +- Open a post whose R&R names 설계팀, then click the team. Sibling posts + that mention the same cataloged team appear as related nodes. +- Click a related team chip the same way you already click a person or + organization chip. +- A later public post in a thread group no longer lists a January run + that could not have known that post. +- A 404 on that hidden row stays generic: do not name the thread or + the cutoff. After that 404, re-read the authorized home list so the + stale row does not stay clickable (ADR 0014). + +## References + +International Organization for Standardization. (2019). *ISO 8601-1:2019: +Date and time—Representations for information interchange—Part 1: Basic +rules* (confirmed 2024; Amendment 1:2022). + +Reynolds, D. (Ed.). (2014). *The organization ontology*. World Wide Web +Consortium. https://www.w3.org/TR/vocab-org/ + +Tong, H., Faloutsos, C., & Pan, J.-Y. (2006). Fast random walk with +restart and its applications. *Proceedings of the Sixth International +Conference on Data Mining (ICDM'06)*, 613–622. +https://doi.org/10.1109/ICDM.2006.70 + +World Wide Web Consortium. (2022). *Time ontology in OWL* (W3C +Recommendation). https://www.w3.org/TR/owl-time/ diff --git a/docs/adr/0019-role-catalog-identity.md b/docs/adr/0019-role-catalog-identity.md new file mode 100644 index 00000000..adc36574 --- /dev/null +++ b/docs/adr/0019-role-catalog-identity.md @@ -0,0 +1,66 @@ +# ADR 0019 — R&R catalog identity lives on the role row + +**Decision status:** Accepted +**Date:** 2026-08-16 + +## Context + +ADR 0009 writes `post_team_mention` and `post_organization_mention` so a +cataloged team or organization can start a related-node walk. ADR 0018 +exposes `catalog_node_id` on the summary payload by joining those +mentions back to `post_summary_role` through `actor_name`. + +`corporate_entity.entity_name` is not unique. Two catalog rows can share +a display name (different `corporate_entity_code`, different parents). +A fetch join on name therefore: + +- attaches a homonym that this post never resolved, or +- duplicates the role when more than one same-named row exists. + +Mention tables are post-scoped, not role-scoped. They cannot reconstruct +which catalog id was chosen for a specific R&R row. That reconstruction +is a transitive dependency on a non-key attribute, so it is not third +normal form (Codd, 1970; Date, 2019). + +Team identity is already unique on +`(team_name, affiliated_organization_name)`. Organization identity is +not. + +## Decision + +`post_summary_role` stores the resolved catalog foreign keys +(`cataloged_team_id`, `cataloged_corporate_entity_id`) written during +`persist_post_summary`. `fetch_persisted_summary` reads those columns. +It does not join `corporate_entity` by `entity_name`. Person identity +is ADR 0027 (`cataloged_person_id`). + +Migration `0019_role_catalog_identity.sql` backfills existing rows from +a post-scoped mention only when the name match is unique on that post. +Two same-named mentions stay unbound rather than guessing. + +## Consequences + +- Open a post whose R&R names an organization that shares a display + name with another catalog row. The button walks the resolved id, not + the homonym. +- Clicking that name still uses `GET /api/corporate-entities/{id}/related` + or `GET /api/teams/{id}/related`. Authz stays person/entity-parity: + a team mentioned only on another corp's private post is 403; an + unknown UUID is 404. + +## References + +Codd, E. F. (1970). A relational model of data for large shared data +banks. *Communications of the ACM, 13*(6), 377–387. +https://doi.org/10.1145/362384.362685 + +Date, C. J. (2019). *Database design and relational theory: Normal forms +and all that jazz* (2nd ed.). Apress. +https://doi.org/10.1007/978-1-4842-5540-7 + +International Organization for Standardization. (2023). *ISO/IEC +11179-1:2023: Information technology—Metadata registries (MDR)—Part 1: +Framework*. + +Reynolds, D. (Ed.). (2014). *The organization ontology*. World Wide Web +Consortium. https://www.w3.org/TR/vocab-org/ diff --git a/docs/adr/0020-analysis-run-retention-purge.md b/docs/adr/0020-analysis-run-retention-purge.md new file mode 100644 index 00000000..5d294c41 --- /dev/null +++ b/docs/adr/0020-analysis-run-retention-purge.md @@ -0,0 +1,110 @@ +# ADR 0020 — Approved retention purge requires a session grant and admin role + +**Decision status:** Accepted on this active PR; not protected-main truth until merge +**Date:** 2026-08-16 +**Depends on:** ADR 0013 normalized analysis-run registry + +## Context + +ADR 0013 and migration `0018` make `analysis_run`, `analysis_run_scope`, and +`analysis_run_status_event` immutable. The 0018 rollback refuses to drop +non-empty registry relations and tells operators to export or delete evidence +under an approved retention procedure. + +After the first `analysis_run` insert, a raw `DELETE` is rejected. Snapshot +delete is then blocked by the foreign key. Operators following the documented +procedure cannot satisfy `analysis_run_registry_not_empty` without a superuser +`DISABLE TRIGGER`. That is not a supported product path (ISO 15489-1:2016 +disposition; NIST SP 800-92 protected audit records). + +A `SECURITY DEFINER` function that `PUBLIC` can execute, or that accepts only +a documented phrase, lets any SQL session wipe analysis-run evidence +(NIST SP 800-53 Rev. 5 AC-3; CWE-250). The phrase is a procedure name, not +an authorization secret. The write API is a separate slice; SQL operators +still need a grant that is independent of application `user_account` rows. + +Landed #122 occupies ADR 0018 / package 0.86.0 for the team and organization +related-node walk. ADR 0019 binds `cataloged_team_id` / +`cataloged_corporate_entity_id` on `post_summary_role` and must not be +reused here. This decision is the next free slot. + +## Decision + +Migration `0020_analysis_run_retention_purge.sql` adds a conjunctive +fail-closed purge: + +- `analysis_run_retention_grant` — one unrevoked row per + `database_role_name`; history of revoked grants is allowed; +- `analysis_run_retention_admin` — `NOLOGIN` role that receives + `EXECUTE`; `PUBLIC` does not; +- `purge_analysis_run_registry(approval_token text)` — `SECURITY DEFINER`, + checks the unrevoked grant, then `pg_has_role(..., 'member')` on the + admin role, then accepts only `approved-retention-purge`, disables the + three immutability delete triggers inside that call, deletes in FK + order, re-enables the triggers, and writes one + `analysis_run_retention_event`; +- `analysis_run_retention_event` — purged run/snapshot counts, the SHA-256 + of the approval token, `invoking_session_role`, `invoking_current_role`, + and optional `client_network_address`. The raw phrase is never stored. + +A session `SET` cannot authorize a raw `DELETE`. A table-DML runtime role +that only knows the public phrase cannot call the function. A member of +the admin role without a grant cannot purge. A grant without admin +membership cannot purge. After purge, export the retention event, delete +those rows, roll back 0020, then roll back 0018. + +This migration does not insert a grant or grant the admin role to the +migrator. Production `DATABASE_URL` must not be a superuser and must not +hold either privilege. + +## Consequences + +- A run-bearing registry can be emptied without superuser trigger disable. +- Retention remains an explicit, audited operator action, not a silent + downgrade. +- 0018 rollback stays fail-closed until the registry tables are empty; + 0020 rollback stays fail-closed until retention events are exported + and deleted. +- Repeated citation-chip and close-button appearance lives in + `frontend/src/styles/tokens.css` and the Storybook inventory. + +## Follow-up + +When the authorized write API exists, bind an administrator +`user_account` to the same grant table. Keep the SQL-role grant for +operators who purge from `psql`. Do not expose purge on a public HTTP +route. Split the application login from the migration owner so the +product role cannot execute the function even as table owner. + +Start reconstruction (ADR 0021) adds `analysis_run_lineage_edge`, +`analysis_run_reconstruction`, and `analysis_source_snapshot_member` +with delete-reject triggers. This procedure already disables those +user triggers when `to_regclass` finds the tables, deletes lineage +edges, then reconstruction, then the 0018 rows, then snapshot +members, then the snapshot, and re-enables the triggers (ADR 0032). +A 0020-only database without those relations still purges. Do not +require a superuser `DISABLE TRIGGER` after a Succeeded start. + +## References — APA 7th + +American Institute of Certified Public Accountants. (2017). *SOC 2®: SOC +for Service Organizations: Trust Services Criteria*. + +International Organization for Standardization. (2016). *ISO 15489-1:2016: +Information and documentation—Records management—Part 1: Concepts and +principles*. + +Kent, K., & Souppaya, M. (2006). *Guide to computer security log management* +(NIST Special Publication 800-92). National Institute of Standards and +Technology. https://doi.org/10.6028/NIST.SP.800-92 + +MITRE. (2026). *CWE-250: Execution with unnecessary privileges*. +https://cwe.mitre.org/data/definitions/250.html + +National Institute of Standards and Technology. (2020). *Security and +privacy controls for information systems and organizations* (NIST Special +Publication 800-53 Rev. 5). https://doi.org/10.6028/NIST.SP.800-53r5 + +PostgreSQL Global Development Group. (2026). *PostgreSQL 18 documentation: +5.8. Privileges*. +https://www.postgresql.org/docs/current/ddl-priv.html diff --git a/docs/adr/0021-authorized-analysis-run-start.md b/docs/adr/0021-authorized-analysis-run-start.md new file mode 100644 index 00000000..5e0921a0 --- /dev/null +++ b/docs/adr/0021-authorized-analysis-run-start.md @@ -0,0 +1,114 @@ +# ADR 0021 — Operators start a pending lineage reconstruction + +**Decision status:** Accepted on this active PR; not protected-main truth until merge +**Date:** 2026-08-16 +**Depends on:** ADR 0013 registry; ADR 0014 authorized read; ADR 0016 cutoff +posts; ADR 0017 authorized create +**Refs:** Issue #79 (Milestone 2 parent); ADR 0013 follow-up 3 (in-process +start; durable outbox is ADR 0023) + +## Context + +ADR 0017 let an operator record a Pending analysis run. The home button +said “Request a lineage reconstruction,” then the row stayed Pending. +Seed still owned the only Succeeded Demo Corp tree. A buyer cannot +treat a request they cannot start as a product. + +ADR 0013 follow-up 3 asked for a PostgreSQL outbox and Valkey worker. +That durable delivery path is ADR 0023. This slice starts +reconstruction so the operator can see the cutoff tree immediately. + +Landed #145 occupies ADR 0020 / package 0.87.0 for granted retention +purge. ADR 0019 binds R&R catalog identity. This decision is the next +free slot. + +## Decision + +`POST /api/analysis-runs/{id}/start` requires `post_read` and, in one +transaction: + +1. loads the authorized run (hidden scopes 404); +2. rejects period-report so this path cannot invent a calibrated + score; TEPP start is ADR 0022 and still cannot invent a theta; +3. replays a Succeeded run (documented no-op; same stored digest); +4. accepts only Pending lineage — Running is 409; +5. locks the run row, re-reads status, appends Running, runs + `lineage_edge_specs` / `reconstruct()` on the frozen + `analysis_source_snapshot_member` bag (or the live cutoff query when + membership was never persisted), persists + `analysis_run_reconstruction` plus `analysis_run_lineage_edge`, then + appends Succeeded. A concurrent start is 409 with a refresh next + action, not a 500. + +```mermaid +sequenceDiagram + participant Operator + participant API + participant ThreadWeave + participant Registry + Operator->>API: POST /api/analysis-runs/{id}/start + API->>Registry: lock visible Pending lineage run + alt TEPP or period-report + API-->>Operator: 422 use the kind's own path + else already Succeeded + Registry-->>API: stored edges + API-->>Operator: 200 replay + else Running + API-->>Operator: 409 refresh + else Pending lineage + Registry->>Registry: Running + API->>ThreadWeave: reconstruct frozen bag + ThreadWeave-->>API: parent choices + Registry->>Registry: reconstruction + edges + Succeeded + API-->>Operator: 200 titled edges + end +``` + +Rules: + +- Edges are run-scoped. This write does not replace live + `post_lineage_edge` (the Event Lineage panel stays a later rebuild). +- The digest hashes parent id, child id, and rounded fused score — never + a post body, DSN, or image. +- Empty cutoff bags Succeed with zero edges. +- Failed TEPP remains a `tepp_client` transport problem. +- Create freezes authorized post ids on + `analysis_source_snapshot_member` so start cannot pick up a later + backfill that shares the cutoff clock. + +The home detail adds **Start reconstruction** on a Pending lineage row +and lists titled parent→child edges after Succeeded. Those titles are +buttons that open the live post (a marked child still shows the +live-body warning). The Result digest +prefix is audible next to Code and Config; hover it to verify the +parent-choice hash. Edge titles stay public-or-affiliated. TEPP and +period-report rows do not show the button. + +## Consequences + +Demo Analyst can request a run, start it, and confirm the designed A-100 +fork (revised quote and delivery question under the pricing follow-up). +`make seed` also persists that fork on the Demo Corp Succeeded row so +open-after-seed is not empty. The durable outbox / Valkey worker is +ADR 0023. Live TEPP transport remains a later persistable-result +slice. Do not stamp Succeeded from a missing reconstruct library, and +do not invent a theta. + +## References — APA 7th + +International Organization for Standardization. (2019). *ISO 8601-1:2019: +Date and time—Representations for information interchange—Part 1: Basic +rules* (confirmed 2024; Amendment 1:2022). + +Jensen, C. S., & Snodgrass, R. T. (1999). Temporal data management. +*IEEE Transactions on Knowledge and Data Engineering, 11*(1), 36–44. +https://doi.org/10.1109/69.755613 + +Moreau, L., & Missier, P. (Eds.). (2013). *PROV-DM: The PROV data model*. +World Wide Web Consortium. https://www.w3.org/TR/prov-dm/ + +World Wide Web Consortium. (2013). *PROV-O: The PROV ontology* (W3C +Recommendation). https://www.w3.org/TR/prov-o/ + +World Wide Web Consortium. (2022). *Time ontology in OWL* (W3C +Recommendation). https://www.w3.org/TR/owl-time/ diff --git a/docs/adr/0022-authorized-tepp-start.md b/docs/adr/0022-authorized-tepp-start.md new file mode 100644 index 00000000..a6517ae5 --- /dev/null +++ b/docs/adr/0022-authorized-tepp-start.md @@ -0,0 +1,96 @@ +# ADR 0022 — Operators start a pending TEPP measurement through tepp_client + +**Decision status:** Accepted on this active PR; not protected-main truth until merge +**Date:** 2026-08-17 +**Depends on:** ADR 0013 registry; ADR 0017 authorized create; ADR 0021 +authorized lineage start +**Refs:** Issue #79 (Milestone 2 parent); ADR 0013 follow-up 4 (live TEPP +through the published client; persistable result remains later) + +## Context + +ADR 0021 starts a Pending lineage reconstruction in-process. The same +`POST /api/analysis-runs/{id}/start` path returned 422 for TEPP so it +could not invent a theta. Create already records a Pending TEPP run. +Seed already records a Failed TEPP run through `tepp_client`. The Failed +row tells the operator to connect the measurement service and re-run, +but Failed is terminal and there was no start path that called +`tepp_client`. + +A buyer who connects a live TEPP transport still could not submit the +frozen snapshot. A 422 that says "do not invent a measurement" is +honest, but it is not a product. The missing work is to submit TEPP's +published `AnalysisRunRequest` and fail closed when the transport is +missing or the envelope is not a persistable measurement. + +## Decision + +`POST /api/analysis-runs/{id}/start` accepts Pending TEPP as well as +Pending lineage. Period-report stays 422. TEPP start, in the same +authorized transaction: + +1. locks the visible Pending TEPP row; +2. appends Running; +3. builds `AnalysisRunRequest` from the run's idempotency key, snapshot + digest, knowledge cutoff, and corporate-entity workspace id — never a + post body or a theta; +4. submits through `TeppClient`. An empty `TEPP_TRANSPORT_URL` keeps the + default unavailable transport. A set URL POSTs the published wire + payload through the http(s)-only helper. File URLs stay unavailable; +5. appends Failed / `tepp_not_available` when the transport is missing + or refused, or Failed / `tepp_result_not_persisted` when TEPP accepts + an envelope this product cannot store yet. + +Succeeded TEPP remains unpublished; ADR 0035 stores accepted +transport evidence without stamping Succeeded. This slice does not persist a local +psychometric substitute, does not call contextual-orchestrator as TEPP, +and does not stamp Succeeded from an `accepted` envelope. Failed remains +terminal. `POST /api/analysis-runs` is lineage-only (ADR 0017) and does +not invent a Pending TEPP row. The operator connects a TEPP transport +from the Failed row, then starts that same measurement. + +```mermaid +sequenceDiagram + participant Operator + participant API + participant TeppClient + participant Registry + Operator->>API: POST /api/analysis-runs/{id}/start + API->>Registry: lock visible Pending TEPP run + alt period-report + API-->>Operator: 422 use the reports panel + else Pending TEPP + Registry->>Registry: Running + API->>TeppClient: AnalysisRunRequest v1 + alt TeppNotAvailable + Registry->>Registry: Failed tepp_not_available + else accepted envelope + Registry->>Registry: Failed tepp_result_not_persisted + end + API-->>Operator: 200 Failed history + end +``` + +A durable outbox / Valkey worker is ADR 0023. Start commits Running +plus the outbox row before `tepp_client` so a crash leaves the work +item instead of rolling back to Pending. + +## Consequences + +Demo Analyst can request a TEPP run, start it, and see Failed / +`tepp_not_available` until a live transport is configured. Connecting +`TEPP_TRANSPORT_URL` submits the same published payload. An accepted +envelope still does not become a calibrated result. Do not invent a +theta. + +## References — APA 7th + +International Organization for Standardization. (2019). *ISO 8601-1:2019: +Date and time—Representations for information interchange—Part 1: Basic +rules* (confirmed 2024; Amendment 1:2022). + +Moreau, L., & Missier, P. (Eds.). (2013). *PROV-DM: The PROV data model*. +World Wide Web Consortium. https://www.w3.org/TR/prov-dm/ + +World Wide Web Consortium. (2013). *PROV-O: The PROV ontology* (W3C +Recommendation). https://www.w3.org/TR/prov-o/ diff --git a/docs/adr/0023-analysis-run-outbox.md b/docs/adr/0023-analysis-run-outbox.md new file mode 100644 index 00000000..e89630ef --- /dev/null +++ b/docs/adr/0023-analysis-run-outbox.md @@ -0,0 +1,91 @@ +# ADR 0023 — Durable start outbox and Valkey wake-up + +**Decision status:** Accepted on this active PR; not protected-main truth until merge +**Date:** 2026-08-17 +**Depends on:** ADR 0013 registry; ADR 0017 authorized create; ADR 0021 +authorized lineage start; ADR 0022 authorized TEPP start +**Refs:** Issue #79 (Milestone 2 parent); ADR 0013 follow-up 3 + +## Context + +ADR 0021 and ADR 0022 start a Pending lineage or TEPP run in the same +request transaction as reconstruct / `tepp_client`. That is honest: a +crash rolls back to Pending. It is not durable. A buyer who clicked +Start and then lost the process cannot tell whether work began. TEPP's +HTTP call also sits inside the registry write. + +ADR 0013 follow-up 3 asked for a normalized PostgreSQL outbox and a +Valkey delivery worker. The activity stream already uses Valkey as an +event queue, not a second database. The missing slice is one immutable +start-work row committed with Running, then a worker that claims that +row and finishes ThreadWeave or `tepp_client`. + +## Decision + +`POST /api/analysis-runs/{id}/start` splits into two authorized +transactions: + +1. lock the visible Pending run, append Running, insert one + `analysis_run_outbox` row whose digest hashes run id, kind, snapshot + digest, and cutoff — never a post body or a theta — then commit; +2. `XADD` the wake-up onto the Valkey stream `analysis-run-outbox` + (a missing Valkey does not roll back the outbox); +3. lock that outbox row, append `analysis_outbox_claimed`, run the same + reconstruct or `tepp_client` path as ADR 0021 / ADR 0022, append the + terminal status, then append `analysis_outbox_delivered`. + +A Succeeded retry still replays the stored digest. A Running restart +with an undelivered outbox is delivery, not a second start. A Running +row without pending work stays 409. Period-report stays 422. Failed +TEPP remains `tepp_not_available` / `tepp_result_not_persisted`. The +HTTP response still waits for delivery so the operator sees the A-100 +fork or the Failed TEPP row without polling. + +```mermaid +sequenceDiagram + participant Operator + participant API + participant Registry + participant Valkey + participant Worker + Operator->>API: POST /api/analysis-runs/{id}/start + API->>Registry: Running + outbox + Registry-->>API: committed work item + API->>Valkey: XADD analysis-run-outbox + API->>Worker: claim outbox + alt lineage + Worker->>Registry: reconstruction + Succeeded + else TEPP + Worker->>Registry: Failed tepp_not_available or tepp_result_not_persisted + end + API-->>Operator: 200 stored result +``` + +`make seed` writes a delivered outbox row on the Demo Corp lineage and +TEPP runs so open-after-seed matches the start path. Retention purge +deletes delivery, outbox, reconstruction, and snapshot membership +before the registry rows. + +## Consequences + +Start survives a crash after Running. Refreshing a queued run finishes +the same work item. Valkey is a wake-up, not a source of truth. Detail +lists labeled Claimed / Delivered events; stream entry ids stay off +the payload. Do not invent a theta, and do not stamp Succeeded from a +missing reconstruct library. + +## References — APA 7th + +Hohpe, G., & Woolf, B. (2003). *Enterprise integration patterns: +Designing, building, and deploying messaging solutions*. +Addison-Wesley. + +International Organization for Standardization. (2019). *ISO 8601-1:2019: +Date and time—Representations for information interchange—Part 1: Basic +rules* (confirmed 2024; Amendment 1:2022). + +Moreau, L., & Missier, P. (Eds.). (2013). *PROV-DM: The PROV data model*. +World Wide Web Consortium. https://www.w3.org/TR/prov-dm/ + +World Wide Web Consortium. (2013). *PROV-O: The PROV ontology* (W3C +Recommendation). https://www.w3.org/TR/prov-o/ diff --git a/docs/adr/0024-seed-period-report-analysis-run.md b/docs/adr/0024-seed-period-report-analysis-run.md new file mode 100644 index 00000000..65df43f1 --- /dev/null +++ b/docs/adr/0024-seed-period-report-analysis-run.md @@ -0,0 +1,78 @@ +# ADR 0024 — Seed records the built period report on the shared snapshot + +**Decision status:** Accepted on this active PR; not protected-main truth until merge +**Date:** 2026-08-17 +**Depends on:** ADR 0013 normalized analysis-run registry; ADR 0003 +fast-mlsirm report integration; ADR 0014 authorized analysis-run read +**Refs:** Issue #79 (Milestone 2 parent). After `make seed`, lineage and +TEPP registry rows were visible on home Analysis runs, but the +calibrated period report lived only on the separate report panel. +ADR 0022 is authorized TEPP start. ADR 0023 is the durable start +outbox. This decision is the next free slot. + +## Context + +Seed already scores Demo Corp week-2/week-3 reports through +`fast-mlsirm` and persists them on the report tables. The analysis-run +registry already has `analysis_run_report`. Operators who opened +Analysis runs after `make seed` could retry a Failed TEPP transport or +inspect a Succeeded lineage tree, then had no registry row for the +report they could already see on the period-report panel. + +A fake Failed report row would contradict the built report. Copying +mean θ onto `analysis_run` would invent a psychometric field the +registry is not allowed to store (ADR 0013). Starting a period-report +run through the lineage/TEPP outbox would invent a calibrated score +on a path that is not allowed to (ADR 0021 / ADR 0022 / ADR 0023). + +## Decision + +- `_seed_demo_period_report` still builds the calibrated report first. +- `_seed_demo_report_run` then inserts `analysis_run_report` on the + same Demo Corp snapshot, scoped to the same corporate entity, with + `scope_key` `2026-W02` so the home detail can open that week. +- The lifecycle is Pending → Running → Succeeded because the report + tables already hold the scored period. The run row stores only + registry digests and counts — never a theta, item bank, or provider + body. +- Home next-action copy for a Succeeded report stays empty. Failed + report fixtures still say rebuild the period report. +- `POST /api/analysis-runs` stays lineage-only (ADR 0017). + `POST /api/analysis-runs/{id}/start` stays 422 for this kind. + This slice does not add a Request period-report button, does not + enqueue outbox work, and does not call TEPP. + +## Consequences + +After `make seed`, Demo Analyst opens Analysis runs and sees +**Period report · Succeeded · Demo Corp** next to the lineage and TEPP +rows. Opening it shows the cutoff posts and **Open period report +2026-W02** (the week stored on `scope_key`). That click also switches +Report grouping to Corporate entity and marks the Demo Corp grouping +current, using the persisted scope grouping key. When the operator is +already on that week, the comparison strip lands on Demo Corp. The +focused chip name contains the visible Corporate entity caption and +the persisted mean θ. The period-report panel names the next action: +read that grouping's mean θ and member posts, then open a post. Those +members land immediately under that next action, ahead of other +groupings and the week strip. Opening a member post names the next +action: read Event Lineage, Keyman, and evaluation on that post. Mean +θ remains on the period-report panel. Re-seed is idempotent on +`demo-report-seed-2026-w02`. + +## References — APA 7th + +American Educational Research Association, American Psychological +Association, & National Council on Measurement in Education. (2014). +*Standards for educational and psychological testing*. American +Educational Research Association. + +International Organization for Standardization. (2019). *ISO 8601-1:2019: +Date and time—Representations for information interchange—Part 1: Basic +rules* (confirmed 2024; Amendment 1:2022). + +Moreau, L., & Missier, P. (Eds.). (2013). *PROV-DM: The PROV data model*. +World Wide Web Consortium. https://www.w3.org/TR/prov-dm/ + +World Wide Web Consortium. (2022). *Time ontology in OWL* (W3C +Recommendation). https://www.w3.org/TR/owl-time/ diff --git a/docs/adr/0025-source-post-revision.md b/docs/adr/0025-source-post-revision.md new file mode 100644 index 00000000..b70aebfa --- /dev/null +++ b/docs/adr/0025-source-post-revision.md @@ -0,0 +1,61 @@ +# ADR 0025 — Source-post revisions keep the cutoff-known body + +**Decision status:** Accepted +**Date:** 2026-08-17 + +## Context + +ADR 0016 marks in-cutoff titles whose live `updated_at` is after +`analysis_run.knowledge_cutoff`. After `make seed`, Demo public post was +marked rewritten while the live sentence stayed the January text, so the +operator was told to compare bodies and was given two clocks, not two +texts. + +The analysis-run registry must not store raw posts (ADR 0013). A missing +cutoff body and a confidently-reconstructed body are different things: +do not invent the earlier sentence on the run detail. + +W3C PROV-O `wasRevisionOf` (Moreau & Missier, 2013), W3C Time Ontology +in OWL (World Wide Web Consortium, 2022), and temporal valid-time +intervals (Jensen & Snodgrass, 1999) keep the write history on the +source row, half-open `[written_at, superseded_at)`. + +Migration 0021 is reconstruction, 0022 is snapshot membership, 0023 is +the start outbox, and ADR 0024 seeds the period-report run. This is the +next free slot. + +## Decision + +Migration `0024_source_post_revision.sql` adds `source_post_revision` +(3NF: one post, one title/body pair, one valid-time interval). A trigger +records a revision on insert and on title or body rewrite. Clock-only +updates do not pretend to be a rewrite. + +`GET /api/posts/{id}?as_of=` returns `known_at` when a revision covers +that clock. The live `post_body` stays the live row. A missing cover is +omitted. Analysis-run detail stays titles and clocks. + +`make seed` writes the January Demo public sentence, then rewrites it on +2026-01-13 so the opened marked title shows both texts. + +## Consequences + +- After `make seed`, open **Lineage reconstruction · Succeeded · Demo + Corp**, then Demo public post: **Body this run knew** is the January + follow-up; the live body names the later delivery window. +- Demo private post stays unmarked and has no second text. +- Roll back `0024` before `0023` / `0022` / `0021` / `0020` / `0018`. +- TEPP stays behind `tepp_client`. This write does not invent a theta. + +## References + +Jensen, C. S., & Snodgrass, R. T. (1999). Temporal data management. +*IEEE Transactions on Knowledge and Data Engineering, 11*(1), 36–44. +https://doi.org/10.1109/69.755613 + +Moreau, L., & Missier, P. (Eds.). (2013). *PROV-O: The PROV ontology* +(W3C Recommendation). World Wide Web Consortium. +https://www.w3.org/TR/prov-o/ + +World Wide Web Consortium. (2022). *Time ontology in OWL* (W3C +Recommendation). https://www.w3.org/TR/owl-time/ diff --git a/docs/adr/0026-tied-organization-similarity.md b/docs/adr/0026-tied-organization-similarity.md new file mode 100644 index 00000000..9225525d --- /dev/null +++ b/docs/adr/0026-tied-organization-similarity.md @@ -0,0 +1,97 @@ +# ADR 0026 — Tied organization similarity stays unbound + +**Decision status:** Accepted +**Date:** 2026-08-17 + +## Context + +Role-and-responsibility and Keyman ingestion resolve free-text organization +names against `corporate_entity`. The former resolver returned either one +catalog id or `None`. That collapsed two materially different outcomes: + +1. no candidate met the minimum similarity threshold; and +2. two or more distinct catalog ids shared the best score. + +A genuine miss may enter ADR 0010's inference-and-corroboration path. A tie +must not. Treating a tie as a miss can create a third, deterministic +`AUTO-...` row for a name that is already represented by multiple catalog +records. It can also bind whichever homonym happened to appear first in an +unordered candidate result. + +Keyman adds another boundary: it may run a verified abbreviation rewrite +before hierarchy resolution. If a raw tied name is rewritten first, the new +string can appear to be a miss and incorrectly enter the creation path. + +Fellegi and Sunter's record-linkage decision framework retains an uncertain +region rather than forcing a match. In this product, an equal top score is +that review state. String similarity remains candidate generation, not proof +of identity. + +## Decision + +`score_corporate_entity` classifies each organization mention as: + +- `unique`: exactly one distinct catalog id has the top score at or above + the threshold; +- `miss`: no candidate reaches the threshold; or +- `tie`: multiple distinct catalog ids share the top qualifying score. + +Only `unique` returns a catalog id. Only `miss` may continue into ADR 0010 +inference and corroborated creation. `tie` returns unbound immediately. + +The same classification is repeated after the advisory creation lock and +catalog reload. If concurrent writes make the refreshed result a tie, no +insert occurs. + +Keyman evaluates the raw organization name before abbreviation rewriting. +A raw tie bypasses name resolution and hierarchy inference, remains text, +and stores no new catalog id. This prevents a resolver rewrite from turning +known ambiguity into an apparent miss. + +Duplicate candidate rows carrying the same `corporate_entity_id` are one +candidate, not a tie. + +```mermaid +flowchart TD + mention[Organization mention] --> raw[Score raw catalog candidates] + raw --> outcome{Resolution outcome} + outcome -->|unique| bind[Bind unique catalog id] + outcome -->|tie| hold[Keep unbound; no AUTO row] + outcome -->|miss| enrich[Optional verified name resolution] + enrich --> score[Score resolved name] + score --> resolved{Resolution outcome} + resolved -->|unique| bind + resolved -->|tie| hold + resolved -->|miss| create[ADR 0010 infer and corroborate] + create --> lock[Lock and reload candidates] + lock --> refreshed{Refreshed outcome} + refreshed -->|unique| bind + refreshed -->|tie| hold + refreshed -->|miss| insert[Insert AUTO row] +``` + +## Consequences + +- Equal top scores are deterministic and fail closed rather than depending + on row order. +- A tied organization name never creates an `AUTO-` catalog row, including + with live resolver, inference, and verification clients. +- Genuine misses retain the existing, corroborated hierarchy creation path. +- Buyers see ambiguous organization names as text until the catalog has a + unique identity decision. +- Future collective entity resolution may use relational context to resolve + ties, but must publish a reviewed unique result before binding. + +## References — APA 7th + +Bhattacharya, I., & Getoor, L. (2007). Collective entity resolution in +relational data. *ACM Transactions on Knowledge Discovery from Data, 1*(1), +Article 5. https://doi.org/10.1145/1217299.1217304 + +Christen, P. (2012). *Data matching: Concepts and techniques for record +linkage, entity resolution, and duplicate detection*. Springer. +https://doi.org/10.1007/978-3-642-31164-2 + +Fellegi, I. P., & Sunter, A. B. (1969). A theory for record linkage. +*Journal of the American Statistical Association, 64*(328), 1183–1210. +https://doi.org/10.2307/2286061 diff --git a/docs/adr/0027-role-person-catalog-identity.md b/docs/adr/0027-role-person-catalog-identity.md new file mode 100644 index 00000000..8ec63f6a --- /dev/null +++ b/docs/adr/0027-role-person-catalog-identity.md @@ -0,0 +1,55 @@ +# ADR 0027 — Persist the R&R person catalog identity on the role row + +**Decision status:** Accepted +**Date:** 2026-08-17 +**Depends on:** ADR 0009 cross-post actor identity; ADR 0019 role catalog +identity + +## Context + +ADR 0019 stores `cataloged_team_id` and `cataloged_corporate_entity_id` +on `post_summary_role` so fetch does not rejoin by display name. Person +actors were still joined at read time by `person_name`, or dropped from +the payload entirely. Two people can share a display name. A later +Keyman row with the same name then steals the chip, and a person named +only in R&R (Keyman not extracted on that post) has no button. + +Fellegi and Sunter (1969) treat a match decision as a binding to one +record, not a later re-search by a non-unique attribute. + +ADR 0021 is authorized analysis-run start. ADR 0026 is tied organization +similarity. This is the next free slot. + +## Decision + +`post_summary_role` stores `cataloged_person_id` written during +`persist_post_summary`. `fetch_persisted_summary` reads that column into +`catalog_node_id` / `node_person`. Person lookup, when it still resolves +by name, orders by `created_at`, then `person_id`. It still does not +create a new `cataloged_person` row (ADR 0009 gap). + +Migration `0025_role_person_catalog_identity.sql` backfills existing +rows from a post-scoped mention only when the name match is unique on +that post (`HAVING count(*) = 1`). Two same-named mentions stay unbound. + +At most one of `cataloged_team_id`, `cataloged_corporate_entity_id`, and +`cataloged_person_id` is set, and the set column must match +`actor_type_code`. + +## Consequences + +- Open a post whose R&R names a cataloged person. The chip is a button + even when Keyman extraction was not run on that post. Click it to walk + that person, not a later same-named row. +- Pre-0025 rows with two same-named mentions stay unbound until an + operator re-persists the summary. Do not guess a UUID at migrate time. + +## References + +Fellegi, I. P., & Sunter, A. B. (1969). A theory for record linkage. +*Journal of the American Statistical Association, 64*(328), 1183–1210. +https://doi.org/10.2307/2286061 + +Bhattacharya, I., & Getoor, L. (2007). Collective entity resolution in +relational data. *ACM Transactions on Knowledge Discovery from Data, +1*(1), Article 5. https://doi.org/10.1145/1217299.1217304 diff --git a/docs/adr/0017-persist-lsirm-leftover-pairs.md b/docs/adr/0028-persist-lsirm-leftover-pairs.md similarity index 94% rename from docs/adr/0017-persist-lsirm-leftover-pairs.md rename to docs/adr/0028-persist-lsirm-leftover-pairs.md index 3ceb7157..ac10ae6e 100644 --- a/docs/adr/0017-persist-lsirm-leftover-pairs.md +++ b/docs/adr/0028-persist-lsirm-leftover-pairs.md @@ -1,4 +1,4 @@ -# ADR 0017 — Persist LSIRM leftover post–criterion pairs +# ADR 0028 — Persist LSIRM leftover post–criterion pairs **Decision status:** Accepted **Date:** 2026-08-17 @@ -40,7 +40,7 @@ the IRT matrix is unusable. A rank-0 residual still emits a stable pair so `make seed` is not empty; the stored distance is then zero, not a fabricated interaction. -The UI contract is ADR 0018. +The UI contract is ADR 0029. ## Consequences @@ -48,7 +48,7 @@ Rebuild and seed write leftover pairs in the same transaction as member scores. `GET /api/reports/{grouping}/{period}` returns `leftover_pairs` with the post title so the buyer can open that post. Hidden posts stay hidden: leftover pairs join `source_post` and use -the same ABAC gate as members. Migration `0012_report_leftover_pair.sql` +the same ABAC gate as members. Migration `0026_report_leftover_pair.sql` upgrades volumes that already applied `0001`. ## References diff --git a/docs/adr/0018-leftover-pair-report-ui.md b/docs/adr/0029-leftover-pair-report-ui.md similarity index 87% rename from docs/adr/0018-leftover-pair-report-ui.md rename to docs/adr/0029-leftover-pair-report-ui.md index 2fcbd896..c7d8a24e 100644 --- a/docs/adr/0018-leftover-pair-report-ui.md +++ b/docs/adr/0029-leftover-pair-report-ui.md @@ -1,11 +1,11 @@ -# ADR 0018 — Leftover pairs sit above the report member list +# ADR 0029 — Leftover pairs sit above the report member list **Decision status:** Accepted **Date:** 2026-08-17 ## Context -ADR 0017 persists closest and farthest leftover post–criterion pairs. +ADR 0028 persists closest and farthest leftover post–criterion pairs. Those pairs only help if a buyer can see them on the Period reports panel and open the named post without hunting through the member list. @@ -38,5 +38,5 @@ next action, not only the distance. ## Related -Depends on [ADR 0017](0017-persist-lsirm-leftover-pairs.md) and +Depends on [ADR 0028](0028-persist-lsirm-leftover-pairs.md) and [ADR 0003](0003-fast-mlsirm-report-integration.md). diff --git a/docs/adr/0024-rankweave-fusion-fail-closed.md b/docs/adr/0030-rankweave-fusion-fail-closed.md similarity index 91% rename from docs/adr/0024-rankweave-fusion-fail-closed.md rename to docs/adr/0030-rankweave-fusion-fail-closed.md index af1de902..4f1a682b 100644 --- a/docs/adr/0024-rankweave-fusion-fail-closed.md +++ b/docs/adr/0030-rankweave-fusion-fail-closed.md @@ -1,4 +1,4 @@ -# ADR 0024 — Fail-closed RankWeave ranking port +# ADR 0030 — Fail-closed RankWeave ranking port **Decision status:** Accepted **Date:** 2026-08-17 @@ -40,9 +40,8 @@ tables, and does not bind the demo IdP to production Keyverse. `RANKWEAVE_DISABLED=1` keeps the fail-closed transport. The default seeded stack uses the in-process library already required by -`reconstruct.py`. Mailbox stays on ADR 0020 / #217. Conversations stay -on ADR 0021 / #219. Leftover pairs stay on #211. TEPP stays on #214. -Keyverse IdP remains a later slice. +`reconstruct.py`. Leftover pairs stay on ADR 0028 / #211. TEPP stays +on ADR 0022 / #214. Keyverse IdP remains a later slice. ## References diff --git a/docs/adr/0031-embedded-image-html-parser.md b/docs/adr/0031-embedded-image-html-parser.md new file mode 100644 index 00000000..6acdfd71 --- /dev/null +++ b/docs/adr/0031-embedded-image-html-parser.md @@ -0,0 +1,78 @@ +# ADR 0031 — Embedded images use an HTML parser and a raster allowlist + +**Decision status:** Accepted +**Date:** 2026-08-17 + +## Context + +The product popup stopped dumping a well-formed +`data:image/png;base64,...` invoice as a base64 wall. The splitter and +`extract_base64_images` still used a `[^>]*` regex. Real invoice HTML +puts `>` inside `alt` or `title` *before* `src`. That shape is legal +HTML (WHATWG, n.d.) and is what `chunk_by_dom` already parses. The regex +missed the picture and put the payload back into the text node. + +The same open MIME class `image/[a-zA-Z0-9.+-]+` accepted +`image/svg+xml`. SVG-as-`` does not run script in current browsers, +but the regex also fed the vision channel. `atob` and +`b64decode(validate=True)` already disagreed on padding. + +ADR 0019 is the R&R catalog-identity decision. This decision is the +viewer/extractor parse contract. Layout clues stay as character offsets +and `chunk_position` rows — never raw HTML in the knowledge graph or in +a persisted post body. + +Persistence of OCR under the figure (Li et al., 2023; Radford et al., +2021) is still the next buyer slice. It must not land on a splitter that +fails the HTML the buyer actually opens. + +## Decision + +The popup (`splitPostBody`), `extract_base64_images`, and `chunk_by_dom` +share one decode helper (`lineageweave.embedded_image_payload`): + +1. Parse with an HTML parser (`DOMParser` in the browser, `html.parser` + in Python). Comments, ` +

Please confirm.

diff --git a/tests/test_abbreviation_tree_corroboration.py b/tests/test_abbreviation_tree_corroboration.py new file mode 100644 index 00000000..e3c4bce9 --- /dev/null +++ b/tests/test_abbreviation_tree_corroboration.py @@ -0,0 +1,130 @@ +"""Tree-constrained Searxng abbreviation cross-check (ADR 0033).""" + +from __future__ import annotations + +import pytest + +from lineageweave.abbreviation_tree_corroboration import ( + AbbreviationTreeMatch, + TreeEntityCandidate, + abbreviation_candidates, + corroborate_abbreviation_against_tree, + exact_catalog_matches, +) +from lineageweave.relation_verification import ( + STATUS_CORROBORATED, + STATUS_PENDING, + STATUS_UNCORROBORATED, + NullRelationVerificationClient, + RelationVerificationResult, +) + +_TREE = ( + TreeEntityCandidate("group-id", "Demo Group"), + TreeEntityCandidate("corp-id", "Demo Corp"), + TreeEntityCandidate("plant-id", "Demo Plant"), +) + + +class _FakeVerificationClient: + available = True + + def __init__(self, hits: dict[tuple[str, str], RelationVerificationResult]) -> None: + self._hits = hits + self.calls: list[tuple[str, str]] = [] + + def verify(self, organization_name: str, relationship_label: str) -> RelationVerificationResult: + self.calls.append((organization_name, relationship_label)) + return self._hits.get( + (organization_name, relationship_label), + RelationVerificationResult(STATUS_UNCORROBORATED, None), + ) + + +class _RaisingVerificationClient: + available = True + + def verify(self, organization_name: str, relationship_label: str) -> RelationVerificationResult: + raise RuntimeError("searxng timeout") + + +def test_exact_catalog_name_is_not_an_abbreviation_candidate() -> None: + assert abbreviation_candidates(("Demo Corp", "DC", " "), _TREE) == ("DC",) + + +def test_tied_exact_catalog_names_stay_candidates() -> None: + twins = ( + TreeEntityCandidate("a", "Demo Twin"), + TreeEntityCandidate("b", "Demo Twin"), + ) + assert abbreviation_candidates(("Demo Twin",), twins) == ("Demo Twin",) + + +def test_unique_searxng_hit_binds_the_tree_node() -> None: + client = _FakeVerificationClient( + { + ("Demo Corp", "DC"): RelationVerificationResult( + STATUS_CORROBORATED, "https://example.test/demo-corp-dc" + ) + } + ) + match = corroborate_abbreviation_against_tree("DC", _TREE, client) + assert match == AbbreviationTreeMatch( + raw_organization_name="DC", + corporate_entity_id="corp-id", + verification_status_code=STATUS_CORROBORATED, + verification_evidence_url="https://example.test/demo-corp-dc", + ) + assert ("Demo Group", "DC") in client.calls + assert ("Demo Plant", "DC") in client.calls + + +def test_no_searxng_hit_stays_unbound() -> None: + match = corroborate_abbreviation_against_tree("ZZ", _TREE, _FakeVerificationClient({})) + assert match.corporate_entity_id is None + assert match.verification_status_code == STATUS_UNCORROBORATED + assert match.verification_evidence_url is None + + +def test_tied_searxng_hits_stay_unbound() -> None: + client = _FakeVerificationClient( + { + ("Demo Corp", "DX"): RelationVerificationResult( + STATUS_CORROBORATED, "https://example.test/demo-corp" + ), + ("Demo Group", "DX"): RelationVerificationResult( + STATUS_CORROBORATED, "https://example.test/demo-group" + ), + } + ) + match = corroborate_abbreviation_against_tree("DX", _TREE, client) + assert match.corporate_entity_id is None + assert match.verification_status_code == STATUS_UNCORROBORATED + + +def test_unavailable_searxng_is_pending_and_does_not_invent_a_parent() -> None: + match = corroborate_abbreviation_against_tree("DC", _TREE, NullRelationVerificationClient()) + assert match.corporate_entity_id is None + assert match.verification_status_code == STATUS_PENDING + assert match.verification_evidence_url is None + + +def test_empty_mention_is_uncorroborated() -> None: + match = corroborate_abbreviation_against_tree(" ", _TREE, _FakeVerificationClient({})) + assert match.corporate_entity_id is None + assert match.verification_status_code == STATUS_UNCORROBORATED + + +def test_search_failure_is_not_recorded_as_uncorroborated() -> None: + with pytest.raises(RuntimeError, match="searxng timeout"): + corroborate_abbreviation_against_tree("DC", _TREE, _RaisingVerificationClient()) + + +def test_exact_catalog_matches_normalize_legal_suffix() -> None: + matches = exact_catalog_matches("Demo Corp.", _TREE) + assert [row.entity_id for row in matches] == ["corp-id"] + + +def test_exact_catalog_matches_ignore_empty_normalized_names() -> None: + assert exact_catalog_matches(" ", _TREE) == () + assert exact_catalog_matches("Corp.", _TREE) == () diff --git a/tests/test_analysis_run_authorization.py b/tests/test_analysis_run_authorization.py new file mode 100644 index 00000000..91de6374 --- /dev/null +++ b/tests/test_analysis_run_authorization.py @@ -0,0 +1,267 @@ +"""SQL authorization for the Milestone 2 analysis-run read projection.""" + +from __future__ import annotations + +import os +import uuid +from pathlib import Path +from urllib.parse import urlsplit, urlunsplit + +import psycopg2 +import pytest +from psycopg2 import sql + +from backend.app.analysis_run_ingestion import _RUN_DETAIL_SQL, _RUN_LIST_SQL + +_ROOT = Path(__file__).resolve().parents[1] +_INITIAL_MIGRATION = _ROOT / "migrations" / "0001_initial_schema.sql" +_REGISTRY_MIGRATION = _ROOT / "migrations" / "0018_analysis_run_registry.sql" +_RETENTION_MIGRATION = _ROOT / "migrations" / "0020_analysis_run_retention_purge.sql" +_ADMIN_DSN = os.environ.get( + "LINEAGEWEAVE_TEST_POSTGRES_ADMIN_DSN", "postgresql://localhost/postgres" +) + + +def test_visible_run_sql_is_parameterized_literals() -> None: + """List and detail queries bind $1/$2/$3; they do not format user SQL.""" + assert "$1" in _RUN_LIST_SQL + assert "$2" in _RUN_LIST_SQL + assert "$3" in _RUN_DETAIL_SQL + assert "{" not in _RUN_LIST_SQL + assert "{" not in _RUN_DETAIL_SQL + + +def _postgres_available() -> bool: + """Return whether the configured administrator DSN is reachable.""" + try: + psycopg2.connect(_ADMIN_DSN, connect_timeout=2).close() + return True + except psycopg2.OperationalError: + return False + + +def _database_dsn(database_name: str) -> str: + """Replace only the database path while preserving DSN query options.""" + parsed = urlsplit(_ADMIN_DSN) + return urlunsplit(parsed._replace(path=f"/{database_name}")) + + +@pytest.fixture +def authz_db(): + """Yield a throwaway database migrated through the registry schema.""" + if not _postgres_available(): + pytest.skip("a reachable PostgreSQL administrator DSN is required") + database_name = f"lineageweave_authz_{uuid.uuid4().hex[:12]}" + admin_connection = psycopg2.connect(_ADMIN_DSN) + admin_connection.autocommit = True + with admin_connection.cursor() as cursor: + cursor.execute( + sql.SQL("create database {}").format(sql.Identifier(database_name)) + ) + try: + connection = psycopg2.connect(_database_dsn(database_name)) + try: + connection.autocommit = True + with connection.cursor() as cursor: + cursor.execute(_INITIAL_MIGRATION.read_text(encoding="utf-8")) + cursor.execute(_REGISTRY_MIGRATION.read_text(encoding="utf-8")) + cursor.execute(_RETENTION_MIGRATION.read_text(encoding="utf-8")) + yield connection + finally: + connection.close() + finally: + with admin_connection.cursor() as cursor: + cursor.execute( + sql.SQL("drop database {}").format(sql.Identifier(database_name)) + ) + admin_connection.close() + + +def _insert_account(cursor, label: str) -> str: + """Insert one synthetic authenticated account and return its UUID.""" + suffix = uuid.uuid4().hex + cursor.execute( + """ + insert into user_account + (external_subject_id, display_name, email_address) + values (%s, %s, %s) + returning user_account_id + """, + (f"{label}-{suffix}", f"{label.title()} User", f"{label}-{suffix}@example.test"), + ) + return str(cursor.fetchone()[0]) + + +def _insert_corp(cursor, code: str, name: str) -> str: + """Insert one synthetic corporate entity.""" + cursor.execute( + """ + insert into common_lookup_value (lookup_category, lookup_code, lookup_label) + values ('corporate_entity_level', 'company', 'Company') + on conflict (lookup_code) do nothing + """ + ) + cursor.execute( + """ + insert into corporate_entity (corporate_entity_code, entity_name, entity_level_code) + values (%s, %s, 'company') + returning corporate_entity_id + """, + (code, name), + ) + return str(cursor.fetchone()[0]) + + +def _complete_run( + cursor, + *, + account_id: str, + digest: str, + idempotency_key: str, + scope_kind: str, + corporate_entity_id: str | None = None, +) -> str: + """Insert one succeeded run with one document-count aggregate.""" + cursor.execute( + """ + insert into analysis_source_snapshot + (snapshot_sha256, source_contract_version, + maximum_available_time, captured_at) + values (%s, 'source-contract-v1', + '2026-01-12T00:00:00Z', '2026-01-12T00:05:00Z') + returning analysis_source_snapshot_id + """, + (digest,), + ) + snapshot_id = cursor.fetchone()[0] + cursor.execute( + """ + insert into analysis_source_count + (analysis_source_snapshot_id, count_type_code, count_value) + values (%s, 'analysis_count_document', 3) + """, + (snapshot_id,), + ) + cursor.execute( + """ + insert into analysis_run + (analysis_source_snapshot_id, run_kind_code, idempotency_key, + requested_by_account_id, knowledge_cutoff, + configuration_schema_version, configuration_sha256, + code_revision_sha, requested_at) + values (%s, 'analysis_run_lineage', %s, %s, + '2026-01-12T12:00:00Z', 'lineage-run-v1', %s, %s, + '2026-01-12T12:30:00Z') + returning analysis_run_id + """, + (snapshot_id, idempotency_key, account_id, "b" * 64, "c" * 40), + ) + run_id = str(cursor.fetchone()[0]) + if scope_kind == "analysis_scope_corporate_entity": + cursor.execute( + """ + insert into analysis_run_scope + (analysis_run_id, scope_kind_code, corporate_entity_id) + values (%s, %s, %s) + """, + (run_id, scope_kind, corporate_entity_id), + ) + else: + cursor.execute( + """ + insert into analysis_run_scope + (analysis_run_id, scope_kind_code) + values (%s, %s) + """, + (run_id, scope_kind), + ) + for ordinal, status, occurred in ( + (1, "analysis_status_pending", "2026-01-12T12:31:00Z"), + (2, "analysis_status_running", "2026-01-12T12:32:00Z"), + (3, "analysis_status_succeeded", "2026-01-12T12:33:00Z"), + ): + cursor.execute( + """ + insert into analysis_run_status_event + (analysis_run_id, status_ordinal, status_code, occurred_at) + values (%s, %s, %s, %s) + """, + (run_id, ordinal, status, occurred), + ) + return run_id + + +def _visible_ids(cursor, account_id: str, entity_ids: list[str]) -> set[str]: + """Apply the same visibility predicate the product API uses.""" + cursor.execute( + """ + select run.analysis_run_id + from analysis_run run + join analysis_run_scope scope on scope.analysis_run_id = run.analysis_run_id + where + run.requested_by_account_id = %s + or ( + scope.scope_kind_code = 'analysis_scope_corporate_entity' + and scope.corporate_entity_id = any(%s::uuid[]) + ) + or ( + scope.scope_kind_code = 'analysis_scope_process_unit' + and exists ( + select 1 from account_affiliation aff + where aff.user_account_id = %s + and aff.process_unit_id = scope.process_unit_id + ) + ) + """, + (account_id, entity_ids, account_id), + ) + return {str(row[0]) for row in cursor.fetchall()} + + +def test_hidden_scope_does_not_leak_through_all_visible_or_other_corp(authz_db) -> None: + """A Demo-Corp viewer never sees another tenant's run or its aggregates.""" + with authz_db.cursor() as cursor: + viewer = _insert_account(cursor, "viewer") + outsider = _insert_account(cursor, "outsider") + own_corp = _insert_corp(cursor, "DEMO-CORP-AUTHZ", "Demo Corp") + other_corp = _insert_corp(cursor, "OTHER-CORP-AUTHZ", "Other Corp") + cursor.execute( + """ + insert into account_affiliation (user_account_id, corporate_entity_id) + values (%s, %s) + """, + (viewer, own_corp), + ) + own_run = _complete_run( + cursor, + account_id=viewer, + digest="a" * 64, + idempotency_key="own-corp", + scope_kind="analysis_scope_corporate_entity", + corporate_entity_id=own_corp, + ) + hidden_all_visible = _complete_run( + cursor, + account_id=outsider, + digest="d" * 64, + idempotency_key="hidden-all", + scope_kind="analysis_scope_all_visible", + ) + hidden_other_corp = _complete_run( + cursor, + account_id=outsider, + digest="e" * 64, + idempotency_key="hidden-other", + scope_kind="analysis_scope_corporate_entity", + corporate_entity_id=other_corp, + ) + + visible = _visible_ids(cursor, viewer, [own_corp]) + assert own_run in visible + assert hidden_all_visible not in visible + assert hidden_other_corp not in visible + + outsider_visible = _visible_ids(cursor, outsider, [other_corp]) + assert hidden_all_visible in outsider_visible + assert hidden_other_corp in outsider_visible + assert own_run not in outsider_visible diff --git a/tests/test_analysis_run_create.py b/tests/test_analysis_run_create.py new file mode 100644 index 00000000..613ddc32 --- /dev/null +++ b/tests/test_analysis_run_create.py @@ -0,0 +1,191 @@ +"""Authorized analysis-run create hashes the cutoff bag, never a score.""" + +import asyncio +from datetime import datetime, timezone + +from backend.app.analysis_run_ingestion import ( + AnalysisRunCreateError, + _require_lineage_create_kind, + _resolve_corporate_entity_id, + create_pending_analysis_run, + live_write_after_cutoff, + plan_analysis_run_capture, +) +import pytest + + +_CUTOFF = datetime(2026, 1, 12, 12, 0, tzinfo=timezone.utc) +_EARLIER = datetime(2026, 1, 10, 9, 0, tzinfo=timezone.utc) + + +def test_capture_digest_is_stable_for_the_same_authorized_bag() -> None: + first = plan_analysis_run_capture( + run_kind_code="analysis_run_lineage", + scope_kind_code="analysis_scope_corporate_entity", + corporate_entity_id="corp-1", + knowledge_cutoff=_CUTOFF, + idempotency_key="client-key-1", + post_ids=["post-b", "post-a"], + thread_keys=["thread-a", "thread-a"], + latest_post_created_at=_EARLIER, + ) + second = plan_analysis_run_capture( + run_kind_code="analysis_run_lineage", + scope_kind_code="analysis_scope_corporate_entity", + corporate_entity_id="corp-1", + knowledge_cutoff=_CUTOFF, + idempotency_key="client-key-1", + post_ids=["post-a", "post-b"], + thread_keys=["thread-a", "thread-a"], + latest_post_created_at=_EARLIER, + ) + assert first.snapshot_sha256 == second.snapshot_sha256 + assert first.configuration_sha256 == second.configuration_sha256 + assert first.document_count == 2 + assert first.thread_count == 1 + assert first.maximum_available_time == _EARLIER + assert "theta" not in first.snapshot_sha256 + assert first.configuration_schema_version == "lineage-run-v1" + + +def test_later_cutoff_or_other_kind_does_not_reuse_the_wrong_digest() -> None: + lineage = plan_analysis_run_capture( + run_kind_code="analysis_run_lineage", + scope_kind_code="analysis_scope_corporate_entity", + corporate_entity_id="corp-1", + knowledge_cutoff=_CUTOFF, + idempotency_key="client-key-1", + post_ids=["post-a"], + thread_keys=["thread-a"], + latest_post_created_at=_EARLIER, + ) + later = plan_analysis_run_capture( + run_kind_code="analysis_run_lineage", + scope_kind_code="analysis_scope_corporate_entity", + corporate_entity_id="corp-1", + knowledge_cutoff=datetime(2026, 1, 13, 12, 0, tzinfo=timezone.utc), + idempotency_key="client-key-1", + post_ids=["post-a"], + thread_keys=["thread-a"], + latest_post_created_at=_EARLIER, + ) + tepp = plan_analysis_run_capture( + run_kind_code="analysis_run_tepp", + scope_kind_code="analysis_scope_corporate_entity", + corporate_entity_id="corp-1", + knowledge_cutoff=_CUTOFF, + idempotency_key="client-key-1", + post_ids=["post-a"], + thread_keys=["thread-a"], + latest_post_created_at=_EARLIER, + ) + assert lineage.snapshot_sha256 != later.snapshot_sha256 + assert lineage.snapshot_sha256 == tepp.snapshot_sha256 + assert lineage.configuration_sha256 != tepp.configuration_sha256 + assert tepp.configuration_schema_version == "tepp-run-v1" + + +def test_omitted_cutoff_keeps_the_same_client_key_stable() -> None: + first = plan_analysis_run_capture( + run_kind_code="analysis_run_lineage", + scope_kind_code="analysis_scope_corporate_entity", + corporate_entity_id="corp-1", + knowledge_cutoff=_CUTOFF, + idempotency_key="client-key-1", + post_ids=["post-a"], + thread_keys=["thread-a"], + latest_post_created_at=_EARLIER, + cutoff_explicit=False, + ) + later_clock = plan_analysis_run_capture( + run_kind_code="analysis_run_lineage", + scope_kind_code="analysis_scope_corporate_entity", + corporate_entity_id="corp-1", + knowledge_cutoff=datetime(2026, 1, 13, 12, 0, tzinfo=timezone.utc), + idempotency_key="client-key-1", + post_ids=["post-a"], + thread_keys=["thread-a"], + latest_post_created_at=_EARLIER, + cutoff_explicit=False, + ) + assert first.configuration_sha256 == later_clock.configuration_sha256 + assert first.snapshot_sha256 == later_clock.snapshot_sha256 + + +def test_empty_corpus_uses_the_cutoff_as_latest_available_time() -> None: + capture = plan_analysis_run_capture( + run_kind_code="analysis_run_lineage", + scope_kind_code="analysis_scope_corporate_entity", + corporate_entity_id="corp-1", + knowledge_cutoff=_CUTOFF, + idempotency_key="client-key-1", + post_ids=[], + thread_keys=[], + latest_post_created_at=None, + ) + assert capture.document_count == 0 + assert capture.thread_count == 0 + assert capture.maximum_available_time == _CUTOFF + + +def test_live_write_clock_is_distinct_from_the_cutoff_admission_clock() -> None: + """An in-cutoff title can still have been rewritten after the run.""" + cutoff = _CUTOFF + assert live_write_after_cutoff(_EARLIER, cutoff) is False + assert live_write_after_cutoff(cutoff, cutoff) is False + assert live_write_after_cutoff( + datetime(2026, 1, 13, 9, 0, tzinfo=timezone.utc), cutoff + ) is True + assert live_write_after_cutoff(datetime(2026, 1, 13, 9, 0), cutoff) is True + + +def test_create_rejects_tepp_and_report_kinds_without_a_fake_score() -> None: + """POST must not record a TEPP row that never called tepp_client.""" + with pytest.raises(AnalysisRunCreateError) as tepp: + _require_lineage_create_kind("analysis_run_tepp") + assert tepp.value.status_code == 422 + assert "invent a measurement" in tepp.value.detail + with pytest.raises(AnalysisRunCreateError) as report: + _require_lineage_create_kind("analysis_run_report") + assert report.value.status_code == 422 + assert "Reports panel" in report.value.detail + with pytest.raises(AnalysisRunCreateError) as unknown: + _require_lineage_create_kind("analysis_run_unknown") + assert unknown.value.status_code == 422 + assert "Only lineage reconstruction" in unknown.value.detail + _require_lineage_create_kind("analysis_run_lineage") + + +def test_create_pending_rejects_tepp_before_touching_the_registry() -> None: + """Kind rejection happens before any snapshot or run insert.""" + + class ForbiddenConnection: + def __getattr__(self, name: str) -> object: + raise AssertionError(f"TEPP create must not touch the registry ({name})") + + async def _run() -> None: + with pytest.raises(AnalysisRunCreateError) as err: + await create_pending_analysis_run( + ForbiddenConnection(), # type: ignore[arg-type] + account_id="acct-1", + affiliated_entity_ids=["corp-1"], + run_kind_code="analysis_run_tepp", + scope_kind_code="analysis_scope_corporate_entity", + corporate_entity_id="corp-1", + knowledge_cutoff=_CUTOFF, + idempotency_key="client-key-1", + ) + assert err.value.status_code == 422 + assert "invent a measurement" in err.value.detail + + asyncio.run(_run()) + + +def test_create_rejects_an_unaffiliated_or_ambiguous_corporate_entity() -> None: + with pytest.raises(AnalysisRunCreateError) as hidden: + _resolve_corporate_entity_id("corp-other", ["corp-1"]) + assert hidden.value.status_code == 404 + with pytest.raises(AnalysisRunCreateError) as ambiguous: + _resolve_corporate_entity_id(None, ["corp-1", "corp-2"]) + assert ambiguous.value.status_code == 422 + assert _resolve_corporate_entity_id(None, ["corp-1"]) == "corp-1" diff --git a/tests/test_analysis_run_outbox.py b/tests/test_analysis_run_outbox.py new file mode 100644 index 00000000..240006ba --- /dev/null +++ b/tests/test_analysis_run_outbox.py @@ -0,0 +1,105 @@ +"""Static and unit contracts for the durable analysis-run start outbox.""" + +from __future__ import annotations + +import re +from datetime import datetime, timezone +from pathlib import Path + +from backend.app.analysis_run_outbox import ( + OUTBOX_STREAM_KEY, + latest_outbox_delivery_is_claimed, + latest_outbox_delivery_is_delivered, + outbox_request_digest, + outbox_stream_fields, +) +from backend.app.analysis_run_start import start_kind_rejection + +_ROOT = Path(__file__).resolve().parents[1] +_OUTBOX_MIGRATION = _ROOT / "migrations" / "0023_analysis_run_outbox.sql" +_OUTBOX_ROLLBACK = _ROOT / "migrations" / "rollback" / "0023_analysis_run_outbox.sql" +_POSTGRES_IMAGE = _ROOT / "docker" / "postgres-init" / "Dockerfile" + + +def test_outbox_migration_is_normalized_and_wired() -> None: + """Static contract: 3NF names, no payload JSON, Dockerfile copy, rollback.""" + migration = _OUTBOX_MIGRATION.read_text(encoding="utf-8") + rollback = _OUTBOX_ROLLBACK.read_text(encoding="utf-8") + dockerfile = _POSTGRES_IMAGE.read_text(encoding="utf-8") + created_tables = set( + re.findall(r"create table if not exists\s+([a-z0-9_]+)", migration, re.I) + ) + assert {"analysis_run_outbox", "analysis_run_outbox_delivery"} <= created_tables + assert "jsonb" not in migration.casefold() + assert "metadata_payload" not in migration + assert "theta" not in migration.casefold() + assert "0023_analysis_run_outbox.sql" in dockerfile + assert "analysis_run_outbox_not_empty" in rollback + assert "reject_analysis_run_outbox_mutation" in migration + assert "reject_analysis_run_outbox_delivery_mutation" in migration + assert "analysis_run_lineage_edge" in migration + assert "analysis_source_snapshot_member" in migration + object_patterns = ( + r"create table if not exists\s+([a-z0-9_]+)", + r"create or replace function\s+([a-z0-9_]+)", + r"create trigger\s+([a-z0-9_]+)", + ) + for pattern in object_patterns: + for object_name in re.findall(pattern, migration, re.I): + assert len(object_name.split("_")) >= 2, object_name + + +def test_outbox_request_digest_is_stable_and_ignores_bodies() -> None: + """The same frozen start hashes the same way and never includes a body.""" + cutoff = datetime(2026, 1, 12, 12, 0, tzinfo=timezone.utc) + digest = outbox_request_digest( + analysis_run_id="11111111-1111-1111-1111-111111111111", + work_kind_code="analysis_run_lineage", + snapshot_sha256="ab" * 32, + knowledge_cutoff=cutoff, + ) + again = outbox_request_digest( + analysis_run_id="11111111-1111-1111-1111-111111111111", + work_kind_code="analysis_run_lineage", + snapshot_sha256="ab" * 32, + knowledge_cutoff=datetime(2026, 1, 12, 12, 0), + ) + other = outbox_request_digest( + analysis_run_id="11111111-1111-1111-1111-111111111111", + work_kind_code="analysis_run_tepp", + snapshot_sha256="ab" * 32, + knowledge_cutoff=cutoff, + ) + assert digest == again + assert digest != other + assert "theta" not in digest + assert "Pricing renegotiation" not in digest + + +def test_outbox_stream_fields_are_the_wake_up_only() -> None: + """Valkey carries the run id and digest, never a measurement.""" + fields = outbox_stream_fields( + analysis_run_id="11111111-1111-1111-1111-111111111111", + work_kind_code="analysis_run_tepp", + request_sha256="cd" * 32, + ) + assert fields["work_kind_code"] == "analysis_run_tepp" + assert fields["request_sha256"] == "cd" * 32 + assert "theta" not in str(fields).casefold() + assert OUTBOX_STREAM_KEY == "analysis-run-outbox" + + +def test_outbox_delivery_helpers_distinguish_claimed_from_done() -> None: + """A claimed row is retryable. A delivered row is finished.""" + assert latest_outbox_delivery_is_claimed("analysis_outbox_claimed") + assert not latest_outbox_delivery_is_claimed("analysis_outbox_delivered") + assert latest_outbox_delivery_is_delivered("analysis_outbox_delivered") + assert not latest_outbox_delivery_is_delivered(None) + + +def test_period_report_never_enters_the_start_outbox() -> None: + """The outbox is for lineage and TEPP start, not a fabricated report.""" + report = start_kind_rejection("analysis_run_report") + assert report is not None + assert report.status_code == 422 + assert "invent a measurement" in report.detail diff --git a/tests/test_analysis_run_reconstruction_schema.py b/tests/test_analysis_run_reconstruction_schema.py new file mode 100644 index 00000000..4d332f57 --- /dev/null +++ b/tests/test_analysis_run_reconstruction_schema.py @@ -0,0 +1,156 @@ +"""Static and optional PostgreSQL contracts for run-scoped reconstruction.""" + +from __future__ import annotations + +import os +import re +import uuid +from pathlib import Path +from urllib.parse import urlsplit, urlunsplit + +import pytest + +_ROOT = Path(__file__).resolve().parents[1] +_INITIAL_MIGRATION = _ROOT / "migrations" / "0001_initial_schema.sql" +_REGISTRY_MIGRATION = _ROOT / "migrations" / "0018_analysis_run_registry.sql" +_RECONSTRUCTION_MIGRATION = _ROOT / "migrations" / "0021_analysis_run_reconstruction.sql" +_RECONSTRUCTION_ROLLBACK = ( + _ROOT / "migrations" / "rollback" / "0021_analysis_run_reconstruction.sql" +) +_SNAPSHOT_MEMBER_MIGRATION = ( + _ROOT / "migrations" / "0022_analysis_source_snapshot_member.sql" +) +_SNAPSHOT_MEMBER_ROLLBACK = ( + _ROOT / "migrations" / "rollback" / "0022_analysis_source_snapshot_member.sql" +) +_POSTGRES_IMAGE = _ROOT / "docker" / "postgres-init" / "Dockerfile" +_ADMIN_DSN = os.environ.get( + "LINEAGEWEAVE_TEST_POSTGRES_ADMIN_DSN", "postgresql://localhost/postgres" +) +_REQUIRED_TABLES = { + "analysis_run_reconstruction", + "analysis_run_lineage_edge", +} + + +def test_reconstruction_migration_is_normalized_and_wired() -> None: + """Static contract: 3NF names, no payload JSON, Dockerfile copy, rollback.""" + migration = _RECONSTRUCTION_MIGRATION.read_text(encoding="utf-8") + rollback = _RECONSTRUCTION_ROLLBACK.read_text(encoding="utf-8") + dockerfile = _POSTGRES_IMAGE.read_text(encoding="utf-8") + created_tables = set( + re.findall(r"create table if not exists\s+([a-z0-9_]+)", migration, re.I) + ) + assert _REQUIRED_TABLES <= created_tables + assert "jsonb" not in migration.casefold() + assert "metadata_payload" not in migration + assert "theta" not in migration.casefold() + assert "0021_analysis_run_reconstruction.sql" in dockerfile + assert "0022_analysis_source_snapshot_member.sql" in dockerfile + assert "0023_analysis_run_outbox.sql" in dockerfile + assert "0024_source_post_revision.sql" in dockerfile + assert "0025_role_person_catalog_identity.sql" in dockerfile + assert "0026_report_leftover_pair.sql" in dockerfile + assert "0028_analysis_run_tepp_result.sql" in dockerfile + assert "0029_analysis_run_tepp_accepted.sql" in dockerfile + assert "analysis_run_reconstruction_not_empty" in rollback + assert "reject_analysis_run_reconstruction_update" in migration + assert "reject_analysis_run_lineage_edge_update" in migration + member_migration = _SNAPSHOT_MEMBER_MIGRATION.read_text(encoding="utf-8") + member_rollback = _SNAPSHOT_MEMBER_ROLLBACK.read_text(encoding="utf-8") + assert "analysis_source_snapshot_member" in member_migration + assert "jsonb" not in member_migration.casefold() + assert "theta" not in member_migration.casefold() + assert "analysis_source_snapshot_member_not_empty" in member_rollback + assert "reject_analysis_source_snapshot_member_update" in member_migration + for object_name in re.findall( + r"create table if not exists\s+([a-z0-9_]+)", + member_migration, + re.I, + ): + assert len(object_name.split("_")) >= 2, object_name + + object_patterns = ( + r"create table if not exists\s+([a-z0-9_]+)", + r"create or replace function\s+([a-z0-9_]+)", + r"create trigger\s+([a-z0-9_]+)", + ) + for pattern in object_patterns: + for object_name in re.findall(pattern, migration, re.I): + assert len(object_name.split("_")) >= 2, object_name + + +def _postgres_available() -> bool: + """Return whether the configured administrator DSN is reachable.""" + try: + import psycopg2 + + psycopg2.connect(_ADMIN_DSN, connect_timeout=2).close() + return True + except Exception: + return False + + +def _database_dsn(database_name: str) -> str: + """Replace only the database path while preserving DSN query options.""" + parsed = urlsplit(_ADMIN_DSN) + return urlunsplit(parsed._replace(path=f"/{database_name}")) + + +@pytest.fixture +def reconstruction_db(): + """Yield a throwaway registry+reconstruction database.""" + if not _postgres_available(): + pytest.skip("a reachable PostgreSQL administrator DSN is required") + import psycopg2 + + database_name = f"lineageweave_recon_{uuid.uuid4().hex[:12]}" + admin = psycopg2.connect(_ADMIN_DSN) + admin.autocommit = True + try: + with admin.cursor() as cursor: + cursor.execute(f'create database "{database_name}"') + finally: + admin.close() + conn = psycopg2.connect(_database_dsn(database_name)) + conn.autocommit = True + try: + with conn.cursor() as cursor: + cursor.execute(_INITIAL_MIGRATION.read_text(encoding="utf-8")) + cursor.execute(_REGISTRY_MIGRATION.read_text(encoding="utf-8")) + cursor.execute(_RECONSTRUCTION_MIGRATION.read_text(encoding="utf-8")) + cursor.execute(_SNAPSHOT_MEMBER_MIGRATION.read_text(encoding="utf-8")) + yield conn + finally: + conn.close() + admin = psycopg2.connect(_ADMIN_DSN) + admin.autocommit = True + try: + with admin.cursor() as cursor: + cursor.execute( + "select pg_terminate_backend(pid) from pg_stat_activity " + "where datname = %s and pid <> pg_backend_pid()", + (database_name,), + ) + cursor.execute(f'drop database "{database_name}"') + finally: + admin.close() + + +def test_empty_reconstruction_rollback_is_replayable(reconstruction_db) -> None: + """An empty reconstruction schema can be rolled back and removed.""" + with reconstruction_db.cursor() as cursor: + cursor.execute( + "select table_name from information_schema.tables " + "where table_schema = 'public' and table_name = any(%s)", + (list(_REQUIRED_TABLES),), + ) + assert {row[0] for row in cursor.fetchall()} == _REQUIRED_TABLES + cursor.execute(_RECONSTRUCTION_ROLLBACK.read_text(encoding="utf-8")) + cursor.execute( + "select table_name from information_schema.tables " + "where table_schema = 'public' and table_name = any(%s)", + (list(_REQUIRED_TABLES),), + ) + assert cursor.fetchall() == [] + cursor.execute(_RECONSTRUCTION_ROLLBACK.read_text(encoding="utf-8")) diff --git a/tests/test_analysis_run_registry_schema.py b/tests/test_analysis_run_registry_schema.py new file mode 100644 index 00000000..083eb8fb --- /dev/null +++ b/tests/test_analysis_run_registry_schema.py @@ -0,0 +1,1305 @@ +"""Real-PostgreSQL contracts for the normalized Milestone 2 run registry.""" + +from __future__ import annotations + +import hashlib +import os +import re +import uuid +from pathlib import Path +from urllib.parse import urlsplit, urlunsplit + +import psycopg2 +import psycopg2.errors +import pytest +from psycopg2 import sql + +_ROOT = Path(__file__).resolve().parents[1] +_INITIAL_MIGRATION = _ROOT / "migrations" / "0001_initial_schema.sql" +_REGISTRY_MIGRATION = _ROOT / "migrations" / "0018_analysis_run_registry.sql" +_REGISTRY_ROLLBACK = _ROOT / "migrations" / "rollback" / "0018_analysis_run_registry.sql" +_RETENTION_MIGRATION = _ROOT / "migrations" / "0020_analysis_run_retention_purge.sql" +_RETENTION_ROLLBACK = ( + _ROOT / "migrations" / "rollback" / "0020_analysis_run_retention_purge.sql" +) +_POSTGRES_IMAGE = _ROOT / "docker" / "postgres-init" / "Dockerfile" +_ADMIN_DSN = os.environ.get( + "LINEAGEWEAVE_TEST_POSTGRES_ADMIN_DSN", "postgresql://localhost/postgres" +) +_REQUIRED_TABLES = { + "analysis_source_snapshot", + "analysis_source_count", + "analysis_run", + "analysis_run_scope", + "analysis_run_status_event", +} +_REQUIRED_LOOKUP_CODES = { + "analysis_run_lineage", + "analysis_run_report", + "analysis_run_tepp", + "analysis_status_pending", + "analysis_status_running", + "analysis_status_succeeded", + "analysis_status_failed", + "analysis_status_cancelled", + "analysis_scope_all_visible", + "analysis_scope_corporate_entity", + "analysis_scope_process_unit", + "analysis_scope_thread_group", + "analysis_count_source_row", + "analysis_count_document", + "analysis_count_thread", + "analysis_count_lineage_node", + "analysis_count_lineage_edge", +} + + +def _postgres_available() -> bool: + """Return whether the configured administrator DSN is reachable.""" + + try: + psycopg2.connect(_ADMIN_DSN, connect_timeout=2).close() + return True + except psycopg2.OperationalError: + return False + + +def _database_dsn(database_name: str) -> str: + """Replace only the database path while preserving DSN query options.""" + + parsed = urlsplit(_ADMIN_DSN) + return urlunsplit(parsed._replace(path=f"/{database_name}")) + + +def _table_definition(migration: str, table_name: str) -> str: + """Return one table definition from the deterministic migration text.""" + + match = re.search( + rf"create table if not exists {re.escape(table_name)}\s*\((.*?)\n\);", + migration, + re.IGNORECASE | re.DOTALL, + ) + assert match is not None, table_name + return match.group(1) + + +@pytest.fixture +def registry_db(): + """Yield a throwaway database migrated through the registry schema.""" + + if not _postgres_available(): + pytest.skip("a reachable PostgreSQL administrator DSN is required") + database_name = f"lineageweave_registry_{uuid.uuid4().hex[:12]}" + admin_connection = psycopg2.connect(_ADMIN_DSN) + admin_connection.autocommit = True + with admin_connection.cursor() as cursor: + cursor.execute( + sql.SQL("create database {}").format(sql.Identifier(database_name)) + ) + try: + connection = psycopg2.connect(_database_dsn(database_name)) + try: + connection.autocommit = True + with connection.cursor() as cursor: + cursor.execute(_INITIAL_MIGRATION.read_text(encoding="utf-8")) + cursor.execute(_REGISTRY_MIGRATION.read_text(encoding="utf-8")) + cursor.execute(_RETENTION_MIGRATION.read_text(encoding="utf-8")) + yield connection + finally: + connection.close() + finally: + with admin_connection.cursor() as cursor: + cursor.execute( + sql.SQL("drop database {}").format(sql.Identifier(database_name)) + ) + admin_connection.close() + + +def _insert_account(cursor, label: str = "operator") -> str: + """Insert one synthetic authenticated account and return its UUID.""" + + suffix = uuid.uuid4().hex + cursor.execute( + """ + insert into user_account + (external_subject_id, display_name, email_address) + values (%s, %s, %s) + returning user_account_id + """, + (f"{label}-{suffix}", f"{label.title()} User", f"{label}-{suffix}@example.test"), + ) + return str(cursor.fetchone()[0]) + + +def _insert_snapshot( + cursor, + *, + digest: str = "a" * 64, + maximum_available_time: str = "2026-08-15T00:00:00Z", + captured_at: str = "2026-08-15T00:05:00Z", +) -> str: + """Insert one immutable source snapshot and return its UUID.""" + + cursor.execute( + """ + insert into analysis_source_snapshot + (snapshot_sha256, source_contract_version, + maximum_available_time, captured_at) + values (%s, 'source-contract-v1', %s, %s) + returning analysis_source_snapshot_id + """, + (digest, maximum_available_time, captured_at), + ) + return str(cursor.fetchone()[0]) + + +def _insert_run( + cursor, + *, + snapshot_id: str, + account_id: str, + idempotency_key: str, + knowledge_cutoff: str = "2026-08-15T00:30:00Z", + run_kind_code: str = "analysis_run_lineage", + requested_at: str = "2026-08-15T00:45:00Z", +) -> str: + """Insert one immutable account-scoped analysis request.""" + + cursor.execute( + """ + insert into analysis_run + (analysis_source_snapshot_id, run_kind_code, idempotency_key, + requested_by_account_id, knowledge_cutoff, + configuration_schema_version, configuration_sha256, + code_revision_sha, requested_at) + values (%s, %s, %s, %s, %s, 'lineage-run-v1', %s, %s, %s) + returning analysis_run_id + """, + ( + snapshot_id, + run_kind_code, + idempotency_key, + account_id, + knowledge_cutoff, + "b" * 64, + "c" * 40, + requested_at, + ), + ) + return str(cursor.fetchone()[0]) + + +def _insert_run_bearing_registry( + cursor, + *, + digest: str, + idempotency_key: str, +) -> None: + """Insert one snapshot, count, run, scope, and pending event.""" + + account_id = _insert_account(cursor) + snapshot_id = _insert_snapshot(cursor, digest=digest) + cursor.execute( + "insert into analysis_source_count values " + "(%s, 'analysis_count_document', 3)", + (snapshot_id,), + ) + run_id = _insert_run( + cursor, + snapshot_id=snapshot_id, + account_id=account_id, + idempotency_key=idempotency_key, + ) + cursor.execute( + "insert into analysis_run_scope " + "(analysis_run_id, scope_kind_code) " + "values (%s, 'analysis_scope_all_visible')", + (run_id,), + ) + cursor.execute( + "insert into analysis_run_status_event " + "(analysis_run_id, status_ordinal, status_code, occurred_at) " + "values (%s, 1, 'analysis_status_pending', " + "'2026-08-15T01:00:00Z')", + (run_id,), + ) + + +def _authorize_session_for_purge(cursor) -> str: + """Grant the current session_user both retention locks and return it.""" + + cursor.execute("select session_user") + session_role = cursor.fetchone()[0] + cursor.execute( + "insert into analysis_run_retention_grant (database_role_name) " + "select %s " + "where not exists (" + " select 1 from analysis_run_retention_grant " + " where database_role_name = %s and revoked_at is null" + ")", + (session_role, session_role), + ) + cursor.execute( + sql.SQL("grant analysis_run_retention_admin to {}").format( + sql.Identifier(session_role) + ) + ) + return session_role + + +def _drop_role_if_exists(cursor, role_name: str) -> None: + """Drop a test role after releasing objects it owns.""" + + cursor.execute("select 1 from pg_roles where rolname = %s", (role_name,)) + if cursor.fetchone() is None: + return + cursor.execute(sql.SQL("drop owned by {}").format(sql.Identifier(role_name))) + cursor.execute(sql.SQL("drop role {}").format(sql.Identifier(role_name))) + + +def test_registry_contract_is_normalized_and_has_one_temporal_authority() -> None: + """Static contract rejects the parallel prototype and duplicated clocks.""" + + migration = _REGISTRY_MIGRATION.read_text(encoding="utf-8") + rollback = _REGISTRY_ROLLBACK.read_text(encoding="utf-8") + dockerfile = _POSTGRES_IMAGE.read_text(encoding="utf-8") + created_tables = set( + re.findall(r"create table if not exists\s+([a-z0-9_]+)", migration, re.I) + ) + assert _REQUIRED_TABLES <= created_tables + assert "analysis_run_records" not in created_tables + assert "metadata_payload" not in migration + assert "jsonb" not in migration.casefold() + assert _REQUIRED_LOOKUP_CODES <= set( + re.findall(r"'(analysis_[a-z0-9_]+)'", migration) + ) + assert "0018_analysis_run_registry.sql" in dockerfile + assert "0019_role_catalog_identity.sql" in dockerfile + assert "0020_analysis_run_retention_purge.sql" in dockerfile + assert "0021_analysis_run_reconstruction.sql" in dockerfile + assert "0022_analysis_source_snapshot_member.sql" in dockerfile + assert "0023_analysis_run_outbox.sql" in dockerfile + assert "0024_source_post_revision.sql" in dockerfile + assert "0025_role_person_catalog_identity.sql" in dockerfile + assert "0026_report_leftover_pair.sql" in dockerfile + assert "0028_analysis_run_tepp_result.sql" in dockerfile + assert "0029_analysis_run_tepp_accepted.sql" in dockerfile + seed = (_ROOT / "scripts" / "seed_demo_data.py").read_text(encoding="utf-8") + assert seed.index("0019_role_catalog_identity.sql") < seed.index( + "0020_analysis_run_retention_purge.sql" + ) + assert seed.index("0020_analysis_run_retention_purge.sql") < seed.index( + "0021_analysis_run_reconstruction.sql" + ) + assert seed.index("0021_analysis_run_reconstruction.sql") < seed.index( + "0022_analysis_source_snapshot_member.sql" + ) + assert seed.index("0022_analysis_source_snapshot_member.sql") < seed.index( + "0023_analysis_run_outbox.sql" + ) + assert seed.index("0023_analysis_run_outbox.sql") < seed.index( + "0024_source_post_revision.sql" + ) + assert seed.index("0024_source_post_revision.sql") < seed.index( + "0025_role_person_catalog_identity.sql" + ) + assert seed.index("0025_role_person_catalog_identity.sql") < seed.index( + "0026_report_leftover_pair.sql" + ) + assert seed.index("0026_report_leftover_pair.sql") < seed.index( + "0027_abbreviation_tree_corroboration.sql" + ) + assert seed.index("0027_abbreviation_tree_corroboration.sql") < seed.index( + "0028_analysis_run_tepp_result.sql" + ) + assert seed.index("0028_analysis_run_tepp_result.sql") < seed.index( + "0029_analysis_run_tepp_accepted.sql" + ) + assert "analysis_run_registry_not_empty" in rollback + retention = _RETENTION_MIGRATION.read_text(encoding="utf-8") + retention_rollback = _RETENTION_ROLLBACK.read_text(encoding="utf-8") + assert "purge_analysis_run_registry" in retention + assert "analysis_run_retention_event" in retention + assert "analysis_run_retention_grant" in retention + assert "analysis_run_retention_admin" in retention + assert "invoking_session_role" in retention + assert "invoking_current_role" in retention + assert "security definer" in retention.casefold() + assert "revoke all" in retention.casefold() + assert "from public" in retention.casefold() + assert "analysis_run_retention_not_approved" in retention + assert "analysis_run_retention_not_granted" in retention + assert "analysis_run_retention_not_admin" in retention + assert "analysis_run_lineage_edge" in retention + assert "analysis_run_reconstruction" in retention + assert "analysis_source_snapshot_member" in retention + assert "to_regclass" in retention + assert retention.index("delete from analysis_run_lineage_edge") < ( + retention.index("delete from analysis_run_reconstruction") + ) + assert retention.index("delete from analysis_run_reconstruction") < ( + retention.index("delete from analysis_run_status_event") + ) + assert retention.index("delete from analysis_source_count") < ( + retention.index("delete from analysis_source_snapshot_member") + ) + assert retention.index("delete from analysis_source_snapshot_member") < ( + retention.index("delete from analysis_source_snapshot;") + ) + outbox_purge = ( + _ROOT / "migrations" / "0023_analysis_run_outbox.sql" + ).read_text(encoding="utf-8") + assert outbox_purge.index("delete from analysis_run_lineage_edge") < ( + outbox_purge.index("delete from analysis_run_reconstruction") + ) + assert outbox_purge.index("delete from analysis_run_reconstruction") < ( + outbox_purge.index("delete from analysis_run_status_event") + ) + assert outbox_purge.index("delete from analysis_source_count") < ( + outbox_purge.index("delete from analysis_source_snapshot_member") + ) + assert outbox_purge.index("delete from analysis_source_snapshot_member") < ( + outbox_purge.index("delete from analysis_source_snapshot;") + ) + assert "analysis_run_retention_event_not_empty" in retention_rollback + assert "jsonb" not in retention.casefold() + for object_name in re.findall( + r"create table if not exists\s+([a-z0-9_]+)" + r"|create or replace function\s+([a-z0-9_]+)" + r"|create role\s+([a-z0-9_]+)", + retention, + re.I, + ): + name = object_name[0] or object_name[1] or object_name[2] + assert len(name.split("_")) >= 2, name + + snapshot_definition = _table_definition(migration, "analysis_source_snapshot") + run_definition = _table_definition(migration, "analysis_run") + assert "maximum_available_time" in snapshot_definition + assert "knowledge_cutoff" not in snapshot_definition + assert "knowledge_cutoff" in run_definition + assert "requested_by_account_id uuid not null" in run_definition + assert "unique (requested_by_account_id, idempotency_key)" in run_definition + assert "enforce_analysis_run_knowledge_cutoff" in migration + assert "reject_analysis_source_snapshot_update" in migration + assert "reject_analysis_run_mutation" in migration + assert "reject_analysis_run_scope_mutation" in migration + assert "analysis_run_scope_required" in migration + assert "enforce_analysis_source_count_freeze" in migration + assert "enforce_analysis_run_status_transition" in migration + assert "analysis_run_current_status" in migration + + object_patterns = ( + r"create table if not exists\s+([a-z0-9_]+)", + r"create(?: unique)? index if not exists\s+([a-z0-9_]+)", + r"create or replace function\s+([a-z0-9_]+)", + r"create trigger\s+([a-z0-9_]+)", + r"create or replace view\s+([a-z0-9_]+)", + ) + for pattern in object_patterns: + for object_name in re.findall(pattern, migration, re.I): + assert len(object_name.split("_")) >= 2, object_name + + +def test_registry_migration_is_idempotent(registry_db) -> None: + """Sequential migration replay preserves one object set.""" + + with registry_db.cursor() as cursor: + cursor.execute(_REGISTRY_MIGRATION.read_text(encoding="utf-8")) + cursor.execute(_RETENTION_MIGRATION.read_text(encoding="utf-8")) + cursor.execute( + "select table_name from information_schema.tables " + "where table_schema = 'public'" + ) + tables = {row[0] for row in cursor.fetchall()} + cursor.execute( + "select table_name from information_schema.views " + "where table_schema = 'public'" + ) + views = {row[0] for row in cursor.fetchall()} + assert _REQUIRED_TABLES <= tables + assert "analysis_run_retention_event" in tables + assert "analysis_run_retention_grant" in tables + assert "analysis_run_current_status" in views + + +def test_registry_persists_scope_counts_and_legal_status_history(registry_db) -> None: + """A valid run keeps normalized scope, counts, and current status.""" + + with registry_db.cursor() as cursor: + account_id = _insert_account(cursor) + snapshot_id = _insert_snapshot(cursor) + cursor.execute( + "insert into analysis_source_count values " + "(%s, 'analysis_count_document', 12)", + (snapshot_id,), + ) + run_id = _insert_run( + cursor, + snapshot_id=snapshot_id, + account_id=account_id, + idempotency_key="synthetic-run-1", + ) + cursor.execute( + "insert into analysis_run_scope " + "(analysis_run_id, scope_kind_code) " + "values (%s, 'analysis_scope_all_visible')", + (run_id,), + ) + cursor.execute( + """ + insert into analysis_run_status_event + (analysis_run_id, status_ordinal, status_code, occurred_at) + values + (%s, 1, 'analysis_status_pending', '2026-08-15T01:00:01Z'), + (%s, 2, 'analysis_status_running', '2026-08-15T01:00:02Z'), + (%s, 3, 'analysis_status_succeeded', '2026-08-15T01:00:03Z') + """, + (run_id, run_id, run_id), + ) + cursor.execute( + "select status_code, status_ordinal from analysis_run_current_status " + "where analysis_run_id = %s", + (run_id,), + ) + assert cursor.fetchone() == ("analysis_status_succeeded", 3) + + +def test_snapshot_supports_multiple_run_owned_cutoffs_and_blocks_future_evidence( + registry_db, +) -> None: + """One capture is reusable, but each run must respect its own cutoff.""" + + with registry_db.cursor() as cursor: + snapshot_id = _insert_snapshot(cursor) + first_account_id = _insert_account(cursor, "first") + second_account_id = _insert_account(cursor, "second") + first_run_id = _insert_run( + cursor, + snapshot_id=snapshot_id, + account_id=first_account_id, + idempotency_key="cutoff-one", + knowledge_cutoff="2026-08-15T00:30:00Z", + ) + second_run_id = _insert_run( + cursor, + snapshot_id=snapshot_id, + account_id=second_account_id, + idempotency_key="cutoff-two", + knowledge_cutoff="2026-08-16T00:00:00Z", + requested_at="2026-08-16T00:30:00Z", + ) + assert first_run_id != second_run_id + with pytest.raises(psycopg2.errors.RaiseException): + _insert_run( + cursor, + snapshot_id=snapshot_id, + account_id=first_account_id, + idempotency_key="future-leakage", + knowledge_cutoff="2026-08-14T23:59:59Z", + ) + + +def test_snapshot_counts_and_run_request_are_immutable(registry_db) -> None: + """Evidence and request configuration freeze before derivation starts.""" + + with registry_db.cursor() as cursor: + snapshot_id = _insert_snapshot(cursor) + cursor.execute( + "insert into analysis_source_count values " + "(%s, 'analysis_count_document', 12)", + (snapshot_id,), + ) + with pytest.raises(psycopg2.errors.RaiseException): + cursor.execute( + "update analysis_source_snapshot set source_contract_version = 'x' " + "where analysis_source_snapshot_id = %s", + (snapshot_id,), + ) + with pytest.raises(psycopg2.errors.RaiseException): + cursor.execute( + "update analysis_source_count set count_value = 13 " + "where analysis_source_snapshot_id = %s", + (snapshot_id,), + ) + account_id = _insert_account(cursor) + run_id = _insert_run( + cursor, + snapshot_id=snapshot_id, + account_id=account_id, + idempotency_key="freeze-evidence", + ) + with pytest.raises(psycopg2.errors.RaiseException): + cursor.execute( + "update analysis_run set knowledge_cutoff = now() " + "where analysis_run_id = %s", + (run_id,), + ) + with pytest.raises(psycopg2.errors.RaiseException): + cursor.execute( + "insert into analysis_source_count values " + "(%s, 'analysis_count_thread', 8)", + (snapshot_id,), + ) + with pytest.raises(psycopg2.errors.RaiseException): + cursor.execute( + "delete from analysis_source_count " + "where analysis_source_snapshot_id = %s", + (snapshot_id,), + ) + + +def test_idempotency_is_scoped_to_the_authenticated_account(registry_db) -> None: + """Two actors may use one opaque key; one actor may not reuse it.""" + + with registry_db.cursor() as cursor: + snapshot_id = _insert_snapshot(cursor) + first_account_id = _insert_account(cursor, "first") + second_account_id = _insert_account(cursor, "second") + _insert_run( + cursor, + snapshot_id=snapshot_id, + account_id=first_account_id, + idempotency_key="shared-key", + ) + _insert_run( + cursor, + snapshot_id=snapshot_id, + account_id=second_account_id, + idempotency_key="shared-key", + ) + with pytest.raises(psycopg2.errors.UniqueViolation): + _insert_run( + cursor, + snapshot_id=snapshot_id, + account_id=first_account_id, + idempotency_key="shared-key", + ) + + +def test_registry_rejects_invalid_evidence_and_missing_actor(registry_db) -> None: + """Database constraints reject malformed audit evidence before persistence.""" + + with registry_db.cursor() as cursor: + with pytest.raises(psycopg2.errors.CheckViolation): + cursor.execute( + "insert into analysis_source_snapshot " + "(snapshot_sha256, source_contract_version, " + "maximum_available_time, captured_at) " + "values ('bad', 'source-contract-v1', now(), now())" + ) + snapshot_id = _insert_snapshot(cursor) + with pytest.raises(psycopg2.errors.CheckViolation): + cursor.execute( + "insert into analysis_source_count values " + "(%s, 'analysis_count_source_row', -1)", + (snapshot_id,), + ) + with pytest.raises(psycopg2.errors.NotNullViolation): + cursor.execute( + """ + insert into analysis_run + (analysis_source_snapshot_id, run_kind_code, idempotency_key, + knowledge_cutoff, configuration_schema_version, + configuration_sha256, code_revision_sha) + values (%s, 'analysis_run_report', 'missing-actor', now(), + 'report-run-v1', %s, %s) + """, + (snapshot_id, "d" * 64, "e" * 40), + ) + + +def test_status_history_enforces_shape_order_time_and_legal_transitions( + registry_db, +) -> None: + """Append-only status evidence is a serialized state machine.""" + + with registry_db.cursor() as cursor: + snapshot_id = _insert_snapshot(cursor) + account_id = _insert_account(cursor) + first_run_id = _insert_run( + cursor, + snapshot_id=snapshot_id, + account_id=account_id, + idempotency_key="first-status", + ) + cursor.execute( + "insert into analysis_run_scope " + "(analysis_run_id, scope_kind_code) " + "values (%s, 'analysis_scope_all_visible')", + (first_run_id,), + ) + with pytest.raises(psycopg2.errors.RaiseException): + cursor.execute( + "insert into analysis_run_status_event " + "(analysis_run_id, status_ordinal, status_code, occurred_at) " + "values (%s, 1, 'analysis_status_running', now())", + (first_run_id,), + ) + + second_run_id = _insert_run( + cursor, + snapshot_id=snapshot_id, + account_id=account_id, + idempotency_key="second-status", + ) + cursor.execute( + "insert into analysis_run_scope " + "(analysis_run_id, scope_kind_code) " + "values (%s, 'analysis_scope_all_visible')", + (second_run_id,), + ) + cursor.execute( + "insert into analysis_run_status_event " + "(analysis_run_id, status_ordinal, status_code, occurred_at) " + "values (%s, 1, 'analysis_status_pending', " + "'2026-08-15T01:00:00Z')", + (second_run_id,), + ) + with pytest.raises(psycopg2.errors.RaiseException): + cursor.execute( + "insert into analysis_run_status_event " + "(analysis_run_id, status_ordinal, status_code, occurred_at) " + "values (%s, 3, 'analysis_status_running', " + "'2026-08-15T01:00:01Z')", + (second_run_id,), + ) + with pytest.raises(psycopg2.errors.RaiseException): + cursor.execute( + "insert into analysis_run_status_event " + "(analysis_run_id, status_ordinal, status_code, occurred_at) " + "values (%s, 2, 'analysis_status_succeeded', " + "'2026-08-15T01:00:01Z')", + (second_run_id,), + ) + cursor.execute( + "insert into analysis_run_status_event " + "(analysis_run_id, status_ordinal, status_code, occurred_at) " + "values (%s, 2, 'analysis_status_running', " + "'2026-08-15T01:00:02Z')", + (second_run_id,), + ) + with pytest.raises(psycopg2.errors.RaiseException): + cursor.execute( + "insert into analysis_run_status_event " + "(analysis_run_id, status_ordinal, status_code, occurred_at) " + "values (%s, 3, 'analysis_status_succeeded', " + "'2026-08-15T01:00:01Z')", + (second_run_id,), + ) + with pytest.raises(psycopg2.errors.CheckViolation): + cursor.execute( + "insert into analysis_run_status_event " + "(analysis_run_id, status_ordinal, status_code, occurred_at) " + "values (%s, 3, 'analysis_status_failed', " + "'2026-08-15T01:00:03Z')", + (second_run_id,), + ) + cursor.execute( + "insert into analysis_run_status_event " + "(analysis_run_id, status_ordinal, status_code, occurred_at) " + "values (%s, 3, 'analysis_status_succeeded', " + "'2026-08-15T01:00:03Z')", + (second_run_id,), + ) + with pytest.raises(psycopg2.errors.RaiseException): + cursor.execute( + "insert into analysis_run_status_event " + "(analysis_run_id, status_ordinal, status_code, occurred_at) " + "values (%s, 4, 'analysis_status_running', " + "'2026-08-15T01:00:04Z')", + (second_run_id,), + ) + with pytest.raises(psycopg2.errors.RaiseException): + cursor.execute( + "update analysis_run_status_event set retryable = true " + "where analysis_run_id = %s and status_ordinal = 3", + (second_run_id,), + ) + + + +def test_run_scope_and_request_evidence_are_immutable(registry_db) -> None: + """Authorization scope and request identity cannot be rewritten or erased.""" + + with registry_db.cursor() as cursor: + snapshot_id = _insert_snapshot(cursor) + account_id = _insert_account(cursor) + run_id = _insert_run( + cursor, + snapshot_id=snapshot_id, + account_id=account_id, + idempotency_key="immutable-run", + ) + cursor.execute( + "insert into analysis_run_scope " + "(analysis_run_id, scope_kind_code) " + "values (%s, 'analysis_scope_all_visible')", + (run_id,), + ) + with pytest.raises(psycopg2.errors.RaiseException): + cursor.execute( + "update analysis_run_scope set scope_kind_code = scope_kind_code " + "where analysis_run_id = %s", + (run_id,), + ) + with pytest.raises(psycopg2.errors.RaiseException): + cursor.execute( + "delete from analysis_run_scope where analysis_run_id = %s", + (run_id,), + ) + with pytest.raises(psycopg2.errors.RaiseException): + cursor.execute( + "delete from analysis_run where analysis_run_id = %s", + (run_id,), + ) + + +def test_status_requires_scope_and_cannot_predate_request(registry_db) -> None: + """Lifecycle evidence starts only after an immutable authorized request.""" + + with registry_db.cursor() as cursor: + snapshot_id = _insert_snapshot(cursor) + account_id = _insert_account(cursor) + run_id = _insert_run( + cursor, + snapshot_id=snapshot_id, + account_id=account_id, + idempotency_key="scoped-status", + ) + with pytest.raises(psycopg2.errors.RaiseException): + cursor.execute( + "insert into analysis_run_status_event " + "(analysis_run_id, status_ordinal, status_code, occurred_at) " + "values (%s, 1, 'analysis_status_pending', " + "'2026-08-15T01:00:00Z')", + (run_id,), + ) + cursor.execute( + "insert into analysis_run_scope " + "(analysis_run_id, scope_kind_code) " + "values (%s, 'analysis_scope_all_visible')", + (run_id,), + ) + with pytest.raises(psycopg2.errors.RaiseException): + cursor.execute( + "insert into analysis_run_status_event " + "(analysis_run_id, status_ordinal, status_code, occurred_at) " + "values (%s, 1, 'analysis_status_pending', " + "'2026-08-15T00:44:59Z')", + (run_id,), + ) + cursor.execute( + "insert into analysis_run_status_event " + "(analysis_run_id, status_ordinal, status_code, occurred_at, recorded_at) " + "values (%s, 1, 'analysis_status_pending', " + "'2026-08-15T01:00:00Z', '2099-01-01T00:00:00Z') " + "returning recorded_at", + (run_id,), + ) + recorded_at = cursor.fetchone()[0] + assert recorded_at.year < 2099 + + +def test_machine_codes_and_canonical_idempotency_are_fail_closed(registry_db) -> None: + """Audit identifiers are canonical and failure details stay machine-safe.""" + + with registry_db.cursor() as cursor: + snapshot_id = _insert_snapshot(cursor) + account_id = _insert_account(cursor) + with pytest.raises(psycopg2.errors.RaiseException): + _insert_run( + cursor, + snapshot_id=snapshot_id, + account_id=account_id, + idempotency_key="future-request", + requested_at="2099-01-01T00:00:00Z", + ) + with pytest.raises(psycopg2.errors.CheckViolation): + _insert_run( + cursor, + snapshot_id=snapshot_id, + account_id=account_id, + idempotency_key=" padded-key ", + ) + run_id = _insert_run( + cursor, + snapshot_id=snapshot_id, + account_id=account_id, + idempotency_key="machine-safe", + ) + cursor.execute( + "insert into analysis_run_scope " + "(analysis_run_id, scope_kind_code) " + "values (%s, 'analysis_scope_all_visible')", + (run_id,), + ) + cursor.execute( + "insert into analysis_run_status_event " + "(analysis_run_id, status_ordinal, status_code, occurred_at) " + "values (%s, 1, 'analysis_status_pending', " + "'2026-08-15T01:00:00Z')", + (run_id,), + ) + cursor.execute( + "insert into analysis_run_status_event " + "(analysis_run_id, status_ordinal, status_code, occurred_at) " + "values (%s, 2, 'analysis_status_running', " + "'2026-08-15T01:00:00Z')", + (run_id,), + ) + with pytest.raises(psycopg2.errors.CheckViolation): + cursor.execute( + "insert into analysis_run_status_event " + "(analysis_run_id, status_ordinal, status_code, occurred_at, " + "failure_code, retryable) " + "values (%s, 3, 'analysis_status_failed', " + "'2026-08-15T01:00:00Z', 'provider timeout', true)", + (run_id,), + ) + cursor.execute( + "insert into analysis_run_status_event " + "(analysis_run_id, status_ordinal, status_code, occurred_at, " + "failure_code, retryable) " + "values (%s, 3, 'analysis_status_failed', " + "'2026-08-15T01:00:00Z', 'provider_timeout', true)", + (run_id,), + ) + +def test_rollback_refuses_data_loss_then_removes_an_empty_registry(registry_db) -> None: + """Downgrade fails closed until audit evidence is explicitly removed.""" + + rollback_sql = _REGISTRY_ROLLBACK.read_text(encoding="utf-8") + with registry_db.cursor() as cursor: + snapshot_id = _insert_snapshot(cursor) + with pytest.raises(psycopg2.errors.RaiseException): + cursor.execute(rollback_sql) + # The rollback script opens an explicit transaction on this + # autocommit connection. A RAISE leaves that transaction aborted, and + # connection.rollback() is a no-op while autocommit is true. + cursor.execute("rollback") + with registry_db.cursor() as cursor: + cursor.execute( + "delete from analysis_source_snapshot " + "where analysis_source_snapshot_id = %s", + (snapshot_id,), + ) + cursor.execute(rollback_sql) + cursor.execute("select to_regclass('public.analysis_run')") + assert cursor.fetchone()[0] is None + cursor.execute(rollback_sql) + + +def test_approved_retention_purge_empties_a_run_bearing_registry(registry_db) -> None: + """Grant plus admin empties expired registry rows; a raw DELETE still fails.""" + + rollback_sql = _REGISTRY_ROLLBACK.read_text(encoding="utf-8") + retention_rollback = _RETENTION_ROLLBACK.read_text(encoding="utf-8") + with registry_db.cursor() as cursor: + _insert_run_bearing_registry( + cursor, + digest="a" * 64, + idempotency_key="retention-purge", + ) + cursor.execute("select analysis_run_id from analysis_run") + run_id = cursor.fetchone()[0] + with pytest.raises( + psycopg2.errors.RaiseException, + match="analysis_run_request_is_immutable", + ): + cursor.execute( + "delete from analysis_run where analysis_run_id = %s", + (run_id,), + ) + with pytest.raises( + psycopg2.errors.RaiseException, + match="analysis_run_registry_not_empty", + ): + cursor.execute(rollback_sql) + cursor.execute("rollback") + session_role = _authorize_session_for_purge(cursor) + with pytest.raises( + psycopg2.errors.RaiseException, + match="analysis_run_retention_not_approved", + ): + cursor.execute("select purge_analysis_run_registry(%s)", ("wrong-token",)) + cursor.execute( + "select purge_analysis_run_registry(%s)", + ("approved-retention-purge",), + ) + cursor.execute("select count(*) from analysis_run") + assert cursor.fetchone()[0] == 0 + cursor.execute("select count(*) from analysis_source_snapshot") + assert cursor.fetchone()[0] == 0 + cursor.execute( + "select purged_run_count, purged_snapshot_count, " + "approval_token_digest, invoking_session_role, " + "invoking_current_role from analysis_run_retention_event" + ) + ( + purged_run_count, + purged_snapshot_count, + token_digest, + invoking_session_role, + invoking_current_role, + ) = cursor.fetchone() + assert purged_run_count == 1 + assert purged_snapshot_count == 1 + assert token_digest == hashlib.sha256( + b"approved-retention-purge" + ).hexdigest() + assert invoking_session_role == session_role + assert invoking_current_role + cursor.execute(rollback_sql) + cursor.execute("select to_regclass('public.analysis_run')") + assert cursor.fetchone()[0] is None + with pytest.raises( + psycopg2.errors.RaiseException, + match="analysis_run_retention_event_not_empty", + ): + cursor.execute(retention_rollback) + cursor.execute("rollback") + cursor.execute("delete from analysis_run_retention_event") + cursor.execute(retention_rollback) + cursor.execute( + "select to_regclass('public.analysis_run_retention_event')" + ) + assert cursor.fetchone()[0] is None + + +def test_retention_purge_empties_optional_reconstruction_children( + registry_db, +) -> None: + """Grant plus admin empties ADR 0021 children despite delete-reject triggers.""" + + with registry_db.cursor() as cursor: + _insert_run_bearing_registry( + cursor, + digest="b" * 64, + idempotency_key="retention-purge-children", + ) + cursor.execute( + "select analysis_run_id, analysis_source_snapshot_id " + "from analysis_run" + ) + run_id, snapshot_id = cursor.fetchone() + cursor.execute( + """ + create table analysis_run_reconstruction ( + analysis_run_id uuid primary key + references analysis_run (analysis_run_id), + result_sha256 text not null, + edge_count integer not null, + reconstructed_at timestamptz not null + ); + create table analysis_run_lineage_edge ( + analysis_run_id uuid not null + references analysis_run_reconstruction (analysis_run_id), + child_post_id uuid not null, + parent_post_id uuid not null, + fused_score double precision not null, + primary key (analysis_run_id, child_post_id) + ); + create table analysis_source_snapshot_member ( + analysis_source_snapshot_id uuid not null + references analysis_source_snapshot + (analysis_source_snapshot_id), + source_post_id uuid not null, + primary key ( + analysis_source_snapshot_id, source_post_id + ) + ); + create function reject_reconstruction_child_delete() + returns trigger language plpgsql as $fn$ + begin + raise exception 'analysis_run_reconstruction_is_immutable'; + end + $fn$; + create trigger analysis_run_reconstruction_update_reject + before update or delete on analysis_run_reconstruction + for each row execute function reject_reconstruction_child_delete(); + create trigger analysis_run_lineage_edge_update_reject + before update or delete on analysis_run_lineage_edge + for each row execute function reject_reconstruction_child_delete(); + create trigger analysis_source_snapshot_member_update_reject + before update or delete on analysis_source_snapshot_member + for each row execute function reject_reconstruction_child_delete(); + """ + ) + cursor.execute( + "insert into analysis_run_reconstruction " + "(analysis_run_id, result_sha256, edge_count, reconstructed_at) " + "values (%s, %s, 1, now())", + (run_id, "c" * 64), + ) + cursor.execute( + "insert into analysis_run_lineage_edge " + "(analysis_run_id, child_post_id, parent_post_id, fused_score) " + "values (%s, %s, %s, 0.91)", + (run_id, str(uuid.uuid4()), str(uuid.uuid4())), + ) + cursor.execute( + "insert into analysis_source_snapshot_member " + "(analysis_source_snapshot_id, source_post_id) " + "values (%s, %s)", + (snapshot_id, str(uuid.uuid4())), + ) + with pytest.raises( + psycopg2.errors.RaiseException, + match="analysis_run_reconstruction_is_immutable", + ): + cursor.execute("delete from analysis_run_reconstruction") + _authorize_session_for_purge(cursor) + cursor.execute( + "select purge_analysis_run_registry(%s)", + ("approved-retention-purge",), + ) + cursor.execute("select count(*) from analysis_run") + assert cursor.fetchone()[0] == 0 + cursor.execute("select count(*) from analysis_source_snapshot") + assert cursor.fetchone()[0] == 0 + cursor.execute("select count(*) from analysis_run_reconstruction") + assert cursor.fetchone()[0] == 0 + cursor.execute("select count(*) from analysis_run_lineage_edge") + assert cursor.fetchone()[0] == 0 + cursor.execute("select count(*) from analysis_source_snapshot_member") + assert cursor.fetchone()[0] == 0 + + +def test_retention_purge_requires_unrevoked_session_grant(registry_db) -> None: + """Admin membership plus the published token cannot purge without a grant.""" + + role_name = f"retention_denied_{uuid.uuid4().hex[:8]}" + with registry_db.cursor() as cursor: + cursor.execute( + "select has_function_privilege(%s, %s, 'execute')", + ("public", "purge_analysis_run_registry(text)"), + ) + assert cursor.fetchone()[0] is False + _insert_run_bearing_registry( + cursor, + digest="d" * 64, + idempotency_key="retention-grant-deny", + ) + cursor.execute( + sql.SQL("create role {} nologin nosuperuser inherit").format( + sql.Identifier(role_name) + ) + ) + cursor.execute( + sql.SQL("grant analysis_run_retention_admin to {}").format( + sql.Identifier(role_name) + ) + ) + try: + cursor.execute( + sql.SQL("set session authorization {}").format( + sql.Identifier(role_name) + ) + ) + except psycopg2.errors.InsufficientPrivilege: + cursor.execute("reset session authorization") + _drop_role_if_exists(cursor, role_name) + pytest.skip("session authorization requires superuser") + with pytest.raises( + psycopg2.errors.RaiseException, + match="analysis_run_retention_not_granted", + ): + cursor.execute( + "select purge_analysis_run_registry(%s)", + ("approved-retention-purge",), + ) + cursor.execute("reset session authorization") + cursor.execute( + "insert into analysis_run_retention_grant (database_role_name) " + "values (%s)", + (role_name,), + ) + cursor.execute( + sql.SQL("set session authorization {}").format( + sql.Identifier(role_name) + ) + ) + cursor.execute( + "select purge_analysis_run_registry(%s)", + ("approved-retention-purge",), + ) + cursor.execute("reset session authorization") + cursor.execute( + "select invoking_session_role from analysis_run_retention_event" + ) + assert cursor.fetchone()[0] == role_name + cursor.execute( + "update analysis_run_retention_grant " + "set revoked_at = clock_timestamp() " + "where database_role_name = %s and revoked_at is null", + (role_name,), + ) + _insert_run_bearing_registry( + cursor, + digest="e" * 64, + idempotency_key="retention-grant-revoked", + ) + cursor.execute( + sql.SQL("set session authorization {}").format( + sql.Identifier(role_name) + ) + ) + with pytest.raises( + psycopg2.errors.RaiseException, + match="analysis_run_retention_not_granted", + ): + cursor.execute( + "select purge_analysis_run_registry(%s)", + ("approved-retention-purge",), + ) + cursor.execute("reset session authorization") + _drop_role_if_exists(cursor, role_name) + + +def test_runtime_role_cannot_purge_with_only_the_public_token(registry_db) -> None: + """Table DML plus the documented phrase is not a retention grant.""" + + runtime_role = f"analysis_run_app_{uuid.uuid4().hex[:12]}" + operator_role = f"analysis_run_operator_{uuid.uuid4().hex[:12]}" + try: + with registry_db.cursor() as cursor: + _insert_run_bearing_registry( + cursor, + digest="f" * 64, + idempotency_key="runtime-denied-purge", + ) + cursor.execute("select analysis_run_id from analysis_run") + run_id = cursor.fetchone()[0] + cursor.execute( + sql.SQL( + "create role {} nologin nosuperuser inherit" + ).format(sql.Identifier(runtime_role)) + ) + cursor.execute( + sql.SQL( + "create role {} nologin nosuperuser inherit" + ).format(sql.Identifier(operator_role)) + ) + cursor.execute( + sql.SQL("grant usage on schema public to {}, {}").format( + sql.Identifier(runtime_role), + sql.Identifier(operator_role), + ) + ) + cursor.execute( + sql.SQL( + "grant select, insert, update, delete on " + "analysis_run, analysis_run_scope, " + "analysis_run_status_event, analysis_source_snapshot, " + "analysis_source_count to {}" + ).format(sql.Identifier(runtime_role)) + ) + cursor.execute( + sql.SQL("grant analysis_run_retention_admin to {}").format( + sql.Identifier(operator_role) + ) + ) + cursor.execute( + "insert into analysis_run_retention_grant (database_role_name) " + "values (%s)", + (operator_role,), + ) + cursor.execute( + sql.SQL("set role {}").format(sql.Identifier(runtime_role)) + ) + with pytest.raises(psycopg2.errors.InsufficientPrivilege): + cursor.execute( + "select purge_analysis_run_registry(%s)", + ("approved-retention-purge",), + ) + with pytest.raises( + psycopg2.errors.RaiseException, + match="analysis_run_request_is_immutable", + ): + cursor.execute( + "delete from analysis_run where analysis_run_id = %s", + (run_id,), + ) + cursor.execute("reset role") + try: + cursor.execute( + sql.SQL("set session authorization {}").format( + sql.Identifier(operator_role) + ) + ) + except psycopg2.errors.InsufficientPrivilege: + cursor.execute("reset session authorization") + pytest.skip("session authorization requires superuser") + cursor.execute( + "select purge_analysis_run_registry(%s)", + ("approved-retention-purge",), + ) + cursor.execute("reset session authorization") + cursor.execute("select count(*) from analysis_run") + assert cursor.fetchone()[0] == 0 + cursor.execute( + "select invoking_session_role, invoking_current_role " + "from analysis_run_retention_event" + ) + invoking_session_role, invoking_current_role = cursor.fetchone() + assert invoking_session_role + assert invoking_current_role + finally: + with registry_db.cursor() as cursor: + cursor.execute("reset role") + cursor.execute("reset session authorization") + for role_name in (runtime_role, operator_role): + _drop_role_if_exists(cursor, role_name) + + +def test_retention_purge_requires_admin_membership_even_with_a_grant( + registry_db, +) -> None: + """A grant without analysis_run_retention_admin cannot empty the registry.""" + + role_name = f"retention_grant_only_{uuid.uuid4().hex[:8]}" + with registry_db.cursor() as cursor: + _insert_run_bearing_registry( + cursor, + digest="c" * 64, + idempotency_key="retention-admin-deny", + ) + cursor.execute( + sql.SQL("create role {} nologin nosuperuser inherit").format( + sql.Identifier(role_name) + ) + ) + cursor.execute( + sql.SQL( + "grant execute on function purge_analysis_run_registry(text) " + "to {}" + ).format(sql.Identifier(role_name)) + ) + cursor.execute( + "insert into analysis_run_retention_grant (database_role_name) " + "values (%s)", + (role_name,), + ) + try: + cursor.execute( + sql.SQL("set session authorization {}").format( + sql.Identifier(role_name) + ) + ) + except psycopg2.errors.InsufficientPrivilege: + cursor.execute("reset session authorization") + _drop_role_if_exists(cursor, role_name) + pytest.skip("session authorization requires superuser") + with pytest.raises( + psycopg2.errors.RaiseException, + match="analysis_run_retention_not_admin", + ): + cursor.execute( + "select purge_analysis_run_registry(%s)", + ("approved-retention-purge",), + ) + cursor.execute("reset session authorization") + cursor.execute("select count(*) from analysis_run") + assert cursor.fetchone()[0] == 1 + _drop_role_if_exists(cursor, role_name) diff --git a/tests/test_analysis_run_scope_grouping_key.py b/tests/test_analysis_run_scope_grouping_key.py new file mode 100644 index 00000000..fb8414ef --- /dev/null +++ b/tests/test_analysis_run_scope_grouping_key.py @@ -0,0 +1,56 @@ +"""Report-open grouping keys stay on the authorized scope, never a week or theta.""" + +from backend.app.analysis_run_ingestion import scope_grouping_key + + +def test_corporate_scope_persists_entity_id_not_the_week_key() -> None: + assert ( + scope_grouping_key( + { + "scope_kind_code": "analysis_scope_corporate_entity", + "corporate_entity_id": "corp-1", + "process_unit_id": None, + "scope_key": "2026-W02", + } + ) + == "corp-1" + ) + + +def test_process_unit_and_thread_scopes_keep_their_grouping_keys() -> None: + assert ( + scope_grouping_key( + { + "scope_kind_code": "analysis_scope_process_unit", + "corporate_entity_id": "corp-1", + "process_unit_id": "pu-high", + "scope_key": None, + } + ) + == "pu-high" + ) + assert ( + scope_grouping_key( + { + "scope_kind_code": "analysis_scope_thread_group", + "corporate_entity_id": None, + "process_unit_id": None, + "scope_key": "A-100", + } + ) + == "A-100" + ) + + +def test_scope_grouping_key_is_never_a_theta() -> None: + key = scope_grouping_key( + { + "scope_kind_code": "analysis_scope_corporate_entity", + "corporate_entity_id": "corp-1", + "process_unit_id": None, + "scope_key": "2026-W02", + } + ) + assert key is not None + assert "theta" not in key.lower() + assert "θ" not in key diff --git a/tests/test_analysis_run_start.py b/tests/test_analysis_run_start.py new file mode 100644 index 00000000..2fcccd0b --- /dev/null +++ b/tests/test_analysis_run_start.py @@ -0,0 +1,445 @@ +"""Start-reconstruction contracts: digest, freeze, 422/409, designed tree.""" + +import asyncio +from datetime import datetime, timezone + +import asyncpg +import pytest + +from backend.app.analysis_run_ingestion import reconstructed_edge_is_visible +from backend.app.analysis_run_start import ( + AnalysisRunStartError, + _deliver_tepp_measurement, + _persist_tepp_accepted, + configured_tepp_client, + reconstruction_member_ids, + reconstruction_result_digest, + start_kind_rejection, + start_write_conflict_error, + tepp_accepted_clocks, + tepp_run_request, + tepp_submit_outcome, +) +from backend.app.lineage_ingestion import records_from_source_posts +from lineageweave.fixtures import sample_records +from lineageweave.lineage_persistence import lineage_edge_specs +from lineageweave.tepp_client import AnalysisRunRequest, TeppClient, TeppNotAvailable +from lineageweave.tepp_result import ( + TeppAcceptedEvidence, + accepted_tepp_seed_envelope, + parse_tepp_accepted_evidence, + persistable_tepp_seed_envelope, +) + + +def test_reconstruction_digest_is_stable_and_ignores_edge_order() -> None: + """The same parent choices hash the same way regardless of insert order.""" + edges = lineage_edge_specs(sample_records()) + reversed_edges = list(reversed(edges)) + assert reconstruction_result_digest(edges) == reconstruction_result_digest(reversed_edges) + assert reconstruction_result_digest([]) == reconstruction_result_digest([]) + assert reconstruction_result_digest(edges) != reconstruction_result_digest([]) + + +def test_start_uses_the_same_parent_choices_as_library_reconstruct() -> None: + """The product start path must recover the designed A-100 fork. + + fixtures.sample_records() is the synthetic gold tree: rec-002 is the + branch point for the revised quote and the delivery question. A start + that dropped an edge or invented a parent would fail this check. + """ + edges = lineage_edge_specs(sample_records()) + children = {edge.child_id for edge in edges if edge.parent_id == "rec-002"} + assert children >= {"rec-003", "rec-004"} + assert all(0.0 <= edge.fused_score <= 1.0 for edge in edges) + assert "theta" not in reconstruction_result_digest(edges) + + +def test_start_wiring_recovers_a100_from_source_post_rows() -> None: + """CI must exercise records_from_source_posts, not only library reconstruct.""" + rows = [ + { + "post_id": record.record_id, + "post_title": record.label, + "created_at": record.occurred_at, + "thread_group_key": record.group_key, + "secondary_grouping_key": record.secondary_key, + "process_unit_id": None, + "corporate_entity_id": "corp-demo", + } + for record in sample_records() + ] + edges = lineage_edge_specs(records_from_source_posts(rows)) + children = {edge.child_id for edge in edges if edge.parent_id == "rec-002"} + assert children >= {"rec-003", "rec-004"} + assert reconstruction_result_digest(edges) == reconstruction_result_digest( + lineage_edge_specs(sample_records()) + ) + + +def test_snapshot_members_exclude_a_later_backfill() -> None: + """Start reconstructs the create-time bag, not a later cutoff re-query.""" + captured = ["rec-001", "rec-002", "rec-003", "rec-004"] + cutoff_with_backfill = [*captured, "rec-backfill"] + assert reconstruction_member_ids(captured, cutoff_with_backfill) == captured + assert reconstruction_member_ids([], cutoff_with_backfill) == cutoff_with_backfill + + +def test_reconstructed_edge_hides_unaffiliated_private_titles() -> None: + """Edge titles use the same public-or-affiliated rule as cutoff posts.""" + affiliated = ["corp-demo"] + assert reconstructed_edge_is_visible( + parent_visibility_code="public", + parent_corporate_entity_id="corp-other", + child_visibility_code="public", + child_corporate_entity_id="corp-other", + affiliated_entity_ids=affiliated, + ) + assert not reconstructed_edge_is_visible( + parent_visibility_code="private", + parent_corporate_entity_id="corp-other", + child_visibility_code="public", + child_corporate_entity_id="corp-demo", + affiliated_entity_ids=affiliated, + ) + + +def test_period_report_start_is_unprocessable_and_tepp_is_allowed() -> None: + """Period-report stays 422. TEPP start is allowed so tepp_client can run.""" + report = start_kind_rejection("analysis_run_report") + assert report is not None + assert report.status_code == 422 + assert "invent a measurement" in report.detail + assert "period report" in report.detail + assert start_kind_rejection("analysis_run_lineage") is None + assert start_kind_rejection("analysis_run_tepp") is None + + +def _tepp_request() -> AnalysisRunRequest: + return tepp_run_request( + idempotency_key="buyer-tepp-2026-w07", + snapshot_sha256="ab" * 32, + knowledge_cutoff=datetime(2026, 1, 12, 12, 0, tzinfo=timezone.utc), + corporate_entity_id="11111111-1111-1111-1111-111111111111", + ) + + +def test_tepp_run_request_is_the_published_wire_shape() -> None: + """Start builds TEPP's seven-field request from the frozen run.""" + request = _tepp_request() + payload = request.to_json() + assert payload["contract_version"] == 1 + assert payload["idempotency_key"] == "buyer-tepp-2026-w07" + assert payload["snapshot_id"] == "ab" * 32 + assert payload["knowledge_cutoff"] == "2026-01-12T12:00:00Z" + assert payload["model_contract_version"] == "tepp-analysis-run-v1" + assert payload["output_profile"] == "calibrated_event_measurement" + assert "theta" not in str(payload).casefold() + + +def test_tepp_submit_outcome_drops_a_missing_transport() -> None: + """A missing TEPP transport is Failed, never a fabricated score.""" + status, failure, result = tepp_submit_outcome(TeppClient(), _tepp_request()) + assert status == "analysis_status_failed" + assert failure == "tepp_not_available" + assert result is None + + +def test_tepp_submit_outcome_does_not_persist_an_empty_envelope() -> None: + """A bare accepted status is not TEPP's published acknowledgement.""" + + class _Accepting(TeppClient): + def __init__(self) -> None: + super().__init__(transport=lambda _payload: {"status": "accepted"}) + + status, failure, result = tepp_submit_outcome(_Accepting(), _tepp_request()) + assert status == "analysis_status_failed" + assert failure == "tepp_result_not_persisted" + assert result is None + + +def test_tepp_submit_outcome_keeps_a_published_accepted_envelope_failed() -> None: + """A published accepted ack is transport evidence, never Succeeded.""" + request = _tepp_request() + + class _Accepted(TeppClient): + def __init__(self) -> None: + super().__init__( + transport=lambda _payload: accepted_tepp_seed_envelope( + idempotency_key=request.idempotency_key + ) + ) + + status, failure, result = tepp_submit_outcome(_Accepted(), request) + assert status == "analysis_status_failed" + assert failure == "tepp_completed_result_unsupported" + assert result is not None + assert result.run_state == "accepted" + assert result.evidence_kind() == "aggregate transport evidence" + assert "theta" not in result.evidence_sha256() + + +def test_tepp_submit_outcome_rejects_a_local_completed_envelope() -> None: + """A LineageWeave-local completed shape must not become Succeeded.""" + + class _Local(TeppClient): + def __init__(self) -> None: + super().__init__(transport=lambda _payload: persistable_tepp_seed_envelope()) + + status, failure, result = tepp_submit_outcome(_Local(), _tepp_request()) + assert status == "analysis_status_failed" + assert failure == "tepp_result_not_persisted" + assert result is None + + +def test_configured_tepp_client_stays_unavailable_without_http() -> None: + """Empty or non-http URLs keep the default dropped channel.""" + assert isinstance(configured_tepp_client(""), TeppClient) + client = configured_tepp_client("file:///tmp/tepp.json") + with pytest.raises(TeppNotAvailable): + client.submit_analysis_run(_tepp_request()) + + +def test_tepp_accepted_clocks_keep_distinct_receipt_and_row_write() -> None: + """Transport receipt and row write stay two values when they differ.""" + started = datetime(2026, 1, 12, 12, 44, tzinfo=timezone.utc) + received = datetime(2026, 1, 12, 12, 45, tzinfo=timezone.utc) + recorded = datetime(2026, 1, 12, 12, 46, tzinfo=timezone.utc) + assert tepp_accepted_clocks( + started_at=started, + received_at=received, + recorded_at=recorded, + ) == (received, recorded) + + +def test_tepp_accepted_clocks_clamp_backward_receipt_to_start() -> None: + """A receipt earlier than start is not stored as a later invention.""" + started = datetime(2026, 1, 12, 12, 45, tzinfo=timezone.utc) + earlier = datetime(2026, 1, 12, 12, 44, tzinfo=timezone.utc) + assert tepp_accepted_clocks( + started_at=started, + received_at=earlier, + recorded_at=earlier, + ) == (started, started) + + +def test_tepp_accepted_clocks_clamp_backward_row_write_to_receipt() -> None: + """A row-write earlier than receipt stays the receipt, not invented later.""" + started = datetime(2026, 1, 12, 12, 44, tzinfo=timezone.utc) + received = datetime(2026, 1, 12, 12, 45, tzinfo=timezone.utc) + earlier = datetime(2026, 1, 12, 12, 44, 30, tzinfo=timezone.utc) + assert tepp_accepted_clocks( + started_at=started, + received_at=received, + recorded_at=earlier, + ) == (received, received) + + +def test_tepp_accepted_clocks_do_not_invent_a_second_instant() -> None: + """Equal receipt and persist stay one stored instant.""" + instant = datetime(2026, 1, 12, 12, 45, tzinfo=timezone.utc) + assert tepp_accepted_clocks( + started_at=instant, + received_at=instant, + recorded_at=instant, + ) == (instant, instant) + + +def _accepted_evidence() -> TeppAcceptedEvidence: + """Published Demo Corp accepted envelope used by persist-path tests.""" + parsed = parse_tepp_accepted_evidence( + accepted_tepp_seed_envelope(idempotency_key="buyer-key"), + expected_idempotency_key="buyer-key", + ) + assert parsed is not None + return parsed + + +def test_persist_tepp_accepted_stores_two_clock_values() -> None: + """The insert binds transport receipt and row-write as distinct values.""" + received = datetime(2026, 1, 12, 12, 45, tzinfo=timezone.utc) + recorded = datetime(2026, 1, 12, 12, 46, tzinfo=timezone.utc) + + class _Conn: + def __init__(self) -> None: + self.bound: tuple[object, ...] | None = None + + async def execute(self, _sql: str, *args: object) -> str: + self.bound = args + return "INSERT 0 1" + + conn = _Conn() + stored = asyncio.run( + _persist_tepp_accepted(conn, "run-id", _accepted_evidence(), received, recorded) + ) + assert stored is True + assert conn.bound is not None + assert conn.bound[6] == received + assert conn.bound[7] == recorded + assert conn.bound[6] != conn.bound[7] + + +def test_persist_tepp_accepted_keeps_equal_clocks_equal() -> None: + """Same-instant receipt and persist are stored once each, not rewritten later.""" + instant = datetime(2026, 1, 12, 12, 45, tzinfo=timezone.utc) + + class _Conn: + def __init__(self) -> None: + self.bound: tuple[object, ...] | None = None + + async def execute(self, _sql: str, *args: object) -> str: + self.bound = args + return "INSERT 0 1" + + conn = _Conn() + stored = asyncio.run( + _persist_tepp_accepted(conn, "run-id", _accepted_evidence(), instant, instant) + ) + assert stored is True + assert conn.bound is not None + assert conn.bound[6] == instant + assert conn.bound[7] == instant + + +def test_persist_tepp_accepted_fails_closed_without_the_table() -> None: + """A missing accepted-evidence table is not success.""" + instant = datetime(2026, 1, 12, 12, 45, tzinfo=timezone.utc) + + class _Conn: + async def execute(self, _sql: str, *_args: object) -> str: + raise asyncpg.UndefinedTableError("undefined_table") + + stored = asyncio.run( + _persist_tepp_accepted(_Conn(), "run-id", _accepted_evidence(), instant, instant) + ) + assert stored is False + + +class _DeliverConn: + """In-memory start connection for TEPP persist and status append.""" + + def __init__(self, *, persist_ok: bool = True) -> None: + self.persist_ok = persist_ok + self.accepted_args: tuple[object, ...] | None = None + self.status_args: tuple[object, ...] | None = None + + async def fetchval(self, _sql: str, *_args: object) -> int: + return 2 + + async def execute(self, sql: str, *args: object) -> str: + if "analysis_run_tepp_accepted" in sql: + if not self.persist_ok: + raise asyncpg.UndefinedTableError("undefined_table") + self.accepted_args = args + return "INSERT 0 1" + if "analysis_run_status_event" in sql: + self.status_args = args + return "INSERT 0 1" + raise AssertionError(sql) + + +def _locked_tepp_row() -> dict[str, object]: + """Frozen Demo Corp TEPP start row. Never invents a theta.""" + return { + "idempotency_key": "buyer-tepp-2026-w07", + "snapshot_sha256": "ab" * 32, + "knowledge_cutoff": datetime(2026, 1, 12, 12, 0, tzinfo=timezone.utc), + "corporate_entity_id": "11111111-1111-1111-1111-111111111111", + } + + +def test_deliver_tepp_measurement_persists_distinct_clocks() -> None: + """Accepted evidence stores receipt then row-write, and stays Failed.""" + request = _tepp_request() + + class _Accepted(TeppClient): + def __init__(self) -> None: + super().__init__( + transport=lambda _payload: accepted_tepp_seed_envelope( + idempotency_key=request.idempotency_key + ) + ) + + conn = _DeliverConn() + asyncio.run( + _deliver_tepp_measurement( + conn, + analysis_run_id="run-id", + locked=_locked_tepp_row(), + tepp_client=_Accepted(), + ) + ) + assert conn.accepted_args is not None + received_at = conn.accepted_args[6] + recorded_at = conn.accepted_args[7] + assert isinstance(received_at, datetime) + assert isinstance(recorded_at, datetime) + assert received_at <= recorded_at + assert conn.status_args is not None + assert conn.status_args[2] == "analysis_status_failed" + assert conn.status_args[4] == "tepp_completed_result_unsupported" + assert conn.status_args[3] == recorded_at + + +def test_deliver_tepp_measurement_fails_closed_when_table_is_missing() -> None: + """A missing accepted table is Failed, never a fabricated measurement.""" + request = _tepp_request() + + class _Accepted(TeppClient): + def __init__(self) -> None: + super().__init__( + transport=lambda _payload: accepted_tepp_seed_envelope( + idempotency_key=request.idempotency_key + ) + ) + + conn = _DeliverConn(persist_ok=False) + asyncio.run( + _deliver_tepp_measurement( + conn, + analysis_run_id="run-id", + locked=_locked_tepp_row(), + tepp_client=_Accepted(), + ) + ) + assert conn.accepted_args is None + assert conn.status_args is not None + assert conn.status_args[2] == "analysis_status_failed" + assert conn.status_args[4] == "tepp_result_not_persisted" + + +def test_deliver_tepp_measurement_does_not_persist_a_missing_transport() -> None: + """A missing TEPP transport writes no accepted evidence row.""" + conn = _DeliverConn() + asyncio.run( + _deliver_tepp_measurement( + conn, + analysis_run_id="run-id", + locked=_locked_tepp_row(), + tepp_client=TeppClient(), + ) + ) + assert conn.accepted_args is None + assert conn.status_args is not None + assert conn.status_args[2] == "analysis_status_failed" + assert conn.status_args[4] == "tepp_not_available" + + +def test_hidden_run_start_is_not_found() -> None: + """Operators get a 404 next action, not an internal exception name.""" + error = AnalysisRunStartError(404, "This analysis run is not visible.") + assert error.status_code == 404 + assert "not visible" in error.detail + + +def test_running_restart_conflicts_and_succeeded_replay_is_documented() -> None: + """Running without pending outbox is 409. Succeeded replay is a no-op.""" + conflict = start_write_conflict_error() + assert conflict.status_code == 409 + assert "Refresh to see the stored tree" in conflict.detail + running = AnalysisRunStartError( + 409, + "Open this run. Start is only for a Pending lineage reconstruction.", + ) + assert running.status_code == 409 + assert "Pending" in running.detail diff --git a/tests/test_analysis_run_tepp_accepted_schema.py b/tests/test_analysis_run_tepp_accepted_schema.py new file mode 100644 index 00000000..18c380ce --- /dev/null +++ b/tests/test_analysis_run_tepp_accepted_schema.py @@ -0,0 +1,157 @@ +"""Static and optional PostgreSQL contracts for TEPP accepted evidence.""" + +from __future__ import annotations + +import os +import re +import uuid +from pathlib import Path +from urllib.parse import urlsplit, urlunsplit + +import pytest + +_ROOT = Path(__file__).resolve().parents[1] +_INITIAL_MIGRATION = _ROOT / "migrations" / "0001_initial_schema.sql" +_REGISTRY_MIGRATION = _ROOT / "migrations" / "0018_analysis_run_registry.sql" +_TEPP_RESULT_MIGRATION = _ROOT / "migrations" / "0028_analysis_run_tepp_result.sql" +_TEPP_ACCEPTED_MIGRATION = _ROOT / "migrations" / "0029_analysis_run_tepp_accepted.sql" +_TEPP_ACCEPTED_ROLLBACK = ( + _ROOT / "migrations" / "rollback" / "0029_analysis_run_tepp_accepted.sql" +) +_POSTGRES_IMAGE = _ROOT / "docker" / "postgres-init" / "Dockerfile" +_ADMIN_DSN = os.environ.get( + "LINEAGEWEAVE_TEST_POSTGRES_ADMIN_DSN", "postgresql://localhost/postgres" +) +_REQUIRED_TABLES = {"analysis_run_tepp_accepted"} + + +def test_tepp_accepted_migration_is_normalized_and_wired() -> None: + """Static contract: 3NF names, additive to 0028, Dockerfile copy, rollback.""" + migration = _TEPP_ACCEPTED_MIGRATION.read_text(encoding="utf-8") + rollback = _TEPP_ACCEPTED_ROLLBACK.read_text(encoding="utf-8") + dockerfile = _POSTGRES_IMAGE.read_text(encoding="utf-8") + seed = (_ROOT / "scripts" / "seed_demo_data.py").read_text(encoding="utf-8") + created_tables = set( + re.findall(r"create table if not exists\s+([a-z0-9_]+)", migration, re.I) + ) + assert _REQUIRED_TABLES <= created_tables + assert "jsonb" not in migration.casefold() + assert "metadata_payload" not in migration + assert "theta" not in migration.casefold() + assert "affiliation_count" not in migration + assert "interval_count" not in migration + assert "validated multilevel estimate" in migration + assert "0029_analysis_run_tepp_accepted.sql" in dockerfile + assert "0029_analysis_run_tepp_accepted.sql" in seed + assert seed.index("0028_analysis_run_tepp_result.sql") < seed.index( + "0029_analysis_run_tepp_accepted.sql" + ) + assert "analysis_run_tepp_accepted_not_empty" in rollback + assert "drop table if exists analysis_run_tepp_result" not in rollback + assert "reject_analysis_run_tepp_accepted_update" in migration + assert "delete from analysis_run_tepp_accepted" in migration + assert migration.index("delete from analysis_run_tepp_accepted") < ( + migration.index("delete from analysis_run_status_event") + ) + for object_name in re.findall( + r"create table if not exists\s+([a-z0-9_]+)", + migration, + re.I, + ): + assert len(object_name.split("_")) >= 2, object_name + for object_name in re.findall( + r"create or replace function\s+([a-z0-9_]+)", + migration, + ): + assert len(object_name.split("_")) >= 2, object_name + for object_name in re.findall(r"create trigger\s+([a-z0-9_]+)", migration, re.I): + assert len(object_name.split("_")) >= 2, object_name + + +def test_tepp_accepted_migration_is_idempotent_sql() -> None: + """Upgrade-safe: create if not exists and replace, never drop 0028.""" + migration = _TEPP_ACCEPTED_MIGRATION.read_text(encoding="utf-8") + assert "create table if not exists analysis_run_tepp_accepted" in migration + assert "create or replace function purge_analysis_run_registry" in migration + assert "drop table" not in migration.casefold() + assert "analysis_run_tepp_result" in migration + + +def _postgres_available() -> bool: + """Return whether the configured administrator DSN is reachable.""" + try: + import psycopg2 + + psycopg2.connect(_ADMIN_DSN, connect_timeout=2).close() + return True + except Exception: + return False + + +def _database_dsn(database_name: str) -> str: + """Replace only the database path while preserving DSN query options.""" + parsed = urlsplit(_ADMIN_DSN) + return urlunsplit(parsed._replace(path=f"/{database_name}")) + + +@pytest.fixture +def tepp_accepted_db(): + """Yield a throwaway registry plus TEPP-accepted database.""" + if not _postgres_available(): + pytest.skip("a reachable PostgreSQL administrator DSN is required") + import psycopg2 + + database_name = f"lineageweave_tepp_acc_{uuid.uuid4().hex[:12]}" + admin = psycopg2.connect(_ADMIN_DSN) + admin.autocommit = True + try: + with admin.cursor() as cursor: + cursor.execute(f'create database "{database_name}"') + finally: + admin.close() + conn = psycopg2.connect(_database_dsn(database_name)) + conn.autocommit = True + try: + with conn.cursor() as cursor: + cursor.execute(_INITIAL_MIGRATION.read_text(encoding="utf-8")) + cursor.execute(_REGISTRY_MIGRATION.read_text(encoding="utf-8")) + cursor.execute(_TEPP_RESULT_MIGRATION.read_text(encoding="utf-8")) + cursor.execute(_TEPP_ACCEPTED_MIGRATION.read_text(encoding="utf-8")) + cursor.execute(_TEPP_ACCEPTED_MIGRATION.read_text(encoding="utf-8")) + yield conn + finally: + conn.close() + admin = psycopg2.connect(_ADMIN_DSN) + admin.autocommit = True + try: + with admin.cursor() as cursor: + cursor.execute( + "select pg_terminate_backend(pid) from pg_stat_activity " + "where datname = %s and pid <> pg_backend_pid()", + (database_name,), + ) + cursor.execute(f'drop database "{database_name}"') + finally: + admin.close() + + +def test_empty_tepp_accepted_rollback_is_replayable(tepp_accepted_db) -> None: + """An empty accepted-evidence schema can be rolled back twice.""" + with tepp_accepted_db.cursor() as cursor: + cursor.execute( + "select table_name from information_schema.tables " + "where table_schema = 'public' and table_name = any(%s)", + (list(_REQUIRED_TABLES | {"analysis_run_tepp_result"}),), + ) + present = {row[0] for row in cursor.fetchall()} + assert _REQUIRED_TABLES <= present + assert "analysis_run_tepp_result" in present + cursor.execute(_TEPP_ACCEPTED_ROLLBACK.read_text(encoding="utf-8")) + cursor.execute( + "select table_name from information_schema.tables " + "where table_schema = 'public' and table_name = any(%s)", + (list(_REQUIRED_TABLES | {"analysis_run_tepp_result"}),), + ) + remaining = {row[0] for row in cursor.fetchall()} + assert remaining == {"analysis_run_tepp_result"} + cursor.execute(_TEPP_ACCEPTED_ROLLBACK.read_text(encoding="utf-8")) diff --git a/tests/test_analysis_run_tepp_result_schema.py b/tests/test_analysis_run_tepp_result_schema.py new file mode 100644 index 00000000..46ec05a4 --- /dev/null +++ b/tests/test_analysis_run_tepp_result_schema.py @@ -0,0 +1,139 @@ +"""Static and optional PostgreSQL contracts for persistable TEPP results.""" + +from __future__ import annotations + +import os +import re +import uuid +from pathlib import Path +from urllib.parse import urlsplit, urlunsplit + +import pytest + +_ROOT = Path(__file__).resolve().parents[1] +_INITIAL_MIGRATION = _ROOT / "migrations" / "0001_initial_schema.sql" +_REGISTRY_MIGRATION = _ROOT / "migrations" / "0018_analysis_run_registry.sql" +_TEPP_RESULT_MIGRATION = _ROOT / "migrations" / "0028_analysis_run_tepp_result.sql" +_TEPP_RESULT_ROLLBACK = ( + _ROOT / "migrations" / "rollback" / "0028_analysis_run_tepp_result.sql" +) +_POSTGRES_IMAGE = _ROOT / "docker" / "postgres-init" / "Dockerfile" +_ADMIN_DSN = os.environ.get( + "LINEAGEWEAVE_TEST_POSTGRES_ADMIN_DSN", "postgresql://localhost/postgres" +) +_REQUIRED_TABLES = {"analysis_run_tepp_result"} + + +def test_tepp_result_migration_is_normalized_and_wired() -> None: + """Static contract: 3NF names, no payload JSON, Dockerfile copy, rollback.""" + migration = _TEPP_RESULT_MIGRATION.read_text(encoding="utf-8") + rollback = _TEPP_RESULT_ROLLBACK.read_text(encoding="utf-8") + dockerfile = _POSTGRES_IMAGE.read_text(encoding="utf-8") + seed = (_ROOT / "scripts" / "seed_demo_data.py").read_text(encoding="utf-8") + created_tables = set( + re.findall(r"create table if not exists\s+([a-z0-9_]+)", migration, re.I) + ) + assert _REQUIRED_TABLES <= created_tables + assert "jsonb" not in migration.casefold() + assert "metadata_payload" not in migration + assert "theta" not in migration.casefold() + assert "0028_analysis_run_tepp_result.sql" in dockerfile + assert "0028_analysis_run_tepp_result.sql" in seed + assert seed.index("0027_abbreviation_tree_corroboration.sql") < seed.index( + "0028_analysis_run_tepp_result.sql" + ) + assert "analysis_run_tepp_result_not_empty" in rollback + assert "reject_analysis_run_tepp_result_update" in migration + assert "delete from analysis_run_tepp_result" in migration + assert migration.index("delete from analysis_run_tepp_result") < ( + migration.index("delete from analysis_run_status_event") + ) + for object_name in re.findall( + r"create table if not exists\s+([a-z0-9_]+)", + migration, + re.I, + ): + assert len(object_name.split("_")) >= 2, object_name + for object_name in re.findall( + r"create or replace function\s+([a-z0-9_]+)", + migration, + re.I, + ): + assert len(object_name.split("_")) >= 2, object_name + for object_name in re.findall(r"create trigger\s+([a-z0-9_]+)", migration, re.I): + assert len(object_name.split("_")) >= 2, object_name + + +def _postgres_available() -> bool: + """Return whether the configured administrator DSN is reachable.""" + try: + import psycopg2 + + psycopg2.connect(_ADMIN_DSN, connect_timeout=2).close() + return True + except Exception: + return False + + +def _database_dsn(database_name: str) -> str: + """Replace only the database path while preserving DSN query options.""" + parsed = urlsplit(_ADMIN_DSN) + return urlunsplit(parsed._replace(path=f"/{database_name}")) + + +@pytest.fixture +def tepp_result_db(): + """Yield a throwaway registry plus TEPP-result database.""" + if not _postgres_available(): + pytest.skip("a reachable PostgreSQL administrator DSN is required") + import psycopg2 + + database_name = f"lineageweave_tepp_{uuid.uuid4().hex[:12]}" + admin = psycopg2.connect(_ADMIN_DSN) + admin.autocommit = True + try: + with admin.cursor() as cursor: + cursor.execute(f'create database "{database_name}"') + finally: + admin.close() + conn = psycopg2.connect(_database_dsn(database_name)) + conn.autocommit = True + try: + with conn.cursor() as cursor: + cursor.execute(_INITIAL_MIGRATION.read_text(encoding="utf-8")) + cursor.execute(_REGISTRY_MIGRATION.read_text(encoding="utf-8")) + cursor.execute(_TEPP_RESULT_MIGRATION.read_text(encoding="utf-8")) + yield conn + finally: + conn.close() + admin = psycopg2.connect(_ADMIN_DSN) + admin.autocommit = True + try: + with admin.cursor() as cursor: + cursor.execute( + "select pg_terminate_backend(pid) from pg_stat_activity " + "where datname = %s and pid <> pg_backend_pid()", + (database_name,), + ) + cursor.execute(f'drop database "{database_name}"') + finally: + admin.close() + + +def test_empty_tepp_result_rollback_is_replayable(tepp_result_db) -> None: + """An empty TEPP-result schema can be rolled back and removed.""" + with tepp_result_db.cursor() as cursor: + cursor.execute( + "select table_name from information_schema.tables " + "where table_schema = 'public' and table_name = any(%s)", + (list(_REQUIRED_TABLES),), + ) + assert {row[0] for row in cursor.fetchall()} == _REQUIRED_TABLES + cursor.execute(_TEPP_RESULT_ROLLBACK.read_text(encoding="utf-8")) + cursor.execute( + "select table_name from information_schema.tables " + "where table_schema = 'public' and table_name = any(%s)", + (list(_REQUIRED_TABLES),), + ) + assert cursor.fetchall() == [] + cursor.execute(_TEPP_RESULT_ROLLBACK.read_text(encoding="utf-8")) diff --git a/tests/test_chunking.py b/tests/test_chunking.py index 4eab1ff6..27e47c23 100644 --- a/tests/test_chunking.py +++ b/tests/test_chunking.py @@ -107,6 +107,18 @@ def test_chunk_by_dom_skips_malformed_image_data() -> None: assert [c.unit_type for c in chunks] == ["dom"] +def test_chunk_by_dom_skips_script_and_style_images() -> None: + tiny_png_b64 = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNk+A8AAQUBAScY42YAAAAASUVORK5CYII=" + html = ( + f'' + f'' + "

Visible.

" + ) + chunks = chunk_by_dom(html) + assert [c.unit_type for c in chunks] == ["dom"] + assert chunks[0].text == "Visible." + + def test_chunk_by_conversation_turn_labels_each_chunk_with_its_sender() -> None: turns = [ ConversationTurn(sender="alice@example.com", text="Can we move the meeting?"), diff --git a/tests/test_corporate_hierarchy_inference.py b/tests/test_corporate_hierarchy_inference.py new file mode 100644 index 00000000..b8795bb0 --- /dev/null +++ b/tests/test_corporate_hierarchy_inference.py @@ -0,0 +1,71 @@ +"""Tests for lineageweave.corporate_hierarchy_inference (ADR 0010). + +Pure parse-function tests, same style as test_organization_name_resolution.py +-- the HTTP mechanics are already covered by test_http_client.py. +""" + +from __future__ import annotations + +from lineageweave.corporate_hierarchy_inference import ( + LEVEL_COMPANY, + LEVEL_GROUP, + LEVEL_PLANT, + HierarchyProposal, + parse_inference_response, +) + + +def test_parses_a_plant_with_a_parent() -> None: + content = '{"level": "plant", "parent_name": "Acme Electronics"}' + assert parse_inference_response(content) == HierarchyProposal( + level_code=LEVEL_PLANT, parent_name="Acme Electronics" + ) + + +def test_parses_a_group_with_no_parent() -> None: + content = '{"level": "group", "parent_name": null}' + assert parse_inference_response(content) == HierarchyProposal(level_code=LEVEL_GROUP, parent_name=None) + + +def test_company_level_recognized() -> None: + content = '{"level": "company", "parent_name": "Some Group"}' + result = parse_inference_response(content) + assert result is not None + assert result.level_code == LEVEL_COMPANY + + +def test_unknown_response_returns_none() -> None: + assert parse_inference_response("UNKNOWN") is None + assert parse_inference_response("unknown\n") is None + + +def test_empty_response_returns_none() -> None: + assert parse_inference_response("") is None + + +def test_malformed_json_returns_none() -> None: + assert parse_inference_response("not json at all") is None + + +def test_invalid_level_code_returns_none() -> None: + """A level outside the three valid codes must not be silently + accepted as if it were a real classification.""" + content = '{"level": "division", "parent_name": null}' + assert parse_inference_response(content) is None + + +def test_blank_parent_name_becomes_none() -> None: + content = '{"level": "company", "parent_name": " "}' + result = parse_inference_response(content) + assert result is not None + assert result.parent_name is None + + +def test_markdown_fenced_json_is_rejected_not_stripped() -> None: + """Unlike post_summary's parser, this one does not strip code + fences -- the prompt asks for raw JSON only; a fenced response + means the model did not follow instructions and should not be + silently repaired into a trusted hierarchy claim. + """ + content = '```json\n{"level": "company", "parent_name": null}\n```' + assert parse_inference_response(content) is None diff --git a/tests/test_corporate_hierarchy_resolution.py b/tests/test_corporate_hierarchy_resolution.py index 218fd4e0..72a7927c 100644 --- a/tests/test_corporate_hierarchy_resolution.py +++ b/tests/test_corporate_hierarchy_resolution.py @@ -1,18 +1,15 @@ -"""Tests for lineageweave.corporate_hierarchy_resolution, against a -synthetic hierarchy fixture structurally identical to the one already used -in tests/test_schema.py's real-database test (Acme Group -> Acme -Electronics Korea -> Acme Electronics Gwangju Plant), so the correct -resolution is known by construction: an abbreviation or trailing legal -suffix of one of these three names must resolve to it, and an unrelated -organization name must not resolve to anything. -""" +"""Tests for deterministic corporate-hierarchy candidate resolution.""" from __future__ import annotations from lineageweave.corporate_hierarchy_resolution import ( + RESOLUTION_MISS, + RESOLUTION_TIE, + RESOLUTION_UNIQUE, CorporateEntityCandidate, normalize_organization_name, resolve_corporate_entity, + score_corporate_entity, ) _CANDIDATES = [ @@ -23,11 +20,17 @@ def test_exact_name_resolves() -> None: - assert resolve_corporate_entity("Acme Electronics Korea", _CANDIDATES) == "korea-id" + assert resolve_corporate_entity( + "Acme Electronics Korea", + _CANDIDATES, + ) == "korea-id" def test_trailing_legal_suffix_still_resolves() -> None: - assert resolve_corporate_entity("Acme Electronics Korea Ltd.", _CANDIDATES) == "korea-id" + assert resolve_corporate_entity( + "Acme Electronics Korea Ltd.", + _CANDIDATES, + ) == "korea-id" def test_abbreviation_still_resolves() -> None: @@ -35,20 +38,25 @@ def test_abbreviation_still_resolves() -> None: def test_resolves_to_the_correct_sibling_not_a_different_one() -> None: - """The whole point of similarity scoring over "any partial match": - a mention close to the Gwangju plant must resolve to the plant, not - accidentally to the parent "Acme Electronics Korea" it shares most of - its name with. - """ - assert resolve_corporate_entity("Acme Gwangju Plant", _CANDIDATES) == "gwangju-id" - - -def test_unrelated_organization_does_not_resolve() -> None: - """A genuine non-match must return None, not the closest-available - guess -- a wrong hierarchy link corrupts every downstream Knowledge - Graph traversal through it. - """ - assert resolve_corporate_entity("Totally Different Company", _CANDIDATES) is None + """A plant-like mention resolves to the plant rather than its parent.""" + assert resolve_corporate_entity( + "Acme Gwangju Plant", + _CANDIDATES, + ) == "gwangju-id" + + +def test_unrelated_organization_is_a_miss() -> None: + """A below-threshold candidate set is distinct from an equal-score tie.""" + outcome = score_corporate_entity( + "Totally Different Company", + _CANDIDATES, + ) + assert outcome.kind == RESOLUTION_MISS + assert outcome.catalog_id is None + assert resolve_corporate_entity( + "Totally Different Company", + _CANDIDATES, + ) is None def test_empty_mention_does_not_resolve() -> None: @@ -60,6 +68,45 @@ def test_no_candidates_does_not_resolve() -> None: assert resolve_corporate_entity("Acme Electronics Korea", []) is None +def test_tied_same_display_name_stays_unbound() -> None: + """Distinct same-named catalog rows are a tie, not a first-wins match.""" + homonyms = [ + CorporateEntityCandidate("homonym-a", "Tied Energy"), + CorporateEntityCandidate("homonym-b", "Tied Energy"), + ] + outcome = score_corporate_entity("Tied Energy", homonyms) + assert outcome.kind == RESOLUTION_TIE + assert outcome.catalog_id is None + assert set(outcome.top_catalog_ids) == {"homonym-a", "homonym-b"} + assert resolve_corporate_entity("Tied Energy", homonyms) is None + + +def test_duplicate_snapshot_rows_for_one_catalog_id_are_unique() -> None: + """Duplicate query rows do not manufacture an identity tie.""" + duplicated = [ + CorporateEntityCandidate("korea-id", "Acme Electronics Korea"), + CorporateEntityCandidate("korea-id", "Acme Electronics Korea"), + ] + outcome = score_corporate_entity("Acme Electronics Korea", duplicated) + assert outcome.kind == RESOLUTION_UNIQUE + assert outcome.catalog_id == "korea-id" + + +def test_unique_exact_name_still_wins_beside_unrelated_homonyms() -> None: + mixed = [ + *_CANDIDATES, + CorporateEntityCandidate("homonym-a", "Tied Energy"), + CorporateEntityCandidate("homonym-b", "Tied Energy"), + ] + assert resolve_corporate_entity( + "Acme Electronics Korea", + mixed, + ) == "korea-id" + assert resolve_corporate_entity("Tied Energy", mixed) is None + + def test_normalize_strips_suffix_punctuation_and_case() -> None: - assert normalize_organization_name("Acme Electronics Korea, Ltd.") == "acme electronics korea" + assert normalize_organization_name( + "Acme Electronics Korea, Ltd." + ) == "acme electronics korea" assert normalize_organization_name(" ACME Group ") == "acme group" diff --git a/tests/test_corporate_hierarchy_resolution_branches.py b/tests/test_corporate_hierarchy_resolution_branches.py new file mode 100644 index 00000000..a2f4287f --- /dev/null +++ b/tests/test_corporate_hierarchy_resolution_branches.py @@ -0,0 +1,29 @@ +"""Branch-complete edge cases for corporate entity resolution.""" + +from lineageweave.corporate_hierarchy_resolution import ( + RESOLUTION_MISS, + CorporateEntityCandidate, + score_corporate_entity, +) + + +def test_zero_similarity_candidate_remains_a_miss() -> None: + """A zero score does not enter the tied-candidate set.""" + outcome = score_corporate_entity( + "aaa", + [CorporateEntityCandidate("bbb-id", "bbb")], + ) + assert outcome.kind == RESOLUTION_MISS + assert outcome.top_score == 0.0 + assert outcome.top_catalog_ids == () + + +def test_zero_threshold_without_candidates_is_still_a_miss() -> None: + """An empty candidate set cannot become a unique zero-score match.""" + outcome = score_corporate_entity( + "Synthetic Energy", + [], + min_similarity=0.0, + ) + assert outcome.kind == RESOLUTION_MISS + assert outcome.catalog_id is None diff --git a/tests/test_customer_group_tree.py b/tests/test_customer_group_tree.py new file mode 100644 index 00000000..852d11ad --- /dev/null +++ b/tests/test_customer_group_tree.py @@ -0,0 +1,69 @@ +"""Authorized customer-group forest: affiliated rows plus ancestors/descendants.""" + +from __future__ import annotations + +from lineageweave.customer_group_tree import ( + CatalogEntityRow, + TreeAbbreviation, + authorized_catalog_ids, + build_customer_group_forest, +) + +_ENTITIES = ( + CatalogEntityRow("group-id", None, "Demo Group", "group"), + CatalogEntityRow("corp-id", "group-id", "Demo Corp", "company"), + CatalogEntityRow("plant-id", "corp-id", "Demo Plant", "plant"), + CatalogEntityRow("other-id", None, "Other Corp", "group"), +) + + +def test_affiliated_company_includes_group_parent_and_plant_child() -> None: + needed = authorized_catalog_ids(_ENTITIES, ("corp-id",)) + assert needed == {"group-id", "corp-id", "plant-id"} + + +def test_unaffiliated_sibling_group_is_omitted() -> None: + forest = build_customer_group_forest(_ENTITIES, ("corp-id",)) + assert [node.entity_name for node in forest] == ["Demo Group"] + group = forest[0] + assert [child.entity_name for child in group.children] == ["Demo Corp"] + assert [child.entity_name for child in group.children[0].children] == ["Demo Plant"] + + +def test_unknown_affiliation_adds_no_invented_parent() -> None: + forest = build_customer_group_forest(_ENTITIES, ("missing-id",)) + assert forest == () + + +def test_broken_parent_pointer_stops_without_inventing_an_ancestor() -> None: + broken = ( + CatalogEntityRow("corp-id", "missing-parent", "Demo Corp", "company"), + ) + assert authorized_catalog_ids(broken, ("corp-id",)) == {"corp-id"} + + +def test_already_included_descendant_is_not_walked_twice() -> None: + needed = authorized_catalog_ids(_ENTITIES, ("plant-id", "group-id")) + assert needed == {"group-id", "corp-id", "plant-id"} + + +def test_corroborated_abbreviation_attaches_only_to_authorized_nodes() -> None: + forest = build_customer_group_forest( + _ENTITIES, + ("corp-id",), + ( + ( + "corp-id", + TreeAbbreviation("DC", "verify_corroborated", "https://example.test/demo-corp-dc"), + ), + ( + "other-id", + TreeAbbreviation("OC", "verify_corroborated", "https://example.test/other"), + ), + ), + ) + company = forest[0].children[0] + assert [alias.raw_organization_name for alias in company.abbreviations] == ["DC"] + assert forest[0].to_dict()["children"][0]["abbreviations"][0]["raw_organization_name"] == "DC" + serialized = forest[0].to_dict() + assert "Other Corp" not in str(serialized) diff --git a/tests/test_customer_group_tree_labels.py b/tests/test_customer_group_tree_labels.py new file mode 100644 index 00000000..592496de --- /dev/null +++ b/tests/test_customer_group_tree_labels.py @@ -0,0 +1,36 @@ +"""Lookup-label helpers for the customer-group JSON forest.""" + +from __future__ import annotations + +from backend.app.customer_group_tree_ingestion import _apply_lookup_labels, _collect_level_codes + + +def test_collect_level_codes_walks_nested_children() -> None: + codes = _collect_level_codes( + [ + { + "entity_level_code": "group", + "children": [ + { + "entity_level_code": "company", + "children": [{"entity_level_code": "plant", "children": []}], + } + ], + } + ] + ) + assert codes == ["group", "company", "plant"] + + +def test_apply_lookup_labels_falls_back_to_the_code() -> None: + forest = [ + { + "entity_level_code": "group", + "children": [{"entity_level_code": "company", "children": []}], + }, + {"entity_level_code": None, "children": []}, + ] + _apply_lookup_labels(forest, {"group": "Group"}) + assert forest[0]["entity_level_label"] == "Group" + assert forest[0]["children"][0]["entity_level_label"] == "company" + assert forest[1]["entity_level_label"] is None diff --git a/tests/test_documentation_hygiene.py b/tests/test_documentation_hygiene.py new file mode 100644 index 00000000..53a93cb2 --- /dev/null +++ b/tests/test_documentation_hygiene.py @@ -0,0 +1,100 @@ +"""Permanent hygiene checks for committed architecture-decision records.""" + +from __future__ import annotations + +import re +from collections import Counter +from pathlib import Path + +_ROOT = Path(__file__).resolve().parents[1] +_ADR_DIRECTORY = _ROOT / "docs" / "adr" +_ROLE_CATALOG_COLUMNS = ( + "cataloged_team_id", + "cataloged_corporate_entity_id", + "cataloged_person_id", +) +_ADR_NAME = re.compile(r"^(?P[0-9]{4})-.+\.md$") +_FORBIDDEN_MARKERS = ( + "PLACEHOLDER_DO_NOT_WRITE", + "TODO_WRITE_ADR", +) + + +def test_adr_numbers_are_unique_and_documents_are_not_placeholders() -> None: + """Every committed ADR number identifies one substantive UTF-8 document.""" + paths = sorted(_ADR_DIRECTORY.glob("*.md")) + assert paths, "the repository must contain architecture-decision records" + + numbered_paths: list[tuple[str, Path]] = [] + for path in paths: + match = _ADR_NAME.fullmatch(path.name) + assert match is not None, f"ADR filename is not numbered: {path.name}" + numbered_paths.append((match.group("number"), path)) + + content = path.read_text(encoding="utf-8") + assert content.strip(), f"ADR is empty: {path.relative_to(_ROOT)}" + for marker in _FORBIDDEN_MARKERS: + assert marker not in content, ( + f"ADR contains forbidden placeholder {marker!r}: " + f"{path.relative_to(_ROOT)}" + ) + + counts = Counter(number for number, _ in numbered_paths) + duplicates = sorted(number for number, count in counts.items() if count > 1) + assert duplicates == [], f"duplicate ADR numbers: {duplicates}" + + +def test_fetch_persisted_summary_reads_stored_catalog_ids() -> None: + """ADR 0019 / 0027: fetch must not rejoin the catalog by a non-unique name.""" + + source = (_ROOT / "backend" / "app" / "post_summary_ingestion.py").read_text( + encoding="utf-8" + ) + assert "org.entity_name = role.actor_name" not in source + assert "role.cataloged_team_id" in source + assert "role.cataloged_corporate_entity_id" in source + assert "role.cataloged_person_id" in source + assert "order by created_at, person_id limit 1" in source + + +def test_role_catalog_identity_migration_is_wired() -> None: + """Fresh stacks and seed must apply the catalog-identity columns.""" + + dockerfile = (_ROOT / "docker" / "postgres-init" / "Dockerfile").read_text( + encoding="utf-8" + ) + seed = (_ROOT / "scripts" / "seed_demo_data.py").read_text(encoding="utf-8") + migration_0019 = (_ROOT / "migrations" / "0019_role_catalog_identity.sql").read_text( + encoding="utf-8" + ) + migration_0025 = ( + _ROOT / "migrations" / "0025_role_person_catalog_identity.sql" + ).read_text(encoding="utf-8") + assert "0019_role_catalog_identity.sql" in dockerfile + assert "0025_role_person_catalog_identity.sql" in dockerfile + assert "0026_report_leftover_pair.sql" in dockerfile + assert "0027_abbreviation_tree_corroboration.sql" in dockerfile + assert "0019_role_catalog_identity.sql" in seed + assert "0025_role_person_catalog_identity.sql" in seed + assert "0026_report_leftover_pair.sql" in seed + assert "0027_abbreviation_tree_corroboration.sql" in seed + assert seed.index("0024_source_post_revision.sql") < seed.index( + "0025_role_person_catalog_identity.sql" + ) + assert seed.index("0025_role_person_catalog_identity.sql") < seed.index( + "0026_report_leftover_pair.sql" + ) + assert seed.index("0026_report_leftover_pair.sql") < seed.index( + "0027_abbreviation_tree_corroboration.sql" + ) + assert "cataloged_person_id" in seed + assert "order by created_at, person_id limit 1" in seed + assert "cataloged_team_id" in migration_0019 + assert "cataloged_corporate_entity_id" in migration_0019 + assert "cataloged_person_id" in migration_0025 + for column_name in _ROLE_CATALOG_COLUMNS: + assert len(column_name.split("_")) >= 2 + assert "having count(*) = 1" in migration_0019 + assert "having count(*) = 1" in migration_0025 + assert "distinct on" not in migration_0019.lower() + assert "distinct on" not in migration_0025.lower() diff --git a/tests/test_embedded_image_payload.py b/tests/test_embedded_image_payload.py new file mode 100644 index 00000000..f5756613 --- /dev/null +++ b/tests/test_embedded_image_payload.py @@ -0,0 +1,109 @@ +from __future__ import annotations + +import base64 + +from lineageweave.embedded_image_payload import ( + decode_data_uri_image, + looks_like_raster_image, + source_offset, +) + +_TINY_PNG_B64 = ( + "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNk+A8AAQUBAScY42YAAAAASUVORK5CYII=" +) +_TINY_PNG = base64.b64decode(_TINY_PNG_B64) +_JPEG_BYTES = b"\xff\xd8\xff\x00" +_GIF87_BYTES = b"GIF87a" + b"\x00" * 2 +_GIF89_BYTES = b"GIF89a" + b"\x00" * 2 +_WEBP_BYTES = b"RIFF\x00\x00\x00\x00WEBP" +_AVIF_BYTES = b"\x00\x00\x00\x00ftypavif\x00\x00\x00\x00" +_AVIS_BYTES = b"\x00\x00\x00\x00ftypavis\x00\x00\x00\x00" +_MIF1_BYTES = b"\x00\x00\x00\x00ftypmif1\x00\x00\x00\x00" + + +def test_looks_like_raster_image_accepts_png_signature() -> None: + assert looks_like_raster_image("image/png", _TINY_PNG) is True + + +def test_looks_like_raster_image_rejects_ascii_labeled_as_png() -> None: + assert looks_like_raster_image("image/png", b"Hello") is False + + +def test_looks_like_raster_image_rejects_empty_payload() -> None: + assert looks_like_raster_image("image/png", b"") is False + + +def test_looks_like_raster_image_accepts_jpeg_gif_webp_avif_signatures() -> None: + assert looks_like_raster_image("image/jpeg", _JPEG_BYTES) is True + assert looks_like_raster_image("image/jpg", _JPEG_BYTES) is True + assert looks_like_raster_image("image/gif", _GIF87_BYTES) is True + assert looks_like_raster_image("image/gif", _GIF89_BYTES) is True + assert looks_like_raster_image("image/webp", _WEBP_BYTES) is True + assert looks_like_raster_image("image/avif", _AVIF_BYTES) is True + assert looks_like_raster_image("image/avif", _AVIS_BYTES) is True + assert looks_like_raster_image("image/avif", _MIF1_BYTES) is True + + +def test_looks_like_raster_image_rejects_wrong_magic_and_unknown_type() -> None: + assert looks_like_raster_image("image/jpeg", b"not-a-jpeg") is False + assert looks_like_raster_image("image/gif", b"GIF8xa") is False + assert looks_like_raster_image("image/webp", b"RIFF....NOTW") is False + assert looks_like_raster_image("image/webp", b"RIFF") is False + assert looks_like_raster_image("image/avif", b"xxxxftypxxxx") is False + assert looks_like_raster_image("image/avif", b"short") is False + assert looks_like_raster_image("image/svg+xml", _TINY_PNG) is False + + +def test_decode_data_uri_image_rejects_svg_and_remote_src() -> None: + assert decode_data_uri_image("https://example.test/invoice.png") is None + assert ( + decode_data_uri_image( + "data:image/svg+xml;base64,PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciPjwvc3ZnPg==" + ) + is None + ) + + +def test_decode_data_uri_image_rejects_missing_comma_or_base64_marker() -> None: + assert decode_data_uri_image("data:image/png;base64") is None + assert decode_data_uri_image(f"data:image/png,{_TINY_PNG_B64}") is None + + +def test_decode_data_uri_image_rejects_unpadded_and_wrong_magic() -> None: + assert decode_data_uri_image("data:image/png;base64,YQ") is None + assert decode_data_uri_image("data:image/png;base64,AAAA") is None + + +def test_decode_data_uri_image_accepts_newlines_inside_png_payload() -> None: + wrapped = f"data:image/png;base64,{_TINY_PNG_B64[:24]}\n{_TINY_PNG_B64[24:]}" + decoded = decode_data_uri_image(wrapped) + assert decoded == ("image/png", _TINY_PNG) + + +def test_decode_data_uri_image_accepts_jpeg_alias() -> None: + encoded = base64.b64encode(_JPEG_BYTES).decode("ascii") + assert decode_data_uri_image(f"data:image/jpg;base64,{encoded}") == ( + "image/jpg", + _JPEG_BYTES, + ) + + +def test_decode_data_uri_image_accepts_gif_webp_and_avif() -> None: + for mime_type, payload in ( + ("image/gif", _GIF89_BYTES), + ("image/webp", _WEBP_BYTES), + ("image/avif", _AVIF_BYTES), + ): + encoded = base64.b64encode(payload).decode("ascii") + assert decode_data_uri_image(f"data:{mime_type};base64,{encoded}") == ( + mime_type, + payload, + ) + + +def test_source_offset_maps_htmlparser_getpos() -> None: + source = "ab\ncd" + assert source_offset(source, 1, 0) == 0 + assert source_offset(source, 2, 1) == 4 + assert source_offset(source, 0, 0) == 0 + assert source_offset(source, 9, 0) == len(source) diff --git a/tests/test_image_content.py b/tests/test_image_content.py index 572909d6..8015c1b6 100644 --- a/tests/test_image_content.py +++ b/tests/test_image_content.py @@ -1,9 +1,12 @@ from __future__ import annotations import base64 +from pathlib import Path import pytest +from lineageweave.chunking import chunk_by_dom +from lineageweave.embedded_image_payload import decode_data_uri_image from lineageweave.image_content import ( ImageContentClient, ImageDescriptionParseError, @@ -55,6 +58,57 @@ def test_extract_base64_images_empty_document_yields_no_images() -> None: assert extract_base64_images("

No images here.

") == [] +def test_extract_base64_images_skips_svg_and_unpadded_payloads() -> None: + svg = ( + '' + ) + assert extract_base64_images(svg) == [] + assert extract_base64_images('') == [] + assert decode_data_uri_image(f"data:image/png;base64,{_TINY_PNG_B64}") is not None + + +def test_extract_base64_images_skips_style_script_and_src_less_tags() -> None: + html = ( + f'' + f'' + "" + f'' + ) + images = extract_base64_images(html) + assert len(images) == 1 + assert images[0].data == base64.b64decode(_TINY_PNG_B64) + + +def test_invoice_fixture_is_one_visible_png_for_every_extractor() -> None: + """Outlook-style invoice HTML must not resurrect the base64 wall. + + The same file is read by the TypeScript popup splitter. All three + extractors must see one raster PNG, ignore the commented copy, the + remote URL, the SVG, and the CSS background, and keep surrounding + sentences readable. + """ + html = (Path(__file__).parent / "fixtures" / "synthetic_invoice_embedded_image.html").read_text( + encoding="utf-8" + ) + images = extract_base64_images(html) + chunks = chunk_by_dom(html) + image_chunks = [chunk for chunk in chunks if chunk.unit_type == "image"] + text = " ".join(chunk.text for chunk in chunks if chunk.unit_type == "dom") + + assert len(images) == 1 + assert len(image_chunks) == 1 + assert images[0].mime_type == "image/png" + assert images[0].data == base64.b64decode(_TINY_PNG_B64) + assert images[0].position == html.find(")
+    assert None: content = "TEXT: Quarterly Budget Report\nCAPTION: A printed report cover page.\nTAGS: document, report, text" description = _parse_description(content) @@ -82,6 +136,56 @@ def test_parse_description_preserves_multiline_ocr_text() -> None: assert description.caption == "A scanned page." +def test_parse_description_tolerates_markdown_emphasis_on_labels() -> None: + """Synthetic provider drift may bold labels without changing content.""" + content = "**TEXT:** LT7\n**CAPTION:** A close-up of a component.\n**TAGS:** component, close-up" + description = _parse_description(content) + assert description.extracted_text == "LT7" + assert description.caption == "A close-up of a component." + assert description.tags == ("component", "close-up") + + +def test_parse_description_strips_balanced_markdown_emphasis_from_values() -> None: + content = "TEXT: **LT7**\nCAPTION: _A synthetic component._\nTAGS: `component`, close-up" + description = _parse_description(content) + assert description.extracted_text == "LT7" + assert description.caption == "A synthetic component." + assert description.tags == ("component", "close-up") + + +def test_parse_description_tolerates_reordered_labels() -> None: + content = "CAPTION: A blue sky.\nTEXT: NONE\nTAGS: sky" + description = _parse_description(content) + assert description.caption == "A blue sky." + assert description.extracted_text == "" + assert description.tags == ("sky",) + + +def test_parse_description_missing_tags_still_recovers_text_and_caption() -> None: + """Missing optional tags must not discard provided TEXT/CAPTION fields.""" + content = "TEXT: Quarterly Budget Report\nCAPTION: A printed report cover page." + description = _parse_description(content) + assert description.extracted_text == "Quarterly Budget Report" + assert description.caption == "A printed report cover page." + assert description.tags == () + + +def test_parse_description_leading_commentary_before_labels_is_ignored() -> None: + content = "Sure, here is the analysis:\n\nTEXT: LT7\nCAPTION: A component.\nTAGS: component" + description = _parse_description(content) + assert description.extracted_text == "LT7" + + +def test_parse_description_does_not_absorb_trailing_commentary() -> None: + content = ( + "TEXT: NONE\nCAPTION: A turbine diagram.\nTAGS: turbine, diagram\n" + "Let me know if you need more detail." + ) + description = _parse_description(content) + assert description.caption == "A turbine diagram." + assert description.tags == ("turbine", "diagram") + + def test_vision_client_rejects_non_http_url_schemes() -> None: with pytest.raises(ValueError, match="unsupported vision client URL scheme: file"): OpenAiCompatibleVisionClient( @@ -146,3 +250,11 @@ def test_image_content_client_protocol_stub_raises() -> None: """ with pytest.raises(NotImplementedError): ImageContentClient.describe(None, b"", "image/png") # type: ignore[arg-type] + + +def test_parse_description_does_not_absorb_unknown_labels_into_tags() -> None: + parsed = _parse_description( + "TEXT: NONE\nCAPTION: A turbine diagram\n" + "TAGS: turbine, diagram\nNOTE: synthetic" + ) + assert parsed.tags == ("turbine", "diagram") diff --git a/tests/test_ingestion_transaction_contracts.py b/tests/test_ingestion_transaction_contracts.py new file mode 100644 index 00000000..55a94ab4 --- /dev/null +++ b/tests/test_ingestion_transaction_contracts.py @@ -0,0 +1,706 @@ +"""Regression contracts for ingestion transactions and review documentation.""" + +from __future__ import annotations + +import asyncio +import uuid +from pathlib import Path +from types import SimpleNamespace +from typing import Any + +from backend.app import corporate_entity_ingestion as corporate_ingestion +from backend.app import keyman_ingestion +from backend.app import post_summary_ingestion as summary_ingestion +from lineageweave.corporate_hierarchy_inference import HierarchyProposal +from lineageweave.keyman_extraction import OUR_SIDE, PersonMention +from lineageweave.knowledge_graph import NODE_PERSON +from lineageweave.post_summary import ( + ACTOR_TYPE_ORGANIZATION, + ACTOR_TYPE_PERSON, + ACTOR_TYPE_TEAM, + PostSummary, + RoleResponsibility, +) +from lineageweave.relation_verification import STATUS_CORROBORATED + + +class _RecordedTransaction: + """Record transaction entry and exit for one fake asyncpg connection.""" + + def __init__(self, events: list[Any], owner: Any | None = None) -> None: + self._events = events + self._owner = owner + + async def __aenter__(self) -> "_RecordedTransaction": + if self._owner is not None: + assert not self._owner.in_transaction + self._owner.in_transaction = True + self._events.append("transaction:enter") + return self + + async def __aexit__(self, exc_type, exc, traceback) -> bool: + self._events.append("transaction:exit") + if self._owner is not None: + self._owner.in_transaction = False + return False + + +class _InferenceClient: + """Return one verified root-company proposal without network access.""" + + available = True + + def __init__(self, events: list[Any]) -> None: + self._events = events + + def infer(self, organization_name: str, context_text: str) -> HierarchyProposal: + self._events.append("inference") + return HierarchyProposal(level_code="company", parent_name=None) + + +class _VerificationClient: + """Corroborate the synthetic proposal while recording call order.""" + + available = True + + def __init__(self, events: list[Any]) -> None: + self._events = events + + def verify(self, subject: str, relation: str) -> SimpleNamespace: + self._events.append("verification") + return SimpleNamespace(status_code=STATUS_CORROBORATED) + + +class _CorporateConnection: + """Minimal asyncpg-compatible connection for creation-lock behavior.""" + + def __init__( + self, + events: list[Any], + *, + reloaded_rows: tuple[dict[str, Any], ...] = (), + inserted_id: uuid.UUID | None = None, + allow_insert: bool = True, + ) -> None: + self._events = events + self._reloaded_rows = reloaded_rows + self._inserted_id = inserted_id or uuid.uuid4() + self._allow_insert = allow_insert + + def transaction(self) -> _RecordedTransaction: + self._events.append("transaction:open") + return _RecordedTransaction(self._events) + + async def execute(self, query: str, *args: Any) -> str: + compact = " ".join(query.split()) + assert "pg_advisory_xact_lock" in compact + self._events.append(("creation_lock", args, compact)) + return "SELECT 1" + + async def fetch(self, query: str, *args: Any) -> list[dict[str, Any]]: + compact = " ".join(query.split()) + assert compact == "select corporate_entity_id, entity_name from corporate_entity" + self._events.append("candidate_reload") + return list(self._reloaded_rows) + + async def fetchrow(self, query: str, *args: Any) -> dict[str, Any]: + assert self._allow_insert, "locked candidate recheck should avoid insertion" + compact = " ".join(query.split()) + assert compact.startswith("insert into corporate_entity") + self._events.append("entity_insert") + return {"corporate_entity_id": self._inserted_id} + + +def test_corporate_entity_creation_locks_rechecks_and_inserts_in_one_transaction() -> None: + """Network verification precedes one transaction-scoped global creation lock.""" + events: list[Any] = [] + inserted_id = uuid.uuid4() + connection = _CorporateConnection(events, inserted_id=inserted_id) + candidates: list[Any] = [] + + result = asyncio.run( + corporate_ingestion.get_or_create_corporate_entity( + connection, + "Synthetic Energy", + "Synthetic context", + _InferenceClient(events), + _VerificationClient(events), + candidates, + ) + ) + + assert result == str(inserted_id) + assert [candidate.entity_name for candidate in candidates] == ["Synthetic Energy"] + assert events[:4] == [ + "inference", + "verification", + "transaction:open", + "transaction:enter", + ] + lock_event = events[4] + assert lock_event[0] == "creation_lock" + assert lock_event[1] == ("lineageweave:corporate_entity_creation",) + assert events[5:] == ["candidate_reload", "entity_insert", "transaction:exit"] + + +def test_locked_candidate_recheck_reuses_concurrently_created_entity() -> None: + """A same-name row committed after inference wins over a duplicate insert.""" + events: list[Any] = [] + existing_id = uuid.uuid4() + connection = _CorporateConnection( + events, + reloaded_rows=( + { + "corporate_entity_id": existing_id, + "entity_name": "Synthetic Energy", + }, + ), + allow_insert=False, + ) + candidates: list[Any] = [] + + result = asyncio.run( + corporate_ingestion.get_or_create_corporate_entity( + connection, + "Synthetic Energy", + "Synthetic context", + _InferenceClient(events), + _VerificationClient(events), + candidates, + ) + ) + + assert result == str(existing_id) + assert [candidate.entity_name for candidate in candidates] == ["Synthetic Energy"] + assert "entity_insert" not in events + assert events[-1] == "transaction:exit" + + +class _SummaryConnection: + """Minimal connection that records every post-summary database operation.""" + + def __init__(self, events: list[Any]) -> None: + self._events = events + self.in_transaction = False + + def transaction(self) -> _RecordedTransaction: + self._events.append("transaction:open") + return _RecordedTransaction(self._events, self) + + async def execute(self, query: str, *args: Any) -> str: + assert self.in_transaction + compact = " ".join(query.split()) + self._events.append(("execute", compact)) + return "OK" + + async def fetchrow(self, query: str, *args: Any) -> dict[str, Any] | None: + compact = " ".join(query.split()) + self._events.append(("fetchrow", compact)) + if compact.startswith("select korean_summary from post_summary_result"): + assert not self.in_transaction + return {"korean_summary": "합성 요약"} + if compact.startswith("select person_id from cataloged_person"): + assert self.in_transaction + return None + raise AssertionError(f"unexpected fetchrow query: {compact}") + + async def fetch(self, query: str, *args: Any) -> list[dict[str, Any]]: + compact = " ".join(query.split()) + self._events.append(("fetch", compact)) + assert not self.in_transaction + if "from post_summary_event" in compact: + return [{"event_text": "검토 완료"}] + if "from post_summary_role" in compact: + assert "entity_name" not in compact + assert "cataloged_corporate_entity_id" in compact + return [ + { + "actor_name": "Synthetic Design Team", + "responsibility": "도면 검토", + "actor_type_code": ACTOR_TYPE_TEAM, + "affiliated_organization_name": "Synthetic Energy", + "cataloged_team_id": None, + "cataloged_corporate_entity_id": None, + "cataloged_person_id": None, + } + ] + raise AssertionError(f"unexpected fetch query: {compact}") + + +def test_post_summary_replacement_mentions_and_edges_share_one_transaction(monkeypatch) -> None: + """Deletion, replacement, mention regeneration, and edges commit atomically.""" + events: list[Any] = [] + connection = _SummaryConnection(events) + team_id = str(uuid.uuid4()) + + async def load_candidates(conn) -> list[Any]: + events.append("candidate_load") + return [] + + async def upsert_team(conn, team_name, organization_name, candidates) -> str: + assert conn.in_transaction + events.append("team_upsert") + return team_id + + async def persist_edges(conn, post_id) -> list[Any]: + assert conn.in_transaction + events.append("edge_persist") + return [] + + monkeypatch.setattr(summary_ingestion, "_load_corporate_entity_candidates", load_candidates) + monkeypatch.setattr(summary_ingestion, "upsert_team", upsert_team) + monkeypatch.setattr(summary_ingestion, "persist_edges_for_post", persist_edges) + + summary = PostSummary( + korean_summary="합성 요약", + key_events=("검토 완료",), + roles_and_responsibilities=( + RoleResponsibility( + actor_name="Synthetic Design Team", + responsibility="도면 검토", + actor_type_code=ACTOR_TYPE_TEAM, + affiliated_organization_name="Synthetic Energy", + ), + ), + ) + + payload = asyncio.run( + summary_ingestion.persist_post_summary( + connection, + str(uuid.uuid4()), + summary, + ) + ) + + enter_index = events.index("transaction:enter") + exit_index = events.index("transaction:exit") + required_sql = ( + "delete from post_summary_person_mention", + "delete from post_team_mention", + "delete from post_organization_mention", + "delete from post_summary_result", + "insert into post_summary_result", + "insert into post_summary_event", + "insert into post_summary_role", + "insert into post_team_mention", + ) + for fragment in required_sql: + operation_index = next( + index + for index, event in enumerate(events) + if isinstance(event, tuple) + and event[0] == "execute" + and fragment in event[1] + ) + assert enter_index < operation_index < exit_index + assert events.index("candidate_load") < enter_index + assert enter_index < events.index("team_upsert") < exit_index + assert enter_index < events.index("edge_persist") < exit_index + assert payload["korean_summary"] == "합성 요약" + + +def test_organization_enrichment_finishes_before_summary_transaction(monkeypatch) -> None: + """LLM verification and the advisory-lock transaction precede summary writes.""" + events: list[Any] = [] + connection = _SummaryConnection(events) + corporate_entity_id = str(uuid.uuid4()) + + async def load_candidates(conn) -> list[Any]: + events.append(("candidate_load", conn.in_transaction)) + return [] + + async def resolve_organization( + conn, + organization_name, + context_text, + inference_client, + verification_client, + candidates, + ) -> str: + events.append(("organization_resolve", conn.in_transaction)) + assert not conn.in_transaction + return corporate_entity_id + + async def persist_edges(conn, post_id) -> list[Any]: + assert conn.in_transaction + events.append("edge_persist") + return [] + + monkeypatch.setattr(summary_ingestion, "_load_corporate_entity_candidates", load_candidates) + monkeypatch.setattr(summary_ingestion, "get_or_create_corporate_entity", resolve_organization) + monkeypatch.setattr(summary_ingestion, "persist_edges_for_post", persist_edges) + + summary = PostSummary( + korean_summary="합성 요약", + roles_and_responsibilities=( + RoleResponsibility( + actor_name="Synthetic Energy", + responsibility="납품 일정 확정", + actor_type_code=ACTOR_TYPE_ORGANIZATION, + ), + ), + ) + + asyncio.run( + summary_ingestion.persist_post_summary( + connection, + str(uuid.uuid4()), + summary, + ) + ) + + assert ("candidate_load", False) in events + assert ("organization_resolve", False) in events + resolve_index = events.index(("organization_resolve", False)) + enter_index = events.index("transaction:enter") + exit_index = events.index("transaction:exit") + mention_index = next( + index + for index, event in enumerate(events) + if isinstance(event, tuple) + and event[0] == "execute" + and "insert into post_organization_mention" in event[1] + ) + role_insert = next( + event[1] + for event in events + if isinstance(event, tuple) + and event[0] == "execute" + and "insert into post_summary_role" in event[1] + ) + assert "cataloged_corporate_entity_id" in role_insert + assert "cataloged_person_id" in role_insert + assert resolve_index < enter_index < mention_index < exit_index + + +class _KeymanConnection: + """Record whether organization enrichment runs outside the write transaction.""" + + def __init__(self, events: list[Any]) -> None: + self._events = events + self.in_transaction = False + + def transaction(self) -> _RecordedTransaction: + self._events.append("transaction:open") + return _RecordedTransaction(self._events, self) + + async def execute(self, query: str, *args: Any) -> str: + assert self.in_transaction + compact = " ".join(query.split()) + self._events.append(("execute", compact)) + return "OK" + + async def fetch(self, query: str, *args: Any) -> list[dict[str, Any]]: + compact = " ".join(query.split()) + self._events.append(("fetch", compact)) + if compact == "select corporate_entity_id, entity_name from corporate_entity": + assert not self.in_transaction + return [] + if compact.startswith("select person_id, last_known_job_title"): + assert self.in_transaction + return [] + raise AssertionError(f"unexpected fetch query: {compact}") + + async def fetchrow(self, query: str, *args: Any) -> dict[str, Any] | None: + assert self.in_transaction + compact = " ".join(query.split()) + self._events.append(("fetchrow", compact)) + if compact.startswith("select person_id, last_known_job_title"): + return None + if compact.startswith("insert into cataloged_person"): + return {"person_id": uuid.uuid4()} + raise AssertionError(f"unexpected fetchrow query: {compact}") + + +def test_keyman_organization_enrichment_finishes_before_write_transaction(monkeypatch) -> None: + """LLM resolution and hierarchy creation must not hold the Keyman write lock.""" + events: list[Any] = [] + connection = _KeymanConnection(events) + corporate_entity_id = str(uuid.uuid4()) + + async def resolve_name(conn, resolution_client, verification_client, organization_name, post_body) -> str: + events.append(("organization_resolve", conn.in_transaction)) + assert not conn.in_transaction + return "Aurora Grid Power" + + async def resolve_organization( + conn, + organization_name, + context_text, + inference_client, + verification_client, + candidates, + ) -> str: + events.append(("organization_create", conn.in_transaction)) + assert not conn.in_transaction + return corporate_entity_id + + class _Client: + available = True + + def extract(self, post_title: str, post_body: str) -> list[PersonMention]: + return [ + PersonMention( + "Ada West", + OUR_SIDE, + affiliated_organization_names=("AGP",), + ) + ] + + monkeypatch.setattr(keyman_ingestion, "resolve_organization_name", resolve_name) + monkeypatch.setattr(keyman_ingestion, "get_or_create_corporate_entity", resolve_organization) + + asyncio.run( + keyman_ingestion.ingest_post_keymen( + connection, + _Client(), + str(uuid.uuid4()), + "Synthetic post", + "Ada West at AGP followed up.", + persist_graph=False, + ) + ) + + assert ("organization_resolve", False) in events + assert ("organization_create", False) in events + resolve_index = events.index(("organization_resolve", False)) + create_index = events.index(("organization_create", False)) + enter_index = events.index("transaction:enter") + mention_index = next( + index + for index, event in enumerate(events) + if isinstance(event, tuple) + and event[0] == "execute" + and "insert into post_person_mention" in event[1] + ) + assert resolve_index < enter_index + assert create_index < enter_index < mention_index + + +def test_release_notes_describe_balanced_outer_emphasis_stripping() -> None: + """Release notes must match the parser's reviewed normalization contract.""" + content = (Path(__file__).resolve().parents[1] / "CHANGELOG.md").read_text( + encoding="utf-8" + ) + assert "strips balanced outer Markdown emphasis from field values" in content + assert "while still accepting emphasized field labels" in content + assert "preserves Markdown emphasis in field values" not in content + + +def test_fetch_persisted_summary_returns_stored_person_catalog_id() -> None: + """A persisted person role keeps catalog_node_id for the chip button.""" + + person_id = str(uuid.uuid4()) + events: list[Any] = [] + + class _PersonFetchConnection: + """Return one stored person catalog id without a live database.""" + + async def fetchrow(self, query: str, *args: Any) -> dict[str, Any] | None: + compact = " ".join(query.split()) + events.append(("fetchrow", compact)) + if compact.startswith("select korean_summary from post_summary_result"): + return {"korean_summary": "합성 요약"} + raise AssertionError(f"unexpected fetchrow query: {compact}") + + async def fetch(self, query: str, *args: Any) -> list[dict[str, Any]]: + compact = " ".join(query.split()) + events.append(("fetch", compact)) + if "from post_summary_event" in compact: + return [] + if "from post_summary_role" in compact: + assert "cataloged_person_id" in compact + return [ + { + "actor_name": "Priya Nair", + "responsibility": "고객 측 수신", + "actor_type_code": ACTOR_TYPE_PERSON, + "affiliated_organization_name": "Northridge Grid", + "cataloged_team_id": None, + "cataloged_corporate_entity_id": None, + "cataloged_person_id": person_id, + } + ] + raise AssertionError(f"unexpected fetch query: {compact}") + + payload = asyncio.run( + summary_ingestion.fetch_persisted_summary(_PersonFetchConnection(), str(uuid.uuid4())) + ) + assert payload is not None + role = payload["roles_and_responsibilities"][0] + assert role["catalog_node_id"] == person_id + assert role["catalog_node_type_code"] == NODE_PERSON + assert role["actor_name"] == "Priya Nair" + + +class _PersonPersistConnection(_SummaryConnection): + """Resolve one existing catalog person during the write transaction.""" + + def __init__(self, events: list[Any], person_id: str) -> None: + super().__init__(events) + self._person_id = person_id + + async def fetchrow(self, query: str, *args: Any) -> dict[str, Any] | None: + compact = " ".join(query.split()) + if compact.startswith("select person_id from cataloged_person"): + assert "order by created_at, person_id limit 1" in compact + assert self.in_transaction + self._events.append(("fetchrow", compact)) + return {"person_id": self._person_id} + return await super().fetchrow(query, *args) + + +def test_persist_stores_earliest_person_catalog_id(monkeypatch) -> None: + """Write-time person lookup stores the catalog id on the role row.""" + + events: list[Any] = [] + person_id = str(uuid.uuid4()) + connection = _PersonPersistConnection(events, person_id) + + async def load_candidates(conn) -> list[Any]: + return [] + + async def persist_edges(conn, post_id) -> list[Any]: + return [] + + monkeypatch.setattr(summary_ingestion, "_load_corporate_entity_candidates", load_candidates) + monkeypatch.setattr(summary_ingestion, "persist_edges_for_post", persist_edges) + + payload = asyncio.run( + summary_ingestion.persist_post_summary( + connection, + str(uuid.uuid4()), + PostSummary( + korean_summary="합성 요약", + roles_and_responsibilities=( + RoleResponsibility( + actor_name="Priya Nair", + responsibility="고객 측 수신", + actor_type_code=ACTOR_TYPE_PERSON, + ), + ), + ), + ) + ) + role_insert = next( + event[1] + for event in events + if isinstance(event, tuple) + and event[0] == "execute" + and "insert into post_summary_role" in event[1] + ) + mention_insert = next( + event[1] + for event in events + if isinstance(event, tuple) + and event[0] == "execute" + and "insert into post_summary_person_mention" in event[1] + ) + assert "cataloged_person_id" in role_insert + assert "post_summary_person_mention" in mention_insert + assert payload["korean_summary"] == "합성 요약" + + +def test_persist_leaves_uncataloged_person_unbound(monkeypatch) -> None: + """A person name with no catalog row stays unbound and has no mention.""" + + events: list[Any] = [] + connection = _SummaryConnection(events) + + async def load_candidates(conn) -> list[Any]: + return [] + + async def persist_edges(conn, post_id) -> list[Any]: + return [] + + monkeypatch.setattr(summary_ingestion, "_load_corporate_entity_candidates", load_candidates) + monkeypatch.setattr(summary_ingestion, "persist_edges_for_post", persist_edges) + + asyncio.run( + summary_ingestion.persist_post_summary( + connection, + str(uuid.uuid4()), + PostSummary( + korean_summary="합성 요약", + roles_and_responsibilities=( + RoleResponsibility( + actor_name="Uncataloged Person", + responsibility="후속", + actor_type_code=ACTOR_TYPE_PERSON, + ), + ), + ), + ) + ) + mention_inserts = [ + event[1] + for event in events + if isinstance(event, tuple) + and event[0] == "execute" + and "insert into post_summary_person_mention" in event[1] + ] + assert mention_inserts == [] + + +def test_hidden_run_copy_stays_generic_and_drops_the_stale_row() -> None: + """ADR 0014/0018: a 404 must not confirm why a row is hidden.""" + + app = ( + Path(__file__).resolve().parents[1] / "frontend" / "src" / "App.tsx" + ).read_text(encoding="utf-8") + alert = ( + Path(__file__).resolve().parents[1] + / "frontend" + / "src" + / "components" + / "StatusAlert.tsx" + ).read_text(encoding="utf-8") + agents = ( + Path(__file__).resolve().parents[1] / "AGENTS.md" + ).read_text(encoding="utf-8") + assert "This analysis run is not visible." not in app + assert "This run is not on your list. Open a visible run from the home list," in app + assert ( + "or request a lineage reconstruction for a corporation you already walk." + in app + ) + assert "do not name the thread or the cutoff" in app + assert "setRuns((await fetchAnalysisRuns(accessToken)).analysis_runs)" in app + assert 'role="alert"' in alert + assert "{error}" in app + assert "re-read the authorized list" in agents + assert "do not name the thread or the cutoff" in agents + + +def test_role_catalog_identity_is_stored_on_the_role_row() -> None: + """ADR 0019: fetch must not reconstruct organization identity by name.""" + root = Path(__file__).resolve().parents[1] + fetch_source = ( + root / "backend" / "app" / "post_summary_ingestion.py" + ).read_text(encoding="utf-8") + initial = (root / "migrations" / "0001_initial_schema.sql").read_text( + encoding="utf-8" + ) + upgrade = (root / "migrations" / "0019_role_catalog_identity.sql").read_text( + encoding="utf-8" + ) + person_upgrade = ( + root / "migrations" / "0025_role_person_catalog_identity.sql" + ).read_text(encoding="utf-8") + dockerfile = ( + root / "docker" / "postgres-init" / "Dockerfile" + ).read_text(encoding="utf-8") + changelog = (root / "CHANGELOG.md").read_text(encoding="utf-8") + fetch_sql = fetch_source.split("async def fetch_persisted_summary", 1)[1] + fetch_sql = fetch_sql.split("async def persist_post_summary", 1)[0] + assert "org.entity_name = role.actor_name" not in fetch_sql + assert "cataloged_corporate_entity_id" in fetch_sql + assert "cataloged_person_id" in fetch_sql + assert "cataloged_team_id" in initial + assert "cataloged_corporate_entity_id" in upgrade + assert "cataloged_person_id" in person_upgrade + assert "0019_role_catalog_identity.sql" in dockerfile + assert "0025_role_person_catalog_identity.sql" in dockerfile + assert "0026_report_leftover_pair.sql" in dockerfile + assert "ADR 0019" in changelog + assert "ADR 0027" in changelog diff --git a/tests/test_keyman_extraction.py b/tests/test_keyman_extraction.py index b1ab49da..d9de4571 100644 --- a/tests/test_keyman_extraction.py +++ b/tests/test_keyman_extraction.py @@ -47,6 +47,24 @@ def test_parses_a_well_formed_json_array() -> None: assert mentions[1].affiliated_organization_names == ("Acme Corp", "Acme Holdings") +def test_job_title_is_captured_when_present() -> None: + content = '[{"name": "Kim Cheolsu", "side": "counterparty", "affiliations": [], "job_title": "Sales Manager"}]' + mentions = parse_keyman_response(content) + assert mentions[0].job_title == "Sales Manager" + + +def test_job_title_is_none_not_empty_string_when_absent() -> None: + content = '[{"name": "Kim Cheolsu", "side": "counterparty", "affiliations": []}]' + mentions = parse_keyman_response(content) + assert mentions[0].job_title is None + + +def test_null_job_title_is_none_not_the_string_null() -> None: + content = '[{"name": "Kim Cheolsu", "side": "counterparty", "affiliations": [], "job_title": null}]' + mentions = parse_keyman_response(content) + assert mentions[0].job_title is None + + def test_strips_a_markdown_code_fence() -> None: content = '```json\n[{"name": "Jo Park", "side": "our_side", "affiliations": []}]\n```' mentions = parse_keyman_response(content) @@ -110,3 +128,11 @@ def test_contextual_orchestrator_extracts_keymen_from_an_ambiguous_post() -> Non assert jordan.person_side_code == OUR_SIDE assert priya.person_side_code == COUNTERPARTY assert len(priya.affiliated_organization_names) >= 2 + + # Sam Okonkwo is named only by role ("our legal counsel, Sam Okonkwo") -- + # a real assertion that job_title extraction reads the text, not a + # synthetic fixture built just to satisfy this one field. + sam = next((m for name, m in by_name.items() if "Sam" in name or "Okonkwo" in name), None) + assert sam is not None + assert sam.job_title is not None + assert "counsel" in sam.job_title.lower() or "legal" in sam.job_title.lower() diff --git a/tests/test_leftover_pairs.py b/tests/test_leftover_pairs.py index 72dea89b..989062ab 100644 --- a/tests/test_leftover_pairs.py +++ b/tests/test_leftover_pairs.py @@ -1,4 +1,4 @@ -"""Leftover post–criterion pairs after the main-effect IRT (ADR 0017). +"""Leftover post–criterion pairs after the main-effect IRT (ADR 0028). Uses a constructed residual matrix so the closest and farthest pair are known without calling ``fit_polytomous``. Loads diff --git a/tests/test_ontology.py b/tests/test_ontology.py index 41423ada..0e611bc8 100644 --- a/tests/test_ontology.py +++ b/tests/test_ontology.py @@ -31,12 +31,31 @@ _SEED_SCRIPT_PATH = Path(__file__).resolve().parents[1] / "scripts" / "seed_demo_data.py" +# Several covered categories add lookup rows via their own migration +# SQL rather than literally embedded in seed_demo_data.py's own source +# text -- read alongside it below so the round-trip still sees them: +# 0012 (ADR 0006: prov_person/prov_organization), 0014 (ADR 0007: +# prov_team), 0016 (ADR 0009: node_team/edge_mention_team/ +# edge_team_affiliation/edge_mention_organization). +_ADDITIONAL_LOOKUP_MIGRATION_PATHS = ( + Path(__file__).resolve().parents[1] / "migrations" / "0012_role_responsibility_agent_type.sql", + Path(__file__).resolve().parents[1] / "migrations" / "0014_role_responsibility_team_actor_type.sql", + Path(__file__).resolve().parents[1] / "migrations" / "0016_cross_post_actor_identity.sql", +) + # The categories this ontology covers (ADR 0004's scope). seed_demo_data.py # also seeds categories this ontology deliberately does not model yet # (post_visibility, voc_type, permission, ticket_status) -- those are # real, expected gaps, not a test bug. _ONTOLOGY_COVERED_CATEGORIES = frozenset( - {"node_type", "edge_type", "entity_relationship_type", "person_side", "corporate_entity_level"} + { + "node_type", + "edge_type", + "entity_relationship_type", + "person_side", + "corporate_entity_level", + "prov_agent_type", + } ) _INSERT_TUPLE_PATTERN = re.compile(r"\('([a-z_]+)',\s*'([a-z_]+)'") @@ -44,12 +63,14 @@ def _seeded_lookup_codes_for_covered_categories() -> set[str]: """Every `(lookup_category, lookup_code)` pair seed_demo_data.py's own - SQL literally inserts, filtered to the categories this ontology - covers. Parsed from source, not executed -- this is a static - consistency check between two committed files, not a live-database - test. + SQL, plus the additional migrations' SQL, literally inserts, filtered + to the categories this ontology covers. Parsed from source, not + executed -- this is a static consistency check between committed + files, not a live-database test. """ - source = _SEED_SCRIPT_PATH.read_text() + source = _SEED_SCRIPT_PATH.read_text() + "".join( + p.read_text() for p in _ADDITIONAL_LOOKUP_MIGRATION_PATHS + ) return { code for category, code in _INSERT_TUPLE_PATTERN.findall(source) @@ -129,6 +150,38 @@ def test_mentions_property_domain_and_range_match_the_schema() -> None: assert (LW.mentions, RDFS.range, LW.Person) in graph +def test_prov_agent_type_terms_resolve_and_subclass_real_prov_o() -> None: + """Beyond the generic round-trip above: the two prov_agent_type terms + must actually subclass the real external W3C PROV-O classes, not + just carry a matching :lookupCode -- the whole point of grounding + this in a standard ontology is that :RoleActorPerson really is a + prov:Person, not a same-named local invention. + """ + from rdflib import URIRef + from rdflib.namespace import Namespace + + prov = Namespace("http://www.w3.org/ns/prov#") + graph = load_ontology() + assert iri_for_lookup_code("prov_person") == str(LW.RoleActorPerson) + assert iri_for_lookup_code("prov_organization") == str(LW.RoleActorOrganization) + assert (LW.RoleActorPerson, RDFS.subClassOf, URIRef(prov.Person)) in graph + assert (LW.RoleActorOrganization, RDFS.subClassOf, URIRef(prov.Organization)) in graph + + +def test_prov_team_type_resolves_and_subclasses_real_org_ontology() -> None: + """ADR 0007: a team actor is grounded in the real external W3C + Organization Ontology's org:OrganizationalUnit, the meso-level + sub-organization concept PROV-O itself has no equivalent for. + """ + from rdflib import URIRef + from rdflib.namespace import Namespace + + org = Namespace("http://www.w3.org/ns/org#") + graph = load_ontology() + assert iri_for_lookup_code("prov_team") == str(LW.RoleActorTeam) + assert (LW.RoleActorTeam, RDFS.subClassOf, URIRef(org.OrganizationalUnit)) in graph + + def test_corporate_entity_level_hierarchy_is_broadest_first() -> None: """Group is broader than Company is broader than Plant -- the Acme Group -> Acme Electronics Korea -> plant direction the @@ -137,3 +190,12 @@ def test_corporate_entity_level_hierarchy_is_broadest_first() -> None: assert (LW.CompanyLevel, SKOS.broader, LW.GroupLevel) in graph assert (LW.PlantLevel, SKOS.broader, LW.CompanyLevel) in graph assert (LW.GroupLevel, SKOS.broader, LW.CompanyLevel) not in graph + + +def test_actor_mentions_follow_stored_edge_direction() -> None: + """Ontology domain/range matches Team/Organization -> Post storage.""" + graph = load_ontology() + assert (LW.mentionsTeam, RDFS.domain, LW.Team) in graph + assert (LW.mentionsTeam, RDFS.range, LW.Post) in graph + assert (LW.mentionsOrganization, RDFS.domain, LW.CorporateEntity) in graph + assert (LW.mentionsOrganization, RDFS.range, LW.Post) in graph diff --git a/tests/test_organization_name_resolution.py b/tests/test_organization_name_resolution.py new file mode 100644 index 00000000..7eb0a96c --- /dev/null +++ b/tests/test_organization_name_resolution.py @@ -0,0 +1,125 @@ +"""Tests for lineageweave.organization_name_resolution (ADR 0008). + +Deterministic fake clients, same style as tests/test_post_summary.py +and tests/test_keyman_extraction.py's pure-parse-function tests -- the +underlying HTTP mechanics (post_json) and SearxngRelationVerificationClient's +own HTTP behavior are already covered in test_http_client.py and +test_relation_verification.py respectively; these tests are for this +module's own resolve-then-verify orchestration logic. +""" + +from __future__ import annotations + +from lineageweave.organization_name_resolution import ( + NullOrganizationNameResolutionClient, + OrganizationNameResolution, + parse_resolution_response, + resolve_and_verify_organization_name, +) +from lineageweave.relation_verification import ( + STATUS_CORROBORATED, + STATUS_PENDING, + STATUS_UNCORROBORATED, + NullRelationVerificationClient, + RelationVerificationResult, +) + + +class _FakeResolutionClient: + available = True + + def __init__(self, candidate: str | None) -> None: + self._candidate = candidate + self.calls: list[tuple[str, str]] = [] + + def resolve(self, raw_name: str, context_text: str) -> str | None: + self.calls.append((raw_name, context_text)) + return self._candidate + + +class _FakeVerificationClient: + available = True + + def __init__(self, result: RelationVerificationResult) -> None: + self._result = result + self.calls: list[tuple[str, str]] = [] + + def verify(self, organization_name: str, relationship_label: str) -> RelationVerificationResult: + self.calls.append((organization_name, relationship_label)) + return self._result + + +def test_parse_resolution_response_extracts_the_first_line() -> None: + assert parse_resolution_response("Aurora Grid Power\n") == "Aurora Grid Power" + + +def test_parse_resolution_response_rejects_unknown() -> None: + assert parse_resolution_response("UNKNOWN") is None + assert parse_resolution_response("unknown\n") is None + + +def test_parse_resolution_response_rejects_empty() -> None: + assert parse_resolution_response("") is None + assert parse_resolution_response(" ") is None + + +def test_no_resolution_when_client_unavailable() -> None: + result = resolve_and_verify_organization_name( + "AGP", "context", NullOrganizationNameResolutionClient(), NullRelationVerificationClient() + ) + assert result is None + + +def test_no_resolution_when_model_proposes_nothing() -> None: + result = resolve_and_verify_organization_name( + "AGP", "context", _FakeResolutionClient(None), NullRelationVerificationClient() + ) + assert result is None + + +def test_no_resolution_when_model_echoes_the_same_name() -> None: + """A "resolution" that just returns the raw name back is not a real + resolution -- must not be persisted as one.""" + result = resolve_and_verify_organization_name( + "AGP", "context", _FakeResolutionClient("AGP"), NullRelationVerificationClient() + ) + assert result is None + + +def test_corroborated_resolution_carries_evidence() -> None: + verification = _FakeVerificationClient( + RelationVerificationResult(status_code=STATUS_CORROBORATED, evidence_url="https://example.org/agp") + ) + resolution_client = _FakeResolutionClient("Aurora Grid Power") + result = resolve_and_verify_organization_name("AGP", "설계팀이 AGP와 회의했다", resolution_client, verification) + assert result == OrganizationNameResolution( + raw_organization_name="AGP", + resolved_organization_name="Aurora Grid Power", + verification_status_code=STATUS_CORROBORATED, + verification_evidence_url="https://example.org/agp", + ) + # The full name and the raw abbreviation are searched together -- + # the specific pairing is what needs corroborating, not just that + # the full name exists as some organization. + assert verification.calls == [("Aurora Grid Power", "AGP")] + + +def test_uncorroborated_resolution_still_returned_with_evidence_none() -> None: + verification = _FakeVerificationClient( + RelationVerificationResult(status_code=STATUS_UNCORROBORATED, evidence_url=None) + ) + result = resolve_and_verify_organization_name( + "AGP", "context", _FakeResolutionClient("Invented Co"), verification + ) + assert result is not None + assert result.verification_status_code == STATUS_UNCORROBORATED + assert result.verification_evidence_url is None + + +def test_verification_unavailable_yields_pending_not_a_fabricated_result() -> None: + result = resolve_and_verify_organization_name( + "AGP", "context", _FakeResolutionClient("Aurora Grid Power"), NullRelationVerificationClient() + ) + assert result is not None + assert result.verification_status_code == STATUS_PENDING + assert result.verification_evidence_url is None diff --git a/tests/test_person_mention_projection.py b/tests/test_person_mention_projection.py new file mode 100644 index 00000000..d24195bb --- /dev/null +++ b/tests/test_person_mention_projection.py @@ -0,0 +1,620 @@ +"""Real-PostgreSQL regressions for source-aware person and graph projections. + +Keyman extraction and post-summary R&R are independent evidence channels. A +replacement in either channel must remove only that channel's stale person +mentions, then reconcile the buyer-facing Knowledge Graph from the currently +supported union. Orphan graph-registry rows must never become visible. +""" + +from __future__ import annotations + +import asyncio +import os +from pathlib import Path +from urllib.parse import urlsplit, urlunsplit +import uuid + +import asyncpg +import psycopg2 +from psycopg2 import sql +import pytest + +from backend.app.keyman_ingestion import ingest_post_keymen +from backend.app.knowledge_graph import ( + hydrate_related_nodes, + load_visible_subgraph, + persist_edges_for_post, + related_for_start, + visible_mention_post_ids, +) +from backend.app import post_summary_ingestion as summary_ingestion +from backend.app.post_summary_ingestion import ( + fetch_persisted_summary, + persist_post_summary, +) +from lineageweave.keyman_extraction import OUR_SIDE, PersonMention +from lineageweave.knowledge_graph import ( + EDGE_MENTION, + EDGE_MENTION_TEAM, + NODE_CORPORATE_ENTITY, + NODE_PERSON, + NODE_POST, + NODE_TEAM, +) +from lineageweave.post_summary import ( + ACTOR_TYPE_ORGANIZATION, + ACTOR_TYPE_PERSON, + PostSummary, + RoleResponsibility, +) + +_ADMIN_DSN = os.environ.get( + "LINEAGEWEAVE_TEST_POSTGRES_ADMIN_DSN", "postgresql://localhost/postgres" +) +_MIGRATION_PATH = Path(__file__).resolve().parents[1] / "migrations" / "0001_initial_schema.sql" + + +def _postgres_available() -> bool: + """Return whether the configured real PostgreSQL test service is reachable.""" + + try: + psycopg2.connect(_ADMIN_DSN, connect_timeout=2).close() + return True + except psycopg2.OperationalError: + return False + + +pytestmark = pytest.mark.skipif( + not _postgres_available(), + reason=f"no reachable PostgreSQL server at {_ADMIN_DSN}", +) + + +class _KeymanClient: + """Mutable deterministic extractor used to model replacement runs.""" + + available = True + + def __init__(self, mentions: list[PersonMention]) -> None: + self.mentions = mentions + + def extract(self, post_title: str, post_body: str) -> list[PersonMention]: + """Return a copy so production code cannot mutate the fixture.""" + + return list(self.mentions) + + +def _database_dsn(database_name: str) -> str: + """Replace only the database path while preserving DSN query parameters.""" + + parsed = urlsplit(_ADMIN_DSN) + return urlunsplit(parsed._replace(path=f"/{database_name}")) + + +@pytest.fixture +def projection_database() -> str: + """Create one freshly migrated PostgreSQL database and seed one post.""" + + database_name = f"lineageweave_projection_{uuid.uuid4().hex[:12]}" + admin = psycopg2.connect(_ADMIN_DSN) + admin.autocommit = True + with admin.cursor() as cursor: + cursor.execute(sql.SQL("create database {}").format(sql.Identifier(database_name))) + try: + database_dsn = _database_dsn(database_name) + connection = psycopg2.connect(database_dsn) + try: + with connection.cursor() as cursor: + cursor.execute(_MIGRATION_PATH.read_text(encoding="utf-8")) + cursor.execute( + """ + insert into common_lookup_value + (lookup_category, lookup_code, lookup_label) + values + ('corporate_entity_level', 'company', 'Company'), + ('post_visibility', 'public', 'Public'), + ('voc_type', 'voc', 'Voice of Customer'), + ('person_side', 'our_side', 'Our side'), + ('person_side', 'counterparty', 'Counterparty'), + ('prov_agent_type', 'prov_person', 'Person'), + ('prov_agent_type', 'prov_organization', 'Organization'), + ('prov_agent_type', 'prov_team', 'Team'), + ('node_type', 'node_person', 'Person node'), + ('node_type', 'node_post', 'Post node'), + ('node_type', 'node_corporate_entity', 'Corporate node'), + ('node_type', 'node_team', 'Team node'), + ('edge_type', 'edge_mention', 'Person mentioned in'), + ('edge_type', 'edge_affiliation', 'Person affiliated with'), + ('edge_type', 'edge_co_mention', 'People co-mentioned'), + ('edge_type', 'edge_mention_team', 'Team mentioned in'), + ('edge_type', 'edge_team_affiliation', 'Team affiliated with'), + ('edge_type', 'edge_mention_organization', 'Organization mentioned in') + """ + ) + cursor.execute( + """ + insert into corporate_entity + (corporate_entity_code, entity_name, entity_level_code) + values ('SYNTH-CORP', 'Synthetic Corp', 'company') + returning corporate_entity_id + """ + ) + corporate_entity_id = cursor.fetchone()[0] + cursor.execute( + """ + insert into user_account + (external_subject_id, display_name, email_address) + values ('projection-subject', 'Projection User', 'projection@example.test') + returning user_account_id + """ + ) + account_id = cursor.fetchone()[0] + cursor.execute( + """ + insert into source_post + (author_account_id, corporate_entity_id, post_title, post_body, + voc_type_code, visibility_code) + values (%s, %s, 'Synthetic post', 'Synthetic body', 'voc', 'public') + returning post_id + """, + (account_id, corporate_entity_id), + ) + post_id = cursor.fetchone()[0] + cursor.execute( + """ + insert into cataloged_person + (person_name, person_side_code, last_known_job_title) + values ('Summary Person', 'counterparty', 'Reviewer') + returning person_id + """ + ) + summary_person_id = cursor.fetchone()[0] + connection.commit() + finally: + connection.close() + yield "|".join((database_dsn, str(post_id), str(summary_person_id))) + finally: + with admin.cursor() as cursor: + cursor.execute(sql.SQL("drop database {}").format(sql.Identifier(database_name))) + admin.close() + + +async def _exercise_projection_contract( + database_dsn: str, + post_id: str, + summary_person_id: str, +) -> None: + """Run Keyman and R&R replacements and prove graph support follows them.""" + + connection = await asyncpg.connect(database_dsn) + try: + keyman = PersonMention("Keyman Person", OUR_SIDE) + client = _KeymanClient([keyman]) + await ingest_post_keymen( + connection, + client, + post_id, + "Synthetic post", + "Synthetic body", + ) + keyman_person_id = str( + await connection.fetchval( + "select person_id from cataloged_person where person_name = 'Keyman Person'" + ) + ) + + await persist_post_summary( + connection, + post_id, + PostSummary( + korean_summary="합성 요약", + roles_and_responsibilities=( + RoleResponsibility( + actor_name="Summary Person", + responsibility="검토", + ), + ), + ), + ) + + keyman_rows = await connection.fetch( + "select person_id from post_person_mention where post_id = $1", + post_id, + ) + summary_rows = await connection.fetch( + "select person_id from post_summary_person_mention where post_id = $1", + post_id, + ) + assert {str(row["person_id"]) for row in keyman_rows} == {keyman_person_id} + assert {str(row["person_id"]) for row in summary_rows} == {summary_person_id} + assert await visible_mention_post_ids( + connection, summary_person_id, lambda row: True + ) == [post_id] + + await persist_post_summary( + connection, + post_id, + PostSummary(korean_summary="역할이 제거된 합성 요약"), + ) + assert await visible_mention_post_ids( + connection, summary_person_id, lambda row: True + ) == [] + assert await visible_mention_post_ids( + connection, keyman_person_id, lambda row: True + ) == [post_id] + visible_edges = await load_visible_subgraph(connection, [post_id]) + visible_person_ids = { + edge.source_node_id + for edge in visible_edges + if edge.source_node_type_code == NODE_PERSON + } | { + edge.target_node_id + for edge in visible_edges + if edge.target_node_type_code == NODE_PERSON + } + assert summary_person_id not in visible_person_ids + assert keyman_person_id in visible_person_ids + + client.mentions = [] + await ingest_post_keymen( + connection, + client, + post_id, + "Synthetic post", + "Synthetic body", + ) + assert await visible_mention_post_ids( + connection, keyman_person_id, lambda row: True + ) == [] + assert await load_visible_subgraph(connection, [post_id]) == [] + + async with connection.transaction(): + await persist_edges_for_post(connection, post_id) + await persist_edges_for_post(connection, post_id) + duplicate_count = await connection.fetchval( + """ + select count(*) + from ( + select source_node_type_code, source_node_id, + target_node_type_code, target_node_id, edge_type_code + from knowledge_graph_edge + group by source_node_type_code, source_node_id, + target_node_type_code, target_node_id, edge_type_code + having count(*) > 1 + ) duplicate_edge + """ + ) + assert duplicate_count == 0 + + orphan_id = await connection.fetchval( + """ + insert into knowledge_graph_edge + (source_node_type_code, source_node_id, target_node_type_code, + target_node_id, edge_type_code, edge_weight) + values ($1, $2::uuid, $3, $4::uuid, $5, 1.0) + on conflict ( + source_node_type_code, source_node_id, + target_node_type_code, target_node_id, edge_type_code + ) do update set edge_weight = excluded.edge_weight + returning knowledge_graph_edge_id + """, + NODE_PERSON, + keyman_person_id, + NODE_POST, + post_id, + EDGE_MENTION, + ) + await connection.execute( + "delete from knowledge_graph_edge_evidence where knowledge_graph_edge_id = $1", + orphan_id, + ) + assert await load_visible_subgraph(connection, [post_id]) == [] + finally: + await connection.close() + + +def test_person_mention_sources_reconcile_without_stale_graph_edges( + projection_database: str, +) -> None: + """Each evidence channel replaces itself and the visible graph follows suit.""" + + database_dsn, post_id, summary_person_id = projection_database.split("|") + asyncio.run( + _exercise_projection_contract(database_dsn, post_id, summary_person_id) + ) + + +def test_cross_post_identity_upgrade_keeps_keyman_mention_context( + projection_database: str, +) -> None: + """Migration 0016 copies R&R names and must not steal Keyman mention_context.""" + + database_dsn, post_id, summary_person_id = projection_database.split("|") + migration = Path(__file__).resolve().parents[1] / "migrations" / "0016_cross_post_actor_identity.sql" + connection = psycopg2.connect(database_dsn) + try: + with connection.cursor() as cursor: + cursor.execute( + """ + insert into post_person_mention (post_id, person_id, mention_context) + values (%s, %s, %s) + """, + ( + post_id, + summary_person_id, + "Keyman extracted this mention from the synthetic body", + ), + ) + cursor.execute( + "insert into post_summary_result (post_id, korean_summary) values (%s, %s)", + (post_id, "합성 요약"), + ) + cursor.execute( + """ + insert into post_summary_role + (post_id, actor_name, responsibility, actor_type_code) + values (%s, 'Summary Person', '검토', 'prov_person') + """, + (post_id,), + ) + cursor.execute(migration.read_text(encoding="utf-8")) + cursor.execute( + """ + select mention_context + from post_person_mention + where post_id = %s and person_id = %s + """, + (post_id, summary_person_id), + ) + keyman_row = cursor.fetchone() + cursor.execute( + """ + select count(*) + from post_summary_person_mention + where post_id = %s and person_id = %s + """, + (post_id, summary_person_id), + ) + summary_count = cursor.fetchone()[0] + connection.commit() + finally: + connection.close() + + assert keyman_row is not None + assert keyman_row[0] == "Keyman extracted this mention from the synthetic body" + assert summary_count == 1 + + +async def _exercise_team_only_related_walk( + database_dsn: str, + first_post_id: str, +) -> None: + """A team mentioned on two posts must walk even when one post has no people.""" + + connection = await asyncpg.connect(database_dsn) + try: + author_id, corporate_entity_id = await connection.fetchrow( + "select author_account_id, corporate_entity_id from source_post where post_id = $1", + first_post_id, + ) + second_post_id = str( + await connection.fetchval( + """ + insert into source_post + (author_account_id, corporate_entity_id, post_title, post_body, + voc_type_code, visibility_code) + values ($1, $2, 'Team-only follow-up', '설계팀이 도면을 재검토했다.', + 'voc', 'public') + returning post_id + """, + author_id, + corporate_entity_id, + ) + ) + team_id = str( + await connection.fetchval( + """ + insert into cataloged_team (team_name, affiliated_organization_name) + values ('설계팀', 'Synthetic Corp') + returning team_id + """ + ) + ) + await connection.execute( + """ + insert into post_team_mention (post_id, team_id) + values ($1, $2), ($3, $2) + """, + first_post_id, + team_id, + second_post_id, + ) + async with connection.transaction(): + await persist_edges_for_post(connection, first_post_id) + await persist_edges_for_post(connection, second_post_id) + + team_only_edges = await load_visible_subgraph(connection, [second_post_id]) + assert any( + edge.edge_type_code == EDGE_MENTION_TEAM + and edge.source_node_id == team_id + and edge.target_node_id == second_post_id + for edge in team_only_edges + ), "a team-only post must still load its mention edge" + + related = await related_for_start( + connection, NODE_TEAM, team_id, [first_post_id, second_post_id] + ) + related_ids = {node["node_id"] for node in related} + assert first_post_id in related_ids + assert second_post_id in related_ids + hydrated = await hydrate_related_nodes( + connection, [(f"{NODE_TEAM}:{team_id}", 1.0)] + ) + assert hydrated[0]["label"] == "설계팀" + assert hydrated[0]["node_type_code"] == NODE_TEAM + finally: + await connection.close() + + +def test_team_only_posts_walk_related_nodes(projection_database: str) -> None: + """ADR 0018: team mention edges must participate in the visible RWR walk.""" + + database_dsn, post_id, _summary_person_id = projection_database.split("|") + asyncio.run(_exercise_team_only_related_walk(database_dsn, post_id)) + + +async def _exercise_homonym_organization_role_binding( + database_dsn: str, + post_id: str, +) -> None: + """A same-named catalog org that this post did not resolve must stay off the role.""" + + connection = await asyncpg.connect(database_dsn) + try: + mentioned_id = str( + await connection.fetchval( + """ + insert into corporate_entity + (corporate_entity_code, entity_name, entity_level_code) + values ('HOMONYM-MENTIONED', 'Homonym Energy', 'company') + returning corporate_entity_id + """ + ) + ) + other_id = str( + await connection.fetchval( + """ + insert into corporate_entity + (corporate_entity_code, entity_name, entity_level_code) + values ('HOMONYM-OTHER', 'Homonym Energy', 'company') + returning corporate_entity_id + """ + ) + ) + + async def resolve_mentioned_organization(*_args, **_kwargs) -> str: + return mentioned_id + + original = summary_ingestion.get_or_create_corporate_entity + summary_ingestion.get_or_create_corporate_entity = resolve_mentioned_organization + try: + payload = await persist_post_summary( + connection, + post_id, + PostSummary( + korean_summary="동명이인 조직이 일정만 확정했다.", + roles_and_responsibilities=( + RoleResponsibility( + actor_name="Homonym Energy", + responsibility="납품 일정 확정", + actor_type_code=ACTOR_TYPE_ORGANIZATION, + ), + ), + ), + ) + finally: + summary_ingestion.get_or_create_corporate_entity = original + + roles = payload["roles_and_responsibilities"] + assert len(roles) == 1 + assert roles[0]["catalog_node_id"] == mentioned_id + assert roles[0]["catalog_node_type_code"] == NODE_CORPORATE_ENTITY + fetched = await fetch_persisted_summary(connection, post_id) + assert fetched is not None + assert fetched["roles_and_responsibilities"][0]["catalog_node_id"] == mentioned_id + assert fetched["roles_and_responsibilities"][0]["catalog_node_id"] != other_id + mention_ids = [ + str(row["corporate_entity_id"]) + for row in await connection.fetch( + "select corporate_entity_id from post_organization_mention " + "where post_id = $1", + post_id, + ) + ] + assert mention_ids == [mentioned_id] + finally: + await connection.close() + + +def test_homonym_organization_role_binds_the_resolved_catalog_id( + projection_database: str, +) -> None: + """ADR 0019: two catalog orgs can share a display name; the role keeps one id.""" + + database_dsn, post_id, _summary_person_id = projection_database.split("|") + asyncio.run(_exercise_homonym_organization_role_binding(database_dsn, post_id)) + + +async def _exercise_same_name_person_catalog_order( + database_dsn: str, + post_id: str, +) -> None: + """Two people with the same name must bind the earlier catalog row.""" + + connection = await asyncpg.connect(database_dsn) + try: + earlier_id = str( + await connection.fetchval( + """ + insert into cataloged_person + (person_name, person_side_code, last_known_job_title, created_at) + values ( + 'Kim Cheolsu', 'our_side', 'Sales Manager', + '2024-01-01T00:00:00+00' + ) + returning person_id + """ + ) + ) + await connection.execute( + """ + insert into cataloged_person + (person_name, person_side_code, last_known_job_title, created_at) + values ( + 'Kim Cheolsu', 'counterparty', 'Purchasing Lead', + '2024-06-01T00:00:00+00' + ) + """ + ) + await persist_post_summary( + connection, + post_id, + PostSummary( + korean_summary="김철수가 후속을 맡았다.", + roles_and_responsibilities=( + RoleResponsibility( + actor_name="Kim Cheolsu", + responsibility="후속", + actor_type_code=ACTOR_TYPE_PERSON, + ), + ), + ), + ) + payload = await fetch_persisted_summary(connection, post_id) + assert payload is not None + roles = payload["roles_and_responsibilities"] + assert len(roles) == 1 + stored_id = str( + await connection.fetchval( + """ + select cataloged_person_id + from post_summary_role + where post_id = $1 and actor_name = 'Kim Cheolsu' + """, + post_id, + ) + ) + assert stored_id == earlier_id + assert roles[0]["catalog_node_id"] == earlier_id + assert roles[0]["catalog_node_type_code"] == NODE_PERSON + finally: + await connection.close() + + +def test_same_name_person_roles_bind_the_earliest_catalog_row( + projection_database: str, +) -> None: + """ADR 0027: R&R person lookup must order by created_at, then person_id.""" + + database_dsn, post_id, _summary_person_id = projection_database.split("|") + asyncio.run(_exercise_same_name_person_catalog_order(database_dsn, post_id)) diff --git a/tests/test_post_summary.py b/tests/test_post_summary.py index 52694bc3..4f863769 100644 --- a/tests/test_post_summary.py +++ b/tests/test_post_summary.py @@ -24,6 +24,7 @@ from lineageweave.post_summary import ( ContextualOrchestratorPostSummaryClient, NullPostSummaryClient, + RoleResponsibility, parse_summary_response, ) @@ -39,13 +40,72 @@ def test_parses_a_well_formed_json_object() -> None: content = ( '{"korean_summary": "회의 후속 조치에 대한 요약입니다.", ' '"key_events": ["입찰 워크숍 진행", "검사 일정 확인 요청"], ' - '"roles_and_responsibilities": [{"person_name": "Jordan Hale", "responsibility": "입찰 일정 안내"}]}' + '"roles_and_responsibilities": [{"actor_name": "Jordan Hale", "responsibility": "입찰 일정 안내", ' + '"actor_type": "person", "affiliated_organization_name": "Westfield Power"}]}' ) summary = parse_summary_response(content) assert summary is not None assert summary.korean_summary == "회의 후속 조치에 대한 요약입니다." assert summary.key_events == ("입찰 워크숍 진행", "검사 일정 확인 요청") - assert summary.roles_and_responsibilities[0].person_name == "Jordan Hale" + role = summary.roles_and_responsibilities[0] + assert role.actor_name == "Jordan Hale" + assert role.actor_type_code == "prov_person" + assert role.affiliated_organization_name == "Westfield Power" + + +def test_organization_actor_is_not_forced_into_a_person_slot() -> None: + """A named actor that is genuinely an organization (e.g. our own + company acting in its own name, not a named individual) must parse + as ``prov_organization``, not silently default to person -- the + default only applies when the model omits ``actor_type`` entirely. + """ + content = ( + '{"korean_summary": "당사가 요청 사항을 확인했습니다.", "key_events": [], ' + '"roles_and_responsibilities": [{"actor_name": "당사", "responsibility": "요청 확인", ' + '"actor_type": "organization", "affiliated_organization_name": null}]}' + ) + summary = parse_summary_response(content) + assert summary is not None + role = summary.roles_and_responsibilities[0] + assert role.actor_name == "당사" + assert role.actor_type_code == "prov_organization" + assert role.affiliated_organization_name is None + + +def test_team_actor_is_meso_level_not_organization() -> None: + """A named sub-unit of a company (e.g. 설계팀, "design team") must + parse as ``prov_team``, distinct from both ``prov_person`` and + ``prov_organization`` -- it is part of a company, not the company + itself (ADR 0007), and its parent company's name must still land in + ``affiliated_organization_name``. + """ + content = ( + '{"korean_summary": "설계팀이 도면을 검토했습니다.", "key_events": [], ' + '"roles_and_responsibilities": [{"actor_name": "설계팀", "responsibility": "도면 검토", ' + '"actor_type": "team", "affiliated_organization_name": "Demo Corp"}]}' + ) + summary = parse_summary_response(content) + assert summary is not None + role = summary.roles_and_responsibilities[0] + assert role.actor_name == "설계팀" + assert role.actor_type_code == "prov_team" + assert role.affiliated_organization_name == "Demo Corp" + + +def test_unknown_actor_type_code_is_rejected() -> None: + with pytest.raises(ValueError, match="actor_type_code"): + RoleResponsibility(actor_name="Ada West", responsibility="후속", actor_type_code="person") + + +def test_missing_actor_type_defaults_to_person() -> None: + content = ( + '{"korean_summary": "요약", "key_events": [], ' + '"roles_and_responsibilities": [{"actor_name": "Ada West", "responsibility": "후속"}]}' + ) + summary = parse_summary_response(content) + assert summary is not None + assert summary.roles_and_responsibilities[0].actor_type_code == "prov_person" + assert summary.roles_and_responsibilities[0].affiliated_organization_name is None def test_missing_korean_summary_returns_none() -> None: @@ -75,7 +135,7 @@ def test_every_sample_record_has_a_seeded_korean_summary() -> None: assert summary.korean_summary not in seen seen.add(summary.korean_summary) cast = fixture_thread_cast(rec.label) - names = {role.person_name for role in summary.roles_and_responsibilities} + names = {role.actor_name for role in summary.roles_and_responsibilities} if cast is not None and cast.person_names: assert set(cast.person_names) <= names else: @@ -91,7 +151,7 @@ def test_every_sample_record_has_a_seeded_korean_summary() -> None: def test_malformed_roles_entries_are_skipped_not_crashed_on() -> None: content = ( '{"korean_summary": "요약", "key_events": [], ' - '"roles_and_responsibilities": [{"person_name": "Only Name"}, "not an object"]}' + '"roles_and_responsibilities": [{"actor_name": "Only Name"}, "not an object"]}' ) summary = parse_summary_response(content) assert summary is not None @@ -119,5 +179,5 @@ def test_contextual_orchestrator_summarizes_a_non_trivial_post() -> None: # block -- not just an English sentence handed back unchanged. assert any("가" <= ch <= "힣" for ch in summary.korean_summary) assert len(summary.key_events) >= 1 - people_named = {rr.person_name for rr in summary.roles_and_responsibilities} + people_named = {rr.actor_name for rr in summary.roles_and_responsibilities} assert any("Jordan" in name or "Priya" in name for name in people_named) diff --git a/tests/test_prov_o.py b/tests/test_prov_o.py new file mode 100644 index 00000000..2a33a9b8 --- /dev/null +++ b/tests/test_prov_o.py @@ -0,0 +1,401 @@ +from __future__ import annotations + +from datetime import datetime, timezone +from pathlib import Path + +import sys + +import pytest +from rdflib import Graph, Literal, Namespace, URIRef +from rdflib.namespace import RDF, XSD + +from lineageweave.prov_o import ( + PROV, + PROV_CLASSES, + PROV_QUALIFICATIONS, + PROV_RELATIONS, + PROV_RECOMMENDED_INVERSES, + ProvAssertion, + ProvGraph, + ProvLiteral, + ProvValidationError, + class_code, + relation_code, +) + +EXPECTED_CLASS_NAMES = { + "Entity", "Activity", "Agent", "Collection", "EmptyCollection", "Bundle", + "Person", "SoftwareAgent", "Organization", "Location", "Influence", + "EntityInfluence", "Usage", "Start", "End", "Derivation", "PrimarySource", + "Quotation", "Revision", "ActivityInfluence", "Generation", "Communication", + "Invalidation", "AgentInfluence", "Attribution", "Association", "Plan", + "Delegation", "InstantaneousEvent", "Role", +} + +EXPECTED_RELATION_NAMES = { + "wasGeneratedBy", "wasDerivedFrom", "wasAttributedTo", "startedAtTime", "used", + "wasInformedBy", "endedAtTime", "wasAssociatedWith", "actedOnBehalfOf", + "alternateOf", "specializationOf", "generatedAtTime", "hadPrimarySource", "value", + "wasQuotedFrom", "wasRevisionOf", "invalidatedAtTime", "wasInvalidatedBy", + "hadMember", "wasStartedBy", "wasEndedBy", "invalidated", "influenced", + "atLocation", "generated", "wasInfluencedBy", "qualifiedInfluence", + "qualifiedGeneration", "qualifiedDerivation", "qualifiedPrimarySource", + "qualifiedQuotation", "qualifiedRevision", "qualifiedAttribution", + "qualifiedInvalidation", "qualifiedStart", "qualifiedUsage", + "qualifiedCommunication", "qualifiedAssociation", "qualifiedEnd", + "qualifiedDelegation", "influencer", "entity", "hadUsage", "hadGeneration", + "activity", "agent", "hadPlan", "hadActivity", "atTime", "hadRole", +} + +EXPECTED_DATATYPE_RELATIONS = { + "startedAtTime", "endedAtTime", "generatedAtTime", "invalidatedAtTime", "value", "atTime" +} + +EXPECTED_QUALIFICATIONS = { + "wasGeneratedBy": ("qualifiedGeneration", "Generation", "activity"), + "wasDerivedFrom": ("qualifiedDerivation", "Derivation", "entity"), + "wasAttributedTo": ("qualifiedAttribution", "Attribution", "agent"), + "used": ("qualifiedUsage", "Usage", "entity"), + "wasInformedBy": ("qualifiedCommunication", "Communication", "activity"), + "wasAssociatedWith": ("qualifiedAssociation", "Association", "agent"), + "actedOnBehalfOf": ("qualifiedDelegation", "Delegation", "agent"), + "wasInfluencedBy": ("qualifiedInfluence", "Influence", "influencer"), + "hadPrimarySource": ("qualifiedPrimarySource", "PrimarySource", "entity"), + "wasQuotedFrom": ("qualifiedQuotation", "Quotation", "entity"), + "wasRevisionOf": ("qualifiedRevision", "Revision", "entity"), + "wasInvalidatedBy": ("qualifiedInvalidation", "Invalidation", "activity"), + "wasStartedBy": ("qualifiedStart", "Start", "entity"), + "wasEndedBy": ("qualifiedEnd", "End", "entity"), +} + + +def test_registry_contains_every_normative_prov_o_class_and_relation() -> None: + assert set(PROV_CLASSES) == EXPECTED_CLASS_NAMES + assert set(PROV_RELATIONS) == EXPECTED_RELATION_NAMES + assert len(PROV_CLASSES) == 30 + assert len(PROV_RELATIONS) == 50 + + +def test_registry_distinguishes_all_six_datatype_properties() -> None: + actual = {name for name, spec in PROV_RELATIONS.items() if spec.property_kind == "datatype"} + assert actual == EXPECTED_DATATYPE_RELATIONS + assert {name for name, spec in PROV_RELATIONS.items() if spec.property_kind == "object"} == ( + EXPECTED_RELATION_NAMES - EXPECTED_DATATYPE_RELATIONS + ) + + +def test_qualification_table_matches_both_normative_tables() -> None: + actual = { + item.unqualified_relation: ( + item.qualification_relation, + item.influence_class, + item.influencer_relation, + ) + for item in PROV_QUALIFICATIONS + } + assert actual == EXPECTED_QUALIFICATIONS + + +def test_every_object_property_has_the_appendix_b_inverse_name() -> None: + object_properties = { + name for name, spec in PROV_RELATIONS.items() if spec.property_kind == "object" + } + assert set(PROV_RECOMMENDED_INVERSES) == object_properties + assert PROV_RECOMMENDED_INVERSES["actedOnBehalfOf"].inverse_local_name == "hadDelegate" + assert PROV_RECOMMENDED_INVERSES["wasDerivedFrom"].inverse_local_name == "hadDerivation" + assert PROV_RECOMMENDED_INVERSES["specializationOf"].inverse_local_name == "generalizationOf" + assert PROV_RECOMMENDED_INVERSES["wasGeneratedBy"].inverse_local_name == "generated" + assert PROV_RECOMMENDED_INVERSES["alternateOf"].inverse_local_name == "alternateOf" + + +def test_codes_are_stable_two_word_snake_case() -> None: + assert class_code("Entity") == "prov_entity" + assert class_code("InstantaneousEvent") == "prov_instantaneous_event" + assert relation_code("wasGeneratedBy") == "prov_was_generated_by" + assert relation_code("qualifiedPrimarySource") == "prov_qualified_primary_source" + for name in PROV_CLASSES: + assert class_code(name).startswith("prov_") and "_" in class_code(name) + for name in PROV_RELATIONS: + assert relation_code(name).startswith("prov_") and "_" in relation_code(name) + + +def _graph_with_core_resources() -> ProvGraph: + graph = ProvGraph() + graph.add_resource("urn:entity:input", "Entity") + graph.add_resource("urn:entity:output", "Entity") + graph.add_resource("urn:activity:transform", "Activity") + graph.add_resource("urn:agent:operator", "Person") + graph.add_resource("urn:agent:principal", "Organization") + graph.add_resource("urn:location:lab", "Location") + graph.add_resource("urn:plan:procedure", "Plan") + graph.add_resource("urn:role:reviewer", "Role") + return graph + + +def test_graph_rejects_wrong_object_kind_and_wrong_domain() -> None: + graph = _graph_with_core_resources() + with pytest.raises(ProvValidationError, match="requires a resource object"): + graph.add_assertion("urn:activity:transform", "used", ProvLiteral("not-a-resource")) + with pytest.raises(ProvValidationError, match="requires a literal object"): + graph.add_assertion("urn:activity:transform", "startedAtTime", "urn:entity:input") + with pytest.raises(ProvValidationError, match="subject.*Entity"): + graph.add_assertion("urn:agent:operator", "wasDerivedFrom", "urn:entity:input") + + +def test_subclass_membership_satisfies_agent_domain() -> None: + graph = _graph_with_core_resources() + graph.add_assertion("urn:agent:operator", "actedOnBehalfOf", "urn:agent:principal") + assert ProvAssertion.resource( + "urn:agent:operator", "actedOnBehalfOf", "urn:agent:principal" + ) in graph.explicit_assertions + + +@pytest.mark.parametrize( + ("unqualified", "qualified", "influence_class", "influencer_relation", "subject", "object_iri"), + [ + ("wasGeneratedBy", "qualifiedGeneration", "Generation", "activity", "urn:entity:output", "urn:activity:transform"), + ("wasDerivedFrom", "qualifiedDerivation", "Derivation", "entity", "urn:entity:output", "urn:entity:input"), + ("wasAttributedTo", "qualifiedAttribution", "Attribution", "agent", "urn:entity:output", "urn:agent:operator"), + ("used", "qualifiedUsage", "Usage", "entity", "urn:activity:transform", "urn:entity:input"), + ("wasInformedBy", "qualifiedCommunication", "Communication", "activity", "urn:activity:transform", "urn:activity:source"), + ("wasAssociatedWith", "qualifiedAssociation", "Association", "agent", "urn:activity:transform", "urn:agent:operator"), + ("actedOnBehalfOf", "qualifiedDelegation", "Delegation", "agent", "urn:agent:operator", "urn:agent:principal"), + ("wasInfluencedBy", "qualifiedInfluence", "Influence", "influencer", "urn:entity:output", "urn:entity:input"), + ("hadPrimarySource", "qualifiedPrimarySource", "PrimarySource", "entity", "urn:entity:output", "urn:entity:input"), + ("wasQuotedFrom", "qualifiedQuotation", "Quotation", "entity", "urn:entity:output", "urn:entity:input"), + ("wasRevisionOf", "qualifiedRevision", "Revision", "entity", "urn:entity:output", "urn:entity:input"), + ("wasInvalidatedBy", "qualifiedInvalidation", "Invalidation", "activity", "urn:entity:output", "urn:activity:transform"), + ("wasStartedBy", "qualifiedStart", "Start", "entity", "urn:activity:transform", "urn:entity:input"), + ("wasEndedBy", "qualifiedEnd", "End", "entity", "urn:activity:transform", "urn:entity:output"), + ], +) +def test_each_qualified_form_implies_its_unqualified_form( + unqualified: str, + qualified: str, + influence_class: str, + influencer_relation: str, + subject: str, + object_iri: str, +) -> None: + graph = _graph_with_core_resources() + graph.add_resource("urn:activity:source", "Activity") + graph.add_resource("urn:influence:q", influence_class) + graph.add_assertion(subject, qualified, "urn:influence:q") + graph.add_assertion("urn:influence:q", influencer_relation, object_iri) + assert ProvAssertion.resource(subject, unqualified, object_iri) in graph.materialized_assertions() + + +def test_specific_derivation_implies_general_derivation_and_influence() -> None: + graph = _graph_with_core_resources() + graph.add_assertion("urn:entity:output", "wasQuotedFrom", "urn:entity:input") + materialized = graph.materialized_assertions() + assert ProvAssertion.resource("urn:entity:output", "wasDerivedFrom", "urn:entity:input") in materialized + assert ProvAssertion.resource("urn:entity:output", "wasInfluencedBy", "urn:entity:input") in materialized + + +def test_defined_inverse_and_symmetric_properties_are_materialized() -> None: + graph = _graph_with_core_resources() + graph.add_assertion("urn:entity:output", "wasGeneratedBy", "urn:activity:transform") + graph.add_assertion("urn:entity:output", "alternateOf", "urn:entity:input") + materialized = graph.materialized_assertions() + assert ProvAssertion.resource("urn:activity:transform", "generated", "urn:entity:output") in materialized + assert ProvAssertion.resource("urn:entity:input", "alternateOf", "urn:entity:output") in materialized + + +def test_reserved_inverse_alias_is_normalized_by_reversing_endpoints() -> None: + graph = _graph_with_core_resources() + graph.add_assertion("urn:entity:input", "hadDerivation", "urn:entity:output") + assert ProvAssertion.resource( + "urn:entity:output", "wasDerivedFrom", "urn:entity:input" + ) in graph.explicit_assertions + + +def test_qualified_event_time_implies_direct_time_property() -> None: + graph = _graph_with_core_resources() + graph.add_resource("urn:influence:generation", "Generation") + instant = ProvLiteral.datetime(datetime(2026, 8, 14, 4, 0, tzinfo=timezone.utc)) + graph.add_assertion("urn:entity:output", "qualifiedGeneration", "urn:influence:generation") + graph.add_assertion("urn:influence:generation", "activity", "urn:activity:transform") + graph.add_assertion("urn:influence:generation", "atTime", instant) + materialized = graph.materialized_assertions() + assert ProvAssertion.literal("urn:entity:output", "generatedAtTime", instant) in materialized + + +def test_rdf_serialization_uses_exact_prov_namespace_and_xsd_datetime() -> None: + graph = _graph_with_core_resources() + instant = ProvLiteral.datetime(datetime(2026, 8, 14, 4, 0, tzinfo=timezone.utc)) + graph.add_assertion("urn:activity:transform", "startedAtTime", instant) + rdf_graph = graph.to_rdflib(materialize=True) + assert (URIRef("urn:entity:input"), RDF.type, PROV.Entity) in rdf_graph + assert ( + URIRef("urn:activity:transform"), + PROV.startedAtTime, + Literal("2026-08-14T04:00:00+00:00", datatype=XSD.dateTime), + ) in rdf_graph + + +def test_sql_migration_seeds_every_class_relation_and_qualification() -> None: + sql_path = Path(__file__).resolve().parents[1] / "migrations" / "0017_prov_o_standard_relations.sql" + sql = sql_path.read_text() + for name in EXPECTED_CLASS_NAMES: + assert class_code(name) in sql + assert f"http://www.w3.org/ns/prov#{name}" in sql + for name in EXPECTED_RELATION_NAMES: + assert relation_code(name) in sql + assert f"http://www.w3.org/ns/prov#{name}" in sql + for unqualified, (qualified, influence_class, influencer) in EXPECTED_QUALIFICATIONS.items(): + assert relation_code(unqualified) in sql + assert relation_code(qualified) in sql + assert class_code(influence_class) in sql + assert relation_code(influencer) in sql + + +def test_sql_migration_uses_only_multiword_snake_case_table_names() -> None: + import re + + sql = (Path(__file__).resolve().parents[1] / "migrations" / "0017_prov_o_standard_relations.sql").read_text() + names = re.findall(r"create table(?: if not exists)?\s+([a-z_]+)", sql, flags=re.IGNORECASE) + assert names + assert all(len(name.split("_")) >= 2 for name in names) + + +def test_registry_spec_accessors_and_inverse_iri_use_exact_namespace() -> None: + assert PROV_CLASSES["Entity"].iri == "http://www.w3.org/ns/prov#Entity" + assert PROV_CLASSES["Entity"].code == "prov_entity" + assert PROV_RELATIONS["used"].iri == "http://www.w3.org/ns/prov#used" + assert PROV_RELATIONS["used"].code == "prov_used" + assert ( + PROV_RECOMMENDED_INVERSES["actedOnBehalfOf"].inverse_iri + == "http://www.w3.org/ns/prov#hadDelegate" + ) + + +def test_literal_contract_rejects_conflicts_invalid_language_and_naive_time() -> None: + with pytest.raises(ProvValidationError, match="both datatype_iri and language_tag"): + ProvLiteral("x", datatype_iri=str(XSD.string), language_tag="en") + with pytest.raises(ProvValidationError, match="language_tag"): + ProvLiteral("x", language_tag="not_a_tag!") + with pytest.raises(ProvValidationError, match="timezone-aware"): + ProvLiteral.datetime(datetime(2026, 8, 14, 4, 0)) + assert ProvLiteral("bonjour", language_tag="fr").to_rdflib() == Literal("bonjour", lang="fr") + + +def test_assertion_requires_exactly_one_object_kind() -> None: + with pytest.raises(ProvValidationError, match="exactly one"): + ProvAssertion("urn:s", "used") + with pytest.raises(ProvValidationError, match="exactly one"): + ProvAssertion( + "urn:s", + "used", + object_resource_iri="urn:o", + object_literal=ProvLiteral("x"), + ) + + +def test_resource_registration_and_name_normalization_fail_closed() -> None: + graph = ProvGraph() + with pytest.raises(ProvValidationError, match="resource_iri"): + graph.add_resource("", "Entity") + with pytest.raises(ProvValidationError, match="at least one"): + graph.add_resource("urn:empty") + with pytest.raises(ProvValidationError, match="unknown PROV-O class"): + graph.add_resource("urn:bad", "NotAClass") + + graph.add_resource("urn:e", "prov:Entity") + graph.add_resource("urn:a", "http://www.w3.org/ns/prov#Activity") + assert graph.resource_types == { + "urn:e": frozenset({"Entity"}), + "urn:a": frozenset({"Activity"}), + } + + +def test_assertion_name_and_endpoint_validation_fail_closed() -> None: + graph = _graph_with_core_resources() + with pytest.raises(ProvValidationError, match="unknown PROV-O relation"): + graph.add_assertion("urn:entity:input", "notARelation", "urn:entity:output") + with pytest.raises(ProvValidationError, match="subject resource"): + graph.add_assertion("urn:missing", "prov:wasDerivedFrom", "urn:entity:input") + with pytest.raises(ProvValidationError, match="object resource"): + graph.add_assertion( + "urn:entity:output", + "http://www.w3.org/ns/prov#wasDerivedFrom", + "urn:missing", + ) + with pytest.raises(ProvValidationError, match="object.*Entity"): + graph.add_assertion("urn:activity:transform", "used", "urn:role:reviewer") + with pytest.raises(ProvValidationError, match="requires datatype"): + graph.add_assertion( + "urn:activity:transform", + "startedAtTime", + ProvLiteral("2026-08-14T04:00:00Z"), + ) + with pytest.raises(ProvValidationError, match="cannot reverse a literal"): + graph.add_assertion( + "urn:entity:input", + "hadDerivation", + ProvLiteral("invalid"), + ) + + +def test_rdf_serialization_covers_resource_and_literal_objects_without_materialization() -> None: + graph = _graph_with_core_resources() + graph.add_assertion("urn:activity:transform", "used", "urn:entity:input") + graph.add_assertion("urn:entity:input", "value", ProvLiteral("raw value")) + rdf_graph = graph.to_rdflib() + assert ( + URIRef("urn:activity:transform"), + PROV.used, + URIRef("urn:entity:input"), + ) in rdf_graph + assert ( + URIRef("urn:entity:input"), + PROV.value, + Literal("raw value"), + ) in rdf_graph + + +def test_every_public_callable_has_a_docstring() -> None: + import inspect + + module = sys.modules["lineageweave.prov_o"] + + missing: list[str] = [] + for name, value in vars(module).items(): + if name.startswith("_"): + continue + if inspect.isfunction(value) or inspect.isclass(value): + if value.__module__ == module.__name__ and not inspect.getdoc(value): + missing.append(name) + if inspect.isclass(value) and value.__module__ == module.__name__: + for member_name, member in vars(value).items(): + if member_name.startswith("_"): + continue + target = member.fget if isinstance(member, property) else member + if callable(target) and not inspect.getdoc(target): + missing.append(f"{name}.{member_name}") + assert missing == [] + + +def test_support_profile_imports_prov_o_and_maps_product_classes() -> None: + from rdflib.namespace import OWL, RDFS + + profile_path = ( + Path(__file__).resolve().parents[1] + / "docs" + / "ontology" + / "prov-o-support-profile.ttl" + ) + profile = Graph().parse(profile_path, format="turtle") + ontology_iri = URIRef( + "https://contextualwisdomlab.github.io/LineageWeave/prov-o-support" + ) + local = Namespace("https://contextualwisdomlab.github.io/LineageWeave/ontology#") + assert ( + ontology_iri, + OWL.imports, + URIRef("http://www.w3.org/ns/prov-o#"), + ) in profile + assert (local.Post, RDFS.subClassOf, PROV.Entity) in profile + assert (local.Person, RDFS.subClassOf, PROV.Person) in profile + assert (local.CorporateEntity, RDFS.subClassOf, PROV.Organization) in profile + assert (local.Team, RDFS.subClassOf, PROV.Organization) in profile diff --git a/tests/test_prov_o_schema.py b/tests/test_prov_o_schema.py new file mode 100644 index 00000000..733c9607 --- /dev/null +++ b/tests/test_prov_o_schema.py @@ -0,0 +1,261 @@ +"""Real-PostgreSQL contract tests for the PROV-O migration. + +The module applies the actual base and PROV-O migration files to a throwaway +database. It self-skips when no local PostgreSQL is reachable, matching the +repository's existing real-database schema tests. +""" + +from __future__ import annotations + +import os +import uuid +from pathlib import Path +from urllib.parse import urlsplit, urlunsplit + +import pytest + +psycopg2 = pytest.importorskip("psycopg2") +sql = pytest.importorskip("psycopg2.sql") + +_ADMIN_DSN = os.environ.get( + "LINEAGEWEAVE_TEST_POSTGRES_ADMIN_DSN", "postgresql://localhost/postgres" +) +_ROOT = Path(__file__).resolve().parents[1] +_MIGRATION_PATHS = ( + _ROOT / "migrations" / "0001_initial_schema.sql", + _ROOT / "migrations" / "0017_prov_o_standard_relations.sql", +) + + +def _postgres_available() -> bool: + """Return whether the configured PostgreSQL admin database is reachable.""" + try: + connection = psycopg2.connect(_ADMIN_DSN, connect_timeout=2) + connection.close() + return True + except psycopg2.OperationalError: + return False + + +def _dsn_for_database(admin_dsn: str, database_name: str) -> str: + """Replace only the database path while preserving DSN query options.""" + parsed_admin_dsn = urlsplit(admin_dsn) + return urlunsplit(parsed_admin_dsn._replace(path=f"/{database_name}")) + + +pytestmark = pytest.mark.skipif( + not _postgres_available(), + reason=f"no reachable PostgreSQL server at {_ADMIN_DSN}", +) + + +@pytest.fixture +def prov_schema_db(): + """Yield a freshly migrated database and drop it after the test.""" + database_name = f"lineageweave_prov_{uuid.uuid4().hex[:12]}" + admin_connection = psycopg2.connect(_ADMIN_DSN) + admin_connection.autocommit = True + with admin_connection.cursor() as cursor: + cursor.execute( + sql.SQL("create database {}").format(sql.Identifier(database_name)) + ) + try: + database_dsn = _dsn_for_database(_ADMIN_DSN, database_name) + connection = psycopg2.connect(database_dsn) + try: + with connection.cursor() as cursor: + for migration_path in _MIGRATION_PATHS: + cursor.execute(migration_path.read_text()) + connection.commit() + yield connection + finally: + connection.close() + finally: + with admin_connection.cursor() as cursor: + cursor.execute( + sql.SQL("drop database {}").format(sql.Identifier(database_name)) + ) + admin_connection.close() + + +def _resource(cursor, iri: str, class_code: str) -> str: + """Insert one typed provenance resource and return its UUID.""" + cursor.execute( + "insert into provenance_resource (resource_iri) values (%s) returning resource_id", + (iri,), + ) + resource_id = cursor.fetchone()[0] + cursor.execute( + "insert into provenance_resource_type (resource_id, class_code) values (%s, %s)", + (resource_id, class_code), + ) + return str(resource_id) + + +def test_catalog_has_every_normative_term(prov_schema_db) -> None: + """The database catalog exactly matches the Recommendation inventory.""" + with prov_schema_db.cursor() as cursor: + cursor.execute("select count(*) from provenance_class_definition") + assert cursor.fetchone()[0] == 30 + cursor.execute("select count(*) from provenance_relation_definition") + assert cursor.fetchone()[0] == 50 + cursor.execute("select count(*) from provenance_qualification_definition") + assert cursor.fetchone()[0] == 14 + cursor.execute("select count(*) from provenance_inverse_definition") + assert cursor.fetchone()[0] == 44 + + +def test_database_accepts_valid_generation_and_rejects_wrong_domain(prov_schema_db) -> None: + """Recursive class-domain checks are enforced by PostgreSQL itself.""" + with prov_schema_db.cursor() as cursor: + entity_id = _resource(cursor, "urn:test:entity", "prov_entity") + activity_id = _resource(cursor, "urn:test:activity", "prov_activity") + agent_id = _resource(cursor, "urn:test:agent", "prov_person") + cursor.execute( + "insert into provenance_assertion " + "(subject_resource_id, relation_code, object_resource_id) " + "values (%s, 'prov_was_generated_by', %s)", + (entity_id, activity_id), + ) + with pytest.raises(psycopg2.errors.RaiseException, match="violates PROV-O domain"): + cursor.execute( + "insert into provenance_assertion " + "(subject_resource_id, relation_code, object_resource_id) " + "values (%s, 'prov_was_derived_from', %s)", + (agent_id, entity_id), + ) + prov_schema_db.rollback() + + +def test_database_rejects_literal_for_object_property(prov_schema_db) -> None: + """Object/datatype shape cannot be bypassed by direct SQL writes.""" + with prov_schema_db.cursor() as cursor: + entity_id = _resource(cursor, "urn:test:shape-entity", "prov_entity") + cursor.execute( + "insert into provenance_literal_value (lexical_value) values ('bad') returning literal_id" + ) + literal_id = cursor.fetchone()[0] + with pytest.raises(psycopg2.errors.RaiseException, match="requires object_resource_id"): + cursor.execute( + "insert into provenance_assertion " + "(subject_resource_id, relation_code, object_literal_id) " + "values (%s, 'prov_was_derived_from', %s)", + (entity_id, literal_id), + ) + prov_schema_db.rollback() + + +def test_database_requires_xsd_datetime_for_event_time(prov_schema_db) -> None: + """Date properties reject untyped lexical strings at the storage boundary.""" + with prov_schema_db.cursor() as cursor: + activity_id = _resource(cursor, "urn:test:time-activity", "prov_activity") + cursor.execute( + "insert into provenance_literal_value (lexical_value) " + "values ('2026-08-14T04:00:00Z') returning literal_id" + ) + literal_id = cursor.fetchone()[0] + with pytest.raises(psycopg2.errors.RaiseException, match="violates datatype"): + cursor.execute( + "insert into provenance_assertion " + "(subject_resource_id, relation_code, object_literal_id) " + "values (%s, 'prov_started_at_time', %s)", + (activity_id, literal_id), + ) + prov_schema_db.rollback() + + +def _literal(cursor, lexical_value: str, datatype_iri: str | None) -> str: + """Insert one RDF literal and return its UUID.""" + cursor.execute( + "insert into provenance_literal_value (lexical_value, datatype_iri) " + "values (%s, %s) returning literal_id", + (lexical_value, datatype_iri), + ) + return str(cursor.fetchone()[0]) + + +@pytest.mark.parametrize( + "lexical_value", + ( + "2026-08-14T04:00:00", + "not-a-date", + "2026-02-31T04:00:00Z", + "2026-08-14T04:00:00+14:01", + ), +) +def test_database_rejects_invalid_xsd_datetime(prov_schema_db, lexical_value: str) -> None: + """Malformed and timezone-less xsd:dateTime values fail closed.""" + with prov_schema_db.cursor() as cursor: + activity_id = _resource(cursor, "urn:test:strict-time", "prov_activity") + literal_id = _literal( + cursor, + lexical_value, + "http://www.w3.org/2001/XMLSchema#dateTime", + ) + with pytest.raises(psycopg2.errors.RaiseException, match="lexical xsd:dateTime"): + cursor.execute( + "insert into provenance_assertion " + "(subject_resource_id, relation_code, object_literal_id) " + "values (%s, 'prov_started_at_time', %s)", + (activity_id, literal_id), + ) + prov_schema_db.rollback() + + +def test_database_accepts_timezone_aware_xsd_datetime(prov_schema_db) -> None: + """A valid timezone-aware dateTime reaches the assertion store.""" + with prov_schema_db.cursor() as cursor: + activity_id = _resource(cursor, "urn:test:valid-time", "prov_activity") + literal_id = _literal( + cursor, + "2026-08-14T04:00:00+09:00", + "http://www.w3.org/2001/XMLSchema#dateTime", + ) + cursor.execute( + "insert into provenance_assertion " + "(subject_resource_id, relation_code, object_literal_id) " + "values (%s, 'prov_started_at_time', %s)", + (activity_id, literal_id), + ) + prov_schema_db.rollback() + + +def test_referenced_contract_rows_are_immutable(prov_schema_db) -> None: + """Reference-table mutation cannot invalidate stored assertions.""" + with prov_schema_db.cursor() as cursor: + entity_id = _resource(cursor, "urn:test:immutable-entity", "prov_entity") + activity_id = _resource(cursor, "urn:test:immutable-activity", "prov_activity") + cursor.execute( + "insert into provenance_assertion " + "(subject_resource_id, relation_code, object_resource_id) " + "values (%s, 'prov_was_generated_by', %s)", + (entity_id, activity_id), + ) + with pytest.raises(psycopg2.errors.RaiseException, match="types are immutable"): + cursor.execute( + "delete from provenance_resource_type " + "where resource_id = %s and class_code = 'prov_activity'", + (activity_id,), + ) + prov_schema_db.rollback() + + with prov_schema_db.cursor() as cursor: + activity_id = _resource(cursor, "urn:test:immutable-time", "prov_activity") + literal_id = _literal( + cursor, + "2026-08-14T04:00:00Z", + "http://www.w3.org/2001/XMLSchema#dateTime", + ) + cursor.execute( + "insert into provenance_assertion " + "(subject_resource_id, relation_code, object_literal_id) " + "values (%s, 'prov_started_at_time', %s)", + (activity_id, literal_id), + ) + with pytest.raises(psycopg2.errors.RaiseException, match="literal values are immutable"): + cursor.execute( + "update provenance_literal_value set datatype_iri = null " + "where literal_id = %s", + (literal_id,), + ) + prov_schema_db.rollback() diff --git a/tests/test_schema.py b/tests/test_schema.py index bd3258bd..3dd99b31 100644 --- a/tests/test_schema.py +++ b/tests/test_schema.py @@ -17,6 +17,7 @@ import os import uuid from pathlib import Path +from urllib.parse import urlsplit, urlunsplit import psycopg2 import psycopg2.errors @@ -52,7 +53,8 @@ def schema_db(): with admin_conn.cursor() as cur: cur.execute(f'create database "{db_name}"') try: - db_dsn = _ADMIN_DSN.rsplit("/", 1)[0] + f"/{db_name}" + parsed_admin_dsn = urlsplit(_ADMIN_DSN) + db_dsn = urlunsplit(parsed_admin_dsn._replace(path=f"/{db_name}")) conn = psycopg2.connect(db_dsn) try: with conn.cursor() as cur: @@ -88,7 +90,9 @@ def test_migration_applies_cleanly(schema_db) -> None: "cataloged_person", "person_affiliation", "post_person_mention", + "post_summary_person_mention", "knowledge_graph_edge", + "knowledge_graph_edge_evidence", "issue_ticket", "post_lineage_edge", "post_evaluation_response", @@ -102,6 +106,7 @@ def test_migration_applies_cleanly(schema_db) -> None: "post_summary_role", "post_chat_result", "post_chat_citation", + "abbreviation_tree_corroboration", } assert expected <= tables @@ -205,14 +210,38 @@ def test_lookup_code_is_unique_across_categories(schema_db) -> None: def test_every_created_table_name_has_at_least_two_words() -> None: - """The project naming rule is enforced on the shipped migration, not - only on tables that happen to be created in a live-Postgres run. - """ + """Enforce naming for ordinary and idempotent table declarations.""" import re sql = _MIGRATION_PATH.read_text() - names = re.findall(r"create table (\w+)", sql) + names = re.findall( + r"create\s+table\s+(?:if\s+not\s+exists\s+)?([a-z][a-z0-9_]*)", + sql, + flags=re.IGNORECASE, + ) assert names, "migration must create at least one table" for name in names: words = name.split("_") assert len(words) >= 2, f"table {name!r} must be two or more snake_case words" + + +def test_cataloged_team_null_affiliation_is_unique(schema_db) -> None: + """Repeated NULL-affiliation upserts return one catalog identity.""" + with schema_db.cursor() as cursor: + ids = [] + for _ in range(2): + cursor.execute( + "insert into cataloged_team (team_name, affiliated_organization_name) " + "values ('Synthetic Design Team', null) " + "on conflict (team_name, affiliated_organization_name) do update " + "set team_name = excluded.team_name returning team_id" + ) + ids.append(cursor.fetchone()[0]) + cursor.execute( + "select count(*) from cataloged_team " + "where team_name = 'Synthetic Design Team' " + "and affiliated_organization_name is null" + ) + count = cursor.fetchone()[0] + assert ids[0] == ids[1] + assert count == 1 diff --git a/tests/test_seed_analysis_run_reconstruction.py b/tests/test_seed_analysis_run_reconstruction.py new file mode 100644 index 00000000..7f73800e --- /dev/null +++ b/tests/test_seed_analysis_run_reconstruction.py @@ -0,0 +1,38 @@ +"""Seeded Succeeded lineage runs persist the designed A-100 fork.""" + +from datetime import datetime, timezone + +from lineageweave.fixtures import sample_records +from scripts.seed_demo_data import seed_reconstruction_edges + + +def _rows_from_fixtures() -> list[dict]: + cutoff = datetime(2026, 1, 12, 12, 0, tzinfo=timezone.utc) + rows: list[dict] = [] + for record in sample_records(): + occurred = record.occurred_at.replace(tzinfo=timezone.utc) + if occurred > cutoff: + continue + rows.append( + { + "post_id": record.record_id, + "post_title": record.label, + "created_at": occurred, + "visibility_code": "public", + "corporate_entity_id": "demo-corp", + "process_unit_id": "demo-pu", + "thread_group_key": record.group_key, + "secondary_grouping_key": record.secondary_key, + } + ) + return rows + + +def test_seed_reconstruction_recovers_the_a100_fork() -> None: + """Seed must persist the same parent choices start uses.""" + edges, digest = seed_reconstruction_edges(_rows_from_fixtures()) + children = {edge.child_id for edge in edges if edge.parent_id == "rec-002"} + assert children >= {"rec-003", "rec-004"} + assert "rec-006" not in {edge.child_id for edge in edges} + assert digest + assert "theta" not in digest diff --git a/tests/test_seed_late_demo_post.py b/tests/test_seed_late_demo_post.py new file mode 100644 index 00000000..14688350 --- /dev/null +++ b/tests/test_seed_late_demo_post.py @@ -0,0 +1,147 @@ +"""Late Demo public post is the ADR 0016 own-corp cutoff counter-example.""" + +from datetime import datetime, timezone +from inspect import getsource +from pathlib import Path + +from scripts.seed_demo_data import ( + DEMO_ANALYSIS_RUN_KNOWLEDGE_CUTOFF, + DEMO_PUBLIC_POST_CREATED_AT, + DEMO_PUBLIC_POST_TITLE, + LATE_DEMO_PUBLIC_POST_BODY, + LATE_DEMO_PUBLIC_POST_CREATED_AT, + LATE_DEMO_PUBLIC_POST_TITLE, + seed, + seed_late_demo_public_post, + tepp_accepted_seed_request, + tepp_seed_outcome, + tepp_seed_request, + _ensure_demo_source_snapshot_members, + _seed_demo_run_reconstruction, +) + + +def _parse_seed_clock(value: str) -> datetime: + """Parse a seeded ISO-8601 Z clock as UTC.""" + return datetime.fromisoformat(value.replace("Z", "+00:00")) + + +class _LateDemoCursor: + """Drive ``seed_late_demo_public_post`` without a live database.""" + + def __init__(self, existing: bool = False) -> None: + self.existing = existing + self.statements: list[str] = [] + self.params: list[object] = [] + + def execute(self, sql: str, params=None) -> None: + self.statements.append(" ".join(sql.split())) + self.params.append(params) + + def fetchone(self): + last = self.statements[-1] + if last.lstrip().startswith("select") and "from source_post" in last: + return ("late-demo-id",) if self.existing else None + return None + + +def test_january_12_run_lists_demo_public_not_late_demo() -> None: + """ADR 0016 ``created_at <= knowledge_cutoff`` keeps Demo public, drops Late Demo.""" + cutoff = _parse_seed_clock(DEMO_ANALYSIS_RUN_KNOWLEDGE_CUTOFF) + listed = { + title: _parse_seed_clock(created_at) <= cutoff + for title, created_at in ( + (DEMO_PUBLIC_POST_TITLE, DEMO_PUBLIC_POST_CREATED_AT), + (LATE_DEMO_PUBLIC_POST_TITLE, LATE_DEMO_PUBLIC_POST_CREATED_AT), + ) + } + assert cutoff == datetime(2026, 1, 12, 12, 0, tzinfo=timezone.utc) + assert listed[DEMO_PUBLIC_POST_TITLE] is True + assert listed[LATE_DEMO_PUBLIC_POST_TITLE] is False + + +def test_listing_still_uses_created_at_not_a_second_cutoff() -> None: + """Visible-post SQL stays the ADR 0016 created_at gate on every scope.""" + listing = ( + Path(__file__).resolve().parents[1] + / "backend" + / "app" + / "analysis_run_ingestion.py" + ).read_text(encoding="utf-8") + start = listing.index("async def fetch_visible_scope_posts") + end = listing.index("\nclass AnalysisRunCreateError") + listing_fn = listing[start:end] + assert listing_fn.count("created_at <= $") == 4 + assert "LATE_DEMO" not in listing_fn + + +def test_reconstruction_seed_uses_the_january_12_cutoff() -> None: + """Run reconstruction persists only posts known at the same analysis clock.""" + source = getsource(_seed_demo_run_reconstruction) + assert "created_at <= %s" in source + assert "DEMO_ANALYSIS_RUN_KNOWLEDGE_CUTOFF" in source + assert LATE_DEMO_PUBLIC_POST_TITLE not in source + + +def test_snapshot_members_exclude_late_demo_created_at() -> None: + """Frozen snapshot membership uses created_at before Late Demo exists.""" + source = getsource(_ensure_demo_source_snapshot_members) + assert "created_at <= '2026-01-12T00:00:00Z'" in source + late = _parse_seed_clock(LATE_DEMO_PUBLIC_POST_CREATED_AT) + snapshot_max = datetime(2026, 1, 12, tzinfo=timezone.utc) + assert late > snapshot_max + + +def test_seed_late_demo_public_post_inserts_after_cutoff() -> None: + """Persist writes the own-corp public counter-example dated 2026-01-13.""" + cursor = _LateDemoCursor() + seed_late_demo_public_post(cursor, "account-1", "corp-1", "pu-1") + inserts = [ + (sql, params) + for sql, params in zip(cursor.statements, cursor.params, strict=True) + if "insert into source_post" in sql + ] + assert inserts, "missing Late Demo must be inserted" + sql, params = inserts[0] + assert params is not None + assert LATE_DEMO_PUBLIC_POST_TITLE in params + assert LATE_DEMO_PUBLIC_POST_BODY in params + assert LATE_DEMO_PUBLIC_POST_CREATED_AT in params + assert params.count(LATE_DEMO_PUBLIC_POST_CREATED_AT) == 2 + assert "public" in sql + assert "theta" not in sql.lower() + assert not any( + isinstance(value, str) and ("theta" in value.lower() or "θ" in value) + for value in params + ) + + +def test_seed_late_demo_public_post_skips_when_already_present() -> None: + """Re-seed must not invent a second Late Demo row.""" + cursor = _LateDemoCursor(existing=True) + seed_late_demo_public_post(cursor, "account-1", "corp-1", "pu-1") + assert not any("insert into source_post" in sql for sql in cursor.statements) + + +def test_seed_calls_late_demo_before_analysis_runs() -> None: + """``seed()`` writes Late Demo, then lineage/TEPP runs on the January 12 clock.""" + source = getsource(seed) + late_at = source.index("seed_late_demo_public_post(") + lineage_at = source.index("_seed_demo_analysis_run(") + tepp_at = source.index("_seed_demo_tepp_run(") + assert late_at < lineage_at < tepp_at + helper = getsource(seed_late_demo_public_post) + insert_sql = helper[helper.index("insert into source_post") :] + assert "created_at <= " not in insert_sql + assert "theta" not in insert_sql.lower() + + +def test_tepp_seed_keeps_the_same_january_12_cutoff() -> None: + """Late Demo does not fork TEPP arithmetic or stamp Succeeded.""" + request = tepp_seed_request() + accepted = tepp_accepted_seed_request() + assert request.knowledge_cutoff == DEMO_ANALYSIS_RUN_KNOWLEDGE_CUTOFF + assert accepted.knowledge_cutoff == DEMO_ANALYSIS_RUN_KNOWLEDGE_CUTOFF + status, failure = tepp_seed_outcome() + assert status == "analysis_status_failed" + assert failure == "tepp_not_available" diff --git a/tests/test_seed_report_run.py b/tests/test_seed_report_run.py new file mode 100644 index 00000000..9ed060fc --- /dev/null +++ b/tests/test_seed_report_run.py @@ -0,0 +1,104 @@ +"""Seeded period-report analysis runs record the built report, never a theta.""" + +import inspect + +from scripts.seed_demo_data import ( + DEMO_REPORT_IDEMPOTENCY_KEY, + _seed_demo_report_run, + seed, +) + + +class _ReportSeedCursor: + """Drive ``_seed_demo_report_run`` without a live database.""" + + def __init__(self) -> None: + self.statements: list[str] = [] + self.params: list[object] = [] + + def execute(self, sql: str, params=None) -> None: + self.statements.append(" ".join(sql.split())) + self.params.append(params) + + def fetchone(self): + last = self.statements[-1] + if last.lstrip().startswith("select") and "from analysis_source_snapshot" in last: + return None + if "insert into analysis_source_snapshot" in last: + return ("snapshot-demo",) + if last.lstrip().startswith("select") and "from analysis_source_count" in last: + return None + if last.lstrip().startswith("select") and "from analysis_run" in last: + return None + if "insert into analysis_run" in last: + return ("run-demo-report",) + return None + + +def test_seed_calls_report_run_after_period_report_tables() -> None: + """``seed()`` must persist scored tables before the Succeeded registry row.""" + source = inspect.getsource(seed) + period_at = source.index("_seed_demo_period_report(") + report_at = source.index("_seed_demo_report_run(") + assert period_at < report_at + assert "theta" not in source[period_at:report_at].lower() + assert "θ" not in source[period_at:report_at] + + +def test_seed_demo_report_run_inserts_succeeded_report_without_a_theta() -> None: + cursor = _ReportSeedCursor() + _seed_demo_report_run(cursor, "account-1", "corp-1") + run_inserts = [ + (sql, params) + for sql, params in zip(cursor.statements, cursor.params, strict=True) + if "insert into analysis_run" in sql + ] + assert run_inserts, "seed must insert the period-report analysis_run row" + sql, params = run_inserts[0] + assert "analysis_run_report" in sql + assert params is not None + assert DEMO_REPORT_IDEMPOTENCY_KEY in params + assert "report-run-v1" in sql + assert not any( + isinstance(value, str) and ("theta" in value.lower() or "θ" in value) + for value in params + ) + status_params = [ + params + for sql, params in zip(cursor.statements, cursor.params, strict=True) + if "insert into analysis_run_status_event" in sql + ] + assert any( + event_params is not None and "analysis_status_succeeded" in event_params + for event_params in status_params + ) + assert not any( + event_params is not None and "analysis_status_failed" in event_params + for event_params in status_params + ) + assert not any( + event_params is not None + and any( + isinstance(value, str) and "theta" in value.lower() + for value in event_params + ) + for event_params in status_params + ) + assert not any("analysis_run_outbox" in sql for sql in cursor.statements) + scope_params = [ + params + for sql, params in zip(cursor.statements, cursor.params, strict=True) + if "insert into analysis_run_scope" in sql + ] + assert any( + event_params is not None and "2026-W02" in event_params + for event_params in scope_params + ) + assert not any( + event_params is not None + and any( + isinstance(value, str) and ("theta" in value.lower() or "θ" in value) + for value in event_params + ) + for event_params in scope_params + ) diff --git a/tests/test_seed_tepp_run.py b/tests/test_seed_tepp_run.py new file mode 100644 index 00000000..69484862 --- /dev/null +++ b/tests/test_seed_tepp_run.py @@ -0,0 +1,183 @@ +"""Seeded TEPP analysis runs go through tepp_client, never a local model.""" + +from lineageweave.tepp_client import AnalysisRunRequest, TeppClient, TeppNotAvailable +from lineageweave.tepp_result import persistable_tepp_seed_envelope +from scripts.seed_demo_data import ( + _ensure_demo_source_counts, + _seed_demo_accepted_tepp_run, + _seed_demo_tepp_run, + demo_source_snapshot_sha256, + tepp_accepted_seed_client, + tepp_accepted_seed_request, + tepp_persistable_seed_client, + tepp_seed_outcome, + tepp_seed_request, +) + + +class _RecordingUnavailableClient(TeppClient): + """Default-path stand-in that records the request then drops the channel.""" + + def __init__(self) -> None: + super().__init__() + self.submitted: list[AnalysisRunRequest] = [] + + def submit_analysis_run(self, request: AnalysisRunRequest) -> dict[str, object]: + self.submitted.append(request) + raise TeppNotAvailable("TEPP has no live HTTP endpoint yet.") + + +class _AcceptingClient(TeppClient): + """Transport that returns an envelope without a persistable measurement.""" + + def __init__(self) -> None: + super().__init__(transport=lambda _payload: {"status": "accepted"}) + + +class _CountCursor: + """Minimal cursor for proving re-seed skips a frozen count insert.""" + + def __init__(self, existing_counts: bool) -> None: + self.existing_counts = existing_counts + self.statements: list[str] = [] + + def execute(self, sql: str, _params=None) -> None: + self.statements.append(" ".join(sql.split())) + + def fetchone(self): + if self.existing_counts and "from analysis_source_count" in self.statements[-1]: + return (1,) + return None + + +def test_tepp_seed_request_targets_the_shared_demo_snapshot() -> None: + request = tepp_seed_request() + assert request.snapshot_id == demo_source_snapshot_sha256() + assert request.idempotency_key == "demo-tepp-seed-2026-w02" + assert request.model_contract_version == "tepp-analysis-run-v1" + assert request.output_profile == "calibrated_event_measurement" + + +def test_tepp_seed_outcome_calls_client_and_does_not_invent_a_score() -> None: + client = _RecordingUnavailableClient() + status, failure = tepp_seed_outcome(client) + assert status == "analysis_status_failed" + assert failure == "tepp_not_available" + assert client.submitted == [tepp_seed_request()] + + +def test_tepp_seed_outcome_default_client_is_unavailable_not_a_fake_score() -> None: + status, failure = tepp_seed_outcome() + assert status == "analysis_status_failed" + assert failure == "tepp_not_available" + + +def test_tepp_seed_outcome_does_not_treat_an_empty_envelope_as_success() -> None: + status, failure = tepp_seed_outcome(_AcceptingClient()) + assert status == "analysis_status_failed" + assert failure == "tepp_result_not_persisted" + + +def test_tepp_seed_outcome_keeps_a_published_accepted_envelope_failed() -> None: + request = tepp_seed_request() + status, failure = tepp_seed_outcome( + tepp_accepted_seed_client(request.idempotency_key), + request, + ) + assert status == "analysis_status_failed" + assert failure == "tepp_completed_result_unsupported" + + +def test_tepp_seed_outcome_rejects_a_local_completed_envelope() -> None: + status, failure = tepp_seed_outcome(tepp_persistable_seed_client()) + assert status == "analysis_status_failed" + assert failure == "tepp_result_not_persisted" + assert persistable_tepp_seed_envelope()["affiliation_count"] == 2 + + +def test_ensure_demo_source_counts_skips_insert_when_counts_exist() -> None: + cursor = _CountCursor(existing_counts=True) + _ensure_demo_source_counts(cursor, "snapshot-1") + assert any("from analysis_source_count" in sql for sql in cursor.statements) + assert not any(sql.lstrip().startswith("insert into analysis_source_count") for sql in cursor.statements) + + +def test_ensure_demo_source_counts_inserts_when_the_snapshot_is_empty() -> None: + cursor = _CountCursor(existing_counts=False) + _ensure_demo_source_counts(cursor, "snapshot-1") + assert any(sql.lstrip().startswith("insert into analysis_source_count") for sql in cursor.statements) + + +class _TeppSeedCursor: + """Drive `_seed_demo_tepp_run` without a live database.""" + + def __init__(self) -> None: + self.statements: list[str] = [] + self.params: list[object] = [] + + def execute(self, sql: str, params=None) -> None: + self.statements.append(" ".join(sql.split())) + self.params.append(params) + + def fetchone(self): + last = self.statements[-1] + if last.lstrip().startswith("select") and "from analysis_source_snapshot" in last: + return None + if "insert into analysis_source_snapshot" in last: + return ("snapshot-demo",) + if last.lstrip().startswith("select") and "from analysis_source_count" in last: + return None + if last.lstrip().startswith("select") and "from analysis_run" in last: + return None + if "insert into analysis_run" in last: + return ("run-demo-tepp",) + return None + + +def test_seed_demo_tepp_run_inserts_failed_tepp_not_available() -> None: + cursor = _TeppSeedCursor() + _seed_demo_tepp_run(cursor, "account-1", "corp-1") + run_inserts = [sql for sql in cursor.statements if "insert into analysis_run" in sql] + assert run_inserts, "seed must insert the TEPP analysis_run row" + assert any("analysis_run_tepp" in sql for sql in run_inserts) + status_params = [ + params + for sql, params in zip(cursor.statements, cursor.params, strict=True) + if "insert into analysis_run_status_event" in sql + ] + assert any( + params is not None and "analysis_status_failed" in params and "tepp_not_available" in params + for params in status_params + ) + assert not any( + params is not None and "analysis_status_succeeded" in params for params in status_params + ) + + +def test_seed_demo_accepted_tepp_run_persists_transport_evidence() -> None: + cursor = _TeppSeedCursor() + _seed_demo_accepted_tepp_run(cursor, "account-1", "corp-1") + assert any("insert into analysis_run_tepp_accepted" in sql for sql in cursor.statements) + assert not any("insert into analysis_run_tepp_result" in sql for sql in cursor.statements) + status_params = [ + params + for sql, params in zip(cursor.statements, cursor.params, strict=True) + if "insert into analysis_run_status_event" in sql + ] + assert any( + params is not None + and "analysis_status_failed" in params + and "tepp_completed_result_unsupported" in params + for params in status_params + ) + assert not any( + params is not None and "analysis_status_succeeded" in params for params in status_params + ) + result_params = [ + params + for sql, params in zip(cursor.statements, cursor.params, strict=True) + if "insert into analysis_run_tepp_accepted" in sql + ] + assert result_params + assert all(params is not None and "theta" not in str(params).casefold() for params in result_params) + assert tepp_accepted_seed_request().idempotency_key == "demo-tepp-seed-2026-w02-succeeded" diff --git a/tests/test_source_post_revision.py b/tests/test_source_post_revision.py new file mode 100644 index 00000000..507ea90e --- /dev/null +++ b/tests/test_source_post_revision.py @@ -0,0 +1,64 @@ +"""Cutoff-known bodies come from source_post_revision, never an invented sentence.""" + +from datetime import datetime, timezone +from pathlib import Path + +from backend.app.source_post_revision import parse_as_of_clock, revision_covers_clock + +_ROOT = Path(__file__).resolve().parents[1] +_MIGRATION = _ROOT / "migrations" / "0024_source_post_revision.sql" +_ROLLBACK = _ROOT / "migrations" / "rollback" / "0024_source_post_revision.sql" +_CUTOFF = datetime(2026, 1, 12, 12, 0, tzinfo=timezone.utc) + + +def test_parse_as_of_clock_treats_z_and_naive_as_utc() -> None: + parsed = parse_as_of_clock("2026-01-12T12:00:00Z") + assert parsed == _CUTOFF + naive = parse_as_of_clock("2026-01-12T12:00:00") + assert naive == _CUTOFF + + +def test_parse_as_of_clock_rejects_empty_or_unparseable() -> None: + try: + parse_as_of_clock(" ") + except ValueError as exc: + assert "empty" in str(exc) + else: + raise AssertionError("empty as_of must fail closed") + try: + parse_as_of_clock("not-a-clock") + except ValueError: + return + raise AssertionError("unparseable as_of must fail closed") + + +def test_revision_interval_is_half_open() -> None: + written = datetime(2026, 1, 10, 12, 0, tzinfo=timezone.utc) + superseded = datetime(2026, 1, 13, 9, 0, tzinfo=timezone.utc) + assert revision_covers_clock(written, superseded, written) is True + assert revision_covers_clock(written, superseded, _CUTOFF) is True + assert revision_covers_clock(written, superseded, superseded) is False + assert revision_covers_clock(written, None, superseded) is True + assert revision_covers_clock(superseded, None, _CUTOFF) is False + + +def test_revision_migration_records_title_or_body_rewrites_only() -> None: + sql = _MIGRATION.read_text(encoding="utf-8") + rollback = _ROLLBACK.read_text(encoding="utf-8") + assert "source_post_revision" in sql + assert "record_source_post_revision" in sql + assert "update of post_title, post_body" in sql + assert "superseded_at" in sql + assert "drop table if exists source_post_revision" in rollback + seed = (_ROOT / "scripts" / "seed_demo_data.py").read_text(encoding="utf-8") + assert "later delivery window" in seed + assert "delayed shipment." in seed + assert seed.index("0023_analysis_run_outbox.sql") < seed.index( + "0024_source_post_revision.sql" + ) + assert seed.index("0024_source_post_revision.sql") < seed.index( + "0025_role_person_catalog_identity.sql" + ) + assert seed.index("0025_role_person_catalog_identity.sql") < seed.index( + "0026_report_leftover_pair.sql" + ) diff --git a/tests/test_tepp_public_content.py b/tests/test_tepp_public_content.py new file mode 100644 index 00000000..54643855 --- /dev/null +++ b/tests/test_tepp_public_content.py @@ -0,0 +1,42 @@ +"""Public-content denylist for the TEPP honesty correction.""" + +from __future__ import annotations + +from pathlib import Path + +_ROOT = Path(__file__).resolve().parents[1] +_SCOPED_PATHS = ( + _ROOT / "lineageweave" / "tepp_result.py", + _ROOT / "lineageweave" / "tepp_client.py", + _ROOT / "backend" / "app" / "analysis_run_start.py", + _ROOT / "backend" / "app" / "analysis_run_ingestion.py", + _ROOT / "migrations" / "0029_analysis_run_tepp_accepted.sql", + _ROOT / "docs" / "adr" / "0035-tepp-accepted-transport-evidence.md", + _ROOT / "CHANGELOG.d" / "2.12.1-tepp-accepted-transport-evidence.md", + _ROOT / "CHANGELOG.d" / "2.12.2-tepp-accepted-clocks.md", + _ROOT / "tests" / "test_tepp_result.py", + _ROOT / "tests" / "test_analysis_run_tepp_accepted_schema.py", +) +_FORBIDDEN_TABLES = ( + "document_record", + "model_artifact", + "topic_prevalence", + "membership_assignment", + "event_instance", +) +_FORBIDDEN_SECRETS = ( + "NVIDIA_NIM_API_KEY", + "postgres://", +) + + +def test_tepp_honesty_files_keep_synthetic_public_content() -> None: + """Changed TEPP files must not leak private tables or credentials.""" + for path in _SCOPED_PATHS: + text = path.read_text(encoding="utf-8") + lowered = text.casefold() + for token in _FORBIDDEN_TABLES: + assert token not in lowered, f"{token} in {path.name}" + for token in _FORBIDDEN_SECRETS: + assert token.casefold() not in lowered, f"{token} in {path.name}" + assert "is a validated multilevel estimate" not in lowered diff --git a/tests/test_tepp_result.py b/tests/test_tepp_result.py new file mode 100644 index 00000000..371bad5c --- /dev/null +++ b/tests/test_tepp_result.py @@ -0,0 +1,73 @@ +"""Published TEPP accepted evidence is not a completed measurement.""" + +from __future__ import annotations + +from lineageweave.tepp_result import ( + accepted_tepp_seed_envelope, + parse_persistable_tepp_result, + parse_tepp_accepted_evidence, + persistable_tepp_seed_envelope, + tepp_accepted_evidence_sha256, +) + + +def test_published_accepted_envelope_is_transport_evidence() -> None: + """TEPP's AnalysisRunAccepted fields are storeable transport evidence.""" + envelope = accepted_tepp_seed_envelope(idempotency_key="demo-tepp-seed-2026-w02") + parsed = parse_tepp_accepted_evidence( + envelope, + expected_idempotency_key="demo-tepp-seed-2026-w02", + ) + assert parsed is not None + assert parsed.contract_version == 1 + assert parsed.run_state == "accepted" + assert parsed.accepted_run_id == "demo-tepp-accepted-opaque" + assert parsed.evidence_kind() == "aggregate transport evidence" + expected = tepp_accepted_evidence_sha256( + contract_version=1, + accepted_run_id="demo-tepp-accepted-opaque", + run_state="accepted", + idempotency_key="demo-tepp-seed-2026-w02", + ) + assert parsed.evidence_sha256() == expected + assert len(expected) == 64 + assert "theta" not in expected + + +def test_accepted_ack_without_published_fields_is_not_evidence() -> None: + """A bare status=accepted object is not TEPP's published envelope.""" + assert parse_tepp_accepted_evidence({"status": "accepted"}) is None + assert parse_tepp_accepted_evidence( + {"contract_version": 1, "status": "accepted"} + ) is None + + +def test_local_completed_envelope_is_not_accepted_evidence() -> None: + """The v2.12.0 LineageWeave-local shape is not a TEPP completed result.""" + local = persistable_tepp_seed_envelope() + assert parse_tepp_accepted_evidence(local) is None + assert parse_persistable_tepp_result(local) is None + + +def test_theta_and_unknown_fields_are_not_accepted_evidence() -> None: + """Unknown fields and psychometric keys fail closed.""" + base = accepted_tepp_seed_envelope(idempotency_key="k") + assert parse_tepp_accepted_evidence({**base, "theta": 0.42}) is None + assert parse_tepp_accepted_evidence({**base, "affiliation_count": 2}) is None + assert parse_tepp_accepted_evidence({**base, "extra": True}) is None + assert parse_tepp_accepted_evidence({**base, "run_state": "completed"}) is None + assert parse_tepp_accepted_evidence({**base, "contract_version": 2}) is None + assert parse_tepp_accepted_evidence( + base, + expected_idempotency_key="other-key", + ) is None + assert parse_tepp_accepted_evidence("accepted") is None + + +def test_persistable_parser_never_succeeds() -> None: + """No unpublished completed envelope becomes a persistable measurement.""" + assert parse_persistable_tepp_result(persistable_tepp_seed_envelope()) is None + assert parse_persistable_tepp_result( + accepted_tepp_seed_envelope(idempotency_key="k") + ) is None + assert parse_persistable_tepp_result({"theta": 1}) is None diff --git a/tests/test_tepp_transport_evidence.py b/tests/test_tepp_transport_evidence.py new file mode 100644 index 00000000..764cc16d --- /dev/null +++ b/tests/test_tepp_transport_evidence.py @@ -0,0 +1,106 @@ +"""Authorized TEPP transport-evidence projection stays fail-closed.""" + +from __future__ import annotations + +from pathlib import Path + +from backend.app.analysis_run_ingestion import project_tepp_transport_evidence +from lineageweave.tepp_result import ( + accepted_tepp_seed_envelope, + parse_tepp_accepted_evidence, + tepp_accepted_evidence_sha256, +) + + +def test_project_tepp_transport_evidence_keeps_digest_independent_of_clocks() -> None: + """Clock split must not change the published accepted-field digest.""" + expected = tepp_accepted_evidence_sha256( + contract_version=1, + accepted_run_id="demo-tepp-accepted-opaque", + run_state="accepted", + idempotency_key="buyer-key", + ) + later = { + "contract_version": 1, + "accepted_run_id": "demo-tepp-accepted-opaque", + "run_state": "accepted", + "idempotency_key": "buyer-key", + "evidence_sha256": expected, + "received_at": "2026-01-12T12:45:00Z", + "recorded_at": "2026-01-12T12:46:00Z", + } + projected = project_tepp_transport_evidence(later) + assert projected is not None + assert projected["tepp_evidence_sha256"] == expected + assert projected["tepp_received_at"] == "2026-01-12T12:45:00Z" + assert projected["tepp_recorded_at"] == "2026-01-12T12:46:00Z" + + +def test_project_tepp_transport_evidence_recomputes_the_exact_digest() -> None: + """The API digest must match an independent SHA-256 recomputation.""" + parsed = parse_tepp_accepted_evidence( + accepted_tepp_seed_envelope(idempotency_key="buyer-key"), + expected_idempotency_key="buyer-key", + ) + assert parsed is not None + expected = tepp_accepted_evidence_sha256( + contract_version=1, + accepted_run_id="demo-tepp-accepted-opaque", + run_state="accepted", + idempotency_key="buyer-key", + ) + row = { + "contract_version": parsed.contract_version, + "accepted_run_id": parsed.accepted_run_id, + "run_state": parsed.run_state, + "idempotency_key": parsed.idempotency_key, + "evidence_sha256": expected, + "received_at": "2026-01-12T12:45:00Z", + "recorded_at": "2026-01-12T12:45:00Z", + } + projected = project_tepp_transport_evidence(row) + assert projected is not None + assert projected["tepp_evidence_sha256"] == expected + assert projected["tepp_evidence_kind"] == "aggregate transport evidence" + assert projected["tepp_completed_artifact_available"] is False + assert "affiliation_count" not in projected + assert "theta" not in str(projected).casefold() + + +def test_project_tepp_transport_evidence_fails_closed_on_digest_mismatch() -> None: + """A substituted digest is omitted rather than shown as evidence.""" + row = { + "contract_version": 1, + "accepted_run_id": "demo-tepp-accepted-opaque", + "run_state": "accepted", + "idempotency_key": "buyer-key", + "evidence_sha256": "0" * 64, + "received_at": "2026-01-12T12:45:00Z", + "recorded_at": "2026-01-12T12:45:00Z", + } + assert project_tepp_transport_evidence(row) is None + + +def test_persist_path_binds_received_and_recorded_as_distinct_arguments() -> None: + """Start must not write one timestamp into both accepted-evidence clocks.""" + source = ( + Path(__file__).resolve().parents[1] + / "backend" + / "app" + / "analysis_run_start.py" + ).read_text(encoding="utf-8") + assert "received_at,\n recorded_at," in source + assert source.count("recorded_at,\n recorded_at,") == 0 + + +def test_tepp_accepted_query_binds_authorized_run_ids_only() -> None: + """Hidden runs never enter the evidence query parameter list.""" + source = ( + Path(__file__).resolve().parents[1] + / "backend" + / "app" + / "analysis_run_ingestion.py" + ).read_text(encoding="utf-8") + assert "from analysis_run_tepp_accepted" in source + assert "where analysis_run_id = any($1::uuid[])" in source + assert "_tepp_accepted_by_run(conn, run_ids)" in source diff --git a/tests/test_tied_organization_no_create.py b/tests/test_tied_organization_no_create.py new file mode 100644 index 00000000..c595fae3 --- /dev/null +++ b/tests/test_tied_organization_no_create.py @@ -0,0 +1,147 @@ +"""Regression tests that keep tied organization names out of AUTO rows.""" + +from __future__ import annotations + +import asyncio +import uuid +from types import SimpleNamespace +from typing import Any + +from backend.app import corporate_entity_ingestion, keyman_ingestion +from lineageweave.corporate_hierarchy_inference import HierarchyProposal +from lineageweave.corporate_hierarchy_resolution import CorporateEntityCandidate +from lineageweave.relation_verification import STATUS_CORROBORATED + + +_TIED_CANDIDATES = [ + CorporateEntityCandidate("tied-a", "Tied Energy"), + CorporateEntityCandidate("tied-b", "Tied Energy"), +] + + +class _LiveInferenceClient: + """Return a creatable root-company proposal if a tie leaks through.""" + + available = True + + def __init__(self) -> None: + self.calls = 0 + + def infer(self, organization_name: str, context_text: str) -> HierarchyProposal: + self.calls += 1 + return HierarchyProposal(level_code="company", parent_name=None) + + +class _LiveVerificationClient: + """Corroborate every proposal if a tie leaks through.""" + + available = True + + def __init__(self) -> None: + self.calls = 0 + + def verify(self, subject: str, relation: str) -> SimpleNamespace: + self.calls += 1 + return SimpleNamespace(status_code=STATUS_CORROBORATED) + + +class _Transaction: + """Minimal async transaction context manager.""" + + async def __aenter__(self) -> "_Transaction": + return self + + async def __aexit__(self, exc_type: Any, exc: Any, traceback: Any) -> bool: + return False + + +class _ReloadTieConnection: + """Expose a tie only after inference and the advisory lock.""" + + def __init__(self) -> None: + self.insert_attempted = False + + def transaction(self) -> _Transaction: + return _Transaction() + + async def execute(self, query: str, *args: Any) -> str: + assert "pg_advisory_xact_lock" in query + return "SELECT 1" + + async def fetch(self, query: str, *args: Any) -> list[dict[str, Any]]: + assert "from corporate_entity" in query + return [ + { + "corporate_entity_id": uuid.uuid4(), + "entity_name": "Tied Energy", + }, + { + "corporate_entity_id": uuid.uuid4(), + "entity_name": "Tied Energy", + }, + ] + + async def fetchrow(self, query: str, *args: Any) -> dict[str, Any]: + self.insert_attempted = True + raise AssertionError("a refreshed tie must not insert an AUTO row") + + +def test_initial_tie_never_reaches_live_inference_or_creation() -> None: + """A known tie returns before any live external client is consulted.""" + inference = _LiveInferenceClient() + verification = _LiveVerificationClient() + + result = asyncio.run( + corporate_entity_ingestion.get_or_create_corporate_entity( + object(), + "Tied Energy", + "Synthetic context", + inference, + verification, + list(_TIED_CANDIDATES), + ) + ) + + assert result is None + assert inference.calls == 0 + assert verification.calls == 0 + + +def test_tie_discovered_under_creation_lock_does_not_insert() -> None: + """Concurrent homonyms discovered after inference still fail closed.""" + connection = _ReloadTieConnection() + inference = _LiveInferenceClient() + verification = _LiveVerificationClient() + + result = asyncio.run( + corporate_entity_ingestion.get_or_create_corporate_entity( + connection, + "Tied Energy", + "Synthetic context", + inference, + verification, + [], + ) + ) + + assert result is None + assert inference.calls == 1 + assert verification.calls == 1 + assert connection.insert_attempted is False + + +def test_keyman_raw_tie_blocks_abbreviation_rewrite_and_auto_creation() -> None: + """Keyman checks the raw tied name before any resolver rewrite.""" + result = asyncio.run( + keyman_ingestion._resolve_affiliated_organization( + object(), + "Tied Energy", + "Synthetic context", + object(), + object(), + object(), + list(_TIED_CANDIDATES), + ) + ) + + assert result == ("Tied Energy", "Tied Energy", None) diff --git a/uv.lock b/uv.lock index 08eab776..1575f180 100644 --- a/uv.lock +++ b/uv.lock @@ -188,6 +188,105 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", size = 25335, upload-time = "2022-10-25T02:36:20.889Z" }, ] +[[package]] +name = "coverage" +version = "7.15.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/be/c3/4f2195f512fb172aa425a8803a874b2baa9ba7f80ff7b6080998761fc701/coverage-7.15.4.tar.gz", hash = "sha256:0548198fff07ccf4faf469520bce1c2eceb1ce3e62891921138dec10907f9d00", size = 936952, upload-time = "2026-08-06T13:50:24.442Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/1d/48/bc8d4ba7b37551a767bd863f15b3f80182b271c2f55975356f5f7dbe94c2/coverage-7.15.4-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:d4fedd1f7f428f9fe83b1ead5e7cc87a43427be31aadafbac3ac0636dc7abb22", size = 222543, upload-time = "2026-08-06T13:47:37.562Z" }, + { url = "https://files.pythonhosted.org/packages/20/dd/88d6f83f1fffc974a3691a34a97951c5b12df7512a6782c5963883cbc058/coverage-7.15.4-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:37e2f0cdf58e2e1fed4e4d5a8f8786ae2f7eb80b478016876667dc4a01d60a97", size = 222905, upload-time = "2026-08-06T13:47:38.927Z" }, + { url = "https://files.pythonhosted.org/packages/bd/5c/54ee0d4748585bb0acab9891cd8d92f2d3593165b4e59fc9de113bfb3140/coverage-7.15.4-cp312-cp312-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:fb55d0e70bb15f2e81477613627286581414693d74ac7963c93a790dd453ca9d", size = 254407, upload-time = "2026-08-06T13:47:40.488Z" }, + { url = "https://files.pythonhosted.org/packages/8c/3f/f0642a372f494bd0d7dad3b497083b910194a5f1c88be2c94fef707c3b59/coverage-7.15.4-cp312-cp312-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:899b9da30f3c6c336566e3707495bb23e8302d39d862f01fa78c48b99b9437e2", size = 257145, upload-time = "2026-08-06T13:47:41.931Z" }, + { url = "https://files.pythonhosted.org/packages/71/17/8b46d0ed68251016002ec972c8fc0119961a765d0984cafb8bf317c43758/coverage-7.15.4-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d15715e8c46552827e5e4f30a35575a2dbcad14454cf3284c54483946bd16931", size = 258257, upload-time = "2026-08-06T13:47:43.527Z" }, + { url = "https://files.pythonhosted.org/packages/30/b8/8498a0e72d0adbe15477dd07463d2b3bb2c9f6a4815e8589e50939e2c3ae/coverage-7.15.4-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:002a438859f7b430bc99afeaf01a6d187dad1d0dc907b64cdeffc632a5db8fd8", size = 260517, upload-time = "2026-08-06T13:47:45.121Z" }, + { url = "https://files.pythonhosted.org/packages/41/e1/7dce19c3bdb1e3dd63e769508216500edad81bd5f69a26d724e32aceaf78/coverage-7.15.4-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:e4193a04b518f7968f3099755f5509ee7cccc6dc2b92a6b14841934d22e222c9", size = 254785, upload-time = "2026-08-06T13:47:46.541Z" }, + { url = "https://files.pythonhosted.org/packages/dd/b1/e1494703c675a2561723cd9b89f45c9168782c31280c611b1f767851e57c/coverage-7.15.4-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:e98dcc55d572b38e69d117da7e8e8efb8500f1f5eaf81ecd460a63220790b839", size = 256176, upload-time = "2026-08-06T13:47:48.155Z" }, + { url = "https://files.pythonhosted.org/packages/73/76/a5629d270fb638a43a4b10466f51e2f49d532c1aa4da2913cbbb150bbe0a/coverage-7.15.4-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:af6c538498ce66c10d3fd541c2a8d5b03da5850355add34e6cba564210cb9e72", size = 254321, upload-time = "2026-08-06T13:47:49.757Z" }, + { url = "https://files.pythonhosted.org/packages/ff/4f/9c44447218435d5766b911534f9d798144a5560f85e9a54ebe5f3f5d19f9/coverage-7.15.4-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:1d10025d96ea89fc2f73714dbc4cbd433fe012c1ac9e23f895d7728b238b6e52", size = 258390, upload-time = "2026-08-06T13:47:51.248Z" }, + { url = "https://files.pythonhosted.org/packages/de/36/c1e127616fb3fa18a9ff71e76c417f2fd7424332a4870015ac224ef4c039/coverage-7.15.4-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:d802e1947603162ded419bff83ac7489820355d2b856dfb09206574e3a37ac0c", size = 253894, upload-time = "2026-08-06T13:47:52.816Z" }, + { url = "https://files.pythonhosted.org/packages/e9/b9/fdb92c8ae7a8bb9b850cc253b7b3b9c8526f68130002048b5671cd510d09/coverage-7.15.4-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:c2de40895718f91951b86712b4c5b694acaf9a0a49be13874896f599a1eed3f4", size = 255763, upload-time = "2026-08-06T13:47:54.296Z" }, + { url = "https://files.pythonhosted.org/packages/6f/c0/a7d51b2587c7bdb76e71b0896d2565bf7d60436b5122fc83e511adb1f7cd/coverage-7.15.4-cp312-cp312-win32.whl", hash = "sha256:5c3431b2161279b7db5c2a1aa58ae02e5cb8c3c42d93a5094be3f5537bd5b11b", size = 224597, upload-time = "2026-08-06T13:47:56.074Z" }, + { url = "https://files.pythonhosted.org/packages/49/b9/5c5f80cc55f5acaaca6dee677626bfcec8c87204a7809b438b08e84f4571/coverage-7.15.4-cp312-cp312-win_amd64.whl", hash = "sha256:6befeab5fb2b51c958ca4ac6c5d141a1e8240f4f76e46350f1911963deda49cd", size = 225135, upload-time = "2026-08-06T13:47:57.52Z" }, + { url = "https://files.pythonhosted.org/packages/47/e4/2a4561f89ff6bf7c925c287d0f2cce8bdf139c3a33735c87e3203401cf94/coverage-7.15.4-cp312-cp312-win_arm64.whl", hash = "sha256:67bc345491ab55b837277d76f5775d057e8c7f1ac44d890d8c2c82adde258c6f", size = 224515, upload-time = "2026-08-06T13:47:58.977Z" }, + { url = "https://files.pythonhosted.org/packages/f1/84/651a9310859673aaa3b3203f1aa1641ca60fcf2494683e1c9474c7172780/coverage-7.15.4-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:c705b28feb2775dc82a25f1d473a370bc37ff93f5177f4e29ce2425f560f6921", size = 222565, upload-time = "2026-08-06T13:48:00.796Z" }, + { url = "https://files.pythonhosted.org/packages/82/f9/4dcf700137e8af550670f4d74d1b63828ce93e1e2b05e5f10710eb2ea987/coverage-7.15.4-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:3ff205ab5e3ecc670f6a4dd19d9cbf12ede53dd41cfc1e15716ec961ea6d314e", size = 222936, upload-time = "2026-08-06T13:48:02.391Z" }, + { url = "https://files.pythonhosted.org/packages/07/4a/612ff1e780b3fbfd637486f542f84adc5503873d8b5d279dec1ffeef9414/coverage-7.15.4-cp313-cp313-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:5172326e861a38b48b48befca15e0f477a26b283337a33a739c8fed229934e36", size = 253926, upload-time = "2026-08-06T13:48:04.382Z" }, + { url = "https://files.pythonhosted.org/packages/b0/04/d1cff1c2ead4708a6a79c01d3736b6a25bd38a36678398f72a8dd33dfad9/coverage-7.15.4-cp313-cp313-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:12b59c90084e3234fb11184886bf4a40f4f16a8c8f867be2e087b81f8e8868d4", size = 256523, upload-time = "2026-08-06T13:48:05.996Z" }, + { url = "https://files.pythonhosted.org/packages/b9/80/d34e13fb4b293cbdb9665838cf5522077b8ad14ef947550631a4bced36a5/coverage-7.15.4-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:349062d66f00b40fa2c1c222438bad25fabf755631b5d82937fe985c8008615c", size = 257759, upload-time = "2026-08-06T13:48:08.036Z" }, + { url = "https://files.pythonhosted.org/packages/0f/e7/2c5fe7636fdb0732fe0f09f308a5b066864078b7fc61f6678e8478554f2e/coverage-7.15.4-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:4256ced708e598e05209bc1a8ab4074e04a51dba4c62fb45926a229af675ace7", size = 259890, upload-time = "2026-08-06T13:48:09.834Z" }, + { url = "https://files.pythonhosted.org/packages/92/28/9689f0858dfff59c2ea688938ab9fa2925631235df67126a42b6c5c70ae1/coverage-7.15.4-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:d80f974b20782d9612c8b4c9beeca867074c7cf4079d1419843fa25a26428b25", size = 254121, upload-time = "2026-08-06T13:48:11.459Z" }, + { url = "https://files.pythonhosted.org/packages/f9/e2/785077c230c157243eb5aa9a26c3be260ecd02001bead54a3cada3df8e03/coverage-7.15.4-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:2e179f19bfe1d31f8eeeaa12990194d761c4f62f0759661000bca6cd8729f40b", size = 255891, upload-time = "2026-08-06T13:48:13.209Z" }, + { url = "https://files.pythonhosted.org/packages/d4/90/e20371b17b40f912f21305c2db2f30efa3de306f7320fc916804872c85a4/coverage-7.15.4-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:8bc16bb47b7679670eceff71d78bfb7d6e5b143f6c2cd117487ec7c75e0d4b78", size = 253859, upload-time = "2026-08-06T13:48:14.736Z" }, + { url = "https://files.pythonhosted.org/packages/05/49/25371987ee459a5f67c0427fb75c74f9358e65f2c71fe75bf41c1b6c5fcb/coverage-7.15.4-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:1cd685005cd2c4200adfc14cf39a603b9320efab3f18a8f7f156d20c9cc3345f", size = 258011, upload-time = "2026-08-06T13:48:16.464Z" }, + { url = "https://files.pythonhosted.org/packages/30/6e/32e67467f6154bf4f1c4f63b05acc5097cba4237d45bbeeea446b52e8ac1/coverage-7.15.4-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:337399ad2c93b3acd2a937627dae8b3e86b66707cd3d3e856347999aadf1ef8d", size = 253676, upload-time = "2026-08-06T13:48:18.493Z" }, + { url = "https://files.pythonhosted.org/packages/03/c1/8b24192e89286399765155251f99ee9f070a9d637109018ac23d99b99f6f/coverage-7.15.4-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:96e257121228ec5cd2bb919276e94ac11074471bc37d68dbae0e8308cce15fff", size = 255453, upload-time = "2026-08-06T13:48:20.057Z" }, + { url = "https://files.pythonhosted.org/packages/16/6f/8b41ebdf67c87854e17c035336a90f1cfbad0c14c2a584301be6ff148718/coverage-7.15.4-cp313-cp313-win32.whl", hash = "sha256:c65a9e0dfc6143491879da4e13b5e30f8be192055de508d737fb14601edbd22c", size = 224605, upload-time = "2026-08-06T13:48:21.655Z" }, + { url = "https://files.pythonhosted.org/packages/e0/e2/2946c7f0b42b152ecb21ff1bdad72e3d301e790c0c487e4a86e8c9f69347/coverage-7.15.4-cp313-cp313-win_amd64.whl", hash = "sha256:2ff8f5e9b8f7a94f0c11c45631eee103dbcb7d63274edd12c56efe1be690b3b4", size = 225148, upload-time = "2026-08-06T13:48:23.376Z" }, + { url = "https://files.pythonhosted.org/packages/9e/83/3f4a69957f48ae7a0aba76c34743f88963d607b19e03f3f8e66f91cae0f9/coverage-7.15.4-cp313-cp313-win_arm64.whl", hash = "sha256:6e0a8a5083b096487d6cfced94cdd514d8f5db6f113610fb36c0620edb1028cf", size = 224536, upload-time = "2026-08-06T13:48:25.117Z" }, + { url = "https://files.pythonhosted.org/packages/ea/ac/748cf29eeb2d6be34a3176ce26a4f49e38085ee08e8935f05f6f26ed7e0f/coverage-7.15.4-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:770e9325ab5ea6d56f77e59b29ecfe0ac20b57a82a601876f90494a4dda0386f", size = 222608, upload-time = "2026-08-06T13:48:26.806Z" }, + { url = "https://files.pythonhosted.org/packages/0b/02/1abbf5c984677b0aa439cdacaccbf38d248939d8ef8fe1cc7a50d73edb77/coverage-7.15.4-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:d12b33a3a50a1676b7784dc8d00a0c6d66a9f2add4b85a041c19b6a7e53ef23c", size = 222940, upload-time = "2026-08-06T13:48:28.432Z" }, + { url = "https://files.pythonhosted.org/packages/eb/e1/ff8f9f53d9fcf586125b55d0b1f04ec1c14955fee41e83d5814bee141bb5/coverage-7.15.4-cp314-cp314-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:5669c8378ebde86f5def7a25d29586631b58acc27ffde04399f678f3dfc6e082", size = 253985, upload-time = "2026-08-06T13:48:29.995Z" }, + { url = "https://files.pythonhosted.org/packages/a1/26/595759762e514e81be1d7d01ed03444303bcd152226a6529998d253f9201/coverage-7.15.4-cp314-cp314-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:ff97a14362eef486483ed44042ca2027ea257df6ff768e62358ee0c9776925ac", size = 256492, upload-time = "2026-08-06T13:48:31.634Z" }, + { url = "https://files.pythonhosted.org/packages/24/68/b79aabac54d482be23b5fcdd4f4662bff24a78edc4ee29201726929936d5/coverage-7.15.4-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5a325e815318638aed1655d9c06e6d7c2d3d46c09231ce988070428a8762d734", size = 257837, upload-time = "2026-08-06T13:48:33.186Z" }, + { url = "https://files.pythonhosted.org/packages/09/0f/bf7f297885a5bf6fd71e5782404e0ff059ca09e8711ceb3a08544abde45a/coverage-7.15.4-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:474223409d88eb20d2d6a0d37ea60e8647a65a90cc008dc1f0410af5f64f1e0d", size = 260152, upload-time = "2026-08-06T13:48:34.75Z" }, + { url = "https://files.pythonhosted.org/packages/fd/f1/296744e854ff8368542343457414380465e9ceefb9192342feb9d3bc461d/coverage-7.15.4-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:7f2f62ae3cd189dd2e13aece758c57b3eecbd27be070dbd4cbd10936049e5dbf", size = 253978, upload-time = "2026-08-06T13:48:36.434Z" }, + { url = "https://files.pythonhosted.org/packages/55/b0/bbdb2e9057493e66220a2e149ca2d301ba0e3a58a83bd6b90de9826d16f3/coverage-7.15.4-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:39ece820e29e0a2ba34b3ecb3be83c27e997eed8926f2ba6fe7ce7a0bda5843b", size = 255846, upload-time = "2026-08-06T13:48:38.317Z" }, + { url = "https://files.pythonhosted.org/packages/96/e4/38015b2b6d21258713bd17e76b59d033b191efb5703589cffd037dfbca20/coverage-7.15.4-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:f21b56dcace11dfe013014201f577dcd592b2a9b72182d930361b47cf6f73f25", size = 253808, upload-time = "2026-08-06T13:48:39.993Z" }, + { url = "https://files.pythonhosted.org/packages/0b/64/0d515c1e60ee6fbfd1a0e79c07cd87d388a233b7adc37758735677203808/coverage-7.15.4-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:93a3a0b662abcc10c73a47cbc72cd60f63618d6989fb2d1286e50eacd974f303", size = 258081, upload-time = "2026-08-06T13:48:41.971Z" }, + { url = "https://files.pythonhosted.org/packages/91/71/04d9e7a3642146c6351338aef4ef85ab11dbbb54744c13245caba1aad1c0/coverage-7.15.4-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:141fae2cabf5569b782c10afc4c850ce10f618c13f8db54765cba99cc839da1f", size = 253624, upload-time = "2026-08-06T13:48:43.731Z" }, + { url = "https://files.pythonhosted.org/packages/b4/a7/6c28b74c81ebff66987b0e2522ba5cffa3e90b0c33cb6a2eb264d4ee8cf1/coverage-7.15.4-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:81294c7e6ab30c5f74c0353b11b2fd6320e72d9bee6ac73b357caa8b916323a5", size = 255280, upload-time = "2026-08-06T13:48:45.58Z" }, + { url = "https://files.pythonhosted.org/packages/52/af/bc19996a7014b98d7bbb0f0939453c67074af65784a3aa16a789a07381fa/coverage-7.15.4-cp314-cp314-win32.whl", hash = "sha256:7bbd7d6418e0dab31a206af5203bd43ae36edb8e7fba1940b055d3e9249290d7", size = 224768, upload-time = "2026-08-06T13:48:47.525Z" }, + { url = "https://files.pythonhosted.org/packages/ee/90/219484e476d6e101ba0a444852579e05f5b75c37c611a42ed1190f73ef62/coverage-7.15.4-cp314-cp314-win_amd64.whl", hash = "sha256:f0204ed122758782970526057093f448051a39db9d810d4e344bb87a3546f425", size = 225259, upload-time = "2026-08-06T13:48:49.513Z" }, + { url = "https://files.pythonhosted.org/packages/b7/66/fa77daf4e383e5f776dac62c2409b6af81910ae6fe326bd5170dba74cc63/coverage-7.15.4-cp314-cp314-win_arm64.whl", hash = "sha256:9e71e7bc71c686a123347ae47a0de33a175e797a85bb57b791492adf4eec8ed8", size = 224684, upload-time = "2026-08-06T13:48:51.235Z" }, + { url = "https://files.pythonhosted.org/packages/58/5b/f03bf0ce362bbf3f785fa5219620d00778d4ac6fc9e407734828e9c672f6/coverage-7.15.4-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:7c922735321eef3f87c280a3d39afff6b646723a2880b862cda4ac7a093b8aa8", size = 223338, upload-time = "2026-08-06T13:48:52.896Z" }, + { url = "https://files.pythonhosted.org/packages/0f/76/e77d0ae22501831cc9f92193e8a957a5caa1dd177f90a6d1d9b106242d92/coverage-7.15.4-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:f41c17c4668a655ce96d090d8d5ffdc24ef64b5a02f9753884d08483e8a4a41a", size = 223609, upload-time = "2026-08-06T13:48:54.688Z" }, + { url = "https://files.pythonhosted.org/packages/82/1a/b1f089da8d38ac612fa2dd6dc7f4a1a7657d12f3e261d2996edd3a838d0b/coverage-7.15.4-cp314-cp314t-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:46822e9b6ff1c6a72b518c162c44a8f45a61a1d609c51084bf5b16c023c5037b", size = 264970, upload-time = "2026-08-06T13:48:56.403Z" }, + { url = "https://files.pythonhosted.org/packages/bf/31/e66d98d6e9c7fcc88470f1e234eaf6b1950dc0dfbf797f7282c1c861da24/coverage-7.15.4-cp314-cp314t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:3d6f4955b73b5445271379a59e3792b0d978f42d4a01e0cf7a67d9c33a3bb0a5", size = 267088, upload-time = "2026-08-06T13:48:58.41Z" }, + { url = "https://files.pythonhosted.org/packages/59/a1/ae94eb2c541add426378408379f233591e069040b1e2cdb33df9498a0682/coverage-7.15.4-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3fc9e047706fb4a9abb54f719d3aa643e80e5bb3818182c40aee01ac0f0247ba", size = 269508, upload-time = "2026-08-06T13:49:00.42Z" }, + { url = "https://files.pythonhosted.org/packages/9c/c7/88a10694a1c6a213569766aba9f25847b28155d4ac731b13226db216356d/coverage-7.15.4-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:05e491d4f3165d62d4f5c8fd48dfeabf2ae8f42cbbd484319af33ea851b78982", size = 270629, upload-time = "2026-08-06T13:49:02.234Z" }, + { url = "https://files.pythonhosted.org/packages/b3/34/d8b8232e5e55169933b59aabcef2fedfa4b9d8897361bb80fcbda146505f/coverage-7.15.4-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:226c66e80ec0598d3b9b4874123df167ccca342aca8714f77cac6829688ee09c", size = 264043, upload-time = "2026-08-06T13:49:04.102Z" }, + { url = "https://files.pythonhosted.org/packages/7e/35/58b009dbf8c471c7224716478b9fed4a7e1af15320e1ed41660978504663/coverage-7.15.4-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:ac41cc14bebda0dbfb0628036b7f75706935c95bcc07fefe9a0f93614aa60a57", size = 266963, upload-time = "2026-08-06T13:49:05.821Z" }, + { url = "https://files.pythonhosted.org/packages/62/aa/57fbda1b42c892968273c56b6ee9dc0f1310850859230a507bc7873b1f65/coverage-7.15.4-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:8af623e5cd92080acddd02b38f2f406a2c3a0893c38950b211890361448fbf26", size = 264569, upload-time = "2026-08-06T13:49:07.706Z" }, + { url = "https://files.pythonhosted.org/packages/98/8a/360e6e7f24d477b7e889703af0afa878d15b6d4d8d2a822b2835c169a879/coverage-7.15.4-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:07545711d4f0f32852a18f18ad11f76f0109909d09e78b9008b4cfc67e829429", size = 268299, upload-time = "2026-08-06T13:49:09.587Z" }, + { url = "https://files.pythonhosted.org/packages/4e/89/6f701261aee21b6b5fa8f7872229406dc917e125069448292223bf213606/coverage-7.15.4-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:a0865421cfdc53654b342d515e5a233187590882d20b95752150e53f65460017", size = 263413, upload-time = "2026-08-06T13:49:11.604Z" }, + { url = "https://files.pythonhosted.org/packages/3f/0f/6f04036edc260ed425af83e834f627fad48941ce97b50bfe6edd8b6fa623/coverage-7.15.4-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:460115e32ee40566476db5048f9bec1e842c127ad8e6f8be745aad3ac9cbc839", size = 265725, upload-time = "2026-08-06T13:49:13.38Z" }, + { url = "https://files.pythonhosted.org/packages/c4/ce/d19b5d4d5c49a7bfb925fd74310fee7d28bc99520ac3367ccbc54e662518/coverage-7.15.4-cp314-cp314t-win32.whl", hash = "sha256:cbde877ef9dd7baf272b9bfef2b8a25edd45d9170fc326951dd20eb480335e85", size = 225079, upload-time = "2026-08-06T13:49:15.265Z" }, + { url = "https://files.pythonhosted.org/packages/26/bb/7aa1b3b173faee0679037ca950bbbe1247273656697994d8d13f80f8d4b4/coverage-7.15.4-cp314-cp314t-win_amd64.whl", hash = "sha256:3da9e92d1c551fd7563833e9ade686efb0c4b7363ab7681a94283958c950bf5e", size = 225911, upload-time = "2026-08-06T13:49:17.279Z" }, + { url = "https://files.pythonhosted.org/packages/81/1c/4ea9e47426d80038d9222db3c4534cb6021a74b237d3ff97ffd33b6600dd/coverage-7.15.4-cp314-cp314t-win_arm64.whl", hash = "sha256:3a54f5a0d85050c73a38f6793090ee83974531e67fe5e57a1da9bee11398aa5e", size = 225219, upload-time = "2026-08-06T13:49:19.293Z" }, + { url = "https://files.pythonhosted.org/packages/2b/c4/dc5d2ac8f9142e7ec7de66e7bf0591db29d78955a040bd915870d9c0e657/coverage-7.15.4-cp315-cp315-macosx_10_15_x86_64.whl", hash = "sha256:2c9872e4d9dc5d3cf616bf4b382f5a00359305a5be666a3dd0b5cdb4e49597f9", size = 222604, upload-time = "2026-08-06T13:49:21.279Z" }, + { url = "https://files.pythonhosted.org/packages/70/39/33e63df81fe2ee100897451841c821467635923e58e37c6bd4b46dd8106c/coverage-7.15.4-cp315-cp315-macosx_11_0_arm64.whl", hash = "sha256:e101dbb4b9b72f0cddd8cdc8c9c5b47f456766f5e0ac82dbfb75e5c55409b78a", size = 222944, upload-time = "2026-08-06T13:49:23.187Z" }, + { url = "https://files.pythonhosted.org/packages/99/1f/ef3ffb5557febc75a0d97aa459d0266d7d741110265121cc6d8539343d44/coverage-7.15.4-cp315-cp315-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:7d1abebdb047729e852b9c77a00497dfbeb11eb3a117e037d7dbc3ac8e5f5c54", size = 254050, upload-time = "2026-08-06T13:49:25.008Z" }, + { url = "https://files.pythonhosted.org/packages/6f/f5/1f0f6f77698c3601ca0ae7431e34b24c62ca2f06fecb23b73ed1f651d2be/coverage-7.15.4-cp315-cp315-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:d28a4a899354d0ea6214cc59b4fa19eefbce1b9ff1688ab579acf49e894bd3fb", size = 256967, upload-time = "2026-08-06T13:49:26.896Z" }, + { url = "https://files.pythonhosted.org/packages/03/7a/2ed9bed79925f4367c83c77f66a89e5ca7229c288d2d19ad5f36d1ca0070/coverage-7.15.4-cp315-cp315-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ffb3c2aacea411cc7e1d27712490c11108e2de1d39019ae32915493a59a8b9ed", size = 258587, upload-time = "2026-08-06T13:49:28.692Z" }, + { url = "https://files.pythonhosted.org/packages/45/8c/fa34044f71b7cc4ecb6da9c2408770959b0591fa9b5fb6fb6bca38f94298/coverage-7.15.4-cp315-cp315-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:a9447978a92f405d301123cfd39ff49895490efb769a758fe2734c7f631bf8ce", size = 260785, upload-time = "2026-08-06T13:49:30.472Z" }, + { url = "https://files.pythonhosted.org/packages/4f/54/d5727ce36b4524a7394ab9f5f1df378e1f23affcdab01037dc8655185cc7/coverage-7.15.4-cp315-cp315-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:050467a7983b8e2fe7dd41a78bb30c3e7f8c0b8cafda14b1c46f8b5e3cf2dd3c", size = 254545, upload-time = "2026-08-06T13:49:32.271Z" }, + { url = "https://files.pythonhosted.org/packages/dc/e6/6e3783e576719590194bdffb6dd6d85490801785b7c331e35a245d8cb8b5/coverage-7.15.4-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:d003b7a5708ddad5c206c79607a6b92abb6fc13c57d99d8a4468cc03a2941ced", size = 256682, upload-time = "2026-08-06T13:49:34.089Z" }, + { url = "https://files.pythonhosted.org/packages/dc/f2/bacdbde18b69ed2de424fcf64d9fb0a4913753d4f0eca8bae9daad69f4bd/coverage-7.15.4-cp315-cp315-musllinux_1_2_i686.whl", hash = "sha256:c38efe30fd74e5c19e9433f11fb1f5dc9c6522770971b7c6145bbaa413dc8800", size = 254560, upload-time = "2026-08-06T13:49:36.052Z" }, + { url = "https://files.pythonhosted.org/packages/6c/a3/1fb927196e3477c1b48831169ab58ba08f451ba87ae311ff1de68b26a616/coverage-7.15.4-cp315-cp315-musllinux_1_2_ppc64le.whl", hash = "sha256:1f4f826d70f772ab8b0c052329580d7fe8b8abd191e4ce0c8f81aec6614665d3", size = 258792, upload-time = "2026-08-06T13:49:38.01Z" }, + { url = "https://files.pythonhosted.org/packages/41/58/30d4c149c69053de0edfe325614c1d28d508f62b1783e0e4a234d2e49136/coverage-7.15.4-cp315-cp315-musllinux_1_2_riscv64.whl", hash = "sha256:4a4bf917c9953f57c957be31c1cd504e3bd2f34d4a352b9d391a3025336f6768", size = 253968, upload-time = "2026-08-06T13:49:39.934Z" }, + { url = "https://files.pythonhosted.org/packages/89/e4/77f639371b918aad30dda4051f95404b43578f7f2e2f87ba73e02ed1ff37/coverage-7.15.4-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:1c9bf40ebef178a45192c75c4964760bb261b0e6ad725da5fc4c93f674f19753", size = 255893, upload-time = "2026-08-06T13:49:41.825Z" }, + { url = "https://files.pythonhosted.org/packages/5c/62/13be29b3ddab35f14c87967a4820a05106d2a3eccb4fa4ff550bf30b75e0/coverage-7.15.4-cp315-cp315-win32.whl", hash = "sha256:43619d04c3671792d2c4706ae8bf45e265dc87bbd4078189ef8b847ea1e74be2", size = 224768, upload-time = "2026-08-06T13:49:44.08Z" }, + { url = "https://files.pythonhosted.org/packages/a1/70/af0c6be0f964af6954f6b74bc109b0dbca02824696d2520fb17fe1ab06e3/coverage-7.15.4-cp315-cp315-win_amd64.whl", hash = "sha256:be619439dbcd31a2eab10b32de9fff62c26ed4bab69dc32b8363fdaaa0882809", size = 225242, upload-time = "2026-08-06T13:49:45.899Z" }, + { url = "https://files.pythonhosted.org/packages/4f/2d/f3bd3aab899fc9efc18b53133ee68f5f98574ef480649b23e12962226387/coverage-7.15.4-cp315-cp315-win_arm64.whl", hash = "sha256:def597967dafc2e8d97c9097ea453c464e0bb8ed38f193a43070f10dc623bb6d", size = 224674, upload-time = "2026-08-06T13:49:48.322Z" }, + { url = "https://files.pythonhosted.org/packages/f5/ca/f69251cd63eabc6438321aea22148754cce758a26bde07dd490e3fe7cfc5/coverage-7.15.4-cp315-cp315t-macosx_10_15_x86_64.whl", hash = "sha256:c7dbc748ac8a1e3e59a2b28bea47675e6e778081dbbf081bde0d75def2fcbe1d", size = 223333, upload-time = "2026-08-06T13:49:50.293Z" }, + { url = "https://files.pythonhosted.org/packages/a7/a7/037b53b2885b0d8447064432491a4d5a1014cd9f97a594d53acd0c04541a/coverage-7.15.4-cp315-cp315t-macosx_11_0_arm64.whl", hash = "sha256:2413074a5ecbb61a01a7888fc72db0ca324d13588c5b38bc0dd8564cdcdfea26", size = 223630, upload-time = "2026-08-06T13:49:52.637Z" }, + { url = "https://files.pythonhosted.org/packages/80/4f/152b8a4779ae90da11bb24f7467df8a59f0be48a5c52acb856325ca48289/coverage-7.15.4-cp315-cp315t-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:4e6f6f632b7b2f714bf7a1346e8f97b650ee71f3c298aaad42a2ab60f0f07645", size = 264489, upload-time = "2026-08-06T13:49:54.52Z" }, + { url = "https://files.pythonhosted.org/packages/10/2d/84b4b9e0e1dd6528a51920ff7031f35b789382e467a28ec6a5a578cb8812/coverage-7.15.4-cp315-cp315t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:8df457da2249d3c75ca2e5e835d59c725abfe92d27fdff6cd99eed85b51d5e9a", size = 267567, upload-time = "2026-08-06T13:49:56.721Z" }, + { url = "https://files.pythonhosted.org/packages/53/fc/ba01cc25299f9f8a2c8b02d3b28c53f3543d9fbfbe4e74fa2760b48f163e/coverage-7.15.4-cp315-cp315t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:050f66a08805acb5b8a23c6d4a517b1ecf82c08e81ed0e4bd727df065e5c6624", size = 270123, upload-time = "2026-08-06T13:49:58.736Z" }, + { url = "https://files.pythonhosted.org/packages/cf/d0/db2647cbf40b14f8c308f94ff7bf89c06d564e59f396906edf50086ec788/coverage-7.15.4-cp315-cp315t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:1587fb771d1ccceef708fdde1e5af8c7ed24b486b61d13a321acb7d8145390aa", size = 271107, upload-time = "2026-08-06T13:50:00.811Z" }, + { url = "https://files.pythonhosted.org/packages/70/ff/4d2d17924552c458bb4f77dd631f0e3bc92fbbdf2d2d916cd4b33bbfd5b1/coverage-7.15.4-cp315-cp315t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:8b4f1c3a69ca580f3fbd6b2046915f536d7f586874f25c1bb23add2a3c88d50f", size = 264955, upload-time = "2026-08-06T13:50:03.023Z" }, + { url = "https://files.pythonhosted.org/packages/ee/de/dc010c7a3691f396d93bbc26bfcafa1c2a3a351cd520470f15faf5795bd5/coverage-7.15.4-cp315-cp315t-musllinux_1_2_aarch64.whl", hash = "sha256:ffb58d7eff5b7f6ecc6fa21d6288ab7f968a212cb67d682c269c09b9eba3b66f", size = 267949, upload-time = "2026-08-06T13:50:05.557Z" }, + { url = "https://files.pythonhosted.org/packages/78/ea/dc96a11375e83c045c2f7c61fb6918277cfe9401db7c0f7b1d111a84b2e5/coverage-7.15.4-cp315-cp315t-musllinux_1_2_i686.whl", hash = "sha256:d9df165544774574ee004b953023d1bebada1894a80b1052a43d798b0f676e67", size = 264421, upload-time = "2026-08-06T13:50:07.612Z" }, + { url = "https://files.pythonhosted.org/packages/c8/86/b77131a0f9503ce461cd577076147d7a9040f0c5dda772686f729e2cc9cb/coverage-7.15.4-cp315-cp315t-musllinux_1_2_ppc64le.whl", hash = "sha256:f9de0a24a4079b53e523b5c5e2c5945ec251ab486652659955187cf255a259bc", size = 269121, upload-time = "2026-08-06T13:50:09.58Z" }, + { url = "https://files.pythonhosted.org/packages/24/24/944bc35007862955e7ebf05754e645419dcf5d7526c52735cfa2715e8ebf/coverage-7.15.4-cp315-cp315t-musllinux_1_2_riscv64.whl", hash = "sha256:150089274bdc9f940628552cb92844e0223c987f1902ab8efe9f45a2ec758d88", size = 264565, upload-time = "2026-08-06T13:50:11.722Z" }, + { url = "https://files.pythonhosted.org/packages/c7/cc/a3bb9f93e7e740659163e2ea584f8196ddcd2c456a5dbe15f6c50105fec1/coverage-7.15.4-cp315-cp315t-musllinux_1_2_x86_64.whl", hash = "sha256:a58a94fed5da6997d258e8f7668c1e195fbd04a691d781b7558f1e468f9e68bc", size = 266522, upload-time = "2026-08-06T13:50:13.786Z" }, + { url = "https://files.pythonhosted.org/packages/49/dd/e0e40f3560d878d888c580698ff5ad1179f5e1c3ac949684ef66b41a3817/coverage-7.15.4-cp315-cp315t-win32.whl", hash = "sha256:ebd5a6d8466ff30836572f3ba2cae8a5e8f85029b1c6d5e2ed338dc472a5166a", size = 225068, upload-time = "2026-08-06T13:50:15.825Z" }, + { url = "https://files.pythonhosted.org/packages/c6/7e/37732ea80eebc30e976e4cdab15c190bc42d96959a42e38ddf6f8c60468f/coverage-7.15.4-cp315-cp315t-win_amd64.whl", hash = "sha256:288bde2a2d7ab6b6c2d7252fcde8b524387f2d970bdba9658fc6f8bbcaef0f9b", size = 225895, upload-time = "2026-08-06T13:50:17.928Z" }, + { url = "https://files.pythonhosted.org/packages/c6/08/1e00f7923eaaba45fb3d51dd794125fc766304b1df264f3a9c6557bfb30e/coverage-7.15.4-cp315-cp315t-win_arm64.whl", hash = "sha256:68be5e1de60ff13c9095bbec0e5a7fa45b33b101752215b91345ea1f61c4a278", size = 225213, upload-time = "2026-08-06T13:50:19.981Z" }, + { url = "https://files.pythonhosted.org/packages/b4/d9/e70c286c979378f061d8266e279b686ab0b0b688e1fe0af864684f23a77d/coverage-7.15.4-py3-none-any.whl", hash = "sha256:964730a1e9de9c0cf11be6a1a3c79ce419c34882842abd256086ba4698705e84", size = 214332, upload-time = "2026-08-06T13:50:22.192Z" }, +] + [[package]] name = "cryptography" version = "50.0.0" @@ -355,7 +454,7 @@ wheels = [ [[package]] name = "lineageweave" -version = "0.75.0" +version = "2.12.3" source = { virtual = "." } dependencies = [ { name = "certifi" }, @@ -374,6 +473,7 @@ backend = [ { name = "uvicorn", extra = ["standard"] }, ] dev = [ + { name = "coverage" }, { name = "httpx" }, { name = "pillow" }, { name = "psycopg2-binary" }, @@ -385,6 +485,7 @@ dev = [ requires-dist = [ { name = "asyncpg", marker = "extra == 'backend'", specifier = ">=0.29.0" }, { name = "certifi", specifier = ">=2024.0.0" }, + { name = "coverage", marker = "extra == 'dev'", specifier = ">=7.6" }, { name = "fast-mlsirm", marker = "extra == 'backend'", git = "https://github.com/ContextualWisdomLab/fast-mlsirm.git?rev=5006c38286a4fa1d81bcf57eeed5ce27ae743f50" }, { name = "fastapi", marker = "extra == 'backend'", specifier = ">=0.115.0" }, { name = "httpx", marker = "extra == 'dev'", specifier = ">=0.27.0" },