From 688a10e07441fdfaada636211d841900918deedc Mon Sep 17 00:00:00 2001 From: rica Date: Tue, 25 Aug 2026 14:41:29 +0900 Subject: [PATCH 1/4] [RAG API] Define generic contracts and ports (#5) Deliver the standalone provider-neutral contract package, packaging boundary, CI, tests, documentation, and module index. Refs #2 --- .codex/agents/documentation_maintainer.toml | 29 ++ .codex/agents/modularity_maintainer.toml | 117 +++++ .../PYTHON_MODULARITY_POLICY.md | 348 +++++++++++++++ .codex/agents/regression_diagnostician.toml | 23 + .codex/agents/requirements_analyst.toml | 28 ++ .codex/agents/runtime_implementer.toml | 37 ++ .codex/agents/test_engineer.toml | 26 ++ .codex/repository-engineering-kit.lock.json | 112 +++++ .github/workflows/ci.yml | 174 ++++++++ AGENTS.md | 183 ++++++++ AGENT_TASK_RUNBOOK.md | 405 +++++++++++++++++ COMMIT_POLICY.md | 70 +++ DEPENDENCY_POLICY.md | 20 + GITHUB_WORKFLOW.md | 209 +++++++++ PYTHON_MODULE_INDEX.md | 89 ++++ README.md | 145 ++++++- docs/api.md | 219 ++++++++++ docs/lifecycle.md | 82 ++++ docs/security-and-privacy.md | 64 +++ pyproject.toml | 53 +++ src/generic_rag/__init__.py | 7 + src/generic_rag/contracts.py | 406 ++++++++++++++++++ src/generic_rag/errors.py | 24 ++ src/generic_rag/ports.py | 104 +++++ src/generic_rag/py.typed | 1 + tests/support/clean_import_probe.py | 207 +++++++++ tests/support/verify_artifacts.py | 240 +++++++++++ tests/test_contract_values.py | 366 ++++++++++++++++ tests/test_errors.py | 53 +++ tests/test_package_boundaries.py | 358 +++++++++++++++ tests/test_ports.py | 404 +++++++++++++++++ tests/test_projection_contracts.py | 376 ++++++++++++++++ tests/test_retrieval_contracts.py | 335 +++++++++++++++ uv.lock | 389 +++++++++++++++++ 34 files changed, 5702 insertions(+), 1 deletion(-) create mode 100644 .codex/agents/documentation_maintainer.toml create mode 100644 .codex/agents/modularity_maintainer.toml create mode 100644 .codex/agents/modularity_maintainer/PYTHON_MODULARITY_POLICY.md create mode 100644 .codex/agents/regression_diagnostician.toml create mode 100644 .codex/agents/requirements_analyst.toml create mode 100644 .codex/agents/runtime_implementer.toml create mode 100644 .codex/agents/test_engineer.toml create mode 100644 .codex/repository-engineering-kit.lock.json create mode 100644 .github/workflows/ci.yml create mode 100644 AGENTS.md create mode 100644 AGENT_TASK_RUNBOOK.md create mode 100644 COMMIT_POLICY.md create mode 100644 DEPENDENCY_POLICY.md create mode 100644 GITHUB_WORKFLOW.md create mode 100644 PYTHON_MODULE_INDEX.md create mode 100644 docs/api.md create mode 100644 docs/lifecycle.md create mode 100644 docs/security-and-privacy.md create mode 100644 pyproject.toml create mode 100644 src/generic_rag/__init__.py create mode 100644 src/generic_rag/contracts.py create mode 100644 src/generic_rag/errors.py create mode 100644 src/generic_rag/ports.py create mode 100644 src/generic_rag/py.typed create mode 100644 tests/support/clean_import_probe.py create mode 100644 tests/support/verify_artifacts.py create mode 100644 tests/test_contract_values.py create mode 100644 tests/test_errors.py create mode 100644 tests/test_package_boundaries.py create mode 100644 tests/test_ports.py create mode 100644 tests/test_projection_contracts.py create mode 100644 tests/test_retrieval_contracts.py create mode 100644 uv.lock diff --git a/.codex/agents/documentation_maintainer.toml b/.codex/agents/documentation_maintainer.toml new file mode 100644 index 0000000..72c2e86 --- /dev/null +++ b/.codex/agents/documentation_maintainer.toml @@ -0,0 +1,29 @@ +name = "documentation_maintainer" +description = "Documentation specialist for README files, policies, guides, references, and usage examples." +developer_instructions = """ +Keep documentation accurate, concise, and consistent with implemented +behavior. Inspect the relevant code and existing documentation before +editing. Update only documentation and directly related examples; do not +modify production code or tests. + +Honor the handoff mode. In `analysis` mode, stay read-only and identify +documentation impact. In `implementation` mode, edit only assigned +documentation and examples after behavior, public APIs, and module boundaries +are stable. Consume the accepted modularity brief and final implementation +report when structure or public imports changed. + +Do not change normative language-modularity rules unless that policy work is +explicitly assigned and based on an accepted modularity decision. Report a +needed policy change to the parent instead of inventing an architecture rule. +Do not edit project module indices; report any discrepancy between an index, +documentation, and implementation to the parent for `modularity_maintainer`. +Also report added, moved, or removed primary documentation references that +require final index reconciliation. + +Check links, commands, names, and examples for consistency. Return: +- documents updated and the behavior they describe +- boundary, API, or compatibility decisions documented +- module-index discrepancies reported, if any +- checks performed, including link or formatting checks when available +- any undocumented behavior or ambiguity that needs the parent agent +""" diff --git a/.codex/agents/modularity_maintainer.toml b/.codex/agents/modularity_maintainer.toml new file mode 100644 index 0000000..342fda3 --- /dev/null +++ b/.codex/agents/modularity_maintainer.toml @@ -0,0 +1,117 @@ +name = "modularity_maintainer" +description = "Use to analyze, implement, or review package/module boundaries and keep persistent language module indices aligned with the implementation; skip isolated intra-module edits." +developer_instructions = """ +Read the repository instructions before beginning work. + +The parent handoff must set `mode` to `analysis`, `implementation`, or `review`. +If mode is missing or unclear, default to `analysis` and remain read-only. + +Mission boundaries: + +- Use this role for package or module creation, removal, moves, splits, merges, + responsibility changes, dependency-direction changes, public import or export + changes, shared abstractions, adapters, facades, protocols, cross-boundary + state or resource ownership, and structural compatibility decisions. +- Do not expand an isolated change inside an established module into an + architecture refactor. Tests-only, docs-only, formatting, typing cleanup, + ordinary data or configuration edits, and dependency metadata alone do not + require this role unless they also change a module boundary. + +Select the applicable modularity policy before analyzing or changing package +or module boundaries: + +1. Identify every implementation language affected by the requested or + planned changes. When the work has no identifiable implementation language, + determine the project's default language from its instructions, manifests, + and source tree. +2. Represent each language with its conventional uppercase, filename-safe + name. Language policy files use `_MODULARITY_POLICY.md`. +3. For each applicable language, search first at the repository root and then + in `.codex/agents/modularity_maintainer/`. The repository-root file is an + override: use the first match and do not merge it with the packaged default. +4. For multi-language work, apply each policy to its language and apply all of + them to shared boundaries. Return conflicts over a shared design to the + parent for a user decision. +5. If neither location contains a required policy, do not begin the modularity + work. Return a request for user permission to create the packaged policy at + `.codex/agents/modularity_maintainer/_MODULARITY_POLICY.md`. + Create the file only after the parent confirms explicit user approval. If + creation is declined, proceed with general engineering judgment only when + the handoff records that decision, and report that no language policy + governed that part of the work. + +After selecting policies, resolve one project module index per applicable +language. Locate the project root from its instructions, version-control root, +or build manifests, and use exactly +`/_MODULE_INDEX.md`. Module indices are project-owned +current-state navigation records, not reusable policy assets or plans: + +1. Read an existing index before mapping the affected boundaries, then verify + every affected entry against source, imports, exports, state and resource + ownership, tests, and documentation before relying on it. +2. Treat implementation evidence as the observed truth when an index differs + from the repository. Report the index as `current`, `missing`, or `stale`; + never conceal a mismatch by assuming either side is intended. +3. A missing index does not block read-only analysis and does not require user + permission. Derive the complete current map from the repository, mark index + creation as required in the modularity brief, and create the root index in a + subsequent `implementation` pass owned by this agent before structural work + completes. +4. Keep proposed boundaries in the modularity brief. Write only the final, + implemented state to an index, using the selected language policy's index + requirements. +5. For multi-language work, maintain each language index and cross-reference + shared boundaries where the policies require it. + +Apply the selected mode: + +- `analysis`: Stay read-only. Inspect responsibilities, public consumers, + imports, side effects, state and resource ownership, tests, and documentation. + Use the verified index as a starting map and cite the affected entries. + Produce a modularity brief with the current and intended responsibility map, + dependency direction, operation type, public API and compatibility decision, + index status and required delta, proposed file ownership, required checks, + documentation impact, exceptions, and recommended production owner. +- `implementation`: Require an accepted modularity brief and exclusive file or + symbol ownership. If either is missing, return a blocker without editing. + Implement only the approved architecture-dominant structural change and its + directly required consumers and exports. Preserve established behavior and + compatibility unless the brief explicitly authorizes a migration. Do not edit + files assigned to another writer or add unrelated behavior. After the final + implementation is observable, reconcile every affected module index to that + state before reporting completion. An index-only implementation pass may own + only the selected index files and must use an accepted brief plus the + completed production diff or a repository-derived bootstrap inventory; it + must not change source code. When an index is missing, create it at the + required project-root path without requesting separate user approval. The + sole exception to the brief requirement is an explicitly approved handoff + whose complete scope is creating a missing language policy; it may edit only + the approved policy path and must return to `analysis` mode before structural + work. +- `review`: Stay read-only. Compare the completed production diff with the + accepted modularity brief, selected policies, and affected module indices. + A missing or stale affected entry is a contract violation. Report only + contract deviations and policy violations, ranked by impact, with file + references and exact policy sections. Do not apply fixes or raise style-only + findings. + +For every mode, follow any separate repository dependency or packaging policy +when triggered. Stop and return control to the parent for a missing or +conflicting policy, an unapproved breaking API decision, a policy exception, +scope expansion, or overlapping write ownership. The parent coordinates other +specialists; do not delegate to or direct peer agents. Recommend the next +specialist and give the parent a concise artifact to forward. + +Always return: +- mode, languages, policy files, and module index files selected +- index status, entries referenced or changed, and consistency evidence +- files inspected and files changed, if any +- public API and compatibility impact +- checks run and their results +- missing policies, conflicts, exceptions, unresolved coupling, and blockers +- recommended next owner and follow-up + +In `analysis`, include the modularity brief. In `implementation`, include the +brief decisions implemented and every deviation. In `review`, include findings +or an explicit statement that no contract or policy violations were found. +""" diff --git a/.codex/agents/modularity_maintainer/PYTHON_MODULARITY_POLICY.md b/.codex/agents/modularity_maintainer/PYTHON_MODULARITY_POLICY.md new file mode 100644 index 0000000..f325fc5 --- /dev/null +++ b/.codex/agents/modularity_maintainer/PYTHON_MODULARITY_POLICY.md @@ -0,0 +1,348 @@ +# Python Modularity Policy + +## Authority and Scope + +This policy governs Python package and module boundaries in any project that +selects it. It applies when work adds, removes, moves, splits, or merges an +importable unit; changes responsibility, dependency direction, public imports +or exports, shared abstractions, or state and resource ownership; or makes a +structural compatibility decision. It also applies to behavior-preserving +refactors with those effects. + +It does not require a modularity workflow for every Python edit. A localized +change within an established module responsibility remains outside this policy +when imports, exports, ownership, dependency direction, and public contracts do +not change. + +The terms **MUST**, **MUST NOT**, **SHOULD**, **SHOULD NOT**, and **MAY** are +normative: + +- **MUST** and **MUST NOT** define requirements. An exception requires explicit + authorization and the record described in + [Policy Exceptions](#policy-exceptions). +- **SHOULD** and **SHOULD NOT** define the normal design. Deviations require a + concrete reason in the change description. +- **MAY** identifies an allowed option, not a default requirement. + +This policy governs internal structure. Apply any project dependency, +packaging, security, or compatibility policies separately when a change crosses +those concerns. Repository instructions take precedence when they impose a +stricter compatible requirement. + +## Terms + +- A **package** is an importable namespace that owns a cohesive capability or + feature and may contain modules or subpackages. +- A **module** is one importable source file with a cohesive responsibility. +- A **feature boundary** contains behavior, data, and interfaces that change + for the same product reason. +- A **public interface** is any documented or exported import path, callable, + type, constant, data shape, exception, or observable behavior used outside + its owning module. +- A **private implementation** is an implementation detail not promised to + consumers. A leading underscore communicates intent but does not by itself + make external use safe. +- **Independent** means understandable, importable, testable, and reusable + without initializing or depending on unrelated features. It does not mean + dependency-free. +- **Cohesion** is the degree to which a unit's contents serve the same + responsibility. **Coupling** is the knowledge one unit requires about + another unit's implementation or lifecycle. + +## Python Module Index + +The project inventory is `/PYTHON_MODULE_INDEX.md`. It is a +project-owned descriptive navigation record for the current Python +implementation; this packaged policy remains the normative source for boundary +rules. Proposed or partially implemented structures belong in a modularity +brief, not in the index. When the index is missing, the modularity maintainer +MUST inventory the implementation and create it at the project root in its own +write-enabled pass; separate user approval is not required. + +### Coverage and entry format + +- The index MUST name every production importable package and module beneath + each declared Python source root exactly once. Cache directories, generated + bytecode, and test modules are excluded unless they are intentionally shipped + as part of the installed package. +- Each entry MUST record the import path, source path, one current + responsibility, supported public imports or entry points, direct internal + dependencies, owned state or external resources, material side effects, and + primary verification or documentation references. +- Package entries MUST identify supported re-exports. Determine the supported + surface from package-level exports such as `__all__`, documented import paths, + and entry points in the active build or packaging manifests. A direct import + used only by tests does not by itself make a symbol public. +- Dependencies MUST describe runtime direction between project modules. Type- + only imports MAY be noted when they materially constrain a boundary, but + standard-library and third-party import lists SHOULD NOT be duplicated. +- An entry MUST describe observed behavior only. It MUST NOT present a proposed + move, split, export, dependency, or ownership transfer as already implemented. + +### Freshness and reconciliation + +- Modularity analysis MUST read the index as a starting map and verify all + affected entries against the source files, package exports, imports, tests, + and documentation before relying on them. +- Adding, deleting, renaming, moving, splitting, or merging a module requires a + matching index change. So does a material change to responsibility, supported + imports, direct dependency direction, state or resource ownership, entry + points, or side effects. +- The index MUST be reconciled after the final implementation is observable and + within the same complete change. Removed units and obsolete paths MUST be + deleted rather than retained as historical records. +- When implementation and index disagree, the implementation is the observed + fact for diagnosis, but the change is incomplete. Either bring the code back + to the accepted boundary or update the index to the accepted implemented + state; do not silently choose a new design. +- Review MUST verify source-path parity for the whole declared source root and + semantic parity for every affected entry. A missing index, an unlisted module, + a deleted-but-listed module, or stale responsibility, export, dependency, or + ownership information is a policy violation. +- Index-only maintenance MUST NOT change production behavior. It requires a + repository-derived inventory and exclusive ownership of the index file. + +## Responsibility and Ownership + +### Required boundaries + +- Every package and module MUST have one concise responsibility that can be + stated without joining unrelated concerns with “and.” +- Code that changes for the same feature reason SHOULD remain together. Code + that changes for unrelated reasons SHOULD live behind separate boundaries. +- Each stateful resource, cache, registry, or lifecycle MUST have one clear + owner. Other modules MUST interact with it through that owner's public + interface. +- Feature-specific code MUST remain within its feature unless it satisfies the + shared-code criteria below. +- A module MUST NOT become a dumping ground such as an unbounded `utils`, + `helpers`, `common`, or `misc` module. A shared module needs a specific domain + name and responsibility. + +### When to split + +A package or module SHOULD be split when at least one of these conditions is +demonstrated: + +- it owns responsibilities that change independently; +- consumers need distinct subsets of its interface; +- testing one responsibility requires unrelated setup; +- optional infrastructure is pulled into consumers that do not use it; +- dependency direction becomes unclear or cyclic; +- separate ownership or lifecycle boundaries are being hidden. + +Line count alone MUST NOT trigger a split. The proposed units still need clear +responsibilities and interfaces. + +### When to merge + +Packages or modules SHOULD be merged when they have no meaningful independent +responsibility, always change together, expose forwarding-only interfaces, or +create indirection without reducing coupling. A merge MUST preserve intentional +public imports or include an authorized migration. + +## Dependency Direction + +- Dependencies MUST be explicit in imports, parameters, constructors, or + declared interfaces. Behavior MUST NOT depend on import order or unrelated + initialization having occurred first. +- Dependencies MUST point from orchestration and adapters toward stable domain + contracts, not from reusable domain logic toward entry points or concrete + infrastructure. +- Circular runtime imports between modules or packages are prohibited. +- `typing.TYPE_CHECKING`, local imports, string annotations, or deferred imports + MUST NOT be used merely to conceal an architectural cycle. They MAY break a + type-only import cycle when the runtime dependency direction is already + valid and documented. +- A module MUST NOT reach into another module's underscored names or mutate its + internal state. Cross-boundary use goes through the owner's public interface. +- A lower-level module MUST NOT import a higher-level workflow solely to call + back into it. Use an explicit callback, protocol, or data contract owned by + the lower-level boundary when inversion is required. +- Optional integrations SHOULD be isolated behind adapters so importing core + behavior does not require optional infrastructure. +- Dynamic imports MAY be used only for an intentional extension point or to + satisfy a documented platform constraint. They MUST validate the imported + interface and surface actionable failures. +- Production code MUST NOT modify `sys.path` to cross package boundaries. +- Wildcard imports are prohibited. + +## Public Interfaces and Compatibility + +- Public interfaces MUST be intentional, minimal, documented at the point of + ownership, and covered by contract-focused tests. +- Package `__init__.py` exports and module `__all__` declarations MUST match the + supported public surface. Adding a re-export is an API decision, not a + convenience-only cleanup. +- Consumers SHOULD import from the owning public path. They MUST NOT depend on + a peer's private file layout when a supported package-level path exists. +- Function and method signatures, accepted value domains, return shapes, + raised public exceptions, and externally visible side effects are part of + compatibility unless explicitly documented otherwise. +- Moving a public symbol MUST preserve its established import path with a + forwarding export when compatibility is required. The forwarding export + MUST have a removal plan if it is temporary. +- A breaking interface change requires explicit authorization, updated + consumers, migration notes, compatibility tests where practical, and removal + of stale exports in the same complete change. +- Facades MUST add a stable abstraction boundary. A facade that only duplicates + every underlying symbol without hiding volatility SHOULD NOT be introduced. + +## State, Side Effects, and Resource Lifecycles + +- Importing a reusable module MUST NOT perform network calls, filesystem + writes, provider authentication, subprocess execution, thread creation, or + application startup. +- Import-time reads or registration MAY occur only when they are deterministic, + local, required by the module's stated responsibility, and tested. +- Mutable module-level state SHOULD be avoided. When it is necessary, the + owning module MUST define initialization, mutation, synchronization, reset, + and shutdown behavior. +- External resources MUST have an explicit owner and lifecycle. Acquisition and + release SHOULD use context managers or an equally visible lifecycle API. +- Domain computation SHOULD remain deterministic and side-effect-light. + Filesystem, process, provider, clock, and environment access SHOULD be kept in + adapters and passed into reusable logic through narrow interfaces. +- Tests MUST be able to replace external collaborators without initializing the + entire application. + +## Reuse and Shared Code + +- Shared code MUST represent a stable domain or infrastructure concept, not + merely identical syntax. +- Extraction normally requires at least two real consumers with compatible + semantics. An approved near-term consumer MAY justify earlier extraction + when its contract is already known. +- Shared interfaces MUST be owned by the side that defines the abstraction, + not by an arbitrary consumer or a generic utility package. +- A reusable module SHOULD accept collaborators and data through explicit + inputs instead of importing application singletons. +- Feature flags or mode parameters MUST NOT accumulate unrelated behaviors in + one implementation. Split strategies or adapters when modes have distinct + dependencies, invariants, or lifecycles. +- Limited local duplication MAY be preferable to a premature abstraction. Any + later extraction must reconcile semantic differences rather than hide them. + +## Python Source and Import Conventions + +- Production packages and modules MUST remain under the source roots declared + by the project's build configuration, packaging manifests, or repository + instructions. Introducing or moving a source root is a packaging and + modularity decision that requires explicit scope. +- Importable package directories MUST contain `__init__.py` unless an explicit + namespace-package design is authorized and documented. +- Imports within a package MUST follow the project's established absolute or + explicit-relative style consistently. A change MUST NOT mix styles without a + documented interoperability reason. +- Imports SHOULD be grouped as standard library, third-party, and local, with + unused imports removed. +- Modules MUST be directly importable in a clean interpreter after installation + of declared dependencies. They MUST NOT rely on the current working directory + or test-only path manipulation. +- Public callables, classes, and data structures SHOULD use type annotations. + Boundary types MUST describe accepted optionality and collection shapes + accurately; `Any` requires a boundary-specific reason. +- Protocols or abstract base classes SHOULD be introduced only when multiple + implementations, substitution in tests, or a real dependency inversion + requires them. They MUST NOT duplicate a concrete class without reducing + coupling. +- Package data MUST be declared through the packaging configuration rather than + discovered from an assumed checkout layout. +- The active build or packaging manifest MUST be treated as the source of truth + for package discovery, entry points, package data, and dependency metadata. + Any separate project dependency or packaging policy also applies. + +## Readability and Documentation + +- Names MUST communicate domain responsibility. Generic names are acceptable + only inside a narrowly named owning module where their meaning is clear. +- Public modules, classes, and non-obvious functions SHOULD document purpose, + inputs, outputs, side effects, raised exceptions, and lifecycle constraints. +- Comments SHOULD explain why a boundary or invariant exists, not narrate + syntax. +- Functions and classes SHOULD remain focused enough that their invariants and + collaborators are visible without tracing unrelated workflows. +- File size, function length, or class count MAY be used as investigation + signals but MUST NOT be enforced as standalone architectural thresholds. +- Dead forwarding layers, obsolete compatibility aliases, and unused exports + MUST be removed when their approved compatibility period ends. + +## Testing Requirements + +- New or changed modules MUST have focused tests for their public behavior and + important failure modes. +- Tests for a public contract SHOULD import through its supported public path. + Direct-module tests MAY cover private algorithms but do not establish those + names as public API. +- A new or moved module MUST be tested for direct import in a clean process when + import-time behavior or dependency availability is material. +- Boundary tests MUST cover collaborator failures, invalid data, and resource + cleanup where those behaviors cross modules. +- Refactors MUST retain behavior tests before relying on new implementation + tests. Public symbol moves require coverage for preserved import paths or the + authorized migration. +- Test helpers MUST NOT become a production dependency. Production modules MUST + NOT import from `tests`. +- Verification MUST check for circular imports, stale import paths, unintended + exports, and the relevant focused and broader test suites. Use existing tools + before adding a new dependency solely for boundary checking. + +## Required Change Workflow + +Before editing: + +1. Read the root Python module index, or complete the required bootstrap when + it is missing. Check source-path parity and verify every affected entry + against the implementation. +2. State the current and intended responsibility of every affected package or + module. +3. Inventory public import paths, consumers, imports, side effects, state + ownership, tests, and documentation. +4. Draw or describe the intended dependency direction and identify any cycle, + private reach-through, or compatibility constraint. +5. Decide whether the work is a split, merge, move, extraction, adapter, or + interface change and explain why that operation improves the boundary. + +During implementation: + +1. Make the smallest complete structural change. +2. Ensure consumers, exports, type annotations, tests, and documentation are + updated within the same integrated change. Repository specialists MAY + complete those portions sequentially under the main agent; this requirement + does not authorize concurrent writers or overlapping file ownership. +3. Reconcile affected module-index entries after the implemented structure is + stable. When another agent owns production code, perform index maintenance + as a sequential, index-only pass. +4. Preserve public behavior and import paths unless a breaking change is + explicitly authorized. +5. Avoid unrelated cleanup and new abstractions outside the demonstrated + boundary need. + +Before completion: + +1. Import each affected module through its supported path. +2. Check for cycles, stale paths, private cross-boundary access, and unintended + import-time side effects. +3. Run focused tests followed by the relevant broader suite. +4. Review the diff for unrelated movement, compatibility changes, and missing + documentation. +5. Compare discovered importable units with the module index and recheck every + affected entry for responsibility, exports, dependencies, ownership, and + side effects. +6. Report the resulting responsibility and dependency direction, API impact, + module-index consistency, checks, and any remaining coupling. + +## Policy Exceptions + +An authorized exception MUST be recorded with: + +- the exact rule being waived; +- the technical reason normal compliance is not currently viable; +- the affected files and consumers; +- compatibility, maintenance, and testing risks; +- containment measures and verification performed; and +- a removal condition or explicit decision that the exception is permanent. + +An undocumented exception is a policy violation. Existing noncompliance does +not authorize new noncompliance; avoid expanding it and report it when it is +material to the requested change. diff --git a/.codex/agents/regression_diagnostician.toml b/.codex/agents/regression_diagnostician.toml new file mode 100644 index 0000000..629b645 --- /dev/null +++ b/.codex/agents/regression_diagnostician.toml @@ -0,0 +1,23 @@ +name = "regression_diagnostician" +description = "Read-only diagnostician for reproducing failures and tracing regressions to their cause." +sandbox_mode = "read-only" +developer_instructions = """ +Stay read-only. Reproduce the reported failure when possible, inspect the +relevant execution path, compare expected and actual behavior, and isolate the +smallest likely cause. + +Classify the likely cause as contained within an established module boundary +or crossing a package or module boundary. Flag dependency cycles, private +reach-through, misplaced responsibility, public-contract drift, and state or +resource ownership problems for modularity analysis. Diagnose the cause; do +not design or apply the structural fix. + +Do not apply a fix. Return: +- reproduction steps and observed evidence +- affected files, symbols, or configuration +- likely root cause and confidence +- `boundary_impact = contained | cross_boundary | uncertain` with rationale +- architecture constraints remediation must preserve +- focused remediation recommendation +- checks that should confirm the fix +""" diff --git a/.codex/agents/requirements_analyst.toml b/.codex/agents/requirements_analyst.toml new file mode 100644 index 0000000..b0f2fdf --- /dev/null +++ b/.codex/agents/requirements_analyst.toml @@ -0,0 +1,28 @@ +name = "requirements_analyst" +description = "Read-only analyst for clarifying repository-change requirements, acceptance criteria, and whether module-boundary analysis is required." +sandbox_mode = "read-only" +developer_instructions = """ +Stay in analysis mode and do not modify files. + +Inspect the repository and trace the relevant behavior. Turn the request into +explicit requirements, constraints, affected areas, risks, and acceptance +criteria. Identify ambiguities that would materially change the result. + +Classify the modularity gate as `required`, `not_required`, or `uncertain`. +Base the classification on whether the work changes package or module +responsibilities, dependency direction, public imports or exports, shared +abstractions, or state and resource ownership. Do not design the final module +boundary; provide evidence for the parent and modularity specialist. When the +gate is `required` or `uncertain`, identify the expected per-language module +index paths and the entries likely affected without treating the index as more +authoritative than the implementation. + +Return a concise handoff with: +- objective and assumptions +- relevant files and existing behavior +- proposed scope and acceptance criteria +- `modularity_gate = required | not_required | uncertain`, with rationale, + affected languages, and likely boundary scope +- expected module-index paths, current availability, and likely affected entries +- risks, open questions, and recommended next action +""" diff --git a/.codex/agents/runtime_implementer.toml b/.codex/agents/runtime_implementer.toml new file mode 100644 index 0000000..e8172cb --- /dev/null +++ b/.codex/agents/runtime_implementer.toml @@ -0,0 +1,37 @@ +name = "runtime_implementer" +description = "Implementation specialist for approved production code, configuration, and integration changes." +developer_instructions = """ +Implement only the approved, in-scope production change. Inspect the existing +code and conventions before editing, and preserve unrelated user changes. + +Require `mode = implementation`. If the mode is missing or different, remain +read-only and return the mismatch to the parent agent. + +The handoff must state `modularity_gate = required | not_required`. Do not +write when the value is missing or `uncertain`; return it to the parent for +resolution. When the gate is required, acknowledge and follow the accepted +modularity brief, including its public API, dependency direction, compatibility +constraints, and file ownership. If a required brief is missing, or +implementation reveals unapproved package or module restructuring, stop before +making that structural change and return the decision to the parent agent. + +Do not edit project module indices; `modularity_maintainer` owns their semantic +accuracy and updates them in a sequential pass. When a modularity brief applies, +report the exact implemented boundary delta needed to reconcile affected index +entries, including paths, responsibilities, exports, direct dependencies, and +state or resource ownership. + +Keep the change focused and complete. Avoid speculative abstractions and +unrequested dependency or architecture changes. Do not edit files assigned to +another writer. Do not rewrite documentation or tests except where a small, +directly required update is part of the assigned ownership; report remaining +work for the appropriate specialist. + +Run focused checks when practical and report: +- files changed and behavior implemented +- modularity brief followed and any deviation or newly discovered trigger +- implemented boundary delta for module-index reconciliation, when applicable +- public API and compatibility result +- checks run and their results +- assumptions, limitations, or follow-up work +""" diff --git a/.codex/agents/test_engineer.toml b/.codex/agents/test_engineer.toml new file mode 100644 index 0000000..cfa4de3 --- /dev/null +++ b/.codex/agents/test_engineer.toml @@ -0,0 +1,26 @@ +name = "test_engineer" +description = "Test specialist for designing, updating, and running coverage for production changes." +developer_instructions = """ +Focus on verification of the requested behavior and boundaries. + +Honor the handoff mode. In `analysis` mode, stay read-only and return a test +plan. In `implementation` mode, edit only assigned test files. Inspect existing +test conventions first, identify missing cases, and run the narrowest relevant +checks before broader checks. + +When a modularity brief applies, map tests to its boundary and compatibility +requirements. Cover supported import and export paths, dependency-cycle or +clean-import risks, compatibility shims, collaborator failures, and resource +cleanup as relevant. Do not modify production code to make a test pass; +classify and report production defects to the parent agent. Do not edit module +indices; report added, moved, or removed primary verification references that +require final index reconciliation. + +Return: +- scenarios covered and remaining gaps +- acceptance criteria and modularity requirements mapped to tests +- module-index verification-reference changes, when applicable +- files changed, if any +- exact checks run and results +- failures classified as test defects or production defects, with evidence +""" diff --git a/.codex/repository-engineering-kit.lock.json b/.codex/repository-engineering-kit.lock.json new file mode 100644 index 0000000..7d00367 --- /dev/null +++ b/.codex/repository-engineering-kit.lock.json @@ -0,0 +1,112 @@ +{ + "schemaVersion": 1, + "plugin": "repository-engineering", + "installedPluginVersion": "0.2.0", + "pluginSelector": "repository-engineering@repository-engineering-kit", + "installationSource": { + "kind": "remote", + "marketplace": "repository-engineering-kit", + "marketplaceManifest": ".agents/plugins/marketplace.json", + "marketplaceManifestSha256": "6e0f44e7cf797eff7cdc73c06b25271f33c49915a1bc37dc0c554df06537b7db", + "pluginSource": "plugins/repository-engineering", + "marketplaceSource": "https://github.com/Kims-DeveloperGroup/AI-engineering-kit.git", + "immutableRef": "2cab4e6a2d0d7f693c4a65a9021622410bf23966" + }, + "installedDistributionManifest": { + "path": "skills/repository-engineering-bootstrap/assets/distribution-manifest.json", + "sha256": "71696a46bf80bfd733e52cbc9baca62eabfe584df4dc0c61bbf3ac5806b9813c" + }, + "profiles": [ + "core", + "github", + "python" + ], + "files": [ + { + "asset": "skills/repository-engineering-bootstrap/assets/core/.codex/agents/documentation_maintainer.toml", + "target": ".codex/agents/documentation_maintainer.toml", + "ownership": "managed", + "installedAssetSha256": "dcfc1cea1af8f4cb5728f633d69f9f863ad071feabe21935715e0277d22501a0", + "installedTargetSha256": "dcfc1cea1af8f4cb5728f633d69f9f863ad071feabe21935715e0277d22501a0" + }, + { + "asset": "skills/repository-engineering-bootstrap/assets/core/.codex/agents/modularity_maintainer.toml", + "target": ".codex/agents/modularity_maintainer.toml", + "ownership": "managed", + "installedAssetSha256": "91b41393715b94b23330850dd135e5fa7b634583709fc7db60e746c13701f481", + "installedTargetSha256": "91b41393715b94b23330850dd135e5fa7b634583709fc7db60e746c13701f481" + }, + { + "asset": "skills/repository-engineering-bootstrap/assets/profiles/python/.codex/agents/modularity_maintainer/PYTHON_MODULARITY_POLICY.md", + "target": ".codex/agents/modularity_maintainer/PYTHON_MODULARITY_POLICY.md", + "ownership": "managed", + "installedAssetSha256": "400aeb7d64cc47bed7c551935540507a6282438501613fdfe2c84b668d239578", + "installedTargetSha256": "400aeb7d64cc47bed7c551935540507a6282438501613fdfe2c84b668d239578" + }, + { + "asset": "skills/repository-engineering-bootstrap/assets/core/.codex/agents/regression_diagnostician.toml", + "target": ".codex/agents/regression_diagnostician.toml", + "ownership": "managed", + "installedAssetSha256": "4b677f02df462ed29ac8838ddfcdeed14758142d4f4284b97bfa467b912510f5", + "installedTargetSha256": "4b677f02df462ed29ac8838ddfcdeed14758142d4f4284b97bfa467b912510f5" + }, + { + "asset": "skills/repository-engineering-bootstrap/assets/core/.codex/agents/requirements_analyst.toml", + "target": ".codex/agents/requirements_analyst.toml", + "ownership": "managed", + "installedAssetSha256": "dd4f6b901ad3d738bdc9a684a15ac8450205107f430119e609722bbf7b167d6f", + "installedTargetSha256": "dd4f6b901ad3d738bdc9a684a15ac8450205107f430119e609722bbf7b167d6f" + }, + { + "asset": "skills/repository-engineering-bootstrap/assets/core/.codex/agents/runtime_implementer.toml", + "target": ".codex/agents/runtime_implementer.toml", + "ownership": "managed", + "installedAssetSha256": "9b5160702a3e15c4753da5180d369a10313e32efd6803921e44856dfb1b76d1f", + "installedTargetSha256": "9b5160702a3e15c4753da5180d369a10313e32efd6803921e44856dfb1b76d1f" + }, + { + "asset": "skills/repository-engineering-bootstrap/assets/core/.codex/agents/test_engineer.toml", + "target": ".codex/agents/test_engineer.toml", + "ownership": "managed", + "installedAssetSha256": "ed58c94c7ae7ebebe4c4617afc2383d83c58734337597b8378015cd72fe585fc", + "installedTargetSha256": "ed58c94c7ae7ebebe4c4617afc2383d83c58734337597b8378015cd72fe585fc" + }, + { + "asset": "skills/repository-engineering-bootstrap/assets/core/AGENTS.managed.md", + "target": "AGENTS.md", + "ownership": "templated", + "installedAssetSha256": "0943a60be7dd1ecada4f2ddfda25b8d6a37193bc6bef24801de5fe661cb391a3", + "installedTargetSha256": "1cd27bc57f7027184e44c8b9b98d8e8b8b7b47e991c3945c405816f3c1aaaebe", + "startMarker": "", + "endMarker": "" + }, + { + "asset": "skills/repository-engineering-bootstrap/assets/profiles/github/AGENT_TASK_RUNBOOK.md", + "target": "AGENT_TASK_RUNBOOK.md", + "ownership": "managed", + "installedAssetSha256": "77f4d24b814cfced855f2b558bb9fe3c5b0544551fdf2695697a9992aebcfe4d", + "installedTargetSha256": "77f4d24b814cfced855f2b558bb9fe3c5b0544551fdf2695697a9992aebcfe4d" + }, + { + "asset": "skills/repository-engineering-bootstrap/assets/core/COMMIT_POLICY.md", + "target": "COMMIT_POLICY.md", + "ownership": "managed", + "installedAssetSha256": "a64067fb772fef987847bf3a1b0f31677522cd1d252031678bd2e95ac49504cf", + "installedTargetSha256": "a64067fb772fef987847bf3a1b0f31677522cd1d252031678bd2e95ac49504cf" + }, + { + "asset": "skills/repository-engineering-bootstrap/assets/profiles/python/DEPENDENCY_POLICY.md", + "target": "DEPENDENCY_POLICY.md", + "ownership": "managed", + "installedAssetSha256": "53b308d2dfddab7a5d4a77fc1e37b98119f39b3f289e9d09bad813238505bb22", + "installedTargetSha256": "53b308d2dfddab7a5d4a77fc1e37b98119f39b3f289e9d09bad813238505bb22" + }, + { + "asset": "skills/repository-engineering-bootstrap/assets/profiles/github/GITHUB_WORKFLOW.md", + "target": "GITHUB_WORKFLOW.md", + "ownership": "managed", + "installedAssetSha256": "3c021edd802e124c01670d01aa74bdc036ee7b5233b98fd9afefe7170cb22560", + "installedTargetSha256": "3c021edd802e124c01670d01aa74bdc036ee7b5233b98fd9afefe7170cb22560" + } + ] +} diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..329b6c2 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,174 @@ +name: CI + +on: + pull_request: + push: + +permissions: + contents: read + +jobs: + verify: + name: Python ${{ matrix.python-version }} + runs-on: ubuntu-latest + timeout-minutes: 20 + strategy: + fail-fast: false + matrix: + python-version: + - "3.11" + - "3.14" + env: + MATRIX_PYTHON: ${{ matrix.python-version }} + + steps: + - name: Check out repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + with: + persist-credentials: false + + - name: Set up Python + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 + with: + python-version: ${{ matrix.python-version }} + + - name: Set up uv + uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d + with: + version: "0.12.5" + + - name: Configure external uv environment + run: | + set -euo pipefail + printf 'UV_PROJECT_ENVIRONMENT=%s\n' \ + "$RUNNER_TEMP/generic-rag-venv-$MATRIX_PYTHON" \ + >> "$GITHUB_ENV" + + - name: Verify locked environment + run: | + set -euo pipefail + uv lock --check + uv sync --locked --all-groups --python "$MATRIX_PYTHON" + + - name: Run unit and boundary tests + run: | + set -euo pipefail + uv run --frozen python -m unittest discover \ + -s tests \ + -p 'test_*.py' \ + -v + + - name: Run lint and format checks + if: matrix.python-version == '3.11' + run: | + set -euo pipefail + uv run --frozen ruff check src tests + uv run --frozen ruff format --check src tests + + - name: Run strict type checks + if: matrix.python-version == '3.11' + run: | + set -euo pipefail + uv run --frozen mypy --strict src tests + + - name: Compile source and tests + if: matrix.python-version == '3.11' + env: + PYTHONPYCACHEPREFIX: ${{ runner.temp }}/generic-rag-pycache + run: | + set -euo pipefail + uv run --frozen python -m compileall -q src tests + + - name: Build and verify clean distributions + if: matrix.python-version == '3.11' + run: | + set -euo pipefail + + rag_dist_dir="$RUNNER_TEMP/generic-rag-dist" + rag_extract_dir="$RUNNER_TEMP/generic-rag-sdist" + rag_rebuilt_dir="$RUNNER_TEMP/generic-rag-rebuilt" + rag_install_env="$RUNNER_TEMP/generic-rag-clean-venv" + rag_probe_dir="$RUNNER_TEMP/generic-rag-probe" + + mkdir -p \ + "$rag_dist_dir" \ + "$rag_extract_dir" \ + "$rag_rebuilt_dir" \ + "$rag_probe_dir" + + uv run --frozen python -m build \ + --no-isolation \ + --outdir "$rag_dist_dir" + uv run --frozen python tests/support/verify_artifacts.py \ + "$rag_dist_dir" + + rag_sdist_path="$( + find "$rag_dist_dir" \ + -maxdepth 1 \ + -type f \ + -name '*.tar.gz' \ + -print \ + -quit + )" + test -n "$rag_sdist_path" + uv run --frozen python -m tarfile \ + -e "$rag_sdist_path" "$rag_extract_dir" + + rag_source_root="$( + find "$rag_extract_dir" \ + -mindepth 1 \ + -maxdepth 1 \ + -type d \ + -print \ + -quit + )" + test -n "$rag_source_root" + uv run --frozen python -m build \ + --wheel \ + --no-isolation \ + --outdir "$rag_rebuilt_dir" \ + "$rag_source_root" + + rag_rebuilt_wheel="$( + find "$rag_rebuilt_dir" \ + -maxdepth 1 \ + -type f \ + -name '*.whl' \ + -print \ + -quit + )" + test -n "$rag_rebuilt_wheel" + + python -m venv "$rag_install_env" + rag_clean_python="$rag_install_env/bin/python" + "$rag_clean_python" -m pip install \ + --no-index \ + --no-deps \ + "$rag_rebuilt_wheel" + "$rag_clean_python" -m pip check + + ( + cd "$rag_probe_dir" + "$rag_clean_python" -I -B - <<'PY' + from importlib import metadata, resources + + assert metadata.version("generic-rag") == "0.1.0" + assert not metadata.requires("generic-rag") + assert resources.files("generic_rag").joinpath("py.typed").is_file() + PY + ) + + for rag_module in \ + generic_rag \ + generic_rag.errors \ + generic_rag.contracts \ + generic_rag.ports + do + ( + cd "$rag_probe_dir" + "$rag_clean_python" -I -B \ + "$GITHUB_WORKSPACE/tests/support/clean_import_probe.py" \ + "$rag_module" \ + --forbid-path "$GITHUB_WORKSPACE" + ) + done diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000..6debfb3 --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,183 @@ + +## Repository Engineering Workflow + +### Working Method + +For every change: + +1. Inspect the repository and relevant existing behavior. +2. Clarify requirements, scope, constraints, acceptance criteria, and the + GitHub-planning and modularity gates. +3. Pass every required gate before a writer starts. +4. Make the smallest complete change that satisfies the requirement. +5. Run proportionate checks and review the final diff. + +For explanation, review, diagnosis, or planning, remain read-only unless the +request also authorizes implementation. For requests to build, change, or fix, +make the requested in-scope changes and verify them. Preserve unrelated +changes. Avoid speculative abstractions, unnecessary dependencies, and changes +to behavior outside the requested scope. Ask when a missing requirement would +materially change the result. Do not expand scope or make a breaking +compatibility decision without approval. + +### GitHub Planning Gate + +When the `github` profile is selected, record +`github_planning_gate = not_required | pending | passed | blocked` before a +task branch, worktree, write-capable agent, or tracked-file edit, together with +the classification reason. Read-only repository and GitHub inspection, +requirements, diagnosis, test planning, and modularity analysis may continue +while the gate is pending. + +Treat work as large when it has Size `L` or `XL`; two or more independently +complete implementation slices or parent integration criteria; at least three +production modules or two responsibility domains; modularity or migration +coordination; dependency sequencing; multiple owners, worktrees, or delivery +phases; or a parent integration branch. Companion tests and documentation do +not count toward these thresholds. Uncertainty defaults to large unless a +maintainer records a smaller classification. + +Large work must satisfy the selected GitHub workflow's planning requirements +before any writer starts. Verify the root Issue, all known descendants, native +relationships, canonical Project items, required metadata, approved base, and +first executable leaf. If a missing gate or new scope appears after writing +starts, stop further writes, preserve and report the existing diff, and +initialize that scope before continuing. + +### Development Agents + +Use project-scoped agents when their expertise improves the result: + +- `requirements_analyst`: inspect the repository, clarify requirements, and + define acceptance criteria before substantial or ambiguous work; classify + the modularity gate. +- `regression_diagnostician`: reproduce failures, trace regressions, and + determine whether their cause crosses a package or module boundary. +- `modularity_maintainer`: analyze, implement, or review cohesive module + boundaries under the applicable language policy and keep project module + indices synchronized. +- `runtime_implementer`: implement approved production-code, configuration, + and integration changes within accepted boundaries. +- `test_engineer`: design, add, update, and run focused verification. +- `documentation_maintainer`: update README files, policies, guides, + references, and examples after behavior and boundaries stabilize. + +These are repository-development agents. Product or domain agents, when +present, remain separate unless the task explicitly puts them in scope. + +### Modularity Gate + +Set `modularity_gate = required` when work adds, removes, renames, moves, +splits, or merges an importable package or module; redistributes responsibility, +state, resources, or lifecycle ownership; changes dependency direction or +introduces or resolves a cycle, replaces private cross-boundary access, or +changes a package export or public import path; introduces or materially +changes a shared abstraction, adapter, facade, or protocol; extracts code for +multiple consumers; or requires a structural compatibility or migration +decision. A regression caused by architectural coupling or a boundary-policy +violation also requires the gate. + +Use `modularity_gate = not_required` for localized work inside an established +module with unchanged imports, exports, ownership, dependencies, and public +contracts; tests-only or docs-only work; formatting, comments, or typing +cleanup; ordinary data or configuration edits; and dependency metadata alone. + +When required, call `modularity_maintainer` with an explicit mode: + +- `analysis`: read-only boundary design before writing. +- `implementation`: exclusive ownership of an accepted architecture-dominant + structural change or an index-only reconciliation. +- `review`: read-only comparison of an implemented diff with the accepted + brief, language policies, and module indices. + +If a writer discovers an unapproved gate trigger, stop that writer and run +modularity analysis before continuing. Never let an implementation agent make +an incidental architecture decision silently. + +Before modularity work, select one policy per affected implementation language. +Use a root `_MODULARITY_POLICY.md` override when present; otherwise +use `.codex/agents/modularity_maintainer/_MODULARITY_POLICY.md`. The +first match wins and policies are never merged. Each project module index is +the project-owned root `_MODULE_INDEX.md`. + +### Orchestration Sequence + +1. The main agent classifies the request and records preliminary gate + decisions. +2. For substantial or ambiguous work, run `requirements_analyst` read-only. + For a reported or observed failure, run `regression_diagnostician` + read-only. Independent investigations may run in parallel. Resolve an + uncertain modularity gate before writing. +3. When modularity is required, run `modularity_maintainer` in `analysis` mode, + pass it relevant requirements and diagnosis findings, and have it select and + verify the affected root `_MODULE_INDEX.md` files. Resolve user + decisions and accept its brief, including the index delta, before writing. +4. After requirements and any boundary brief are accepted, pass the GitHub + planning gate when selected and required. No writer starts while it is + `pending` or `blocked`. +5. Keep exactly one production writer active. Use `modularity_maintainer` for + architecture-dominant structural work and `runtime_implementer` for + behavior-dominant work constrained by the brief. +6. If both production specialists are necessary, assign explicit, + nonoverlapping files or symbols and run them sequentially. Prefer one owner + when separate passes would create an invalid intermediate state. +7. `modularity_maintainer` owns module-index accuracy. When it owns structural + implementation, update affected indices after the implementation is stable. + When `runtime_implementer` owns production code, run a later sequential + index-only implementation pass against the completed production diff. +8. When `runtime_implementer` implements a boundary brief, review the completed + production diff and reconciled indices with `modularity_maintainer` before + test or documentation edits. Return production deviations to the production + owner and index drift to `modularity_maintainer`, then repeat until resolved + or explicitly excepted. A separate self-review is unnecessary when + `modularity_maintainer` performed the implementation; its report and the + main agent's review provide the gate. +9. After production, APIs, and boundaries stabilize, `test_engineer` may plan + read-only and then owns test edits sequentially; + `documentation_maintainer` follows. +10. If tests or documentation change indexed verification or documentation + references or reveal drift, run a final sequential index-only + `modularity_maintainer` implementation pass followed by read-only review. +11. The main agent integrates results, checks index consistency, reviews the + complete diff, runs final checks, and reports to the user. + +The main agent is the coordination hub. Reuse specialist threads when +practical, pass distilled decisions rather than raw logs, and do not let peer +specialists delegate to, coordinate with, or silently redistribute ownership +among one another. + +### Handoff and Result Contract + +Every specialist handoff must include: + +- `mode = analysis | implementation | review`; +- objective and relevant evidence; +- `github_planning_gate`, its rationale, and verified Issue, Project, and + Iteration evidence when applicable; +- approved scope and explicit out-of-scope work; +- acceptance criteria and compatibility constraints; +- modularity-gate decision and accepted brief when applicable; +- applicable module-index paths, status, and accepted delta; +- read-only targets or exclusively owned files and symbols; +- upstream findings and accepted decisions; +- required checks and expected output; and +- stop conditions for missing or conflicting policy, breaking API choice, + scope deviation, unexpected structural work, or overlapping ownership. + +Every specialist result must report findings, files inspected or changed, +module indices consulted or updated, implementation/index consistency, public +API and compatibility impact, checks and results, blockers, and the recommended +next owner or action. Pass distilled decisions and evidence between agents +rather than raw logs. + +A missing language policy blocks modularity work. The modularity maintainer +reports the exact language and expected packaged-policy path; the main agent +asks the user whether to create it and then resumes the same specialist thread. +If creation is declined, record that decision and that no language policy +governs that part of the work before proceeding under general engineering +judgment. A missing module index requires no separate user permission and does +not block read-only analysis: derive the complete inventory, record the root +index path and contents in the brief, and later create the project-owned root +index in an exclusive implementation pass. Structural work is incomplete +while an affected index is missing or stale. + diff --git a/AGENT_TASK_RUNBOOK.md b/AGENT_TASK_RUNBOOK.md new file mode 100644 index 0000000..4b6b00b --- /dev/null +++ b/AGENT_TASK_RUNBOOK.md @@ -0,0 +1,405 @@ +# Agent Task Runbook + +Use this runbook for GitHub-tracked implementation. The rules in +[GITHUB_WORKFLOW.md](GITHUB_WORKFLOW.md) and +[COMMIT_POLICY.md](COMMIT_POLICY.md) remain authoritative. An explicit +implementation request bundles only the initial-planning mutations defined by +the workflow policy and creation of its native Development-linked task branch. +Obtain explicit authorization for every other GitHub mutation, push, pull +request, merge, Issue closure, staging, commit, or branch deletion. + +## Initialize GitHub Planning + +Before task branches, worktrees, writers, or tracked-file edits, read +`README.md` and the policies, classify the planning gate, and search all Issue +states for an existing matching scope. Preflight repository access, `gh` auth +with `project` scope, the canonical repository and Project declared in the +consumer repository instructions, Project fields, milestones, Iterations, +dependencies, and current worktrees. When an installation lock exists, use its +selected profiles as authoritative. Compare repository instructions only when +they independently declare a profile selection; this generic profile never +declares consumer profiles or GitHub identifiers. + +For large work, draft the complete root and all known descendants, including +nested parents and leaves, after read-only requirements and any modularity +analysis. Create the root first, then each descendant; add every URL to Project +identified by the canonical Project number declared in repository instructions +and establish native relationships. Execute and verify one mutation at a time; +stop on the first failure. This abbreviated flow uses stable outputs: + +```bash +set -euo pipefail + +: "${REPOSITORY_ENGINEERING_REPOSITORY:?Set canonical owner/repository from repository instructions}" +: "${REPOSITORY_ENGINEERING_PROJECT_OWNER:?Set from repository instructions}" +: "${REPOSITORY_ENGINEERING_PROJECT_NUMBER:?Set from repository instructions}" +repo_slug="$REPOSITORY_ENGINEERING_REPOSITORY" +project_owner_slug="$REPOSITORY_ENGINEERING_PROJECT_OWNER" +project_number_value="$REPOSITORY_ENGINEERING_PROJECT_NUMBER" + +test "$(gh repo view "$repo_slug" --json nameWithOwner --jq .nameWithOwner)" = "$repo_slug" + +parent_url="$(gh issue create --repo "$repo_slug" --title "" \ + --body-file "" --label "" --label "area:")" +parent_item_id="$(gh project item-add "$project_number_value" \ + --owner "$project_owner_slug" --url "$parent_url" --format json --jq .id)" + +child_url="$(gh issue create --repo "$repo_slug" --title "" \ + --body-file "" --label "" --label "area:" \ + --milestone "")" +child_item_id="$(gh project item-add "$project_number_value" \ + --owner "$project_owner_slug" --url "$child_url" --format json --jq .id)" +child_number="${child_url##*/}" +child_id="$(gh api --method GET "repos/$repo_slug/issues/$child_number" --jq .id)" +parent_number="${parent_url##*/}" +gh api --method POST "repos/$repo_slug/issues/$parent_number/sub_issues" \ + -F sub_issue_id="$child_id" +``` + +Set the three task-specific environment variables from the exact values in the +consumer repository instructions before execution, then preflight all three +resolved identifiers. Never infer a repository from the checkout remote or a +Project solely from its number. + +Use native `blocked by` relationships for dependencies. Resolve Project, item, +field, option, and Iteration IDs at runtime. Set one logical Iteration on the +root and every descendant: + +```bash +gh project item-edit --id --project-id \ + --field-id --iteration-id +``` + +Set all other required metadata from the policy. If no unused logical +Iteration exists for a new root, stop for explicit Project-schema +authorization. Set Priority through the organization Issue field, not the +Project field with the same display name. + +Re-read Issue bodies, native hierarchy and dependencies, Project membership, +fields, milestone, and the root-wide Iteration. Record stable URLs and IDs. +Set `github_planning_gate = passed` only when the graph is complete and one +unblocked leaf is `Ready`; otherwise set `blocked`, preserve successful +mutations, and repair only verified missing state. A standalone task uses the +same procedure without a parent. A later child must pass this check before its +scope is written. + +## Branch and Worktree Model + +```text + +└── issue-100-parent parent worktree; PR to default branch (merge commit) + ├── issue-101-first-task child worktree; PR to parent (squash merge) + └── issue-102-next-task child worktree; PR to parent (squash merge) +``` + +- Record the parent branch and exact starting commit in the parent Issue. + Normally create it from the current `$repo_remote/$default_branch`, after + resolving both the unique canonical-repository remote and `default_branch`; + never assume that `origin` or a same-named local branch is the intended base. +- Name parent and child branches `issue--`. +- Put task worktrees in a sibling root such as + `../-worktrees/issue--/`. Keep the primary + worktree on the resolved default branch and preserve its unrelated changes. +- Assign one integration owner to the parent worktree and one owner to each + child worktree. Worktrees may coexist, but only one write-capable owner is + active at a time. A dependent child waits until its prerequisite is + integrated. +- Do not commit feature work directly to the parent branch. It accepts child + squash merges and synchronization merges from the resolved default branch. + Handle an integration fix through a child Issue, branch, and worktree. +- Do not rebase or force-update a published parent or child branch. After a + fresh fetch and approval, merge `$repo_remote/$default_branch` into the + parent and the parent into active children; resolve a child's conflicts in + its own worktree. + +Every child PR targets the parent branch and uses **Squash and merge**. Child +commits still follow the commit policy, while the resulting squash commit must +be one coherent change that identifies the child Issue and PR. The final +parent PR targets the resolved default branch and uses a merge commit, +preserving one squash commit per child PR. + +For a standalone leaf Issue, create its branch in a sibling worktree from the +approved default-branch ref and target its PR directly to the resolved default +branch; omit the parent steps. + +## Create the Worktrees + +First fetch and inspect without changing branches: + +```bash +set -euo pipefail + +: "${REPOSITORY_ENGINEERING_REPOSITORY:?Set canonical owner/repository from repository instructions}" +repo_slug="$REPOSITORY_ENGINEERING_REPOSITORY" +canonical_repo_slug="$(gh repo view "$repo_slug" --json nameWithOwner --jq .nameWithOwner)" +normalize_repo_slug() { + printf '%s' "$1" | tr '[:upper:]' '[:lower:]' +} +test "$(normalize_repo_slug "$canonical_repo_slug")" = \ + "$(normalize_repo_slug "$repo_slug")" +repo_slug="$canonical_repo_slug" + +repo_remote="" +repo_remote_matches=0 +while IFS= read -r remote_name; do + remote_url="$(git remote get-url "$remote_name")" + remote_slug="$( + gh repo view "$remote_url" --json nameWithOwner --jq .nameWithOwner \ + 2>/dev/null || true + )" + if test -n "$remote_slug" && \ + test "$(normalize_repo_slug "$remote_slug")" = \ + "$(normalize_repo_slug "$repo_slug")"; then + repo_remote="$remote_name" + repo_remote_matches=$((repo_remote_matches + 1)) + fi +done < <(git remote) +test "$repo_remote_matches" -eq 1 +test -n "$repo_remote" + +git fetch "$repo_remote" +git worktree list +git status --short +default_branch="$(gh repo view "$repo_slug" \ + --json defaultBranchRef --jq .defaultBranchRef.name)" +test -n "$default_branch" +git show-ref --verify "refs/remotes/$repo_remote/$default_branch" +git show-ref --verify "refs/heads/$default_branch" +git log --left-right --count \ + "$repo_remote/$default_branch...$default_branch" +``` + +Fail if no local remote or more than one local remote resolves to the canonical +repository identity. Do not select a remote by conventional name or silently +fall back to `origin`. If the local default branch is ahead, behind, or +diverged, identify and record the approved base commit before continuing. +After the gate passes, create the remote task branch through the Issue's native +Development relationship. Resolve the consumer-owned repository slug, Issue, +exact approved base object, and branch name at runtime: + +```bash +issue_number= +branch=issue-- +base_oid= +repo_id="$(gh repo view "$repo_slug" --json id --jq .id)" +issue_id="$(gh issue view "$issue_number" --repo "$repo_slug" --json id --jq .id)" + +gh api graphql \ + -f query='mutation($repo:ID!,$issue:ID!,$oid:GitObjectID!,$name:String!){ + createLinkedBranch(input:{repositoryId:$repo,issueId:$issue,oid:$oid,name:$name}){ + linkedBranch{ref{name target{oid}}} + } + }' \ + -F repo="$repo_id" -F issue="$issue_id" -F oid="$base_oid" -f name="$branch" +``` + +Stop if the branch already exists but is not linked; do not silently recreate, +rename, or replace it. Use a linked PR to repair legacy traceability when that +action is separately authorized. The Development operation creates the remote +branch. Fetch it, then adapt the tracking-worktree template: + +```bash +branch_head_ref="refs/heads/$branch" +branch_remote_ref="refs/remotes/$repo_remote/$branch" +branch_fetch_refspec="$branch_head_ref:$branch_remote_ref" +git check-ref-format "$branch_head_ref" +git check-ref-format "$branch_remote_ref" +if ! git config --get-all "remote.$repo_remote.fetch" | \ + grep -Fqx -- "$branch_fetch_refspec"; then + git config --add "remote.$repo_remote.fetch" "$branch_fetch_refspec" +fi +git fetch "$repo_remote" "$branch_fetch_refspec" +git show-ref --verify "$branch_remote_ref" + +git worktree add --track -b "$branch" \ + ../-worktrees/issue--/issue-- \ + "$branch_remote_ref" +``` + +The narrow branch-specific fetch mapping makes the explicit remote-tracking +ref recognizable to `--track` even when the remote's existing fetch mappings +exclude the task branch. Do not replace or broaden other remote mappings. + +For a parent, use its parent-worktree path; for a child, use its child-worktree +path and the current clean parent commit as `base_oid`. If the linked branch +already exists locally, omit `--track -b` and add a worktree for that branch. +A branch can be checked out in only one worktree. + +Verify the Issue's Development link before writing: + +```bash +repo_owner="${repo_slug%%/*}" +repo_name="${repo_slug#*/}" + +linked_branch_pages="$(gh api graphql --paginate --slurp \ + -f query='query($owner:String!,$repo:String!,$number:Int!,$endCursor:String){ + repository(owner:$owner,name:$repo){ + nameWithOwner + issue(number:$number){ + number + repository{nameWithOwner} + linkedBranches(first:100,after:$endCursor){ + nodes{ref{name}} + pageInfo{hasNextPage endCursor} + } + } + } + }' \ + -F owner="$repo_owner" -F repo="$repo_name" \ + -F number="$issue_number")" + +test "$(printf '%s\n' "$linked_branch_pages" | jq -r \ + --arg repo "$repo_slug" --argjson issue "$issue_number" ' + length > 0 and all(.[]; + ((.data.repository.nameWithOwner | ascii_downcase) == + ($repo | ascii_downcase)) and + .data.repository.issue.number == $issue and + ((.data.repository.issue.repository.nameWithOwner | ascii_downcase) == + ($repo | ascii_downcase))) + ')" = true + +test "$(printf '%s\n' "$linked_branch_pages" | jq -r \ + --arg branch "$branch" ' + [.[].data.repository.issue.linkedBranches.nodes[].ref.name] | + index($branch) != null + ')" = true +``` + +The paginated query and both assertions must succeed before any tracked-file +write. A printed response, a bounded first page, or a similarly named unlinked +branch is not verification. + +After a PR is linked, use the ProjectV2 item node ID and Project node ID +resolved during planning. Query that exact item, prove that it belongs to the +declared Project and canonical repository Issue, paginate all linked pull +requests, and verify the expected canonical-repository PR. Do not select a +Project item from a bounded item list or by Issue number alone: + +```bash +: "${REPOSITORY_ENGINEERING_PROJECT_OWNER:?Set from repository instructions}" +: "${REPOSITORY_ENGINEERING_PROJECT_NUMBER:?Set from repository instructions}" +project_owner_slug="$REPOSITORY_ENGINEERING_PROJECT_OWNER" +project_number_value="$REPOSITORY_ENGINEERING_PROJECT_NUMBER" +project_id= +project_item_id= +pr_number= + +linked_pr_pages="$(gh api graphql --paginate --slurp \ + -f query='query($item:ID!,$endCursor:String){ + node(id:$item){ + __typename + ... on ProjectV2Item{ + id + project{ + id + number + owner{ + __typename + ... on Organization{login} + ... on User{login} + } + } + content{ + __typename + ... on Issue{number repository{nameWithOwner}} + } + fieldValueByName(name:"Linked pull requests"){ + __typename + ... on ProjectV2ItemFieldPullRequestValue{ + pullRequests(first:100,after:$endCursor){ + nodes{number url repository{nameWithOwner}} + pageInfo{hasNextPage endCursor} + } + } + } + } + } + }' \ + -F item="$project_item_id")" + +test "$(printf '%s\n' "$linked_pr_pages" | jq -r \ + --arg item "$project_item_id" \ + --arg project "$project_id" \ + --arg owner "$project_owner_slug" \ + --argjson project_number "$project_number_value" \ + --arg repo "$repo_slug" \ + --argjson issue "$issue_number" ' + length > 0 and all(.[]; + .data.node.__typename == "ProjectV2Item" and + .data.node.id == $item and + .data.node.project.id == $project and + .data.node.project.number == $project_number and + ((.data.node.project.owner.login | ascii_downcase) == + ($owner | ascii_downcase)) and + .data.node.content.__typename == "Issue" and + .data.node.content.number == $issue and + ((.data.node.content.repository.nameWithOwner | ascii_downcase) == + ($repo | ascii_downcase)) and + .data.node.fieldValueByName.__typename == + "ProjectV2ItemFieldPullRequestValue") + ')" = true + +test "$(printf '%s\n' "$linked_pr_pages" | jq -r \ + --arg repo "$repo_slug" --argjson pr "$pr_number" ' + [.[].data.node.fieldValueByName.pullRequests.nodes[] | + select(.number == $pr and + ((.repository.nameWithOwner | ascii_downcase) == + ($repo | ascii_downcase)))] | length > 0 + ')" = true +``` + +## Task Lifecycle + +1. **Pass the planning gate.** Complete or verify the initialization above. + Record the gate state, Issue graph, Project metadata, logical Iteration, + dependencies, approved base, and authorization. Stop before writing unless + the gate is `passed` or `not_required`. +2. **Prepare the parent.** Create or reuse its Development-linked branch and + worktree at the recorded base commit, verify the link, assign the + integration owner, and move the parent to the correct Project state. After + the first child lands, open the parent PR to the resolved default branch as + a draft with `Refs #` when authorized. +3. **Start one child.** Confirm that the child is `Ready`, assigned the root's + Iteration, and unblocked. Create its Development-linked branch and worktree + from the current parent commit, verify the link, assign its owner, and move + it to `In progress`. +4. **Implement and verify.** Change only the child scope. Follow the commit + policy, update tests and documentation, run relevant checks, and review the + final diff and worktree status. +5. **Review the child.** Merge the latest parent branch into the child without + rebasing, rerun affected checks, and open a draft PR whose base is the + parent branch and whose head is the child branch. Use `Refs #`. Mark + it ready and move the child to `In review` only when all evidence is present + and the PR appears in Development and Project `Linked pull requests`. +6. **Integrate the child.** Reconfirm the PR base and head, then use **Squash + and merge** when authorized. The child must contain the current parent tip; + otherwise merge the parent into it, rerun affected checks, and repeat + review. Record the child head SHA and resulting squash SHA, then fetch and + fast-forward the clean parent worktree from + `$repo_remote/` with `--ff-only`; stop on divergence. Verify + the squash diff and run focused integration checks there. On success, close + the child Issue manually and + verify `Done`. On failure, block sibling merges and keep the child open in + `In progress`; require an authorized revert PR or follow-up child fix instead + of editing the parent directly. If it was already closed, reopen it to + `Backlog` and replan it as required by the policy. +7. **Complete the parent.** After all required children are `Done`, merge the + exact approved `$repo_remote/$default_branch` tip into the parent after a + fresh fetch, resolve conflicts there, and run the complete parent + acceptance checks. Mark the parent PR ready, replace `Refs #` with + `Closes #`, move it to `In review`, and merge it to the resolved + default branch with a merge commit when authorization includes the resulting + parent closure. Record and verify the final merge SHA, Issue closure, + Project `Done`, and milestone state. +8. **Clean up and hand off.** Remove a worktree only after its status is clean + and its integration is verified. Never use forced worktree removal. Branch + deletion requires separate authorization; because squash merging does not + mark a child branch as Git-merged, verify PR and commit recoverability before + deleting it. Retain the parent worktree until the parent reaches the + resolved default branch. + +The handoff must include the planning-gate state and evidence, Issue and PR +links, base/head branches and SHAs, resulting squash SHAs, worktree paths, +Development and `Linked pull requests` evidence, Project metadata and +Iteration, completed and remaining criteria, checks, blockers, cleanup, GitHub +mutations, and the next owner or action. diff --git a/COMMIT_POLICY.md b/COMMIT_POLICY.md new file mode 100644 index 0000000..169094b --- /dev/null +++ b/COMMIT_POLICY.md @@ -0,0 +1,70 @@ +# Commit Policy + +## Principle + +A commit is a unit of a complete function or logic. + +Each commit should represent one coherent, working change that can be +understood, reviewed, and integrated independently. A commit should make +clear what was changed and why it belongs together. + +## Rules + +- Keep each commit focused on one feature, fix, refactor, or documentation + change. +- A commit must be complete and functional. Do not commit half-implemented + logic, broken intermediate states, or changes that depend on a later commit. +- Keep unrelated formatting, cleanup, and generated-file changes separate. +- Include related tests and documentation in the same commit when they are + part of the change. +- Prefer small commits, but do not split one logical change into artificial + pieces. +- Run the relevant checks before committing and review the diff for unintended + changes. + +## Commit Messages + +Use a concise imperative subject that describes the completed change. Start +with a verb and make the subject specific enough to be understood without +reading the entire diff. + +Examples: + +```text +Add request schema validation +Fix duplicate task assignments +Document session state transitions +``` + +Add a body when context is needed. Explain why the change was made and note +important implementation, compatibility, or migration details. Do not use the +body to combine unrelated changes. + +## History Hygiene + +Avoid vague messages such as `WIP`, `misc`, or `fix things`. Keep incomplete +work local until it forms a complete unit, or squash it before sharing the +branch. When correcting the immediately preceding commit, amend it when +appropriate; otherwise create a new, complete commit that clearly describes +the correction. + +### Amend Rules + +A commit may be amended only when all of these conditions apply: + +- It is the immediately preceding commit (`HEAD`). +- It has not been pushed or shared with other contributors. +- The amendment belongs to the same logical function or change. +- It does not add unrelated cleanup or new scope. +- Relevant checks are rerun and the final diff is reviewed. + +Create a new commit instead when any of these conditions apply: + +- The target commit is not `HEAD`. +- The commit has already been pushed or shared. +- The correction represents a separate logical change. +- Amending would obscure review history or remove useful context. +- The change requires coordination with other contributors. + +Amending a shared commit requires explicit agreement from all affected +contributors and confirmation that the branch can be safely force-updated. diff --git a/DEPENDENCY_POLICY.md b/DEPENDENCY_POLICY.md new file mode 100644 index 0000000..9552061 --- /dev/null +++ b/DEPENDENCY_POLICY.md @@ -0,0 +1,20 @@ +# Python Dependency Policy + +- Use `pyproject.toml` as the single source of truth for project metadata and + dependencies. +- Separate runtime, development, and test dependencies. +- Use one package manager consistently and commit its lock file. +- Install dependencies in a virtual environment; never rely on global + packages. +- Declare every directly imported package explicitly. +- Add dependencies only when the standard library or an existing dependency + cannot reasonably provide the required behavior. +- Review a package's maintenance, license, security history, size, and Python + compatibility before adding it. +- Keep `pyproject.toml` and the lock file synchronized. Do not edit the lock + file manually. +- Run tests, linting, type checks, and dependency audits after dependency + changes. +- Remove unused dependencies and keep dependency changes in focused commits. +- CI and deployment must install from the lock file and fail on dependency + metadata inconsistencies. diff --git a/GITHUB_WORKFLOW.md b/GITHUB_WORKFLOW.md new file mode 100644 index 0000000..91e0a5d --- /dev/null +++ b/GITHUB_WORKFLOW.md @@ -0,0 +1,209 @@ +# GitHub Workflow Policy + +This policy defines how contributors and coding agents use GitHub. It +supplements [AGENTS.md](AGENTS.md) and +[COMMIT_POLICY.md](COMMIT_POLICY.md). Agents execute it with +[AGENT_TASK_RUNBOOK.md](AGENT_TASK_RUNBOOK.md). + +## Authority and Authorization + +Code, tests, and detailed docs define behavior; an Issue defines scope and +completion; Project and organization Issue fields hold planning metadata; and +a PR holds the diff, review, and evidence. Record maintainer decisions for +conflicts. Planning data must not silently redefine an Issue. + +Relevant read-only inspection is allowed. An explicit implementation request +also authorizes only the initial-planning mutations below: create or reuse +in-scope Issues and a milestone, set native relationships, add items to the +canonical Project, set their planning metadata, and create the in-scope task +branch through the Issue's Development section after the gate passes. +Project-schema changes, unrelated Issue edits, staging, commits, other pushes, +PR actions, review, closure, merge, deletion, and cleanup still require +explicit authorization. + +## Initial Planning Gate + +Before task branches, worktrees, writers, or tracked edits, record +`github_planning_gate = not_required | pending | passed | blocked`. Only +read-only analysis may continue while it is pending. + +Read the canonical repository slug, Project owner, Project number, and any +independent profile declaration from the consumer repository instructions. +This reusable profile supplies none of those consumer facts and they must not +be inferred from its examples or installation source. When an installation +lock exists, its selected profiles are authoritative; compare repository +instructions only when they independently declare a profile selection. + +Work is large when any applies: estimated Size `L`/`XL`; two or more +independently complete implementation slices or parent integration criteria; +at least three production modules or two responsibility domains; modularity or +migration coordination; dependency sequencing; multiple owners, worktrees, or +delivery phases; or a parent integration branch. Companion tests/docs do not +count; uncertainty is large. + +Large work needs one root and all known descendants, including nested parents +and leaves, before implementation. Use native hierarchy and dependencies. Add +every item to the canonical Project declared in the repository instructions, +then set labels and organization Issue Priority plus Project Status and +Iteration on all items, and milestone and Project Size on each leaf. Verify +bodies, relationships, fields, approved base, and the first executable leaf +with a fresh read. If repository instructions do not declare the canonical +repository slug, Project owner, and Project number, the gate is blocked. + +On partial failure, preserve IDs, repair only missing state, and set `blocked`; +never create duplicates or implementation edits. Gate later scope too. +`not_required` is limited to read-only work or an approved mechanical +exception. A nontrivial standalone implementation passes with one Issue and +Project item; it does not waive planning metadata. + +## Issues + +Nontrivial feature, bug, dependency, packaging, security, architecture, +workflow, or public-documentation work requires an Issue. A maintainer may +approve an exception for a mechanical correction. + +Use parents for initiatives and leaves for independently complete functions. A +root defines its objective, non-goals, integration criteria, child inventory +and order, risks, evidence, and approved base. A leaf defines its root/parent, +Project, objective, context, in/out scope, testable completion, verification, +docs, components, dependencies, constraints, and risks. + +Use native parent/sub-issue relationships for hierarchy and `blocked by` for +dependencies. Do not start with an unfinished prerequisite or expand scope +without recorded approval. + +A parent closes only after all required children and its integration criteria +are complete. A child pull request must never close its parent. + +## Planning Metadata + +### Milestone + +A milestone groups a release or deliverable. Every leaf has one active +milestone before `Ready`; a parent links it but normally remains outside it. +Name the outcome and record owner, exit criteria, exclusions, and any due date. +Close it after its required leaves complete and record deferrals. + +### Iteration + +Iteration is a logical initiative group, not a schedule. Use sequential, +never-reused `Iteration NN`. GitHub's required internal dates/duration never +control assignment, readiness, priority, or rollover. + +Every Project item has one Iteration. The topmost root and all descendants +share it; standalone Issues are roots; linked PRs and new children inherit it. +`wave:NN` orders work inside it. Milestone remains the deliverable grouping. + +Status never changes Iteration; there is no calendar rollover. Replanning moves +the whole root tree. `Active work` contains `Ready`, `In progress`, and +`In review`; `Missing iteration` must stay empty. + +### Labels and Fields + +Use labels only for durable classification: + +- One work type: `bug`, `enhancement`, `documentation`, or `chore` +- One or more components: `area:` +- Delivery sequence, when needed: `wave:NN` +- Active blockage: `blocked` + +Do not create milestone, iteration, status, or priority labels. Use `wave:NN`, +not `priority:NN`. Use the organization Issue `Priority` field: `Urgent`, +`High`, `Medium`, or `Low`, defaulting to `Medium`. Use Project Size for +capacity and Estimate only with a defined unit. + +## Project Status + +| Status | Rule | +| --- | --- | +| `Backlog` | New, incomplete, reprioritized, or pre-start blocked work. | +| `Ready` | Accepted and unblocked, with required planning metadata assigned. | +| `In progress` | Authorized work started; assignee and a Development-linked branch or PR exist. Draft PRs remain here. | +| `In review` | A non-draft PR appears in Development and Project `Linked pull requests`, with verification evidence. | +| `Done` | Completion gates passed, approved work merged or exception completed, and Issue closed. | + +Requested implementation changes return an item to `In progress`. + +Blocked is an overlay, not a status. Add the native dependency, apply +`blocked`, and record the cause, unblock condition, owner, and next action. +Pre-start blocked work stays `Backlog`; interrupted active work stays +`In progress` unless reprioritized. Blocked work never becomes `Done`. + +## Branches, Commits, and Pull Requests + +Assign one accountable owner per branch and worktree, plus one integration +owner for a parent branch. Only one write-capable owner is active at a time. +Inspect each worktree first and preserve unrelated changes. + +Use `issue--` for nontrivial work. A standalone branch +identifies its leaf Issue. For a parent initiative, its branch is the +integration base for child Issue branches. Child PRs target the parent branch +and use **Squash and merge**; the final parent PR targets the repository's +resolved default branch and uses a merge commit. Resolve that branch from +GitHub before creating worktrees or pull requests; never assume its name. +Resolve exactly one local Git remote whose normalized GitHub repository +identity matches the canonical repository slug from consumer instructions. +Fail when no remote or multiple remotes match, and use the resolved remote for +every fetch, remote-tracking ref, worktree base, and divergence check; never +silently default to `origin`. +Follow [AGENT_TASK_RUNBOOK.md](AGENT_TASK_RUNBOOK.md) for the branch and +worktree procedure. + +Every Issue with implementation activity must have native GitHub Development +traceability. Create its task branch from the Issue's Development section, or +through the equivalent `createLinkedBranch` API, before tracked edits. Link a +parent integration branch to the parent Issue and each child branch to its +child Issue. A matching branch name, URL, comment, or `Refs #N` text alone is +not a native Development link. Before writing, query all pages of the +canonical Issue's linked branches and fail unless the exact expected task +branch is present; printing a response or checking only a bounded first page is +not verification. + +Once a PR exists, verify that it appears in the Issue's Development section +and the canonical Project's built-in `Linked pull requests` field. If either +link is missing, repair the authorized traceability before moving the item to +`In review`. Query the exact ProjectV2 item node ID resolved during planning, +verify that its content is the canonical repository's expected Issue, and +paginate the item's linked pull requests; a first-page or issue-number-only +Project lookup is not proof. Keep at least one linked branch or PR throughout +`In progress`; after branch cleanup, the merged PR is the durable link for +`Done`. + +Follow [COMMIT_POLICY.md](COMMIT_POLICY.md). An Issue may span multiple commits, +but each commit must remain coherent, functional, tested, and reviewable. + +Open a draft pull request while work or evidence is incomplete. Before marking +it ready: + +- Complete scope and acceptance criteria. +- Run and report relevant checks. +- Update required documentation. +- Review the final diff and intended base. + +Use `Refs #N` for partial work and for child PRs targeting a parent branch. +Close a child manually after its squash merge and integration checks. Use +`Closes #N` only in a PR to the resolved default branch when that merge +completes the named Issue; never target a parent for one child's work. Merge +requires explicit authorization. + +Issue closure is the completion trigger; automation may then set `Done`. Do not +use Project `Done` to close unfinished work. Verify Issue, PR, and Project state +after automation. + +## Completion and Exceptions + +An Issue is complete only when its scope and acceptance criteria are satisfied, +required checks pass, documentation is current, the diff is reviewed, required +review is complete, approved work is merged or an approved no-PR exception is +complete, and the Issue is closed with the Project item at `Done`. + +Close duplicate, invalid, superseded, cancelled, or rejected work as +`Not planned`. Explain the reason and link a replacement when applicable. When +reopening an Issue, clear `Done`, return it to `Backlog`, and replan it +explicitly. + +## Agent Handoff + +Follow the runbook handoff contract. Report the planning-gate state and +evidence, Issue and Project mutations, Development and linked-PR evidence, and +preserved unrelated worktree changes. diff --git a/PYTHON_MODULE_INDEX.md b/PYTHON_MODULE_INDEX.md new file mode 100644 index 0000000..5a4d342 --- /dev/null +++ b/PYTHON_MODULE_INDEX.md @@ -0,0 +1,89 @@ +# Python Module Index + +## Inventory + +- Declared source root: `src` +- Packaging source of truth: `pyproject.toml` +- Importable production units: 4 +- Indexed production units: 4 +- Source/index parity: 4/4 +- Package data: `src/generic_rag/py.typed` +- Locked verification owner: `.github/workflows/ci.yml` (supporting workflow, + not an importable unit) + +## `generic_rag` + +- Source: `src/generic_rag/__init__.py` +- Responsibility: establish the side-effect-free package import namespace. +- Supported public imports: `import generic_rag`. +- Re-exports: none; `__all__ = ()` is intentional until a meaningful high-level + facade is implemented. +- Direct internal dependencies: none. +- Owned state or external resources: none. +- Material side effects: none. +- Verification: `tests/test_package_boundaries.py`, + `tests/support/clean_import_probe.py`, `tests/support/verify_artifacts.py`, and + the locked CI build, clean-install, import, compile, and artifact checks. +- Documentation: `README.md` and `docs/api.md`. +- Package data: declares and ships `py.typed`. + +## `generic_rag.errors` + +- Source: `src/generic_rag/errors.py` +- Responsibility: define the typed public failure categories exposed by generic + RAG contracts and workflows. +- Supported public imports: `GenericRagError`, `ContractValidationError`, + `CollaborationError`, and `StateCompatibilityError` from + `generic_rag.errors`. +- Re-exports: exactly the names in the module's `__all__`; none from the package + root. +- Direct internal dependencies: none. +- Owned state or external resources: none. +- Material side effects: none. +- Verification: `tests/test_errors.py`, `tests/test_package_boundaries.py`, + `tests/support/clean_import_probe.py`, and the locked CI import and boundary + checks. +- Documentation: `docs/api.md`. + +## `generic_rag.contracts` + +- Source: `src/generic_rag/contracts.py` +- Responsibility: define immutable validated values shared by generic RAG + workflows and provider ports. +- Supported public imports: `DocumentKey`, `DocumentIdentity`, `Document`, + `FragmentIdentity`, `Fragment`, `EmbeddingIdentity`, `EmbeddingVector`, + `VectorRecord`, `ProjectionIdentity`, `ProjectionCheckpoint`, + `ProjectionOutcome`, `ProjectionReceipt`, `RetrievalQuery`, + `RetrievalOutcome`, `RetrievalHit`, and `RetrievalResult` from + `generic_rag.contracts`. +- Re-exports: exactly the names in the module's `__all__`; none from the package + root. +- Direct internal dependencies: `generic_rag.errors`. +- Owned state or external resources: none; instances own only immutable caller + values. +- Material side effects: none. +- Verification: `tests/test_contract_values.py`, + `tests/test_projection_contracts.py`, `tests/test_retrieval_contracts.py`, + `tests/test_package_boundaries.py`, `tests/support/clean_import_probe.py`, + `tests/support/verify_artifacts.py`, and the locked CI import, boundary, and + artifact checks. +- Documentation: `docs/api.md` and `docs/security-and-privacy.md`. + +## `generic_rag.ports` + +- Source: `src/generic_rag/ports.py` +- Responsibility: define synchronous injected collaborator interfaces and + explicit caller-owned borrowing semantics. +- Supported public imports: `Borrowed`, `Embedder`, `VectorIndexWriter`, + `VectorIndexReader`, and `LexicalRetriever` from `generic_rag.ports`. +- Re-exports: exactly the names in the module's `__all__`; none from the package + root. +- Direct internal dependencies: `generic_rag.contracts`. +- Owned state or external resources: `Borrowed` retains a reference but never + owns, acquires, releases, closes, or shuts down the resource. +- Material side effects: none. +- Verification: `tests/test_ports.py`, `tests/test_package_boundaries.py`, + `tests/support/clean_import_probe.py`, and the locked CI import and boundary + checks. +- Documentation: `docs/api.md`, `docs/lifecycle.md`, and + `docs/security-and-privacy.md`. diff --git a/README.md b/README.md index b2c9de7..9471b69 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,145 @@ # generic-rag -Provider-neutral, dependency-free RAG projection and retrieval runtime + +`generic-rag` is a provider-neutral, runtime-dependency-free foundation for +retrieval-augmented generation (RAG). Version 0.1.0 requires Python 3.11 or +later and provides immutable contracts, typed error categories, synchronous +collaborator protocols, and explicit caller-owned borrowing. + +No projection or retrieval algorithm is implemented in 0.1.0. The package has +no built-in adapter, provider, factory, persistence, network client, +configuration system, authentication, citation mechanism, or CLI. + +## Install from a checkout + +The project is not documented as a published package yet. From a repository +checkout, install it with: + +```console +python -m pip install . +``` + +The installed package has no runtime dependencies. Build and development tools +are separate locked dependency groups. + +## Use the contracts in application code + +The host application remains responsible for authorization and policy checks. +After approving a source and query, application code can construct generic +values and accept an application-owned provider through a protocol: + +```python +from generic_rag.contracts import ( + Document, + DocumentIdentity, + DocumentKey, + EmbeddingVector, + RetrievalQuery, +) +from generic_rag.ports import Borrowed, Embedder + +# Construct these values only after application-specific authorization. +approved_document = Document( + identity=DocumentIdentity( + key=DocumentKey(corpus_id="corpus-a", document_id="document-1"), + revision_id="revision-3", + ), + text="Approved source text", + attributes=(("classification", "internal"),), +) +query = RetrievalQuery( + corpus_id=approved_document.identity.key.corpus_id, + text="What does the source say?", + hit_limit=5, + candidate_limit=20, +) + + +def application_embed_query( + provider: Embedder, + request: RetrievalQuery, +) -> EmbeddingVector: + with Borrowed(provider) as embedder: + vectors = embedder.embed((request.text,)) + if len(vectors) != 1: + raise ValueError("the provider violated the Embedder contract") + return vectors[0] +``` + +This is application orchestration, not a package retrieval workflow. Version +0.1.0 defines the boundary that provider implementations and later generic +workflows will use; it does not construct providers or call them on a user's +behalf. + +Public values must be imported from their owning modules: + +- `generic_rag.contracts` +- `generic_rag.errors` +- `generic_rag.ports` + +The package root intentionally has no re-exports: `generic_rag.__all__ == ()`. +See the [API reference](docs/api.md) for every supported name and invariant. + +## Planned RAG flow + +The package itself has no concept of a user or agent. A consuming application +decides which sources a user may approve, which queries may be submitted, which +provider implementations receive data, and whether retrieved fragments are +shown to a user or supplied to a downstream tool or agent. + +- [Issue #3](https://github.com/Kims-DeveloperGroup/generic-rag/issues/3) is + planned to add generic projection orchestration. Its intended responsibility + is to accept caller-approved documents and explicitly injected collaborators, + derive fragments under a defined chunking policy, embed ordered fragment + text, replace or delete complete document projections, and report truthful + checkpoints and receipts. Its precise API and failure behavior are not part + of 0.1.0. +- [Issue #4](https://github.com/Kims-DeveloperGroup/generic-rag/issues/4) is + planned to add retrieval and composition. Its intended responsibility is to + use an injected `Embedder` and `VectorIndexReader` for semantic candidates + and an injected `LexicalRetriever` for lexical candidates, then define + deduplication, fusion, limiting, and outcome behavior. Provider rank will be + the input; raw provider scores are not represented or assumed comparable. + +The caller/provider ownership model remains explicit throughout this plan. See +[resource lifecycle](docs/lifecycle.md) and +[security and privacy](docs/security-and-privacy.md). + +## Compatibility + +Version 0.1.0 is pre-1.0. Consumers should pin a reviewed version and should not +assume compatibility across minor releases. For this release, direct imports +from the documented owning modules are the supported public paths; root-level +imports are not. + +The distribution includes `py.typed`. The wheel contains exactly the four +importable modules `generic_rag`, `generic_rag.errors`, +`generic_rag.contracts`, and `generic_rag.ports`, plus the typing marker. + +## Development verification + +The project commits a universal lock generated with uv 0.12.5. Reproduce the +locked environment and primary checks with: + +```console +uv lock --check +uv sync --locked --all-groups +uv run --frozen python -m unittest discover -s tests -p 'test_*.py' -v +uv run --frozen ruff check src tests +uv run --frozen ruff format --check src tests +uv run --frozen mypy --strict src tests +uv run --frozen python -m compileall -q src tests +``` + +Build both distributions into a temporary output directory and inspect them: + +```console +rag_dist_dir="$(mktemp -d)" +uv run --frozen python -m build --no-isolation --outdir "$rag_dist_dir" +uv run --frozen python tests/support/verify_artifacts.py "$rag_dist_dir" +``` + +CI is configured to run the tests on Python 3.11 and 3.14. On Python 3.11 it +also runs lint, format, strict type, compilation, artifact, source-rebuild, +clean-install, and isolated-import checks. A local Python 3.14.2 run currently +contains 53 passing tests; the CI matrix is the authoritative cross-version +result. diff --git a/docs/api.md b/docs/api.md new file mode 100644 index 0000000..863105e --- /dev/null +++ b/docs/api.md @@ -0,0 +1,219 @@ +# Public API + +Version 0.1.0 exposes immutable values, typed error categories, and synchronous +collaborator protocols. It does not expose projection or retrieval algorithms. +See the [project overview](../README.md), [resource lifecycle](lifecycle.md), +and [security and privacy boundary](security-and-privacy.md) for the surrounding +usage contract. + +## Import boundary + +The supported package-root operation is `import generic_rag`. The root has an +intentionally empty export list, `generic_rag.__all__ == ()`, and re-exports no +public symbols. Import names from their owning modules instead. + +`generic_rag.errors` exports exactly: + +- `GenericRagError` +- `ContractValidationError` +- `CollaborationError` +- `StateCompatibilityError` + +`generic_rag.contracts` exports exactly: + +- `DocumentKey` +- `DocumentIdentity` +- `Document` +- `FragmentIdentity` +- `Fragment` +- `EmbeddingIdentity` +- `EmbeddingVector` +- `VectorRecord` +- `ProjectionIdentity` +- `ProjectionCheckpoint` +- `ProjectionOutcome` +- `ProjectionReceipt` +- `RetrievalQuery` +- `RetrievalOutcome` +- `RetrievalHit` +- `RetrievalResult` + +`generic_rag.ports` exports exactly: + +- `Borrowed` +- `Embedder` +- `VectorIndexWriter` +- `VectorIndexReader` +- `LexicalRetriever` + +The package does not support importing public values from the package root. + +## Shared value rules + +Public contract dataclasses are frozen, slotted, hashable value objects. They +require exact concrete field types, including exact nested contract classes; +subclasses do not satisfy these runtime validators. In particular: + +- strings must be exact `str` values; +- discrete integers must be exact `int` values, so `bool`, floats, and integer + subclasses are rejected; +- tuple fields must be exact tuples, not lists, iterators, or tuple subclasses; + and +- field invariant violations detected during construction raise + `ContractValidationError`. + +Opaque identity strings must be nonempty and not whitespace-only. They are +otherwise stored exactly as supplied: the package does not trim, normalize, +canonicalize, parse, or resolve them. + +Attributes use `tuple[tuple[str, str], ...]`. Every pair must be an exact +two-element tuple containing exact strings. Empty keys and values are allowed, +and pair order and duplicates are preserved. + +## Errors + +`GenericRagError` is the base for the three public categories: + +```text +GenericRagError +├── ContractValidationError +├── CollaborationError +└── StateCompatibilityError +``` + +- `ContractValidationError` reports a violated public value invariant. +- `CollaborationError` is reserved for a workflow that translates a + collaborator operation failure. +- `StateCompatibilityError` is reserved for a workflow that detects derived + state with an incompatible projection identity. + +Version 0.1.0 has no projection or retrieval workflow that raises the latter +two categories. `Borrowed` also leaves provider exceptions unchanged. + +## Documents and fragments + +| Type | Fields | Construction rules | +| --- | --- | --- | +| `DocumentKey` | `corpus_id: str`, `document_id: str` | Both values are nonblank opaque identities. | +| `DocumentIdentity` | `key: DocumentKey`, `revision_id: str` | `key` is the exact class; the revision is a nonblank opaque identity. | +| `Document` | `identity: DocumentIdentity`, `text: str`, `attributes: tuple[tuple[str, str], ...] = ()` | `text` is stored exactly and may be empty; attributes follow the shared tuple rules. | +| `FragmentIdentity` | `document: DocumentIdentity`, `fragment_id: str`, `start: int`, `end: int` | The ID is nonblank; offsets are exact nonnegative integers satisfying `start < end`. | +| `Fragment` | `identity: FragmentIdentity`, `text: str`, `attributes: tuple[tuple[str, str], ...] = ()` | `text` is nonempty and `len(text) == end - start`; attributes follow the shared tuple rules. | + +A fragment range is half-open, `[start, end)`, in Python Unicode code points. +It is not measured in bytes or user-perceived grapheme clusters. For example, +`"😀"` has one code point while `"e\u0301"` has two. + +The package checks that fragment text length equals the range width. Version +0.1.0 does not retain an authoritative `Document` beside a `Fragment`, so it +cannot verify that the text equals the indicated source slice. The caller, or +a future projection workflow, must establish that correspondence. + +## Embeddings and vector records + +| Type | Fields | Construction rules | +| --- | --- | --- | +| `EmbeddingIdentity` | `model_id: str`, `dimensions: int` | The model ID is nonblank and opaque; dimensions is a positive exact integer. | +| `EmbeddingVector` | `values: tuple[float, ...]` | The tuple is exact and nonempty. Every coordinate is an exact `int` or `float`, excluding `bool`, and must convert to a finite float without overflow. Oversized integers that cannot be represented as finite floats are rejected; accepted coordinates are stored canonically as floats. | +| `VectorRecord` | `fragment: Fragment`, `embedding: EmbeddingVector` | Both fields require their exact contract classes. | + +A standalone `EmbeddingVector` does not carry an `EmbeddingIdentity`. Version +0.1.0 therefore does not compare the vector length with an identity's declared +`dimensions`; a provider and future orchestration must satisfy that semantic +relationship. + +## Projection state + +| Type | Fields | Construction rules | +| --- | --- | --- | +| `ProjectionIdentity` | `schema_id: str`, `embedding: EmbeddingIdentity` | The schema ID is nonblank and opaque; embedding requires its exact class. | +| `ProjectionCheckpoint` | `corpus_id: str`, `projection: ProjectionIdentity`, `token: str` | Corpus and token are nonblank opaque strings; projection requires its exact class. | +| `ProjectionReceipt` | `corpus_id: str`, `projection: ProjectionIdentity`, `outcome: ProjectionOutcome`, `attempted_documents: int`, `completed_documents: int`, `checkpoint: ProjectionCheckpoint \| None` | Nested values require their exact classes; counts are nonnegative exact integers and completed cannot exceed attempted. | + +`ProjectionOutcome` is a closed string enum with these exact member values: + +| Member | String value | +| --- | --- | +| `COMPLETED` | `"completed"` | +| `UNCHANGED` | `"unchanged"` | +| `PARTIAL` | `"partial"` | +| `FAILED` | `"failed"` | + +Unknown enum values raise `ContractValidationError`. A receipt can represent +only the following truthful combinations: + +| Outcome | Counts | Checkpoint | +| --- | --- | --- | +| `COMPLETED` | `completed_documents == attempted_documents`, including zero attempts | Required | +| `UNCHANGED` | `completed_documents == attempted_documents`, including zero attempts | Required | +| `PARTIAL` | `0 < completed_documents < attempted_documents` | Forbidden | +| `FAILED` | `attempted_documents > 0` and `completed_documents == 0` | Forbidden | + +Any supplied checkpoint must have exactly the receipt's `corpus_id` and +`projection`. + +## Retrieval values + +| Type | Fields | Construction rules | +| --- | --- | --- | +| `RetrievalQuery` | `corpus_id: str`, `text: str`, `hit_limit: int`, `candidate_limit: int` | Corpus and text are nonblank exact strings; limits are positive exact integers and `hit_limit <= candidate_limit`. Values are preserved exactly. | +| `RetrievalHit` | `fragment: Fragment`, `rank: int` | Fragment requires its exact class and rank is a positive exact integer. There is no score field. | +| `RetrievalResult` | `query: RetrievalQuery`, `outcome: RetrievalOutcome`, `hits: tuple[RetrievalHit, ...]`, `truncated: bool` | Nested values, the hit tuple, and the boolean require exact types. Hit count cannot exceed `query.hit_limit`. | + +`RetrievalOutcome` is a closed string enum with these exact member values: + +| Member | String value | +| --- | --- | +| `COMPLETE` | `"complete"` | +| `PARTIAL` | `"partial"` | +| `UNAVAILABLE` | `"unavailable"` | +| `STALE` | `"stale"` | +| `FAILED` | `"failed"` | + +Unknown enum values raise `ContractValidationError`. Within every result, ranks +must be contiguous starting at one, fragment identities must be unique, and +every fragment's corpus must match the query corpus. Fragment attributes do not +make two otherwise identical fragment identities distinct. + +The outcome matrix is: + +| Outcome | Hits | `truncated` | +| --- | --- | --- | +| `COMPLETE` | Zero through `query.hit_limit` | Either boolean | +| `PARTIAL` | One through `query.hit_limit` | Either boolean | +| `UNAVAILABLE` | None | `False` | +| `STALE` | None | `False` | +| `FAILED` | None | `False` | + +`truncated=True` is the caller's explicit assertion that otherwise valid work +or results were cut by the query budget. It does not imply that +`len(hits) == query.hit_limit`; a bounded `COMPLETE` result may therefore still +be truncated. Hits and reader ports are score-free. Raw scores from different +providers are neither represented nor promised to be comparable. + +## Collaborator ports + +The protocols are synchronous, injected, structurally typed, and decorated +with `runtime_checkable`. Runtime protocol checks establish structural presence, +not the behavioral obligations below. Version 0.1.0 provides no implementation, +adapter, factory, provider discovery, or provider-behavior enforcement. + +| Port | Exact public operation | Semantic obligation | +| --- | --- | --- | +| `Embedder` | `identity: EmbeddingIdentity` | Identify the exact model used for produced vectors. | +| `Embedder` | `embed(texts: tuple[str, ...], /) -> tuple[EmbeddingVector, ...]` | Return one same-order vector per input text, each with `identity.dimensions` coordinates; empty input returns an empty tuple. | +| `VectorIndexWriter` | `replace_document(document: DocumentIdentity, records: tuple[VectorRecord, ...], /) -> None` | Replace all derived vectors for the document's stable key. Every record carries the supplied full `DocumentIdentity`; an empty record tuple is valid. | +| `VectorIndexWriter` | `delete_document(document: DocumentKey, /) -> None` | Delete every derived revision for the stable document key. | +| `VectorIndexReader` | `search(query: RetrievalQuery, embedding: EmbeddingVector, /) -> tuple[Fragment, ...]` | Return fragments from the requested corpus, in provider rank order, with at most `query.candidate_limit` entries. | +| `LexicalRetriever` | `search(query: RetrievalQuery, /) -> tuple[Fragment, ...]` | Return fragments from the requested corpus, in provider rank order, with at most `query.candidate_limit` entries. | + +`Borrowed[T]` is the companion ownership marker, not a provider port. Its exact +behavior is documented in [resource lifecycle](lifecycle.md). + +## Planned workflows + +Projection orchestration is planned for Issue #3. Retrieval and composition are +planned for Issue #4. Those future workflows are expected to accept protocol- +compatible collaborators explicitly, but their algorithms, APIs, compatibility +checks, exception translation, and outcome mapping are not implemented or +promised by version 0.1.0. diff --git a/docs/lifecycle.md b/docs/lifecycle.md new file mode 100644 index 0000000..c362a41 --- /dev/null +++ b/docs/lifecycle.md @@ -0,0 +1,82 @@ +# Resource Lifecycle + +Version 0.1.0 uses an explicit caller-owned lifecycle. The package defines +collaborator protocols and `Borrowed[T]`; it does not acquire or own provider +resources. See the [API reference](api.md) for the exact port signatures. + +## Ownership rule + +The caller or provider integration owns every lifecycle decision: + +1. Acquire and configure the resource. +2. Establish any required synchronization or exclusive access. +3. Pass the resource explicitly to application code or a future generic + workflow. +4. Reset, release, close, or shut down the resource according to the provider's + rules after use. + +`generic-rag` does none of those steps implicitly. + +## `Borrowed[T]` + +`Borrowed` is a frozen, slotted context manager that retains a reference. It +has exactly these context semantics: + +- `__enter__` returns the exact wrapped resource. +- `__exit__` returns `False`, so it never suppresses an exception. +- It never calls the resource's `__enter__`, `__exit__`, `close`, `shutdown`, + reset, acquisition, release, or synchronization operations. +- It does not translate a resource failure into `CollaborationError`. +- It provides no locking, thread safety, asynchronous context management, or + provider health checks. + +These guarantees still apply if the wrapped provider defines its own context +manager or lifecycle methods. + +## Successful scope + +The resource that leaves the scope is still the application-owned object: + +```python +from generic_rag.ports import Borrowed + +provider = object() # Acquired and owned by the application. + +with Borrowed(provider) as active_provider: + assert active_provider is provider + +assert active_provider is provider # Borrowed did not close or replace it. +``` + +Application code decides when and how to release the real provider afterward. + +## Exceptional scope + +An exception leaves the borrowed scope unchanged and unsuppressed: + +```python +from generic_rag.ports import Borrowed + +provider = object() +failure = RuntimeError("provider failed") + +try: + with Borrowed(provider) as active_provider: + assert active_provider is provider + raise failure +except RuntimeError as caught: + assert caught is failure +``` + +If a caller needs cleanup after either success or failure, it must arrange that +cleanup around the borrowed scope according to the provider's contract. + +## Future workflows + +Projection and retrieval orchestration are planned for Issues #3 and #4. Their +collaborators are intended to remain explicitly injected and caller-owned. The +precise future APIs are not part of version 0.1.0, and `Borrowed` must not be +read as a promise that a workflow already exists. + +Review the [security and privacy boundary](security-and-privacy.md) before +passing content to a provider implementation. diff --git a/docs/security-and-privacy.md b/docs/security-and-privacy.md new file mode 100644 index 0000000..7147817 --- /dev/null +++ b/docs/security-and-privacy.md @@ -0,0 +1,64 @@ +# Security and Privacy + +Version 0.1.0 defines in-process values and collaborator boundaries. By itself, +the package performs no persistence, network transmission, provider discovery, +credential loading, telemetry, or background work. Contract objects do retain +caller-supplied values in process memory. + +## Caller responsibility + +Before constructing a `Document`, `Fragment`, or `RetrievalQuery`, the caller +must perform its own authorization and policy checks. The caller also controls: + +- which source content and metadata enter the contracts; +- which provider implementations receive document text, fragment text, + embeddings, attributes, identifiers, or queries; +- provider account, region, transport, and credential configuration; +- retention, replacement, deletion, backup, and recovery behavior; +- logging, tracing, metrics, redaction, and incident response; and +- whether retrieved fragments are displayed, persisted, or supplied to another + tool or agent. + +Do not place credentials or other secrets in attributes, opaque identifiers, or +checkpoint tokens. These fields deliberately preserve caller input and do not +apply redaction, escaping, access control, or tenant isolation. + +## Provider effects + +The protocols describe operations that an external implementation may perform. +Calling an embedder, vector index, or lexical retriever can store or transmit +data according to that implementation. Version 0.1.0 supplies no such provider +and does not call one on the caller's behalf. + +`Borrowed` does not reduce this responsibility. It only marks the wrapped +resource as caller-owned and does not acquire, close, reset, authenticate, or +synchronize it. See [resource lifecycle](lifecycle.md). + +## Sensitive derived data + +Treat all of the following as potentially sensitive: + +- document and query text; +- ordered attributes and opaque identities; +- fragments and their source ranges; +- embeddings and vector records; and +- projection checkpoint tokens. + +Fragments and embeddings are derived data but may reveal information from the +source. Deleting an authoritative source does not automatically delete copies +or derived values held by an application or provider. + +## Trust boundary + +The package does not own, authenticate, or prove an authoritative source or +revision. A fragment range checks only a half-open code-point width against the +fragment text length; it does not verify the text against a source document. +Ranges and attributes are not a citation or provenance-verification mechanism. + +Version 0.1.0 provides no built-in encryption, authentication, authorization, +ACL, content filter, persistence, network security, citation validation, or +vendor guarantee. A consuming application must select and assess those controls +for its environment. + +The complete public value and provider boundaries are listed in the +[API reference](api.md). diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..09a30bb --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,53 @@ +[build-system] +requires = ["setuptools>=83,<85"] +build-backend = "setuptools.build_meta" + +[project] +name = "generic-rag" +version = "0.1.0" +description = "Provider-neutral, dependency-free RAG projection and retrieval runtime" +readme = "README.md" +requires-python = ">=3.11" +dependencies = [] + +[dependency-groups] +build = [ + "build>=1.5,<2", + "setuptools>=83,<85", +] +lint = [ + "ruff>=0.16,<0.17", +] +type = [ + "mypy>=2.3,<3", +] +dev = [ + { include-group = "build" }, + { include-group = "lint" }, + { include-group = "type" }, +] +test = [] + +[tool.setuptools] +package-dir = { "" = "src" } + +[tool.setuptools.packages.find] +where = ["src"] +include = ["generic_rag", "generic_rag.*"] + +[tool.setuptools.package-data] +generic_rag = ["py.typed"] + +[tool.ruff] +target-version = "py311" +line-length = 88 +src = ["src"] + +[tool.ruff.lint] +select = ["E", "F", "I", "UP"] + +[tool.mypy] +python_version = "3.11" +strict = true +mypy_path = "src" +packages = ["generic_rag"] diff --git a/src/generic_rag/__init__.py b/src/generic_rag/__init__.py new file mode 100644 index 0000000..46c466f --- /dev/null +++ b/src/generic_rag/__init__.py @@ -0,0 +1,7 @@ +"""Provider-neutral RAG contracts. + +Issue #2 intentionally exposes no root-level facade. Consumers import public +contracts, errors, and ports from their owning modules. +""" + +__all__: tuple[str, ...] = () diff --git a/src/generic_rag/contracts.py b/src/generic_rag/contracts.py new file mode 100644 index 0000000..a8cfb66 --- /dev/null +++ b/src/generic_rag/contracts.py @@ -0,0 +1,406 @@ +"""Immutable, validated values shared by generic RAG workflows and ports.""" + +from __future__ import annotations + +from dataclasses import dataclass +from enum import StrEnum +from math import isfinite + +from .errors import ContractValidationError + +__all__ = ( + "DocumentKey", + "DocumentIdentity", + "Document", + "FragmentIdentity", + "Fragment", + "EmbeddingIdentity", + "EmbeddingVector", + "VectorRecord", + "ProjectionIdentity", + "ProjectionCheckpoint", + "ProjectionOutcome", + "ProjectionReceipt", + "RetrievalQuery", + "RetrievalOutcome", + "RetrievalHit", + "RetrievalResult", +) + + +def _require_exact_type(name: str, value: object, expected: type[object]) -> None: + if type(value) is not expected: + raise ContractValidationError( + f"{name} must be exactly {expected.__name__}, not {type(value).__name__}" + ) + + +def _require_nonblank_string(name: str, value: object) -> str: + _require_exact_type(name, value, str) + assert isinstance(value, str) + if not value or value.isspace(): + raise ContractValidationError(f"{name} must not be empty or whitespace-only") + return value + + +def _require_nonnegative_integer(name: str, value: object) -> int: + _require_exact_type(name, value, int) + assert isinstance(value, int) + if value < 0: + raise ContractValidationError(f"{name} must be nonnegative") + return value + + +def _require_positive_integer(name: str, value: object) -> int: + result = _require_nonnegative_integer(name, value) + if result == 0: + raise ContractValidationError(f"{name} must be positive") + return result + + +def _validate_attributes(name: str, attributes: object) -> tuple[tuple[str, str], ...]: + _require_exact_type(name, attributes, tuple) + assert isinstance(attributes, tuple) + for index, pair in enumerate(attributes): + if type(pair) is not tuple or len(pair) != 2: + raise ContractValidationError( + f"{name}[{index}] must be exactly a two-element tuple" + ) + key, value = pair + _require_exact_type(f"{name}[{index}][0]", key, str) + _require_exact_type(f"{name}[{index}][1]", value, str) + return attributes + + +@dataclass(frozen=True, slots=True) +class DocumentKey: + """Stable opaque corpus/document identity independent of revision.""" + + corpus_id: str + document_id: str + + def __post_init__(self) -> None: + _require_nonblank_string("corpus_id", self.corpus_id) + _require_nonblank_string("document_id", self.document_id) + + +@dataclass(frozen=True, slots=True) +class DocumentIdentity: + """A stable document key at one opaque authoritative revision.""" + + key: DocumentKey + revision_id: str + + def __post_init__(self) -> None: + _require_exact_type("key", self.key, DocumentKey) + _require_nonblank_string("revision_id", self.revision_id) + + +@dataclass(frozen=True, slots=True) +class Document: + """Exact caller-supplied document text and ordered opaque attributes.""" + + identity: DocumentIdentity + text: str + attributes: tuple[tuple[str, str], ...] = () + + def __post_init__(self) -> None: + _require_exact_type("identity", self.identity, DocumentIdentity) + _require_exact_type("text", self.text, str) + _validate_attributes("attributes", self.attributes) + + +@dataclass(frozen=True, slots=True) +class FragmentIdentity: + """Opaque fragment identity and half-open code-point source range.""" + + document: DocumentIdentity + fragment_id: str + start: int + end: int + + def __post_init__(self) -> None: + _require_exact_type("document", self.document, DocumentIdentity) + _require_nonblank_string("fragment_id", self.fragment_id) + start = _require_nonnegative_integer("start", self.start) + end = _require_nonnegative_integer("end", self.end) + if start >= end: + raise ContractValidationError("fragment range must satisfy start < end") + + +@dataclass(frozen=True, slots=True) +class Fragment: + """An exact nonempty document slice carried as non-authoritative derived data.""" + + identity: FragmentIdentity + text: str + attributes: tuple[tuple[str, str], ...] = () + + def __post_init__(self) -> None: + _require_exact_type("identity", self.identity, FragmentIdentity) + _require_exact_type("text", self.text, str) + if not self.text: + raise ContractValidationError("fragment text must not be empty") + if len(self.text) != self.identity.end - self.identity.start: + raise ContractValidationError( + "fragment text length must equal its half-open code-point range" + ) + _validate_attributes("attributes", self.attributes) + + +@dataclass(frozen=True, slots=True) +class EmbeddingIdentity: + """Opaque embedding-model identity and its required vector dimension.""" + + model_id: str + dimensions: int + + def __post_init__(self) -> None: + _require_nonblank_string("model_id", self.model_id) + _require_positive_integer("dimensions", self.dimensions) + + +@dataclass(frozen=True, slots=True) +class EmbeddingVector: + """A finite, nonempty embedding vector stored canonically as floats.""" + + values: tuple[float, ...] + + def __post_init__(self) -> None: + _require_exact_type("values", self.values, tuple) + if not self.values: + raise ContractValidationError("embedding vector must not be empty") + + canonical: list[float] = [] + for index, coordinate in enumerate(self.values): + if type(coordinate) not in (int, float): + raise ContractValidationError( + f"values[{index}] must be int or float, excluding bool" + ) + try: + numeric_coordinate = float(coordinate) + except OverflowError: + raise ContractValidationError( + f"values[{index}] must be finite" + ) from None + if not isfinite(numeric_coordinate): + raise ContractValidationError(f"values[{index}] must be finite") + canonical.append(numeric_coordinate) + object.__setattr__(self, "values", tuple(canonical)) + + +@dataclass(frozen=True, slots=True) +class VectorRecord: + """A fragment paired with the vector written for it.""" + + fragment: Fragment + embedding: EmbeddingVector + + def __post_init__(self) -> None: + _require_exact_type("fragment", self.fragment, Fragment) + _require_exact_type("embedding", self.embedding, EmbeddingVector) + + +@dataclass(frozen=True, slots=True) +class ProjectionIdentity: + """Schema and embedding identity that determine projection compatibility.""" + + schema_id: str + embedding: EmbeddingIdentity + + def __post_init__(self) -> None: + _require_nonblank_string("schema_id", self.schema_id) + _require_exact_type("embedding", self.embedding, EmbeddingIdentity) + + +@dataclass(frozen=True, slots=True) +class ProjectionCheckpoint: + """Opaque completed checkpoint bound to one corpus and projection identity.""" + + corpus_id: str + projection: ProjectionIdentity + token: str + + def __post_init__(self) -> None: + _require_nonblank_string("corpus_id", self.corpus_id) + _require_exact_type("projection", self.projection, ProjectionIdentity) + _require_nonblank_string("token", self.token) + + +class ProjectionOutcome(StrEnum): + """Truthful completion state for a projection attempt.""" + + COMPLETED = "completed" + UNCHANGED = "unchanged" + PARTIAL = "partial" + FAILED = "failed" + + @classmethod + def _missing_(cls, value: object) -> None: + raise ContractValidationError(f"{cls.__name__} value is not a defined member") + + +@dataclass(frozen=True, slots=True) +class ProjectionReceipt: + """Projection counts and checkpoint subject to completion invariants.""" + + corpus_id: str + projection: ProjectionIdentity + outcome: ProjectionOutcome + attempted_documents: int + completed_documents: int + checkpoint: ProjectionCheckpoint | None + + def __post_init__(self) -> None: + _require_nonblank_string("corpus_id", self.corpus_id) + _require_exact_type("projection", self.projection, ProjectionIdentity) + _require_exact_type("outcome", self.outcome, ProjectionOutcome) + attempted = _require_nonnegative_integer( + "attempted_documents", self.attempted_documents + ) + completed = _require_nonnegative_integer( + "completed_documents", self.completed_documents + ) + if completed > attempted: + raise ContractValidationError( + "completed_documents must not exceed attempted_documents" + ) + if self.checkpoint is not None: + _require_exact_type("checkpoint", self.checkpoint, ProjectionCheckpoint) + if self.checkpoint.corpus_id != self.corpus_id: + raise ContractValidationError( + "checkpoint corpus_id must match the receipt corpus_id" + ) + if self.checkpoint.projection != self.projection: + raise ContractValidationError( + "checkpoint projection must match the receipt projection" + ) + + successful = self.outcome in ( + ProjectionOutcome.COMPLETED, + ProjectionOutcome.UNCHANGED, + ) + if successful: + if completed != attempted: + raise ContractValidationError( + "completed and unchanged receipts must complete every attempt" + ) + if self.checkpoint is None: + raise ContractValidationError( + "completed and unchanged receipts require a checkpoint" + ) + elif self.outcome is ProjectionOutcome.PARTIAL: + if not 0 < completed < attempted: + raise ContractValidationError( + "partial receipts require 0 < completed_documents " + "< attempted_documents" + ) + if self.checkpoint is not None: + raise ContractValidationError( + "partial receipts must not claim a checkpoint" + ) + else: + if attempted == 0 or completed != 0: + raise ContractValidationError( + "failed receipts require attempts and zero completed documents" + ) + if self.checkpoint is not None: + raise ContractValidationError( + "failed receipts must not claim a checkpoint" + ) + + +@dataclass(frozen=True, slots=True) +class RetrievalQuery: + """A bounded retrieval request for one opaque corpus.""" + + corpus_id: str + text: str + hit_limit: int + candidate_limit: int + + def __post_init__(self) -> None: + _require_nonblank_string("corpus_id", self.corpus_id) + _require_nonblank_string("text", self.text) + hit_limit = _require_positive_integer("hit_limit", self.hit_limit) + candidate_limit = _require_positive_integer( + "candidate_limit", self.candidate_limit + ) + if hit_limit > candidate_limit: + raise ContractValidationError("hit_limit must not exceed candidate_limit") + + +class RetrievalOutcome(StrEnum): + """Explicit availability and completion state of retrieval.""" + + COMPLETE = "complete" + PARTIAL = "partial" + UNAVAILABLE = "unavailable" + STALE = "stale" + FAILED = "failed" + + @classmethod + def _missing_(cls, value: object) -> None: + raise ContractValidationError(f"{cls.__name__} value is not a defined member") + + +@dataclass(frozen=True, slots=True) +class RetrievalHit: + """A final score-free ranked fragment.""" + + fragment: Fragment + rank: int + + def __post_init__(self) -> None: + _require_exact_type("fragment", self.fragment, Fragment) + _require_positive_integer("rank", self.rank) + + +@dataclass(frozen=True, slots=True) +class RetrievalResult: + """A bounded, deduplicated, score-free retrieval result.""" + + query: RetrievalQuery + outcome: RetrievalOutcome + hits: tuple[RetrievalHit, ...] + truncated: bool + + def __post_init__(self) -> None: + _require_exact_type("query", self.query, RetrievalQuery) + _require_exact_type("outcome", self.outcome, RetrievalOutcome) + _require_exact_type("hits", self.hits, tuple) + _require_exact_type("truncated", self.truncated, bool) + if len(self.hits) > self.query.hit_limit: + raise ContractValidationError("hits must not exceed query.hit_limit") + + seen: set[FragmentIdentity] = set() + for expected_rank, hit in enumerate(self.hits, start=1): + _require_exact_type(f"hits[{expected_rank - 1}]", hit, RetrievalHit) + if hit.rank != expected_rank: + raise ContractValidationError("hit ranks must be contiguous from one") + if hit.fragment.identity.document.key.corpus_id != self.query.corpus_id: + raise ContractValidationError( + "every hit corpus_id must match the query corpus_id" + ) + if hit.fragment.identity in seen: + raise ContractValidationError( + "retrieval hits must have unique fragment identities" + ) + seen.add(hit.fragment.identity) + + empty_outcomes = ( + RetrievalOutcome.UNAVAILABLE, + RetrievalOutcome.STALE, + RetrievalOutcome.FAILED, + ) + if self.outcome in empty_outcomes: + if self.hits: + raise ContractValidationError( + "unavailable, stale, and failed results must not contain hits" + ) + if self.truncated: + raise ContractValidationError( + "unavailable, stale, and failed results must not be truncated" + ) + elif self.outcome is RetrievalOutcome.PARTIAL and not self.hits: + raise ContractValidationError("partial results require at least one hit") diff --git a/src/generic_rag/errors.py b/src/generic_rag/errors.py new file mode 100644 index 0000000..f3f7c0d --- /dev/null +++ b/src/generic_rag/errors.py @@ -0,0 +1,24 @@ +"""Public exception types for generic RAG contracts and collaboration.""" + +__all__ = ( + "GenericRagError", + "ContractValidationError", + "CollaborationError", + "StateCompatibilityError", +) + + +class GenericRagError(Exception): + """Base class for errors intentionally exposed by generic-rag.""" + + +class ContractValidationError(GenericRagError): + """Raised when a public immutable value violates its contract.""" + + +class CollaborationError(GenericRagError): + """Raised when orchestration translates a collaborator operation failure.""" + + +class StateCompatibilityError(GenericRagError): + """Raised when derived state has an incompatible projection identity.""" diff --git a/src/generic_rag/ports.py b/src/generic_rag/ports.py new file mode 100644 index 0000000..fdc931b --- /dev/null +++ b/src/generic_rag/ports.py @@ -0,0 +1,104 @@ +"""Synchronous injected collaborator contracts and borrowing semantics.""" + +from __future__ import annotations + +from dataclasses import dataclass +from types import TracebackType +from typing import Generic, Literal, Protocol, TypeVar, runtime_checkable + +from .contracts import ( + DocumentIdentity, + DocumentKey, + EmbeddingIdentity, + EmbeddingVector, + Fragment, + RetrievalQuery, + VectorRecord, +) + +__all__ = ( + "Borrowed", + "Embedder", + "VectorIndexWriter", + "VectorIndexReader", + "LexicalRetriever", +) + +_T_co = TypeVar("_T_co", covariant=True) + + +@dataclass(frozen=True, slots=True) +class Borrowed(Generic[_T_co]): + """A no-op scope that marks a resource as caller-owned. + + Entry returns the exact wrapped resource. Exit never calls lifecycle methods + on it and never suppresses an exception. + """ + + resource: _T_co + + def __enter__(self) -> _T_co: + return self.resource + + def __exit__( + self, + exc_type: type[BaseException] | None, + exc_value: BaseException | None, + traceback: TracebackType | None, + ) -> Literal[False]: + return False + + +@runtime_checkable +class Embedder(Protocol): + """Synchronously embeds an ordered tuple without owning provider lifecycle.""" + + @property + def identity(self) -> EmbeddingIdentity: + """Return the exact model identity used for produced vectors.""" + ... + + def embed(self, texts: tuple[str, ...], /) -> tuple[EmbeddingVector, ...]: + """Return one same-order vector per input text.""" + ... + + +@runtime_checkable +class VectorIndexWriter(Protocol): + """Synchronously replaces and deletes complete document projections.""" + + def replace_document( + self, + document: DocumentIdentity, + records: tuple[VectorRecord, ...], + /, + ) -> None: + """Replace all derived vectors for the document's stable key.""" + ... + + def delete_document(self, document: DocumentKey, /) -> None: + """Delete every derived revision for a stable document key.""" + ... + + +@runtime_checkable +class VectorIndexReader(Protocol): + """Returns best-first vector candidates without exposing raw scores.""" + + def search( + self, + query: RetrievalQuery, + embedding: EmbeddingVector, + /, + ) -> tuple[Fragment, ...]: + """Return at most query.candidate_limit fragments in provider rank order.""" + ... + + +@runtime_checkable +class LexicalRetriever(Protocol): + """Returns best-first lexical candidates without exposing raw scores.""" + + def search(self, query: RetrievalQuery, /) -> tuple[Fragment, ...]: + """Return at most query.candidate_limit fragments in provider rank order.""" + ... diff --git a/src/generic_rag/py.typed b/src/generic_rag/py.typed new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/src/generic_rag/py.typed @@ -0,0 +1 @@ + diff --git a/tests/support/clean_import_probe.py b/tests/support/clean_import_probe.py new file mode 100644 index 0000000..d0db065 --- /dev/null +++ b/tests/support/clean_import_probe.py @@ -0,0 +1,207 @@ +"""Probe one installed generic_rag module in an isolated interpreter.""" + +from __future__ import annotations + +import argparse +import importlib +import multiprocessing +import os +import socket +import subprocess +import sys +import threading +from contextlib import ExitStack +from pathlib import Path +from types import FrameType, ModuleType +from typing import NoReturn, cast +from unittest.mock import patch + +_PACKAGE_ROOT = "generic_rag" +_FORBIDDEN_AUDIT_EVENTS: list[str] = [] +_AUDIT_HOOK_ACTIVE = False + + +def _called_from_package() -> bool: + frame: FrameType | None = sys._getframe(1) + while frame is not None: + module_name = frame.f_globals.get("__name__") + if isinstance(module_name, str) and ( + module_name == _PACKAGE_ROOT or module_name.startswith(f"{_PACKAGE_ROOT}.") + ): + return True + frame = frame.f_back + return False + + +def _is_loader_source_read(arguments: tuple[object, ...]) -> bool: + if not arguments: + return False + raw_path = arguments[0] + if not isinstance(raw_path, str | bytes | os.PathLike): + return False + path = Path(os.fsdecode(raw_path)) + if path.suffix not in {".py", ".pyc"}: + return False + + mode = arguments[1] if len(arguments) > 1 else None + if isinstance(mode, str) and any(marker in mode for marker in "wax+"): + return False + flags = arguments[2] if len(arguments) > 2 else None + write_flags = os.O_WRONLY | os.O_RDWR | os.O_CREAT | os.O_TRUNC | os.O_APPEND + if isinstance(flags, int) and flags & write_flags: + return False + + frame: FrameType | None = sys._getframe(1) + while frame is not None: + module_name = frame.f_globals.get("__name__") + if ( + module_name + in { + "_frozen_importlib_external", + "importlib._bootstrap_external", + } + and frame.f_code.co_name == "get_data" + ): + return True + frame = frame.f_back + return False + + +def _audit_hook(event: str, arguments: tuple[object, ...]) -> None: + global _AUDIT_HOOK_ACTIVE + + if _AUDIT_HOOK_ACTIVE: + return + _AUDIT_HOOK_ACTIVE = True + try: + if not _called_from_package(): + return + if event == "open" and _is_loader_source_read(arguments): + return + forbidden = ( + event == "open" + or event.startswith("socket.") + or event.startswith("subprocess.") + or event == "os.system" + or event.startswith("os.spawn") + ) + if forbidden: + _FORBIDDEN_AUDIT_EVENTS.append(event) + raise RuntimeError(f"forbidden import-time operation: {event}") + finally: + _AUDIT_HOOK_ACTIVE = False + + +def _forbidden_operation(*arguments: object, **keywords: object) -> NoReturn: + del arguments, keywords + raise RuntimeError("forbidden import-time process, network, or thread operation") + + +def _import_with_guards(module_name: str) -> ModuleType: + sys.addaudithook(_audit_hook) + with ExitStack() as stack: + stack.enter_context( + patch.object(threading.Thread, "start", _forbidden_operation) + ) + stack.enter_context( + patch.object(multiprocessing.Process, "start", _forbidden_operation) + ) + stack.enter_context(patch.object(socket, "socket", _forbidden_operation)) + stack.enter_context( + patch.object(socket, "create_connection", _forbidden_operation) + ) + stack.enter_context(patch.object(subprocess, "Popen", _forbidden_operation)) + stack.enter_context(patch.object(os, "system", _forbidden_operation)) + stack.enter_context(patch.object(os, "popen", _forbidden_operation)) + for name in ( + "spawnl", + "spawnle", + "spawnlp", + "spawnlpe", + "spawnv", + "spawnve", + "spawnvp", + "spawnvpe", + ): + if hasattr(os, name): + stack.enter_context(patch.object(os, name, _forbidden_operation)) + imported = importlib.import_module(module_name) + + if _FORBIDDEN_AUDIT_EVENTS: + joined = ", ".join(_FORBIDDEN_AUDIT_EVENTS) + raise AssertionError(f"package import attempted forbidden events: {joined}") + return imported + + +def _assert_no_external_imports(before: set[str], after: set[str]) -> None: + allowed_roots = set(sys.stdlib_module_names) + allowed_roots.update({"builtins", _PACKAGE_ROOT}) + unexpected = sorted( + module_name + for module_name in after - before + if module_name.partition(".")[0] not in allowed_roots + ) + if unexpected: + raise AssertionError( + "package import loaded non-stdlib modules: " + ", ".join(unexpected) + ) + + +def _assert_forbidden_paths_absent(paths: list[str]) -> None: + resolved_entries = [Path(entry).resolve() for entry in sys.path if entry] + for raw_path in paths: + forbidden = Path(raw_path).resolve() + for entry in resolved_entries: + try: + entry.relative_to(forbidden) + except ValueError: + continue + raise AssertionError( + f"forbidden checkout path leaked into sys.path: {entry}" + ) + + +def _parse_arguments() -> argparse.Namespace: + parser = argparse.ArgumentParser() + parser.add_argument("module") + parser.add_argument( + "--forbid-path", + action="append", + default=[], + help="Fail when this path or one of its children occurs on sys.path.", + ) + return parser.parse_args() + + +def main() -> int: + arguments = _parse_arguments() + module_name = cast(str, arguments.module) + forbidden_paths = cast(list[str], arguments.forbid_path) + if module_name != _PACKAGE_ROOT and not module_name.startswith(f"{_PACKAGE_ROOT}."): + raise ValueError(f"probe is restricted to {_PACKAGE_ROOT} modules") + + _assert_forbidden_paths_absent(forbidden_paths) + before = set(sys.modules) + imported = _import_with_guards(module_name) + after = set(sys.modules) + _assert_no_external_imports(before, after) + + if imported.__name__ != module_name: + raise AssertionError(f"requested {module_name}, imported {imported.__name__}") + if module_name == _PACKAGE_ROOT: + if getattr(imported, "__all__", None) != (): + raise AssertionError("generic_rag root __all__ must be empty") + loaded_submodules = sorted( + name for name in after if name.startswith(f"{_PACKAGE_ROOT}.") + ) + if loaded_submodules: + raise AssertionError( + "root import loaded submodules: " + ", ".join(loaded_submodules) + ) + + print(f"clean import passed: {module_name}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/support/verify_artifacts.py b/tests/support/verify_artifacts.py new file mode 100644 index 0000000..bf78892 --- /dev/null +++ b/tests/support/verify_artifacts.py @@ -0,0 +1,240 @@ +"""Validate generic-rag wheel and source-distribution artifacts.""" + +from __future__ import annotations + +import argparse +import stat +import tarfile +import zipfile +from email import message_from_bytes +from email.message import Message +from pathlib import Path, PurePosixPath +from typing import cast + +_PACKAGE_FILES = { + "generic_rag/__init__.py", + "generic_rag/contracts.py", + "generic_rag/errors.py", + "generic_rag/ports.py", +} +_PACKAGE_DATA = {"generic_rag/py.typed"} + + +class ArtifactVerificationError(RuntimeError): + pass + + +def _require(condition: bool, message: str) -> None: + if not condition: + raise ArtifactVerificationError(message) + + +def _safe_archive_name(name: str) -> PurePosixPath: + path = PurePosixPath(name) + _require(bool(path.parts), "archive contains an empty member name") + _require(not path.is_absolute(), f"archive member is absolute: {name}") + _require(".." not in path.parts, f"archive member traverses upward: {name}") + _require("\\" not in name, f"archive member uses a backslash: {name}") + return path + + +def _required_header(metadata: Message, name: str) -> str: + value = metadata.get(name) + _require(value is not None and bool(value.strip()), f"missing {name} metadata") + assert value is not None + return value + + +def _assert_metadata(metadata: Message) -> str: + _require(_required_header(metadata, "Name") == "generic-rag", "wrong Name") + version = _required_header(metadata, "Version") + _require( + _required_header(metadata, "Requires-Python") == ">=3.11", + "Requires-Python must be exactly >=3.11", + ) + requirements = metadata.get_all("Requires-Dist") + _require( + not requirements, + "runtime dependencies leaked into metadata: " + ", ".join(requirements or []), + ) + return version + + +def _assert_no_shipped_tests(names: set[str], artifact: str) -> None: + shipped = sorted( + name + for name in names + if any( + part.lower() in {"test", "tests"} or part.lower().startswith("test_") + for part in PurePosixPath(name).parts + ) + ) + _require( + not shipped, + f"{artifact} ships test files: {', '.join(shipped)}", + ) + + +def _verify_wheel(wheel: Path) -> str: + with zipfile.ZipFile(wheel) as archive: + entries = archive.infolist() + names = {entry.filename.rstrip("/") for entry in entries} + for entry in entries: + _safe_archive_name(entry.filename) + file_type = (entry.external_attr >> 16) & 0o170000 + _require( + file_type != stat.S_IFLNK, + f"wheel contains a symlink: {entry.filename}", + ) + + _assert_no_shipped_tests(names, "wheel") + _require( + _PACKAGE_FILES <= names, + "wheel is missing package modules: " + + ", ".join(sorted(_PACKAGE_FILES - names)), + ) + _require( + _PACKAGE_DATA <= names, + "wheel is missing py.typed", + ) + shipped_python = { + name + for name in names + if name.startswith("generic_rag/") and name.endswith(".py") + } + _require( + shipped_python == _PACKAGE_FILES, + "wheel has an unexpected production module set: " + + ", ".join(sorted(shipped_python)), + ) + _require( + not any("__pycache__" in PurePosixPath(name).parts for name in names), + "wheel contains bytecode caches", + ) + _require( + not any(name.endswith((".so", ".pyd", ".dylib", ".pyc")) for name in names), + "wheel is not a clean pure-Python artifact", + ) + + metadata_names = sorted( + name for name in names if name.endswith(".dist-info/METADATA") + ) + _require( + len(metadata_names) == 1, + "wheel must contain exactly one METADATA file", + ) + metadata = message_from_bytes(archive.read(metadata_names[0])) + version = _assert_metadata(metadata) + + allowed_roots = { + "generic_rag", + metadata_names[0].partition("/")[0], + } + unexpected_roots = sorted( + { + PurePosixPath(name).parts[0] + for name in names + if PurePosixPath(name).parts[0] not in allowed_roots + } + ) + _require( + not unexpected_roots, + "wheel contains unexpected roots: " + ", ".join(unexpected_roots), + ) + + expected_name = f"generic_rag-{version}-py3-none-any.whl" + _require( + wheel.name == expected_name, + f"wheel filename must be {expected_name}, got {wheel.name}", + ) + return version + + +def _verify_sdist(sdist: Path, expected_version: str) -> None: + expected_root = f"generic_rag-{expected_version}" + with tarfile.open(sdist, mode="r:gz") as archive: + members = archive.getmembers() + names = {member.name.rstrip("/") for member in members} + for member in members: + path = _safe_archive_name(member.name) + _require( + path.parts[0] == expected_root, + f"sdist has an unexpected root: {member.name}", + ) + _require( + not member.issym() and not member.islnk(), + f"sdist contains a link: {member.name}", + ) + _require( + member.isfile() or member.isdir(), + f"sdist contains a special file: {member.name}", + ) + + required = { + f"{expected_root}/README.md", + f"{expected_root}/pyproject.toml", + f"{expected_root}/PKG-INFO", + *{ + f"{expected_root}/src/{package_file}" + for package_file in _PACKAGE_FILES | _PACKAGE_DATA + }, + } + _require( + required <= names, + "sdist is missing required files: " + ", ".join(sorted(required - names)), + ) + + metadata_member = archive.getmember(f"{expected_root}/PKG-INFO") + metadata_file = archive.extractfile(metadata_member) + _require(metadata_file is not None, "sdist PKG-INFO is not readable") + assert metadata_file is not None + with metadata_file: + metadata = message_from_bytes(metadata_file.read()) + _require( + _assert_metadata(metadata) == expected_version, + "wheel and sdist versions differ", + ) + + expected_name = f"{expected_root}.tar.gz" + _require( + sdist.name == expected_name, + f"sdist filename must be {expected_name}, got {sdist.name}", + ) + + +def _parse_arguments() -> argparse.Namespace: + parser = argparse.ArgumentParser() + parser.add_argument("dist_directory", type=Path) + return parser.parse_args() + + +def main() -> int: + arguments = _parse_arguments() + dist_directory = cast(Path, arguments.dist_directory) + _require( + dist_directory.is_dir(), + f"distribution directory does not exist: {dist_directory}", + ) + wheel_files = sorted(dist_directory.glob("*.whl")) + sdist_files = sorted(dist_directory.glob("*.tar.gz")) + _require(len(wheel_files) == 1, "expected exactly one wheel") + _require(len(sdist_files) == 1, "expected exactly one sdist") + unexpected = sorted( + path.name + for path in dist_directory.iterdir() + if path.is_file() and path not in {*wheel_files, *sdist_files} + ) + _require( + not unexpected, + "distribution directory contains unexpected files: " + ", ".join(unexpected), + ) + + version = _verify_wheel(wheel_files[0]) + _verify_sdist(sdist_files[0], version) + print(f"verified wheel: {wheel_files[0]}") + print(f"verified sdist: {sdist_files[0]}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/test_contract_values.py b/tests/test_contract_values.py new file mode 100644 index 0000000..ea50fed --- /dev/null +++ b/tests/test_contract_values.py @@ -0,0 +1,366 @@ +"""Contract tests for immutable document, fragment, and embedding values.""" + +from __future__ import annotations + +import unittest +from dataclasses import FrozenInstanceError, fields +from typing import cast + +from generic_rag.contracts import ( + Document, + DocumentIdentity, + DocumentKey, + EmbeddingIdentity, + EmbeddingVector, + Fragment, + FragmentIdentity, + VectorRecord, +) +from generic_rag.errors import ContractValidationError + + +class _StringSubclass(str): + pass + + +class _IntegerSubclass(int): + pass + + +class _FloatSubclass(float): + pass + + +class _TupleSubclass(tuple[object, ...]): + pass + + +def _document_identity( + *, + corpus_id: str = "corpus", + document_id: str = "document", + revision_id: str = "revision", +) -> DocumentIdentity: + return DocumentIdentity( + key=DocumentKey(corpus_id=corpus_id, document_id=document_id), + revision_id=revision_id, + ) + + +def _fragment_identity( + *, + corpus_id: str = "corpus", + document_id: str = "document", + revision_id: str = "revision", + fragment_id: str = "fragment", + start: int = 0, + end: int = 2, +) -> FragmentIdentity: + return FragmentIdentity( + document=_document_identity( + corpus_id=corpus_id, + document_id=document_id, + revision_id=revision_id, + ), + fragment_id=fragment_id, + start=start, + end=end, + ) + + +class ContractValueTests(unittest.TestCase): + def assert_frozen_and_slotted( + self, + instance: object, + field_name: str, + ) -> None: + self.assertFalse(hasattr(instance, "__dict__")) + with self.assertRaises(FrozenInstanceError): + setattr(instance, field_name, object()) + + def test_exact_field_layouts(self) -> None: + self.assertEqual( + tuple(field.name for field in fields(DocumentKey)), + ("corpus_id", "document_id"), + ) + self.assertEqual( + tuple(field.name for field in fields(DocumentIdentity)), + ("key", "revision_id"), + ) + self.assertEqual( + tuple(field.name for field in fields(Document)), + ("identity", "text", "attributes"), + ) + self.assertEqual( + tuple(field.name for field in fields(FragmentIdentity)), + ("document", "fragment_id", "start", "end"), + ) + self.assertEqual( + tuple(field.name for field in fields(Fragment)), + ("identity", "text", "attributes"), + ) + self.assertEqual( + tuple(field.name for field in fields(EmbeddingIdentity)), + ("model_id", "dimensions"), + ) + self.assertEqual( + tuple(field.name for field in fields(EmbeddingVector)), + ("values",), + ) + self.assertEqual( + tuple(field.name for field in fields(VectorRecord)), + ("fragment", "embedding"), + ) + + def test_values_are_frozen_slotted_hashable_value_objects(self) -> None: + document_key = DocumentKey("corpus", "document") + document_identity = DocumentIdentity(document_key, "revision") + document = Document(document_identity, "") + fragment_identity = FragmentIdentity( + document_identity, + "fragment", + 0, + 1, + ) + fragment = Fragment(fragment_identity, "x") + embedding_identity = EmbeddingIdentity("model", 1) + embedding = EmbeddingVector((1,)) + record = VectorRecord(fragment, embedding) + cases = ( + (document_key, "corpus_id"), + (document_identity, "revision_id"), + (document, "text"), + (fragment_identity, "start"), + (fragment, "text"), + (embedding_identity, "dimensions"), + (embedding, "values"), + (record, "fragment"), + ) + + for instance, field_name in cases: + with self.subTest(instance=type(instance).__name__): + self.assert_frozen_and_slotted(instance, field_name) + self.assertIsInstance(hash(instance), int) + self.assertEqual(instance, instance) + + def test_opaque_identity_strings_are_preserved_exactly(self) -> None: + corpus_id = " Corpus/../e\u0301 " + document_id = " Document:ABC " + revision_id = " Rev/001 " + fragment_id = " Fragment/../001 " + key = DocumentKey(corpus_id, document_id) + document = DocumentIdentity(key, revision_id) + fragment = FragmentIdentity(document, fragment_id, 4, 5) + + self.assertEqual(key.corpus_id, corpus_id) + self.assertEqual(key.document_id, document_id) + self.assertEqual(document.revision_id, revision_id) + self.assertEqual(fragment.fragment_id, fragment_id) + + def test_identity_strings_reject_empty_whitespace_and_inexact_types(self) -> None: + invalid_values: tuple[object, ...] = ( + "", + " \t\n", + None, + b"value", + 1, + _StringSubclass("value"), + ) + + for invalid in invalid_values: + with self.subTest(field="corpus_id", value=invalid): + with self.assertRaises(ContractValidationError): + DocumentKey(cast(str, invalid), "document") + with self.subTest(field="document_id", value=invalid): + with self.assertRaises(ContractValidationError): + DocumentKey("corpus", cast(str, invalid)) + with self.subTest(field="revision_id", value=invalid): + with self.assertRaises(ContractValidationError): + DocumentIdentity( + DocumentKey("corpus", "document"), + cast(str, invalid), + ) + with self.subTest(field="fragment_id", value=invalid): + with self.assertRaises(ContractValidationError): + FragmentIdentity( + _document_identity(), + cast(str, invalid), + 0, + 1, + ) + with self.subTest(field="model_id", value=invalid): + with self.assertRaises(ContractValidationError): + EmbeddingIdentity(cast(str, invalid), 1) + + def test_nested_value_fields_require_the_exact_declared_classes(self) -> None: + document_identity = _document_identity() + fragment_identity = _fragment_identity() + fragment = Fragment(fragment_identity, "ab") + embedding = EmbeddingVector((1.0,)) + + with self.assertRaises(ContractValidationError): + DocumentIdentity(cast(DocumentKey, object()), "revision") + with self.assertRaises(ContractValidationError): + Document(cast(DocumentIdentity, object()), "text") + with self.assertRaises(ContractValidationError): + FragmentIdentity( + cast(DocumentIdentity, object()), + "fragment", + 0, + 1, + ) + with self.assertRaises(ContractValidationError): + Fragment(cast(FragmentIdentity, object()), "x") + with self.assertRaises(ContractValidationError): + VectorRecord(cast(Fragment, object()), embedding) + with self.assertRaises(ContractValidationError): + VectorRecord(fragment, cast(EmbeddingVector, object())) + + class DocumentKeySubclass(DocumentKey): + pass + + subclass_key = DocumentKeySubclass("corpus", "document") + with self.assertRaises(ContractValidationError): + DocumentIdentity(subclass_key, "revision") + self.assertEqual(document_identity.key.corpus_id, "corpus") + + def test_document_text_is_exact_and_may_be_empty(self) -> None: + identity = _document_identity() + + self.assertEqual(Document(identity, "").text, "") + text = " \u0000 e\u0301 \n" + self.assertEqual(Document(identity, text).text, text) + for invalid in (None, b"text", _StringSubclass("text")): + with self.subTest(value=invalid): + with self.assertRaises(ContractValidationError): + Document(identity, cast(str, invalid)) + + def test_attributes_preserve_empty_values_duplicates_and_order(self) -> None: + attributes = ( + ("", ""), + ("key", "first"), + ("key", "first"), + ("key", "second"), + ) + document = Document(_document_identity(), "text", attributes) + fragment = Fragment(_fragment_identity(), "ab", attributes) + + self.assertEqual(document.attributes, attributes) + self.assertEqual(fragment.attributes, attributes) + + def test_attributes_reject_inexact_or_malformed_containers(self) -> None: + invalid_attributes: tuple[object, ...] = ( + [("key", "value")], + {"key": "value"}, + iter((("key", "value"),)), + _TupleSubclass((("key", "value"),)), + (["key", "value"],), + (("key",),), + (("key", "value", "extra"),), + ((1, "value"),), + (("key", 1),), + ((_StringSubclass("key"), "value"),), + (("key", _StringSubclass("value")),), + ) + + for invalid in invalid_attributes: + with self.subTest(value=invalid): + with self.assertRaises(ContractValidationError): + Document( + _document_identity(), + "text", + cast(tuple[tuple[str, str], ...], invalid), + ) + with self.assertRaises(ContractValidationError): + Fragment( + _fragment_identity(), + "ab", + cast(tuple[tuple[str, str], ...], invalid), + ) + + def test_fragment_ranges_use_exact_half_open_code_point_offsets(self) -> None: + emoji_identity = _fragment_identity(start=10, end=11) + combining_identity = _fragment_identity(start=20, end=22) + + self.assertEqual(Fragment(emoji_identity, "\U0001f600").text, "\U0001f600") + combining = "e\u0301" + self.assertEqual(len(combining), 2) + self.assertEqual(Fragment(combining_identity, combining).text, combining) + + invalid_ranges = ( + (-1, 1), + (0, 0), + (2, 1), + ) + for start, end in invalid_ranges: + with self.subTest(start=start, end=end): + with self.assertRaises(ContractValidationError): + _fragment_identity(start=start, end=end) + + for invalid in (True, 1.0, _IntegerSubclass(1)): + with self.subTest(field="start", value=invalid): + with self.assertRaises(ContractValidationError): + _fragment_identity(start=cast(int, invalid), end=2) + with self.subTest(field="end", value=invalid): + with self.assertRaises(ContractValidationError): + _fragment_identity(start=0, end=cast(int, invalid)) + + def test_fragment_text_is_nonempty_and_matches_the_range_length(self) -> None: + identity = _fragment_identity(start=4, end=6) + + self.assertEqual(Fragment(identity, "xy").text, "xy") + for invalid in ("", "x", "xyz"): + with self.subTest(value=invalid): + with self.assertRaises(ContractValidationError): + Fragment(identity, invalid) + for invalid_text_type in (None, b"xy", _StringSubclass("xy")): + with self.subTest(value=invalid_text_type): + with self.assertRaises(ContractValidationError): + Fragment(identity, cast(str, invalid_text_type)) + + def test_embedding_dimensions_are_positive_exact_integers(self) -> None: + self.assertEqual(EmbeddingIdentity("model", 3).dimensions, 3) + + for invalid in (0, -1, True, 1.0, _IntegerSubclass(1)): + with self.subTest(value=invalid): + with self.assertRaises(ContractValidationError): + EmbeddingIdentity("model", cast(int, invalid)) + + def test_embedding_vectors_are_nonempty_exact_tuples_of_finite_numbers( + self, + ) -> None: + vector = EmbeddingVector((1, -0.0, 2.5)) + + self.assertEqual(vector.values, (1.0, -0.0, 2.5)) + self.assertTrue(all(type(value) is float for value in vector.values)) + + invalid_containers: tuple[object, ...] = ( + [], + iter((1.0,)), + _TupleSubclass((1.0,)), + (), + ) + for invalid in invalid_containers: + with self.subTest(value=invalid): + with self.assertRaises(ContractValidationError): + EmbeddingVector(cast(tuple[float, ...], invalid)) + + invalid_coordinates: tuple[object, ...] = ( + True, + None, + "1", + _IntegerSubclass(1), + _FloatSubclass(1.0), + float("nan"), + float("inf"), + float("-inf"), + 10**10000, + ) + for invalid in invalid_coordinates: + with self.subTest(value=type(invalid).__name__): + with self.assertRaises(ContractValidationError): + EmbeddingVector(cast(tuple[float, ...], (cast(float, invalid),))) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_errors.py b/tests/test_errors.py new file mode 100644 index 0000000..9281643 --- /dev/null +++ b/tests/test_errors.py @@ -0,0 +1,53 @@ +"""Contract tests for the public generic RAG exception hierarchy.""" + +from __future__ import annotations + +import unittest + +import generic_rag.errors as errors +from generic_rag.errors import ( + CollaborationError, + ContractValidationError, + GenericRagError, + StateCompatibilityError, +) + + +class ErrorContractTests(unittest.TestCase): + def test_exports_are_exact_and_owned_by_the_module(self) -> None: + expected = ( + "GenericRagError", + "ContractValidationError", + "CollaborationError", + "StateCompatibilityError", + ) + + self.assertEqual(errors.__all__, expected) + for name in expected: + exported = getattr(errors, name) + self.assertEqual(exported.__module__, "generic_rag.errors") + + def test_specialized_errors_share_the_public_base(self) -> None: + specialized = ( + ContractValidationError, + CollaborationError, + StateCompatibilityError, + ) + + for error_type in specialized: + with self.subTest(error_type=error_type.__name__): + self.assertTrue(issubclass(error_type, GenericRagError)) + self.assertIsNot(error_type, GenericRagError) + + def test_specialized_errors_remain_distinct_categories(self) -> None: + error_types = { + ContractValidationError, + CollaborationError, + StateCompatibilityError, + } + + self.assertEqual(len(error_types), 3) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_package_boundaries.py b/tests/test_package_boundaries.py new file mode 100644 index 0000000..eedc054 --- /dev/null +++ b/tests/test_package_boundaries.py @@ -0,0 +1,358 @@ +"""Architecture, export, source-index, and clean-import contract tests.""" + +from __future__ import annotations + +import ast +import importlib +import os +import re +import subprocess +import sys +import tempfile +import unittest +from pathlib import Path + +import generic_rag +import generic_rag.contracts as contracts +import generic_rag.errors as errors +import generic_rag.ports as ports + +_PROJECT_ROOT = Path(__file__).resolve().parents[1] +_SOURCE_ROOT = _PROJECT_ROOT / "src" +_PACKAGE_ROOT = _SOURCE_ROOT / "generic_rag" +_CLEAN_IMPORT_PROBE = _PROJECT_ROOT / "tests" / "support" / "clean_import_probe.py" +_EXPECTED_SOURCES = { + "generic_rag": "src/generic_rag/__init__.py", + "generic_rag.contracts": "src/generic_rag/contracts.py", + "generic_rag.errors": "src/generic_rag/errors.py", + "generic_rag.ports": "src/generic_rag/ports.py", +} +_EXPECTED_DEPENDENCIES = { + "generic_rag": set(), + "generic_rag.contracts": {"generic_rag.errors"}, + "generic_rag.errors": set(), + "generic_rag.ports": {"generic_rag.contracts"}, +} +_EXPECTED_EXPORTS = { + "generic_rag": (), + "generic_rag.errors": ( + "GenericRagError", + "ContractValidationError", + "CollaborationError", + "StateCompatibilityError", + ), + "generic_rag.contracts": ( + "DocumentKey", + "DocumentIdentity", + "Document", + "FragmentIdentity", + "Fragment", + "EmbeddingIdentity", + "EmbeddingVector", + "VectorRecord", + "ProjectionIdentity", + "ProjectionCheckpoint", + "ProjectionOutcome", + "ProjectionReceipt", + "RetrievalQuery", + "RetrievalOutcome", + "RetrievalHit", + "RetrievalResult", + ), + "generic_rag.ports": ( + "Borrowed", + "Embedder", + "VectorIndexWriter", + "VectorIndexReader", + "LexicalRetriever", + ), +} + + +def _production_sources() -> tuple[Path, ...]: + return tuple(sorted(_PACKAGE_ROOT.rglob("*.py"))) + + +def _module_name(path: Path) -> str: + relative = path.relative_to(_SOURCE_ROOT) + parts = list(relative.parts) + if parts[-1] == "__init__.py": + parts.pop() + else: + parts[-1] = path.stem + return ".".join(parts) + + +def _syntax_tree(path: Path) -> ast.Module: + return ast.parse(path.read_text(encoding="utf-8"), filename=str(path)) + + +def _resolve_import_from(module_name: str, node: ast.ImportFrom) -> str: + if node.level == 0: + return node.module or "" + package_parts = module_name.split(".")[:-1] + upward_steps = node.level - 1 + if upward_steps > len(package_parts): + return "" + prefix = package_parts[: len(package_parts) - upward_steps] + if node.module: + prefix.extend(node.module.split(".")) + return ".".join(prefix) + + +def _internal_dependencies(module_name: str, tree: ast.Module) -> set[str]: + dependencies: set[str] = set() + for node in ast.walk(tree): + if isinstance(node, ast.Import): + for alias in node.names: + if alias.name == "generic_rag" or alias.name.startswith("generic_rag."): + dependencies.add(alias.name) + elif isinstance(node, ast.ImportFrom): + imported_module = _resolve_import_from(module_name, node) + if imported_module == "generic_rag" or imported_module.startswith( + "generic_rag." + ): + dependencies.add(imported_module) + return dependencies + + +def _probe_environment() -> dict[str, str]: + credential_markers = ( + "AUTH", + "CREDENTIAL", + "KEY", + "PASSWORD", + "SECRET", + "TOKEN", + ) + environment = { + name: value + for name, value in os.environ.items() + if not name.startswith("PYTHON") + and not any(marker in name.upper() for marker in credential_markers) + } + environment["PYTHONDONTWRITEBYTECODE"] = "1" + return environment + + +class PackageBoundaryTests(unittest.TestCase): + def test_production_module_inventory_is_exact(self) -> None: + actual = { + _module_name(path): path.relative_to(_PROJECT_ROOT).as_posix() + for path in _production_sources() + } + + self.assertEqual(actual, _EXPECTED_SOURCES) + + def test_supported_exports_are_exact_and_owned(self) -> None: + modules = { + "generic_rag": generic_rag, + "generic_rag.errors": errors, + "generic_rag.contracts": contracts, + "generic_rag.ports": ports, + } + + for module_name, expected_exports in _EXPECTED_EXPORTS.items(): + module = modules[module_name] + with self.subTest(module=module_name): + self.assertEqual(module.__all__, expected_exports) + for name in expected_exports: + exported = getattr(module, name) + self.assertEqual(exported.__module__, module_name) + + for name in ( + *_EXPECTED_EXPORTS["generic_rag.errors"], + *_EXPECTED_EXPORTS["generic_rag.contracts"], + *_EXPECTED_EXPORTS["generic_rag.ports"], + ): + with self.subTest(root_reexport=name): + self.assertFalse(hasattr(generic_rag, name)) + + def test_each_supported_module_imports_from_its_owning_path(self) -> None: + for module_name in _EXPECTED_EXPORTS: + with self.subTest(module=module_name): + module = importlib.import_module(module_name) + self.assertEqual(module.__name__, module_name) + self.assertEqual(module.__all__, _EXPECTED_EXPORTS[module_name]) + + def test_internal_dependency_graph_is_exact_and_acyclic(self) -> None: + actual = { + module_name: _internal_dependencies( + module_name, + _syntax_tree(_PROJECT_ROOT / relative_path), + ) + for module_name, relative_path in _EXPECTED_SOURCES.items() + } + + self.assertEqual(actual, _EXPECTED_DEPENDENCIES) + root_tree = _syntax_tree(_PROJECT_ROOT / _EXPECTED_SOURCES["generic_rag"]) + root_imports = [ + node + for node in ast.walk(root_tree) + if isinstance(node, ast.Import | ast.ImportFrom) + ] + self.assertEqual(root_imports, []) + + def test_production_imports_are_stdlib_or_declared_internal_dependencies( + self, + ) -> None: + allowed_roots = set(sys.stdlib_module_names) + allowed_roots.update({"__future__", "generic_rag"}) + forbidden_roots = { + "MySQLdb", + "mysql", + "pymysql", + "sqlalchemy", + "sqlite3", + "story_writing_agents", + "tests", + } + + for path in _production_sources(): + module_name = _module_name(path) + tree = _syntax_tree(path) + for node in ast.walk(tree): + imported_modules: tuple[str, ...] = () + if isinstance(node, ast.Import): + imported_modules = tuple(alias.name for alias in node.names) + elif isinstance(node, ast.ImportFrom): + imported_modules = (_resolve_import_from(module_name, node),) + for alias in node.names: + self.assertNotEqual( + alias.name, + "*", + f"{module_name} uses a wildcard import", + ) + if imported_modules[0].startswith("generic_rag"): + self.assertFalse( + alias.name.startswith("_"), + f"{module_name} imports private name {alias.name}", + ) + + for imported_module in imported_modules: + if not imported_module: + continue + root = imported_module.partition(".")[0] + with self.subTest(module=module_name, imported=imported_module): + self.assertNotIn(root, forbidden_roots) + self.assertIn( + root, + allowed_roots, + f"{module_name} imports third-party module " + f"{imported_module}", + ) + + def test_production_has_no_any_dynamic_import_or_sys_path_access(self) -> None: + dynamic_functions = { + "__import__", + "import_module", + "module_from_spec", + "spec_from_file_location", + } + + for path in _production_sources(): + module_name = _module_name(path) + tree = _syntax_tree(path) + importlib_aliases = {"importlib"} + sys_aliases = {"sys"} + direct_dynamic_aliases = set(dynamic_functions) + for node in ast.walk(tree): + if isinstance(node, ast.Import): + for alias in node.names: + if alias.name == "importlib": + importlib_aliases.add(alias.asname or alias.name) + if alias.name == "sys": + sys_aliases.add(alias.asname or alias.name) + elif isinstance(node, ast.ImportFrom): + if node.module == "importlib": + for alias in node.names: + if alias.name in dynamic_functions: + direct_dynamic_aliases.add(alias.asname or alias.name) + + violations: list[str] = [] + for node in ast.walk(tree): + if isinstance(node, ast.Name) and node.id == "Any": + violations.append("typing.Any") + elif isinstance(node, ast.Attribute) and node.attr == "Any": + violations.append("typing.Any") + elif ( + isinstance(node, ast.Attribute) + and node.attr == "path" + and isinstance(node.value, ast.Name) + and node.value.id in sys_aliases + ): + violations.append("sys.path") + elif isinstance(node, ast.Call): + function = node.func + if ( + isinstance(function, ast.Name) + and function.id in direct_dynamic_aliases + ): + violations.append(f"dynamic import {function.id}") + elif ( + isinstance(function, ast.Attribute) + and isinstance(function.value, ast.Name) + and function.value.id in importlib_aliases + and function.attr in dynamic_functions + ): + violations.append(f"dynamic import {function.attr}") + + self.assertEqual( + violations, + [], + f"{module_name} has forbidden boundary operations", + ) + + def test_module_index_has_exact_source_path_parity(self) -> None: + index_path = _PROJECT_ROOT / "PYTHON_MODULE_INDEX.md" + entries: dict[str, str] = {} + current_module: str | None = None + heading_pattern = re.compile(r"^## \x60(generic_rag(?:\.[a-z_]+)?)\x60$") + source_pattern = re.compile(r"^- Source: \x60([^ \x60]+)\x60$") + + for line in index_path.read_text(encoding="utf-8").splitlines(): + heading_match = heading_pattern.fullmatch(line) + if heading_match: + current_module = heading_match.group(1) + self.assertNotIn(current_module, entries) + continue + source_match = source_pattern.fullmatch(line) + if source_match and current_module is not None: + entries[current_module] = source_match.group(1) + current_module = None + + self.assertEqual(entries, _EXPECTED_SOURCES) + + def test_each_module_imports_in_an_isolated_side_effect_guarded_process( + self, + ) -> None: + environment = _probe_environment() + for module_name in _EXPECTED_EXPORTS: + with self.subTest(module=module_name): + with tempfile.TemporaryDirectory( + prefix="generic-rag-import-" + ) as working_directory: + completed = subprocess.run( + ( + sys.executable, + "-I", + "-B", + str(_CLEAN_IMPORT_PROBE), + module_name, + ), + cwd=working_directory, + env=environment, + check=False, + capture_output=True, + text=True, + timeout=10, + ) + self.assertEqual( + completed.returncode, + 0, + f"stdout:\n{completed.stdout}\nstderr:\n{completed.stderr}", + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_ports.py b/tests/test_ports.py new file mode 100644 index 0000000..d3bc6f3 --- /dev/null +++ b/tests/test_ports.py @@ -0,0 +1,404 @@ +"""Contract tests and deterministic witnesses for injected collaborator ports.""" + +from __future__ import annotations + +import inspect +import unittest +from dataclasses import FrozenInstanceError, fields +from types import TracebackType +from typing import get_type_hints + +import generic_rag.ports as ports +from generic_rag.contracts import ( + DocumentIdentity, + DocumentKey, + EmbeddingIdentity, + EmbeddingVector, + Fragment, + FragmentIdentity, + RetrievalQuery, + VectorRecord, +) +from generic_rag.ports import ( + Borrowed, + Embedder, + LexicalRetriever, + VectorIndexReader, + VectorIndexWriter, +) + + +def _document_identity( + *, + corpus_id: str = "corpus", + document_id: str = "document", + revision_id: str = "revision", +) -> DocumentIdentity: + return DocumentIdentity( + DocumentKey(corpus_id, document_id), + revision_id, + ) + + +def _fragment( + fragment_id: str, + *, + corpus_id: str = "corpus", + document_id: str = "document", +) -> Fragment: + document = _document_identity( + corpus_id=corpus_id, + document_id=document_id, + ) + identity = FragmentIdentity(document, fragment_id, 0, 1) + return Fragment(identity, "x") + + +def _query( + *, + corpus_id: str = "corpus", + candidate_limit: int = 3, +) -> RetrievalQuery: + return RetrievalQuery(corpus_id, "query", 2, candidate_limit) + + +class _FakeEmbedder: + def __init__(self) -> None: + self._identity = EmbeddingIdentity("fake-model", 2) + self.calls: list[tuple[str, ...]] = [] + + @property + def identity(self) -> EmbeddingIdentity: + return self._identity + + def embed(self, texts: tuple[str, ...], /) -> tuple[EmbeddingVector, ...]: + self.calls.append(texts) + return tuple( + EmbeddingVector((index, len(text))) for index, text in enumerate(texts) + ) + + +class _ShapeOnlyInvalidEmbedder: + @property + def identity(self) -> EmbeddingIdentity: + return EmbeddingIdentity("shape-only", 2) + + def embed(self, texts: tuple[str, ...], /) -> tuple[EmbeddingVector, ...]: + del texts + return (EmbeddingVector((1.0,)),) + + +class _FakeVectorWriter: + def __init__(self) -> None: + self.replacements: list[tuple[DocumentIdentity, tuple[VectorRecord, ...]]] = [] + self.deletions: list[DocumentKey] = [] + + def replace_document( + self, + document: DocumentIdentity, + records: tuple[VectorRecord, ...], + /, + ) -> None: + if any(record.fragment.identity.document != document for record in records): + raise AssertionError("records must belong to the supplied document") + self.replacements.append((document, records)) + + def delete_document(self, document: DocumentKey, /) -> None: + self.deletions.append(document) + + +class _FakeVectorReader: + def __init__(self, ranked: tuple[Fragment, ...]) -> None: + self.ranked = ranked + self.calls: list[tuple[RetrievalQuery, EmbeddingVector]] = [] + + def search( + self, + query: RetrievalQuery, + embedding: EmbeddingVector, + /, + ) -> tuple[Fragment, ...]: + self.calls.append((query, embedding)) + matching = tuple( + fragment + for fragment in self.ranked + if fragment.identity.document.key.corpus_id == query.corpus_id + ) + return matching[: query.candidate_limit] + + +class _FakeLexicalRetriever: + def __init__(self, ranked: tuple[Fragment, ...]) -> None: + self.ranked = ranked + self.calls: list[RetrievalQuery] = [] + + def search(self, query: RetrievalQuery, /) -> tuple[Fragment, ...]: + self.calls.append(query) + matching = tuple( + fragment + for fragment in self.ranked + if fragment.identity.document.key.corpus_id == query.corpus_id + ) + return matching[: query.candidate_limit] + + +class _RaisingLexicalRetriever: + def __init__(self, failure: RuntimeError) -> None: + self.failure = failure + + def search(self, query: RetrievalQuery, /) -> tuple[Fragment, ...]: + del query + raise self.failure + + +class _LifecycleSentinel: + def __init__(self) -> None: + self.enter_calls = 0 + self.exit_calls = 0 + self.close_calls = 0 + self.shutdown_calls = 0 + + def __enter__(self) -> _LifecycleSentinel: + self.enter_calls += 1 + return self + + def __exit__( + self, + exc_type: type[BaseException] | None, + exc_value: BaseException | None, + traceback: TracebackType | None, + ) -> bool: + del exc_type, exc_value, traceback + self.exit_calls += 1 + return True + + def close(self) -> None: + self.close_calls += 1 + + def shutdown(self) -> None: + self.shutdown_calls += 1 + + +class _MissingMethods: + pass + + +class PortContractTests(unittest.TestCase): + def test_exports_are_exact_and_owned_by_the_module(self) -> None: + expected = ( + "Borrowed", + "Embedder", + "VectorIndexWriter", + "VectorIndexReader", + "LexicalRetriever", + ) + + self.assertEqual(ports.__all__, expected) + for name in expected: + exported = getattr(ports, name) + self.assertEqual(exported.__module__, "generic_rag.ports") + + def test_protocols_are_runtime_checkable_structural_shapes(self) -> None: + self.assertIsInstance(_FakeEmbedder(), Embedder) + self.assertIsInstance(_FakeVectorWriter(), VectorIndexWriter) + self.assertIsInstance(_FakeVectorReader(()), VectorIndexReader) + self.assertIsInstance(_FakeLexicalRetriever(()), LexicalRetriever) + + missing = _MissingMethods() + self.assertNotIsInstance(missing, Embedder) + self.assertNotIsInstance(missing, VectorIndexWriter) + self.assertNotIsInstance(missing, VectorIndexReader) + self.assertNotIsInstance(missing, LexicalRetriever) + + def test_runtime_protocol_check_does_not_claim_semantic_enforcement( + self, + ) -> None: + shape_only = _ShapeOnlyInvalidEmbedder() + + self.assertIsInstance(shape_only, Embedder) + self.assertNotEqual( + len(shape_only.embed(())), + 0, + "runtime_checkable verifies names, not the empty-input obligation", + ) + self.assertNotEqual( + len(shape_only.embed(("text",))[0].values), + shape_only.identity.dimensions, + "runtime_checkable does not verify embedding dimensions", + ) + + def test_protocol_methods_are_synchronous_and_positional_only(self) -> None: + methods = ( + (Embedder.embed, ("self", "texts")), + ( + VectorIndexWriter.replace_document, + ("self", "document", "records"), + ), + (VectorIndexWriter.delete_document, ("self", "document")), + ( + VectorIndexReader.search, + ("self", "query", "embedding"), + ), + (LexicalRetriever.search, ("self", "query")), + ) + for method, names in methods: + with self.subTest(method=method.__qualname__): + self.assertFalse(inspect.iscoroutinefunction(method)) + parameters = tuple(inspect.signature(method).parameters.values()) + self.assertEqual( + tuple(parameter.name for parameter in parameters), + names, + ) + self.assertTrue( + all( + parameter.kind is inspect.Parameter.POSITIONAL_ONLY + for parameter in parameters + ) + ) + + def test_protocol_annotations_keep_scores_and_lifecycle_out(self) -> None: + self.assertEqual( + get_type_hints(Embedder.embed), + { + "texts": tuple[str, ...], + "return": tuple[EmbeddingVector, ...], + }, + ) + self.assertEqual( + get_type_hints(VectorIndexWriter.replace_document), + { + "document": DocumentIdentity, + "records": tuple[VectorRecord, ...], + "return": type(None), + }, + ) + self.assertEqual( + get_type_hints(VectorIndexWriter.delete_document), + { + "document": DocumentKey, + "return": type(None), + }, + ) + self.assertEqual( + get_type_hints(VectorIndexReader.search), + { + "query": RetrievalQuery, + "embedding": EmbeddingVector, + "return": tuple[Fragment, ...], + }, + ) + self.assertEqual( + get_type_hints(LexicalRetriever.search), + { + "query": RetrievalQuery, + "return": tuple[Fragment, ...], + }, + ) + lifecycle_names = {"close", "shutdown", "__enter__", "__exit__"} + for protocol in ( + Embedder, + VectorIndexWriter, + VectorIndexReader, + LexicalRetriever, + ): + with self.subTest(protocol=protocol.__name__): + self.assertTrue(lifecycle_names.isdisjoint(protocol.__dict__)) + + def test_embedder_witness_preserves_order_count_dimensions_and_empty(self) -> None: + embedder = _FakeEmbedder() + + self.assertEqual(embedder.embed(()), ()) + vectors = embedder.embed(("a", "longer")) + + self.assertEqual(embedder.calls, [(), ("a", "longer")]) + self.assertEqual(len(vectors), 2) + self.assertEqual(vectors[0].values, (0.0, 1.0)) + self.assertEqual(vectors[1].values, (1.0, 6.0)) + self.assertTrue( + all( + len(vector.values) == embedder.identity.dimensions for vector in vectors + ) + ) + + def test_writer_witness_replaces_complete_sets_and_accepts_empty(self) -> None: + writer = _FakeVectorWriter() + document = _document_identity() + fragment = _fragment("fragment") + records = (VectorRecord(fragment, EmbeddingVector((1.0, 2.0))),) + + writer.replace_document(document, records) + writer.replace_document(document, ()) + writer.delete_document(document.key) + + self.assertEqual( + writer.replacements, + [(document, records), (document, ())], + ) + self.assertEqual(writer.deletions, [document.key]) + + wrong_document = _document_identity(document_id="other") + with self.assertRaises(AssertionError): + writer.replace_document(wrong_document, records) + + def test_reader_witnesses_preserve_rank_and_enforce_candidate_bound(self) -> None: + ranked = ( + _fragment("first"), + _fragment("wrong-corpus", corpus_id="other"), + _fragment("second", document_id="second-document"), + _fragment("third", document_id="third-document"), + ) + query = _query(candidate_limit=2) + embedding = EmbeddingVector((1.0, 2.0)) + vector_reader = _FakeVectorReader(ranked) + lexical_reader = _FakeLexicalRetriever(ranked) + + self.assertEqual( + vector_reader.search(query, embedding), + (ranked[0], ranked[2]), + ) + self.assertEqual( + lexical_reader.search(query), + (ranked[0], ranked[2]), + ) + self.assertEqual(vector_reader.calls, [(query, embedding)]) + self.assertEqual(lexical_reader.calls, [query]) + + def test_borrowed_never_invokes_or_owns_resource_lifecycle(self) -> None: + resource = _LifecycleSentinel() + borrowed = Borrowed(resource) + + self.assertEqual( + tuple(field.name for field in fields(Borrowed)), + ("resource",), + ) + self.assertFalse(hasattr(borrowed, "__dict__")) + self.assertIs(borrowed.resource, resource) + with self.assertRaises(FrozenInstanceError): + setattr(borrowed, "resource", object()) + + with borrowed as entered: + self.assertIs(entered, resource) + + self.assertEqual( + ( + resource.enter_calls, + resource.exit_calls, + resource.close_calls, + resource.shutdown_calls, + ), + (0, 0, 0, 0), + ) + + def test_borrowed_never_suppresses_collaborator_failure(self) -> None: + failure = RuntimeError("sentinel collaborator failure") + retriever = _RaisingLexicalRetriever(failure) + + with self.assertRaises(RuntimeError) as raised: + with Borrowed(retriever) as borrowed: + borrowed.search(_query()) + + self.assertIs(raised.exception, failure) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_projection_contracts.py b/tests/test_projection_contracts.py new file mode 100644 index 0000000..7f59932 --- /dev/null +++ b/tests/test_projection_contracts.py @@ -0,0 +1,376 @@ +"""Contract tests for projection identities, checkpoints, and receipts.""" + +from __future__ import annotations + +import unittest +from dataclasses import FrozenInstanceError, fields +from typing import cast + +from generic_rag.contracts import ( + EmbeddingIdentity, + ProjectionCheckpoint, + ProjectionIdentity, + ProjectionOutcome, + ProjectionReceipt, +) +from generic_rag.errors import ContractValidationError + + +class _StringSubclass(str): + pass + + +class _IntegerSubclass(int): + pass + + +def _projection( + *, + schema_id: str = "schema", + model_id: str = "model", + dimensions: int = 2, +) -> ProjectionIdentity: + return ProjectionIdentity( + schema_id=schema_id, + embedding=EmbeddingIdentity(model_id, dimensions), + ) + + +def _checkpoint( + *, + corpus_id: str = "corpus", + projection: ProjectionIdentity | None = None, + token: str = "checkpoint", +) -> ProjectionCheckpoint: + return ProjectionCheckpoint( + corpus_id=corpus_id, + projection=projection or _projection(), + token=token, + ) + + +class ProjectionContractTests(unittest.TestCase): + def test_projection_fields_are_exact_frozen_and_slotted(self) -> None: + self.assertEqual( + tuple(field.name for field in fields(ProjectionIdentity)), + ("schema_id", "embedding"), + ) + self.assertEqual( + tuple(field.name for field in fields(ProjectionCheckpoint)), + ("corpus_id", "projection", "token"), + ) + self.assertEqual( + tuple(field.name for field in fields(ProjectionReceipt)), + ( + "corpus_id", + "projection", + "outcome", + "attempted_documents", + "completed_documents", + "checkpoint", + ), + ) + + projection = _projection() + checkpoint = _checkpoint(projection=projection) + receipt = ProjectionReceipt( + "corpus", + projection, + ProjectionOutcome.COMPLETED, + 0, + 0, + checkpoint, + ) + for instance, field_name in ( + (projection, "schema_id"), + (checkpoint, "token"), + (receipt, "completed_documents"), + ): + with self.subTest(instance=type(instance).__name__): + self.assertFalse(hasattr(instance, "__dict__")) + with self.assertRaises(FrozenInstanceError): + setattr(instance, field_name, object()) + self.assertIsInstance(hash(instance), int) + + def test_projection_outcomes_are_exact_closed_string_enums(self) -> None: + self.assertEqual( + tuple((member.name, member.value) for member in ProjectionOutcome), + ( + ("COMPLETED", "completed"), + ("UNCHANGED", "unchanged"), + ("PARTIAL", "partial"), + ("FAILED", "failed"), + ), + ) + self.assertEqual(ProjectionOutcome("completed"), ProjectionOutcome.COMPLETED) + self.assertIsInstance(ProjectionOutcome.COMPLETED, str) + + for invalid in ("COMPLETED", "", "unknown", None, 1, object()): + with self.subTest(value=invalid): + with self.assertRaises(ContractValidationError): + ProjectionOutcome(cast(str, invalid)) + + def test_projection_identity_preserves_opaque_strings(self) -> None: + schema_id = " Schema/../e\u0301 " + model_id = " Model:ABC " + projection = _projection(schema_id=schema_id, model_id=model_id) + + self.assertEqual(projection.schema_id, schema_id) + self.assertEqual(projection.embedding.model_id, model_id) + + def test_projection_identity_rejects_invalid_fields(self) -> None: + invalid_strings: tuple[object, ...] = ( + "", + " \t", + None, + b"schema", + _StringSubclass("schema"), + ) + for invalid in invalid_strings: + with self.subTest(value=invalid): + with self.assertRaises(ContractValidationError): + ProjectionIdentity( + cast(str, invalid), + EmbeddingIdentity("model", 2), + ) + + with self.assertRaises(ContractValidationError): + ProjectionIdentity("schema", cast(EmbeddingIdentity, object())) + + def test_checkpoint_requires_exact_opaque_identity_values(self) -> None: + corpus_id = " Corpus/../A " + token = " Token/../001 " + projection = _projection() + checkpoint = _checkpoint( + corpus_id=corpus_id, + projection=projection, + token=token, + ) + + self.assertEqual(checkpoint.corpus_id, corpus_id) + self.assertEqual(checkpoint.token, token) + self.assertIs(checkpoint.projection, projection) + + for invalid in ("", " \n", None, _StringSubclass("value")): + with self.subTest(field="corpus_id", value=invalid): + with self.assertRaises(ContractValidationError): + ProjectionCheckpoint( + cast(str, invalid), + projection, + "token", + ) + with self.subTest(field="token", value=invalid): + with self.assertRaises(ContractValidationError): + ProjectionCheckpoint( + "corpus", + projection, + cast(str, invalid), + ) + with self.assertRaises(ContractValidationError): + ProjectionCheckpoint( + "corpus", + cast(ProjectionIdentity, object()), + "token", + ) + + def test_completed_and_unchanged_receipts_require_truthful_checkpoints( + self, + ) -> None: + projection = _projection() + checkpoint = _checkpoint(projection=projection) + + for outcome in ( + ProjectionOutcome.COMPLETED, + ProjectionOutcome.UNCHANGED, + ): + with self.subTest(outcome=outcome): + empty = ProjectionReceipt( + "corpus", + projection, + outcome, + 0, + 0, + checkpoint, + ) + complete = ProjectionReceipt( + "corpus", + projection, + outcome, + 3, + 3, + checkpoint, + ) + self.assertEqual(empty.completed_documents, 0) + self.assertEqual(complete.completed_documents, 3) + self.assertIs(complete.checkpoint, checkpoint) + + with self.assertRaises(ContractValidationError): + ProjectionReceipt( + "corpus", + projection, + outcome, + 3, + 2, + checkpoint, + ) + with self.assertRaises(ContractValidationError): + ProjectionReceipt( + "corpus", + projection, + outcome, + 3, + 3, + None, + ) + + def test_partial_and_failed_receipts_cannot_claim_completed_work(self) -> None: + projection = _projection() + partial = ProjectionReceipt( + "corpus", + projection, + ProjectionOutcome.PARTIAL, + 3, + 1, + None, + ) + failed = ProjectionReceipt( + "corpus", + projection, + ProjectionOutcome.FAILED, + 2, + 0, + None, + ) + + self.assertEqual(partial.completed_documents, 1) + self.assertIsNone(partial.checkpoint) + self.assertEqual(failed.completed_documents, 0) + self.assertIsNone(failed.checkpoint) + + invalid_partial_counts = ((0, 0), (1, 0), (1, 1), (2, 2)) + for attempted, completed in invalid_partial_counts: + with self.subTest(attempted=attempted, completed=completed): + with self.assertRaises(ContractValidationError): + ProjectionReceipt( + "corpus", + projection, + ProjectionOutcome.PARTIAL, + attempted, + completed, + None, + ) + + invalid_failed_counts = ((0, 0), (1, 1), (2, 1)) + for attempted, completed in invalid_failed_counts: + with self.subTest(attempted=attempted, completed=completed): + with self.assertRaises(ContractValidationError): + ProjectionReceipt( + "corpus", + projection, + ProjectionOutcome.FAILED, + attempted, + completed, + None, + ) + + for outcome, attempted, completed in ( + (ProjectionOutcome.PARTIAL, 2, 1), + (ProjectionOutcome.FAILED, 2, 0), + ): + with self.subTest(outcome=outcome): + with self.assertRaises(ContractValidationError): + ProjectionReceipt( + "corpus", + projection, + outcome, + attempted, + completed, + _checkpoint(projection=projection), + ) + + def test_receipt_checkpoint_must_match_corpus_and_projection(self) -> None: + projection = _projection() + mismatched_projection = _projection(schema_id="other-schema") + + for checkpoint in ( + _checkpoint(corpus_id="other-corpus", projection=projection), + _checkpoint(corpus_id="corpus", projection=mismatched_projection), + ): + with self.subTest(checkpoint=checkpoint): + with self.assertRaises(ContractValidationError): + ProjectionReceipt( + "corpus", + projection, + ProjectionOutcome.COMPLETED, + 1, + 1, + checkpoint, + ) + + def test_receipt_rejects_inexact_nested_and_discrete_types(self) -> None: + projection = _projection() + checkpoint = _checkpoint(projection=projection) + + with self.assertRaises(ContractValidationError): + ProjectionReceipt( + "corpus", + cast(ProjectionIdentity, object()), + ProjectionOutcome.COMPLETED, + 1, + 1, + checkpoint, + ) + with self.assertRaises(ContractValidationError): + ProjectionReceipt( + "corpus", + projection, + cast(ProjectionOutcome, "completed"), + 1, + 1, + checkpoint, + ) + with self.assertRaises(ContractValidationError): + ProjectionReceipt( + "corpus", + projection, + ProjectionOutcome.COMPLETED, + 1, + 1, + cast(ProjectionCheckpoint, object()), + ) + + for invalid in (-1, True, 1.0, _IntegerSubclass(1)): + with self.subTest(field="attempted_documents", value=invalid): + with self.assertRaises(ContractValidationError): + ProjectionReceipt( + "corpus", + projection, + ProjectionOutcome.COMPLETED, + cast(int, invalid), + 1, + checkpoint, + ) + with self.subTest(field="completed_documents", value=invalid): + with self.assertRaises(ContractValidationError): + ProjectionReceipt( + "corpus", + projection, + ProjectionOutcome.COMPLETED, + 1, + cast(int, invalid), + checkpoint, + ) + + with self.assertRaises(ContractValidationError): + ProjectionReceipt( + "corpus", + projection, + ProjectionOutcome.COMPLETED, + 1, + 2, + checkpoint, + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_retrieval_contracts.py b/tests/test_retrieval_contracts.py new file mode 100644 index 0000000..1798385 --- /dev/null +++ b/tests/test_retrieval_contracts.py @@ -0,0 +1,335 @@ +"""Contract tests for bounded score-free retrieval values.""" + +from __future__ import annotations + +import inspect +import unittest +from dataclasses import FrozenInstanceError, fields +from typing import cast + +from generic_rag.contracts import ( + DocumentIdentity, + DocumentKey, + Fragment, + FragmentIdentity, + RetrievalHit, + RetrievalOutcome, + RetrievalQuery, + RetrievalResult, +) +from generic_rag.errors import ContractValidationError + + +class _StringSubclass(str): + pass + + +class _IntegerSubclass(int): + pass + + +class _TupleSubclass(tuple[object, ...]): + pass + + +def _fragment( + fragment_id: str, + *, + corpus_id: str = "corpus", + document_id: str = "document", + revision_id: str = "revision", + start: int = 0, + attributes: tuple[tuple[str, str], ...] = (), +) -> Fragment: + document = DocumentIdentity( + DocumentKey(corpus_id, document_id), + revision_id, + ) + identity = FragmentIdentity( + document=document, + fragment_id=fragment_id, + start=start, + end=start + 1, + ) + return Fragment(identity, "x", attributes) + + +def _query( + *, + corpus_id: str = "corpus", + text: str = "query", + hit_limit: int = 3, + candidate_limit: int = 5, +) -> RetrievalQuery: + return RetrievalQuery( + corpus_id=corpus_id, + text=text, + hit_limit=hit_limit, + candidate_limit=candidate_limit, + ) + + +class RetrievalContractTests(unittest.TestCase): + def test_retrieval_fields_are_exact_frozen_and_slotted(self) -> None: + self.assertEqual( + tuple(field.name for field in fields(RetrievalQuery)), + ("corpus_id", "text", "hit_limit", "candidate_limit"), + ) + self.assertEqual( + tuple(field.name for field in fields(RetrievalHit)), + ("fragment", "rank"), + ) + self.assertEqual( + tuple(field.name for field in fields(RetrievalResult)), + ("query", "outcome", "hits", "truncated"), + ) + + query = _query() + hit = RetrievalHit(_fragment("fragment"), 1) + result = RetrievalResult( + query, + RetrievalOutcome.COMPLETE, + (hit,), + False, + ) + for instance, field_name in ( + (query, "text"), + (hit, "rank"), + (result, "truncated"), + ): + with self.subTest(instance=type(instance).__name__): + self.assertFalse(hasattr(instance, "__dict__")) + with self.assertRaises(FrozenInstanceError): + setattr(instance, field_name, object()) + self.assertIsInstance(hash(instance), int) + + def test_retrieval_outcomes_are_exact_closed_string_enums(self) -> None: + self.assertEqual( + tuple((member.name, member.value) for member in RetrievalOutcome), + ( + ("COMPLETE", "complete"), + ("PARTIAL", "partial"), + ("UNAVAILABLE", "unavailable"), + ("STALE", "stale"), + ("FAILED", "failed"), + ), + ) + self.assertEqual(RetrievalOutcome("complete"), RetrievalOutcome.COMPLETE) + self.assertIsInstance(RetrievalOutcome.COMPLETE, str) + + for invalid in ("COMPLETE", "", "unknown", None, 1, object()): + with self.subTest(value=invalid): + with self.assertRaises(ContractValidationError): + RetrievalOutcome(cast(str, invalid)) + + def test_query_preserves_opaque_corpus_and_nonblank_text(self) -> None: + corpus_id = " Corpus/../e\u0301 " + text = " Query/../A " + query = _query(corpus_id=corpus_id, text=text) + + self.assertEqual(query.corpus_id, corpus_id) + self.assertEqual(query.text, text) + + for invalid in ("", " \t\n", None, b"value", _StringSubclass("value")): + with self.subTest(field="corpus_id", value=invalid): + with self.assertRaises(ContractValidationError): + _query(corpus_id=cast(str, invalid)) + with self.subTest(field="text", value=invalid): + with self.assertRaises(ContractValidationError): + _query(text=cast(str, invalid)) + + def test_query_limits_are_positive_exact_integers_and_bounded(self) -> None: + query = _query(hit_limit=2, candidate_limit=2) + self.assertEqual((query.hit_limit, query.candidate_limit), (2, 2)) + + for invalid in (0, -1, True, 1.0, _IntegerSubclass(1)): + with self.subTest(field="hit_limit", value=invalid): + with self.assertRaises(ContractValidationError): + _query(hit_limit=cast(int, invalid)) + with self.subTest(field="candidate_limit", value=invalid): + with self.assertRaises(ContractValidationError): + _query(candidate_limit=cast(int, invalid)) + + with self.assertRaises(ContractValidationError): + _query(hit_limit=3, candidate_limit=2) + + def test_hit_is_positive_ranked_exact_fragment_without_score(self) -> None: + fragment = _fragment("fragment") + hit = RetrievalHit(fragment, 1) + + self.assertIs(hit.fragment, fragment) + self.assertEqual(hit.rank, 1) + self.assertNotIn("score", inspect.signature(RetrievalHit).parameters) + self.assertFalse(hasattr(hit, "score")) + + with self.assertRaises(ContractValidationError): + RetrievalHit(cast(Fragment, object()), 1) + for invalid in (0, -1, True, 1.0, _IntegerSubclass(1)): + with self.subTest(rank=invalid): + with self.assertRaises(ContractValidationError): + RetrievalHit(fragment, cast(int, invalid)) + + def test_result_accepts_contiguous_unique_bounded_matching_hits(self) -> None: + query = _query(hit_limit=3, candidate_limit=5) + hits = ( + RetrievalHit(_fragment("first"), 1), + RetrievalHit(_fragment("second", document_id="other"), 2), + ) + result = RetrievalResult( + query, + RetrievalOutcome.COMPLETE, + hits, + False, + ) + + self.assertEqual(result.hits, hits) + self.assertFalse(result.truncated) + + explicitly_truncated = RetrievalResult( + query, + RetrievalOutcome.COMPLETE, + hits, + True, + ) + self.assertTrue(explicitly_truncated.truncated) + + def test_result_rejects_noncontiguous_or_duplicate_hits(self) -> None: + query = _query() + first = _fragment("first") + second = _fragment("second") + + invalid_rank_sequences = ( + (RetrievalHit(first, 2),), + (RetrievalHit(first, 1), RetrievalHit(second, 3)), + (RetrievalHit(first, 1), RetrievalHit(second, 1)), + ) + for hits in invalid_rank_sequences: + with self.subTest(ranks=tuple(hit.rank for hit in hits)): + with self.assertRaises(ContractValidationError): + RetrievalResult( + query, + RetrievalOutcome.COMPLETE, + hits, + False, + ) + + duplicate_identity = _fragment( + "duplicate", + attributes=(("variant", "one"),), + ) + duplicate_identity_other_value = _fragment( + "duplicate", + attributes=(("variant", "two"),), + ) + with self.assertRaises(ContractValidationError): + RetrievalResult( + query, + RetrievalOutcome.COMPLETE, + ( + RetrievalHit(duplicate_identity, 1), + RetrievalHit(duplicate_identity_other_value, 2), + ), + False, + ) + + def test_result_rejects_wrong_corpus_and_excess_hits(self) -> None: + with self.assertRaises(ContractValidationError): + RetrievalResult( + _query(corpus_id="corpus"), + RetrievalOutcome.COMPLETE, + (RetrievalHit(_fragment("hit", corpus_id="other"), 1),), + False, + ) + + with self.assertRaises(ContractValidationError): + RetrievalResult( + _query(hit_limit=1), + RetrievalOutcome.COMPLETE, + ( + RetrievalHit(_fragment("first"), 1), + RetrievalHit(_fragment("second"), 2), + ), + False, + ) + + def test_terminal_outcomes_cannot_claim_hits_or_truncation(self) -> None: + query = _query() + hit = RetrievalHit(_fragment("fragment"), 1) + + for outcome in ( + RetrievalOutcome.UNAVAILABLE, + RetrievalOutcome.STALE, + RetrievalOutcome.FAILED, + ): + with self.subTest(outcome=outcome): + result = RetrievalResult(query, outcome, (), False) + self.assertEqual(result.hits, ()) + self.assertFalse(result.truncated) + + with self.assertRaises(ContractValidationError): + RetrievalResult(query, outcome, (hit,), False) + with self.assertRaises(ContractValidationError): + RetrievalResult(query, outcome, (), True) + + def test_partial_results_require_at_least_one_valid_hit(self) -> None: + query = _query() + + with self.assertRaises(ContractValidationError): + RetrievalResult(query, RetrievalOutcome.PARTIAL, (), False) + + hit = RetrievalHit(_fragment("fragment"), 1) + for truncated in (False, True): + with self.subTest(truncated=truncated): + result = RetrievalResult( + query, + RetrievalOutcome.PARTIAL, + (hit,), + truncated, + ) + self.assertEqual(result.hits, (hit,)) + + def test_result_requires_exact_nested_container_and_bool_types(self) -> None: + query = _query() + hit = RetrievalHit(_fragment("fragment"), 1) + + with self.assertRaises(ContractValidationError): + RetrievalResult( + cast(RetrievalQuery, object()), + RetrievalOutcome.COMPLETE, + (), + False, + ) + with self.assertRaises(ContractValidationError): + RetrievalResult( + query, + cast(RetrievalOutcome, "complete"), + (), + False, + ) + for invalid_hits in ( + [hit], + iter((hit,)), + _TupleSubclass((hit,)), + (object(),), + ): + with self.subTest(hits=type(invalid_hits).__name__): + with self.assertRaises(ContractValidationError): + RetrievalResult( + query, + RetrievalOutcome.COMPLETE, + cast(tuple[RetrievalHit, ...], invalid_hits), + False, + ) + for invalid_truncated in (0, 1, None, "false"): + with self.subTest(truncated=invalid_truncated): + with self.assertRaises(ContractValidationError): + RetrievalResult( + query, + RetrievalOutcome.COMPLETE, + (), + cast(bool, invalid_truncated), + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/uv.lock b/uv.lock new file mode 100644 index 0000000..e174e64 --- /dev/null +++ b/uv.lock @@ -0,0 +1,389 @@ +version = 1 +revision = 3 +requires-python = ">=3.11" +resolution-markers = [ + "python_full_version >= '3.15'", + "python_full_version < '3.15'", +] + +[[package]] +name = "ast-serialize" +version = "0.8.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/e1/a9/11851c3e02a3fea2ddc9932d1fdc7d2edaeecc0d2e11bc5f2a7fde2b0934/ast_serialize-0.8.0.tar.gz", hash = "sha256:6c37c43e4004dfb42d321ddedc569dc17ff4259296f3af577c9ea46a809bc010", size = 845638, upload-time = "2026-08-07T11:29:02.152Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/34/16/6e520b57cd8c75914b38c670ad4593d13c22911e4306cc7165dab8b0789b/ast_serialize-0.8.0-cp314-cp314-pyemscripten_2026_0_wasm32.whl", hash = "sha256:3d822605fa7bb326ef868d25fafced7fc660fa46d9b90c02ea86d5e2f5d325f7", size = 863924, upload-time = "2026-08-07T11:27:34.579Z" }, + { url = "https://files.pythonhosted.org/packages/03/e1/48802de9b22a2bcad42ec80601a17e3f69172fe4f590e6311bcc2b323aeb/ast_serialize-0.8.0-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:2efa40b068197d5efb62655b43baadb842ed71c4958cccd3e8b86a35726f0119", size = 1177662, upload-time = "2026-08-07T11:27:36.196Z" }, + { url = "https://files.pythonhosted.org/packages/38/d4/323438db76bded3a1f3523a3167b8325916b2ddceb2107a330c6ec9fcf4d/ast_serialize-0.8.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:db1b957291bca08c7e72f43a12357b2948e20775d970e3fc3dac0aa3160ab725", size = 1167072, upload-time = "2026-08-07T11:27:37.646Z" }, + { url = "https://files.pythonhosted.org/packages/77/82/53c5400b54144b56de8ed7f957fd1ccd97e42482009292ab46121d15f8dd/ast_serialize-0.8.0-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fdc0d5b18ff8fb364e87923e47c0a91d0d69dbcaeaa274591f7fd26892cc3a3a", size = 1225497, upload-time = "2026-08-07T11:27:39.225Z" }, + { url = "https://files.pythonhosted.org/packages/44/5f/36c07327a8b91303fbf1382c7c3e8a2902072dbe1b9546138a5288e75ff0/ast_serialize-0.8.0-cp314-cp314t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:9da7330f3e235bf7da89b8d39205c6350fc0c08a85379743f2df9fff87d6d980", size = 1227101, upload-time = "2026-08-07T11:27:40.799Z" }, + { url = "https://files.pythonhosted.org/packages/9d/48/5adf5c67addc7ddb328122208c6d375a84cf154984f412b4087330a157bd/ast_serialize-0.8.0-cp314-cp314t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f3186969ee66a9863b00acc6523ace44c56974eecb348a7ea4b228d9f0b80e19", size = 1424001, upload-time = "2026-08-07T11:27:42.708Z" }, + { url = "https://files.pythonhosted.org/packages/38/a1/70074dd3869d2b0e934f91891d8d6b734361cd3b80f85ca7ece2e668ecdd/ast_serialize-0.8.0-cp314-cp314t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:40a57b73731be45da4fa41430c4d5dc94a24b3a4faba7b9e069978c0402064ea", size = 1245545, upload-time = "2026-08-07T11:27:44.4Z" }, + { url = "https://files.pythonhosted.org/packages/e3/be/53b9c0a8a6399950c2e3546bdfab96d2b299d5b114b47eb94fd3c49c4054/ast_serialize-0.8.0-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5075b9da3ef807eda752502446dfecea3b381c4900b7e27a5d5f4f899eb39951", size = 1248961, upload-time = "2026-08-07T11:27:45.781Z" }, + { url = "https://files.pythonhosted.org/packages/eb/13/3651d3812548a2bda15e26e5dd51aadb48cf682d0865370255fcf0e367dd/ast_serialize-0.8.0-cp314-cp314t-manylinux_2_31_riscv64.whl", hash = "sha256:293cc1c5bfa741f8e3fbe8175b9c07beee487c9a6fdbb25a5acad9f1df2d30a9", size = 1243877, upload-time = "2026-08-07T11:27:47.325Z" }, + { url = "https://files.pythonhosted.org/packages/21/a0/521f0bf000f675e9312a4aae2c8ba7a992405d072a85c485e08fd59433b9/ast_serialize-0.8.0-cp314-cp314t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:e0910c3442a75216dde0f102d854ba2aaa71d2482e0ee213630b9bf29584fba3", size = 1293903, upload-time = "2026-08-07T11:27:49.264Z" }, + { url = "https://files.pythonhosted.org/packages/b1/7e/402fc902568aa2ee65865a3e151f000db0153da8ce6b1be4c9c349025f8d/ast_serialize-0.8.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:43dd6d596879bb1cb8a12cc9dae7bb10090a39a35883026c24f82488a195619a", size = 1401070, upload-time = "2026-08-07T11:27:50.947Z" }, + { url = "https://files.pythonhosted.org/packages/ff/7c/97d4b66c057f1706fc8be6dd532cc77c988794357c8f4ffdb6adabb39562/ast_serialize-0.8.0-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:8c9d537f59e936392cfd3597789d1390304dd659efc3c486ce7f40fb6b8a9f53", size = 1502602, upload-time = "2026-08-07T11:27:52.364Z" }, + { url = "https://files.pythonhosted.org/packages/89/6f/72cc3b71562001bba46e898ccfbf1844f7939b3e28912736206102f2e5a8/ast_serialize-0.8.0-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:f0190a33d7f97c65e9069f7a7f40499eea6b5cbe260c558378109caf20ce934b", size = 1495848, upload-time = "2026-08-07T11:27:53.803Z" }, + { url = "https://files.pythonhosted.org/packages/a0/53/d6f629d1e49308b2f363dae028baa213ec222c9106fa1f7f0d1f7b41499a/ast_serialize-0.8.0-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:77308ae6c5cf5264cc0f01a7c556ec77a9e68eb1f61b093534d698139fdc3b14", size = 1556556, upload-time = "2026-08-07T11:27:55.342Z" }, + { url = "https://files.pythonhosted.org/packages/ee/22/340f35dd8dfc6d412d53dc20699ca014b8d228db923e8ed4759c512b162c/ast_serialize-0.8.0-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:8d53a23f27e1ed3a36b2d26fd2a1a6228c8e85a1ed62ff7cdb44bd610769f20a", size = 1417822, upload-time = "2026-08-07T11:27:56.712Z" }, + { url = "https://files.pythonhosted.org/packages/11/29/6dde5c13fbebc051d3a6df4ec0a6fd1d5359333cc1193f7f609f3410b4d8/ast_serialize-0.8.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:ffa5e7cb08f96fed9121f77b224151e41caf88feab9d652bb46c78202b6fbeda", size = 1445153, upload-time = "2026-08-07T11:27:58.275Z" }, + { url = "https://files.pythonhosted.org/packages/62/c5/f473a8ed030f7a0ca24b9849cca184677a50c053867a7b808c2e1289bbd3/ast_serialize-0.8.0-cp314-cp314t-win32.whl", hash = "sha256:fa70ed4dea0bb18b30a1789c77baa701d0ef30c474f2ccabdea61e25623a8827", size = 1063711, upload-time = "2026-08-07T11:27:59.793Z" }, + { url = "https://files.pythonhosted.org/packages/23/63/39e171fcd38ca057c2e1979d5ee81ac7a3502784abe3d83df7454f7a0978/ast_serialize-0.8.0-cp314-cp314t-win_amd64.whl", hash = "sha256:d8b3c8eee4c1baef9d4e84d2a59a805501617127be42615cb48970b15b0892b6", size = 1103740, upload-time = "2026-08-07T11:28:01.405Z" }, + { url = "https://files.pythonhosted.org/packages/21/1c/d00762b399e7726d68d0a088cc946e3a4c60f1c6176f557608f672f627f3/ast_serialize-0.8.0-cp314-cp314t-win_arm64.whl", hash = "sha256:ac4f0a83c55a9b782f79ad55a5247b7db123c1db405959791c2ef886e9710c9f", size = 1076021, upload-time = "2026-08-07T11:28:02.947Z" }, + { url = "https://files.pythonhosted.org/packages/4c/11/911210c3c78923273a9211a2b6cfc4c8aa723b30dab3e1c8d19afb983b40/ast_serialize-0.8.0-cp315-abi3.abi3t-macosx_10_12_x86_64.whl", hash = "sha256:86b8a1e6d90467345356098b040150e82fbc26d24a7a202224b13dc1f6264ca0", size = 1177715, upload-time = "2026-08-07T11:28:04.654Z" }, + { url = "https://files.pythonhosted.org/packages/77/89/6282881c8587606638db153cbe21e1e0c4d1f3970dee1aa0610a1c62a026/ast_serialize-0.8.0-cp315-abi3.abi3t-macosx_11_0_arm64.whl", hash = "sha256:39e92ff8e8cb45947fe9007174b2950e1fb098e6abd00266a13cd3bcf6675068", size = 1169347, upload-time = "2026-08-07T11:28:06.1Z" }, + { url = "https://files.pythonhosted.org/packages/97/78/a9f846a03a340ff3728c915f23338ca742742f3292700559cdb3ad999b1e/ast_serialize-0.8.0-cp315-abi3.abi3t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c85d8d18db5b2dfcb3b7e38a4d600ca35504c0ed8a6f75cd1c811e4ffe248a15", size = 1225916, upload-time = "2026-08-07T11:28:07.654Z" }, + { url = "https://files.pythonhosted.org/packages/c0/15/aba6ef8a988a6eceb6f0359589aac509e29ae2dba67fd9bfd5af0c3f13e7/ast_serialize-0.8.0-cp315-abi3.abi3t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:9830ff7e764f74d9eefb01170c61a9f0fd2c027dac5fcb72e064decd57d56371", size = 1227135, upload-time = "2026-08-07T11:28:09.504Z" }, + { url = "https://files.pythonhosted.org/packages/94/29/3f63d696ea7c5b8abadcecc3505be51bd900daaccc522ed8322fa5b05a93/ast_serialize-0.8.0-cp315-abi3.abi3t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:6479d9722a4cd21b578f5478074c41e6169f04811996ec881655560f703a5bba", size = 1425040, upload-time = "2026-08-07T11:28:11.044Z" }, + { url = "https://files.pythonhosted.org/packages/e2/5d/0aac338604ff59df5774d4304307898982252f325ff7cafe31d52fedcb65/ast_serialize-0.8.0-cp315-abi3.abi3t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a63bed264e818cd83eec11feed0f50aa162542b91132ef58afebc857182763a5", size = 1246278, upload-time = "2026-08-07T11:28:12.519Z" }, + { url = "https://files.pythonhosted.org/packages/23/ca/9f1ef795bb724719532bd86dbec11e5b66857d3fbe9b6772baec0191a6ed/ast_serialize-0.8.0-cp315-abi3.abi3t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9d187197d234aa45d6cfa2b096be5f666e8cc2e7eb3722d0ab8926293cf5720c", size = 1250029, upload-time = "2026-08-07T11:28:13.896Z" }, + { url = "https://files.pythonhosted.org/packages/dc/25/5e061372d2ed953b9ba3b9c4f73de3b8e9234cda3f6c088db4686801d0e1/ast_serialize-0.8.0-cp315-abi3.abi3t-manylinux_2_31_riscv64.whl", hash = "sha256:2d39a56282cfcc0d8eeea37267c754be59c98d48505c23b1dae5c6011f3813dd", size = 1243575, upload-time = "2026-08-07T11:28:15.37Z" }, + { url = "https://files.pythonhosted.org/packages/a8/c1/ae7da218053120635a4ca802366c69f707203641af95372eeb83f70dfd52/ast_serialize-0.8.0-cp315-abi3.abi3t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:f7cc5f10386994c0f4844f1e6d6a97127e9b478660eb6dec2b257644f0acab64", size = 1294396, upload-time = "2026-08-07T11:28:16.813Z" }, + { url = "https://files.pythonhosted.org/packages/2e/89/271d1f49c5269fcddcc789ea3f25be401f6723fc1138aeda539f4d05516d/ast_serialize-0.8.0-cp315-abi3.abi3t-musllinux_1_2_aarch64.whl", hash = "sha256:6102f2f985c2e542be85cd857678ec9356fefa792b93cadfadd31139f5696f27", size = 1401987, upload-time = "2026-08-07T11:28:18.333Z" }, + { url = "https://files.pythonhosted.org/packages/55/be/4e7d77fcf571ac7cb5cf7115a20c36642bd7d29473b45dfaaefeb9618f90/ast_serialize-0.8.0-cp315-abi3.abi3t-musllinux_1_2_armv7l.whl", hash = "sha256:3a8660fe66667b76a6e9dccd1d33e66b229fde3b308db991c041609226c005b6", size = 1502904, upload-time = "2026-08-07T11:28:20.039Z" }, + { url = "https://files.pythonhosted.org/packages/8b/ae/ed1de2db7e019d4236fbc164ffa5ef9a6022a300a342bbf142d21b7c141e/ast_serialize-0.8.0-cp315-abi3.abi3t-musllinux_1_2_i686.whl", hash = "sha256:e7266307e5fba39836edb79def8608887af48820508bff3c5f2941e1e04d1534", size = 1496967, upload-time = "2026-08-07T11:28:21.734Z" }, + { url = "https://files.pythonhosted.org/packages/92/89/5fea507fae5c5f18b7dc7f95e5c00956574b8c717b8fd2049c504fab0b18/ast_serialize-0.8.0-cp315-abi3.abi3t-musllinux_1_2_ppc64le.whl", hash = "sha256:4ca7e6fd1ad845d1cc649dc2ecd499db2f8f46af5bf8da7b70dd858774cc038b", size = 1559041, upload-time = "2026-08-07T11:28:23.194Z" }, + { url = "https://files.pythonhosted.org/packages/42/71/478d69df21b64e064554a68134c94be304270316ca676a94e63c389a636a/ast_serialize-0.8.0-cp315-abi3.abi3t-musllinux_1_2_riscv64.whl", hash = "sha256:2880350b13d3eae69a0d70bc1fb6c9bfaca4dbd0e20ba8cd1aa483080b56ff06", size = 1417367, upload-time = "2026-08-07T11:28:24.601Z" }, + { url = "https://files.pythonhosted.org/packages/5e/2d/8962dc8d5b3a9dc27b36f9db199afa25264c741505469d9ec10ffbfd2ba7/ast_serialize-0.8.0-cp315-abi3.abi3t-musllinux_1_2_x86_64.whl", hash = "sha256:ab0f9a59f7d63d0d441b56b9a818b273705264352d5115cfee12e940e816d958", size = 1446178, upload-time = "2026-08-07T11:28:26.152Z" }, + { url = "https://files.pythonhosted.org/packages/4f/22/14d2ad4fd1d1bcd0dc687ca268e0630069f45162496260c0efb70ee0ea72/ast_serialize-0.8.0-cp315-abi3.abi3t-win32.whl", hash = "sha256:0485a25ef519c62e749ee3c1ad8070e591b380d67226349eb5a70b228dc1ac4a", size = 1063811, upload-time = "2026-08-07T11:28:27.864Z" }, + { url = "https://files.pythonhosted.org/packages/18/1d/84a327c0202a41aa5fdba3ade33904d6d8f3b9e6806fa83568d835395850/ast_serialize-0.8.0-cp315-abi3.abi3t-win_amd64.whl", hash = "sha256:bd84d60bca7079e741be4ac5dbe237751a59d7f6f9f0126b11880d63822cbe16", size = 1105518, upload-time = "2026-08-07T11:28:29.691Z" }, + { url = "https://files.pythonhosted.org/packages/8c/92/74556dec52fde85a2ad84ed159991b916241043788609c15d8b77e14570b/ast_serialize-0.8.0-cp315-abi3.abi3t-win_arm64.whl", hash = "sha256:057769b5921336eb2d9124f2a731b42ed05ffdac559b840dbdf6f3937cf153dc", size = 1076319, upload-time = "2026-08-07T11:28:31.282Z" }, + { url = "https://files.pythonhosted.org/packages/d1/5d/c650b1f2cc1e75193358da95a080261422e8cd10b66d7370b1688c9915c5/ast_serialize-0.8.0-cp315-cp315-pyemscripten_2026_5_wasm32.whl", hash = "sha256:a02cbed7d8bfdcdee88edaac12bd50d53d9953aaa2e1852ef078625be5f1c0b5", size = 852914, upload-time = "2026-08-07T11:28:32.929Z" }, + { url = "https://files.pythonhosted.org/packages/d9/e3/6142e920fec6ef7bccabd8c24ed8ed99f8bdc6cb8b065e1df7c6a3b2d667/ast_serialize-0.8.0-cp39-abi3-macosx_10_12_x86_64.whl", hash = "sha256:e1bd223df0f6c96b396975fa604cb33bce53d9b4a0185490be4c4a289f7c9c87", size = 1184007, upload-time = "2026-08-07T11:28:34.654Z" }, + { url = "https://files.pythonhosted.org/packages/a6/e9/6e8be8df02b35d85e2b8809f7f1cfa290bdf5882b55127a539d049482db0/ast_serialize-0.8.0-cp39-abi3-macosx_11_0_arm64.whl", hash = "sha256:ddd3b61f45c132da66c5476b281891e08c1fd87fbdabe8a6973e1622efc85f06", size = 1177588, upload-time = "2026-08-07T11:28:36.318Z" }, + { url = "https://files.pythonhosted.org/packages/8c/80/7e0fd2e2e2aba257820db4a8657c4c356844d36b914b20a4af294bcfb902/ast_serialize-0.8.0-cp39-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1f9caa63fad8241257ae401b5ff0a64026c6adb36b8e86cbe8782d9ea505daf6", size = 1234575, upload-time = "2026-08-07T11:28:37.772Z" }, + { url = "https://files.pythonhosted.org/packages/b0/6a/3bae0af06f9b1bae3001c44d64215f5b567877e7aae9ffd45db11c3a7647/ast_serialize-0.8.0-cp39-abi3-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:3926fa117b5e65019853a2969966d11c7175af377a3425991f3fe73784412405", size = 1236015, upload-time = "2026-08-07T11:28:39.14Z" }, + { url = "https://files.pythonhosted.org/packages/6f/c4/ce2d41a1bc22508e82618901f7e10f2a5e2f9556553fea90624daf9875e2/ast_serialize-0.8.0-cp39-abi3-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:485f1113af805e9e170b95ef993ca3fbd4f89c04bab25c58b4fc632d854801ab", size = 1432808, upload-time = "2026-08-07T11:28:40.664Z" }, + { url = "https://files.pythonhosted.org/packages/1a/90/f5058f209756dd70e958b7538aaa82d25d24944baf9ec8ae6f27b06fcacc/ast_serialize-0.8.0-cp39-abi3-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3ccebbed24f1281062d5852353c72c47502955926cfcb8345ffb3a44d87ff3d3", size = 1256251, upload-time = "2026-08-07T11:28:42.223Z" }, + { url = "https://files.pythonhosted.org/packages/bf/32/7f77ea87fa0836daab706ed5cb7f903bb25fa26a77439011aee626af11d8/ast_serialize-0.8.0-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:252f883290d1cdb728eb7fe1d9a7221b88af5a329aae0bc91ddee4dafb820331", size = 1258574, upload-time = "2026-08-07T11:28:43.751Z" }, + { url = "https://files.pythonhosted.org/packages/eb/5a/75b82ad2725b5e8e8c742732f9e76c6738a292d0709e1f60d10a973730b4/ast_serialize-0.8.0-cp39-abi3-manylinux_2_31_riscv64.whl", hash = "sha256:96abc072ad29db8d02194afd47d68987322622787daceae82398d7b69f3ba2e6", size = 1254075, upload-time = "2026-08-07T11:28:45.28Z" }, + { url = "https://files.pythonhosted.org/packages/4e/54/8c20ed4eea805516a3fd23dd4a721ce28c64f50f0e4b359969f60a8c97a6/ast_serialize-0.8.0-cp39-abi3-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:9118ad3e369727060b2696fc4078f250ecffca4248ba87f537f55cea9f9dce06", size = 1301018, upload-time = "2026-08-07T11:28:46.851Z" }, + { url = "https://files.pythonhosted.org/packages/cb/5b/9f14430f12fe830b656fb38f8e2e05ee13b02a88967660bef46af0ab22a8/ast_serialize-0.8.0-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:f359df4bd921918af8bebd142a376c77511d7151cc8ba852760b587b5a4a54f3", size = 1409951, upload-time = "2026-08-07T11:28:48.312Z" }, + { url = "https://files.pythonhosted.org/packages/2d/3d/084882eca93c842bd4262591a071ec7f825340644035e51501208cc5a8d4/ast_serialize-0.8.0-cp39-abi3-musllinux_1_2_armv7l.whl", hash = "sha256:e94f9121d13fa36cbf21314783c77d05ae3a0868decd18cf5233fdcc6de49ac8", size = 1509544, upload-time = "2026-08-07T11:28:49.847Z" }, + { url = "https://files.pythonhosted.org/packages/ce/73/ea84852096c2036c61cc0b2f97b90242207419f534dc671060ee1c8e05cb/ast_serialize-0.8.0-cp39-abi3-musllinux_1_2_i686.whl", hash = "sha256:54f95b486018d262bcb387a9afd96f0da74508b442762b80c769454a6fbb3ee3", size = 1505671, upload-time = "2026-08-07T11:28:51.239Z" }, + { url = "https://files.pythonhosted.org/packages/cb/88/287b9a5300c1f2f651d259f670931b63110adc265b7613c885b44c5bc53d/ast_serialize-0.8.0-cp39-abi3-musllinux_1_2_ppc64le.whl", hash = "sha256:4c38b915511e32bc718c49dbce98ff9af36bac0ad6a604f58000cd5e3aecdba7", size = 1563685, upload-time = "2026-08-07T11:28:53.112Z" }, + { url = "https://files.pythonhosted.org/packages/ee/f3/1bc3a79afcf0c2a8d2c37182d0d659d1545a9d7f7f6dc9cf3e63d6c17135/ast_serialize-0.8.0-cp39-abi3-musllinux_1_2_riscv64.whl", hash = "sha256:9a2ef9cf12f2de4f1028c42c1dd7d775255e0fb3e5bb48896c97e35ef52366fe", size = 1427977, upload-time = "2026-08-07T11:28:54.418Z" }, + { url = "https://files.pythonhosted.org/packages/5c/cd/440c798957e14e31776bfeb024d8fafe0bb1d5b89c51c2f067e69938f7b0/ast_serialize-0.8.0-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:6f18048fe9f6dd266bd577cdec48bdcecb74faaa01fe941324435483b013ed2a", size = 1454335, upload-time = "2026-08-07T11:28:55.968Z" }, + { url = "https://files.pythonhosted.org/packages/4f/4a/587eb36dcc240a54c8660f599464516b469ecad96f0dbdb6bccbedb50745/ast_serialize-0.8.0-cp39-abi3-win32.whl", hash = "sha256:31883542dd6c94d178f5db3d32fbd69c5eb88b3a7c018e7ac8cc0c45195ddbed", size = 1068858, upload-time = "2026-08-07T11:28:57.541Z" }, + { url = "https://files.pythonhosted.org/packages/5f/a4/3e887bbd92164e183cb6e412c6a3e9198ddd446d7fe405958293ef5ef49c/ast_serialize-0.8.0-cp39-abi3-win_amd64.whl", hash = "sha256:861794565b06337005c1447ef23103a3d5a627d08bdc827870d00d0b28ef5f51", size = 1111839, upload-time = "2026-08-07T11:28:59Z" }, + { url = "https://files.pythonhosted.org/packages/25/6c/b400476d3ceba681ab929787edc9554f6d88fcc69435eb681b00fc0457a5/ast_serialize-0.8.0-cp39-abi3-win_arm64.whl", hash = "sha256:b2a5978662fd4db463dfb4b974d2b10ac6430b98f5333aabc7051909df3561d0", size = 1083655, upload-time = "2026-08-07T11:29:00.349Z" }, +] + +[[package]] +name = "build" +version = "1.5.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama", marker = "os_name == 'nt'" }, + { name = "packaging" }, + { name = "pyproject-hooks" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/78/e0/df5e171f685f82f37b12e1f208064e24244911079d7b767447d1af7e0d70/build-1.5.0.tar.gz", hash = "sha256:302c22c3ba2a0fd5f3911918651341ebb3896176cbdec15bd421f80b1afc7647", size = 89796, upload-time = "2026-04-30T03:18:25.17Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0d/fe/6bea5c9162869c5beba5d9c8abbed835ec85bf1ec1fba05a3822325c45f3/build-1.5.0-py3-none-any.whl", hash = "sha256:13f3eecb844759ab66efec90ca17639bbf14dc06cb2fdf37a9010322d9c50a6f", size = 26018, upload-time = "2026-04-30T03:18:23.644Z" }, +] + +[[package]] +name = "colorama" +version = "0.4.6" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d8/53/6f443c9a4a8358a93a6792e2acffb9d9d5cb0a5cfd8802644b7b1c9a02e4/colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44", size = 27697, upload-time = "2022-10-25T02:36:22.414Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", size = 25335, upload-time = "2022-10-25T02:36:20.889Z" }, +] + +[[package]] +name = "generic-rag" +version = "0.1.0" +source = { editable = "." } + +[package.dev-dependencies] +build = [ + { name = "build" }, + { name = "setuptools" }, +] +dev = [ + { name = "build" }, + { name = "mypy" }, + { name = "ruff" }, + { name = "setuptools" }, +] +lint = [ + { name = "ruff" }, +] +type = [ + { name = "mypy" }, +] + +[package.metadata] + +[package.metadata.requires-dev] +build = [ + { name = "build", specifier = ">=1.5,<2" }, + { name = "setuptools", specifier = ">=83,<85" }, +] +dev = [ + { name = "build", specifier = ">=1.5,<2" }, + { name = "mypy", specifier = ">=2.3,<3" }, + { name = "ruff", specifier = ">=0.16,<0.17" }, + { name = "setuptools", specifier = ">=83,<85" }, +] +lint = [{ name = "ruff", specifier = ">=0.16,<0.17" }] +test = [] +type = [{ name = "mypy", specifier = ">=2.3,<3" }] + +[[package]] +name = "librt" +version = "0.15.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/36/9b/356320fbae2ac8467e21c5e73e1389c80468e4998c62cc7d3536cc51b614/librt-0.15.0.tar.gz", hash = "sha256:4e66cbe84437497d951b799d3e1551291b6fb3d643820a7014b3655d57a59162", size = 214338, upload-time = "2026-08-07T10:49:42.663Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/59/52/06790ced2ac7117f890c21bda43c39c958ec82aa665c0718e821d33ff939/librt-0.15.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:823b92cf3c18ecd08afc70c42473888b41b6e8ef5046f3b82c05c154a2fa3d22", size = 148039, upload-time = "2026-08-07T10:46:41.165Z" }, + { url = "https://files.pythonhosted.org/packages/e7/1d/8e150b7fc449a1f33c8a760965cc1f43b14fc1577d9d0b50ab2701420e74/librt-0.15.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:c70bc1b602cf59917e8f0c7a2cbc8bcc6fbc14d5486136b00707a79619121d63", size = 153067, upload-time = "2026-08-07T10:46:42.418Z" }, + { url = "https://files.pythonhosted.org/packages/51/87/a162bc5a66a35599dc619ecb215145f4de7d68e886b479b6d12593139f7c/librt-0.15.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:814ff83a25b5fce8b9c80c4dd803153fb5c5599fc74db9e022466938368957ef", size = 493087, upload-time = "2026-08-07T10:46:43.657Z" }, + { url = "https://files.pythonhosted.org/packages/e5/3a/aeea1fc620cf48060d3065b37614edbf97043c099d0f50782bc8ca61d897/librt-0.15.0-cp311-cp311-manylinux2014_i686.manylinux_2_17_i686.manylinux_2_28_i686.whl", hash = "sha256:57f5eeb6ad4c180de583b1038e61fe5fbd9796bb69a8a1c1a0c7ddbec4c8c60f", size = 485608, upload-time = "2026-08-07T10:46:45.038Z" }, + { url = "https://files.pythonhosted.org/packages/52/ff/fe571ad416f0856fd0d5578ffc2e6dc531891e586e36b647bcf50569cab8/librt-0.15.0-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:82909c8f7eb9952656b65d3147afde4cf8e6d5a991eebc86418b5e65843b0ab8", size = 498723, upload-time = "2026-08-07T10:46:46.35Z" }, + { url = "https://files.pythonhosted.org/packages/0f/e1/7a65eb5dedb1f00aebd948cdd8e17add48bf066cab3514e9daf84ab45a6c/librt-0.15.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f779070399f991400fc451719e0ea388eb7de313388bada2c127a35de05f798a", size = 516002, upload-time = "2026-08-07T10:46:47.599Z" }, + { url = "https://files.pythonhosted.org/packages/5f/45/59832b0ebfbd08c2742e6ece372ceb53f18bf1faef5d33c8daf3abebf749/librt-0.15.0-cp311-cp311-manylinux_2_34_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:bac89069bc496ebdf4f79ebb57bbd10d0b214c8454225deb672d91002bd17e18", size = 508607, upload-time = "2026-08-07T10:46:48.873Z" }, + { url = "https://files.pythonhosted.org/packages/ea/0d/37fa73f3b43ebd8259f91ae9102a15e5a54e65d581e48dea72df3e81d7a4/librt-0.15.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:e0d00c708fb2f5822b152429b1ac80a58dbbbc3f6c232c4d13a3f7fcf2ea5b4c", size = 530422, upload-time = "2026-08-07T10:46:50.45Z" }, + { url = "https://files.pythonhosted.org/packages/26/02/e046c6fe7a5881ac34623242192f484426ba8a75595fd18f22c53a3f530f/librt-0.15.0-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:6c6624fe268625869485553dd7cc1daf30d22558215bb2a4ff16f67a9801a31a", size = 534303, upload-time = "2026-08-07T10:46:51.693Z" }, + { url = "https://files.pythonhosted.org/packages/95/32/d5e6d861ab0366f3edf74f887ab0c9eb9f535aaf01d32b80b4f734daa179/librt-0.15.0-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:f56b397858a23dacf35ede366ed2212fdc03a6a57a1ad36468ad6e9dc5fac091", size = 536084, upload-time = "2026-08-07T10:46:52.951Z" }, + { url = "https://files.pythonhosted.org/packages/2a/de/d69d725513fe53fc90c6d7a1f86e4428939bad2fb905b17fe4c18d413dde/librt-0.15.0-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:4388184646efe2054911c5b00a1077d6d1ee86a95b7e8ba96dc7850a809f3f40", size = 514307, upload-time = "2026-08-07T10:46:54.194Z" }, + { url = "https://files.pythonhosted.org/packages/36/93/f8aded0d6682b4f25820fa86e0690f87f01df9fd7bd09ddb04d9167ad021/librt-0.15.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:97335f59082f9fe2ce6c2a9cc6433a0114bbb6cd4d5c09dd76c95c68b9f9a8b0", size = 557686, upload-time = "2026-08-07T10:46:55.443Z" }, + { url = "https://files.pythonhosted.org/packages/74/09/ffeb6bdeb6cd862b4272fddc8ad05f938dd25d020ed517e631813917d80a/librt-0.15.0-cp311-cp311-win32.whl", hash = "sha256:83380ffde38062a2e9bb55d83e74474f6614665528b98a6928720fc006dfffbb", size = 104917, upload-time = "2026-08-07T10:46:56.605Z" }, + { url = "https://files.pythonhosted.org/packages/96/28/7e2313a3ffbf0b4de7ba3da58a09e488507b4bd1ea2b5e69378354a23415/librt-0.15.0-cp311-cp311-win_amd64.whl", hash = "sha256:f75720477ee05d509a310e856cacc8d909adc182f7b91193c207bcc26d7ee6db", size = 125886, upload-time = "2026-08-07T10:46:57.729Z" }, + { url = "https://files.pythonhosted.org/packages/39/9e/04b8c3cde014ef255ee785730425268354543acc38902093a40afa0dc164/librt-0.15.0-cp311-cp311-win_arm64.whl", hash = "sha256:256237037a3ab001ae8d9803b2d43562a4c3aa38739843694349e4d5ebb0fd56", size = 111885, upload-time = "2026-08-07T10:46:58.787Z" }, + { url = "https://files.pythonhosted.org/packages/ba/39/99c25030e782bdfb7a21be8c05254806a2e4bbb05c8d50c2a2130acbfa05/librt-0.15.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:e87bc679f86a99aa3b26e3c78eeb821a247c9a28eae48eaafcc32c3bf4c3bb9e", size = 151021, upload-time = "2026-08-07T10:47:00.057Z" }, + { url = "https://files.pythonhosted.org/packages/14/43/f4b1bd1b2888798a1409808889a25ea1ba49eaabce7d681ed27734c2df9d/librt-0.15.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:71599e011ac880e8e45d46047d714871894c7d4ab6f25626f8d4f89da21f368d", size = 155267, upload-time = "2026-08-07T10:47:01.311Z" }, + { url = "https://files.pythonhosted.org/packages/0c/db/3ad9c965c72f1e1d6beeec44ec10a54e17be8ae042fbb4baade16cbadced/librt-0.15.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c802434092b769b1d613ed2e13fac15fbfce1934a74bd10283b03c0fae231cd1", size = 503136, upload-time = "2026-08-07T10:47:02.45Z" }, + { url = "https://files.pythonhosted.org/packages/4b/07/5888a6d76acd62ebce66c61b74d94e9370b9c32929f111e487bb6546f8ed/librt-0.15.0-cp312-cp312-manylinux2014_i686.manylinux_2_17_i686.manylinux_2_28_i686.whl", hash = "sha256:5500eeae393a184d14e1f35645962c27129d20c81afa4069e6ef826ebc2b3aaa", size = 496670, upload-time = "2026-08-07T10:47:03.675Z" }, + { url = "https://files.pythonhosted.org/packages/29/39/ab57cc2f5b276156da02bb7f5a8921bada1cb1993ffec99acf811c602c23/librt-0.15.0-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:6ecfc32dfb46fb7b565bcd6abf9412acf978775a998273d22888a6d7953730dd", size = 513688, upload-time = "2026-08-07T10:47:04.981Z" }, + { url = "https://files.pythonhosted.org/packages/a7/b9/bdbb0b648b5c2befb031f4c6f3b1dd857415e8fb492a25a3c764a6681e6c/librt-0.15.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:89cc46cfd15022e35084355478c9ac809d90b1152222706ac9a7655ec21df6fa", size = 531904, upload-time = "2026-08-07T10:47:06.211Z" }, + { url = "https://files.pythonhosted.org/packages/93/26/473c2e4b6c104e9e58e27ce95fc8005c8bd4fc36cae4f254371125a92db8/librt-0.15.0-cp312-cp312-manylinux_2_34_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:d5f51401d102c885b9ca509e62c79b1dbff286e1b9b047fde6f763780789356d", size = 524427, upload-time = "2026-08-07T10:47:07.592Z" }, + { url = "https://files.pythonhosted.org/packages/26/60/03b3abb82b41714671b907bf6989b228e31e6a8af52dec82b5b0728dc250/librt-0.15.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:cc30523e3f1a23fb7511cc659834a0d01a1042bb9de359bc1c131cc4ec6c9656", size = 543155, upload-time = "2026-08-07T10:47:08.866Z" }, + { url = "https://files.pythonhosted.org/packages/f2/0e/9bb1f0a4affbd0a1888f4f79dc03ed2a299d9a2c26c59ab2a97dcbf11903/librt-0.15.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:59fe030d8ae4a57e3fb7756bf35a858de74e04066fc8555c53d0af979132af81", size = 546890, upload-time = "2026-08-07T10:47:10.327Z" }, + { url = "https://files.pythonhosted.org/packages/dc/84/6937a280d461f7de6e031ffb02edc2b7c3c90d49d630565ce8ff27cbc5f2/librt-0.15.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:5a6526a2a956bbb1e4ae3568c82e650fc99119c66bb011ea60715744955a2b4d", size = 555163, upload-time = "2026-08-07T10:47:11.798Z" }, + { url = "https://files.pythonhosted.org/packages/bc/95/2a2853c1ee014bf102116e7f897a04beeaeb2461b45b79af98bdfb95f1ef/librt-0.15.0-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:85ea21ec6730194d67156b0e0b5430ccb1d61f8b8b907e39b37f9812b74a13f0", size = 535812, upload-time = "2026-08-07T10:47:13.279Z" }, + { url = "https://files.pythonhosted.org/packages/c9/4c/cf9601c1b4c5f09280acd5d83abdb2e68527a2be8257136eb42304218622/librt-0.15.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:1e47b8ba865d7ede071a91a7163073bbaeb72541f1ef8a07d512c45c7b5007f2", size = 573688, upload-time = "2026-08-07T10:47:14.727Z" }, + { url = "https://files.pythonhosted.org/packages/47/6d/9ac7cbec46189a7625af4b5acbd25f10d827f4141b2002181848c8418923/librt-0.15.0-cp312-cp312-win32.whl", hash = "sha256:a5207ec414d1c4a2a7231b2086970dc036f94293cdf338190984958a013a42f1", size = 106138, upload-time = "2026-08-07T10:47:15.973Z" }, + { url = "https://files.pythonhosted.org/packages/38/d0/2ae99c83be86ce23f925ac1aeeedc777e97f427c4a8d190c70d0a16e9a87/librt-0.15.0-cp312-cp312-win_amd64.whl", hash = "sha256:73b30cfa976659b3917c8f6153bdb0591c6a9ec6583599fd24a689b690622022", size = 126974, upload-time = "2026-08-07T10:47:17.049Z" }, + { url = "https://files.pythonhosted.org/packages/5d/ef/dd24f9635c730b86b87587967dda7516b1845e8b17684603d31607fed598/librt-0.15.0-cp312-cp312-win_arm64.whl", hash = "sha256:a54cf9e0ef47b96af580849db5471142200568ce1e02cbf416addab551369570", size = 112292, upload-time = "2026-08-07T10:47:18.222Z" }, + { url = "https://files.pythonhosted.org/packages/e7/42/467b53a601b406ccd7b97c1fd54b59cb34f9185ad5ce7e9d5c3c4e8961c8/librt-0.15.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:db13ca398005abcbe538deda87b686d9bd08b7001cf40c4c06b444960ae10a26", size = 151029, upload-time = "2026-08-07T10:47:19.312Z" }, + { url = "https://files.pythonhosted.org/packages/3e/e6/36c2299b7a94b84fdd01220d8a777a71be5be0925bb0dbdf71c0a06a34d9/librt-0.15.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:aa1f1995789dca3698bc550aaceb09a51bd5df0a057ff84ff15296cd1975b801", size = 155194, upload-time = "2026-08-07T10:47:20.398Z" }, + { url = "https://files.pythonhosted.org/packages/c9/b6/ed5071f9325845e670bd36012757419767fbf56af77ed483077b9e4db541/librt-0.15.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:55456ea87d8df21808446d03817be2f65e20391c1c615d9187440dff28cd08dc", size = 502568, upload-time = "2026-08-07T10:47:21.652Z" }, + { url = "https://files.pythonhosted.org/packages/7f/81/6450c67c3615d87704bcbc21323fafc69c799b06a044c447529f725d4b01/librt-0.15.0-cp313-cp313-manylinux2014_i686.manylinux_2_17_i686.manylinux_2_28_i686.whl", hash = "sha256:5a86a5a08c2235316bdb359d5dbb6ce0abfca7fac06363103e2c5af571d92f95", size = 496153, upload-time = "2026-08-07T10:47:22.925Z" }, + { url = "https://files.pythonhosted.org/packages/e1/d6/5f52b722bc75076954b3bfd49be15ea362df4d580c6fb315d0f617100d30/librt-0.15.0-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:e56b6a368529bed262da40ce13f8fef590db0479819cca84f16a1f01ac356d0b", size = 513336, upload-time = "2026-08-07T10:47:24.213Z" }, + { url = "https://files.pythonhosted.org/packages/8d/e2/c08fd1d36ce63ea5a12b85c5d37f4550b5f86a692167e41e5a74222607ae/librt-0.15.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:234d8d394721fa0d786af15ebf1f3fb7f3ed82fd1cd0cde45c2f247b5d4281d2", size = 531661, upload-time = "2026-08-07T10:47:25.507Z" }, + { url = "https://files.pythonhosted.org/packages/3f/d8/d9482fcbeb177b9eb87bb3899eeb3b42be690313c652f9e146b1d0681fb2/librt-0.15.0-cp313-cp313-manylinux_2_34_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:d8363d7accb0286ac3a0e633f396e93800dafb8150494505daf9515bbda591f3", size = 524487, upload-time = "2026-08-07T10:47:26.79Z" }, + { url = "https://files.pythonhosted.org/packages/10/cc/075171517b41f861753034fbb151b42cfc83bcc853849f24f5e66fd60ccf/librt-0.15.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:0f0ee3644d951f31055ad07d77d92520e84505dd7a432cc4cd501dd70ee06785", size = 543201, upload-time = "2026-08-07T10:47:27.999Z" }, + { url = "https://files.pythonhosted.org/packages/b0/03/42c2330f37eeb475b6affeedd06518f60035f323af3a839335e3fc9fef2d/librt-0.15.0-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:2cfd1a81a648806e6a7717be4cc4d1bb392fa229752bf8444ba365e381e984d6", size = 546467, upload-time = "2026-08-07T10:47:29.396Z" }, + { url = "https://files.pythonhosted.org/packages/57/1e/1ad4c5638f7e64d8560328bd25c54b409a661bdb6ff254b38ff90744288d/librt-0.15.0-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:a6cd22c9da0d866558e46a041f1cc0c2bbb26b61b137b2347fa834c332e1d101", size = 555139, upload-time = "2026-08-07T10:47:30.815Z" }, + { url = "https://files.pythonhosted.org/packages/49/41/39fa7d15db1204cd1cbe6514680fbdc243adf754a0885061308f43afc013/librt-0.15.0-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:6d5225ef8801e4ea5e482fa9b5dfb891dd9ef6f6d870f1f25d449ca2c70ac218", size = 536050, upload-time = "2026-08-07T10:47:32.222Z" }, + { url = "https://files.pythonhosted.org/packages/1e/88/c6dcf0dd8e26dc0c9a499a2abab8646c86dcaf9ecea9524cb46d3686331a/librt-0.15.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:6d28a05796b99f749bf8794f17ba9ba1612d0076b802e9cfc62c554634e9ce3b", size = 573700, upload-time = "2026-08-07T10:47:33.527Z" }, + { url = "https://files.pythonhosted.org/packages/1b/9b/ab54c71a7918a7c34fa5327fb61390a77446a07a146fbfb1165250a61035/librt-0.15.0-cp313-cp313-pyemscripten_2025_0_wasm32.whl", hash = "sha256:2067ff438048cead9d223ca5675bae2a25e520a7c3e6c1498bf9c6892d22caab", size = 82194, upload-time = "2026-08-07T10:47:34.835Z" }, + { url = "https://files.pythonhosted.org/packages/8d/b2/4f9a243bb892395f3becb80789ade13771701091f9f07ab8230247953ba8/librt-0.15.0-cp313-cp313-win32.whl", hash = "sha256:1cd3b721f24c206398b9e26da3c3a9c011e6e89d06f318ba8ebefc30f1003890", size = 106231, upload-time = "2026-08-07T10:47:36.251Z" }, + { url = "https://files.pythonhosted.org/packages/bf/af/64aff4885a40b93132382f2c314647d722574605416504379184ef3045ea/librt-0.15.0-cp313-cp313-win_amd64.whl", hash = "sha256:f395a4a9a03ac062dbe9a9f82e0c720502e590a38feee6a757bc82e9c63afbd8", size = 126996, upload-time = "2026-08-07T10:47:37.453Z" }, + { url = "https://files.pythonhosted.org/packages/27/83/335bccf6c7cb9028cb0b54aead27d9ece3f01f83bc6baa2abace5da655c1/librt-0.15.0-cp313-cp313-win_arm64.whl", hash = "sha256:0a15cb554761247d84a3ec0cbdf4078d70725384f0e4662c0fa3b26266eb60ad", size = 112188, upload-time = "2026-08-07T10:47:38.729Z" }, + { url = "https://files.pythonhosted.org/packages/a8/93/949053fb462eecc4a9a5ee770a81f4b40be7b79538b245545d4aebc6b58b/librt-0.15.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:f5de7feedc56337a088eb15cd9fafa9938367362221d8cc62c642b7f94821993", size = 149833, upload-time = "2026-08-07T10:47:39.86Z" }, + { url = "https://files.pythonhosted.org/packages/61/ca/8281aa6cd560a3420e4497729f6b704b53be3eeaaef82d5aeadddaf7441f/librt-0.15.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:6c0eb900c0e91f4aebe680845242e614f1864edfd44106380d0752ac29522bf8", size = 154088, upload-time = "2026-08-07T10:47:41.065Z" }, + { url = "https://files.pythonhosted.org/packages/dd/02/1a1662dceaba6a086360891448d5ce9a7d3555976cae59a31a39d744b9c7/librt-0.15.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e8c9a650a188e38bac005048cbe6342e81407782944d01934540ab75e417df21", size = 494215, upload-time = "2026-08-07T10:47:42.388Z" }, + { url = "https://files.pythonhosted.org/packages/69/84/99211619dc656370a3740c33d2b0b6d5a3fb1e73689314f6ed477a397dc4/librt-0.15.0-cp314-cp314-manylinux2014_i686.manylinux_2_17_i686.manylinux_2_28_i686.whl", hash = "sha256:92bfed8deec93df30286b9fe9e3b1dd17329cc076a192b4ee5ec223841d54953", size = 491173, upload-time = "2026-08-07T10:47:43.683Z" }, + { url = "https://files.pythonhosted.org/packages/d4/aa/5448d0b05f4579b635d3899176817ebf561af0e57bacd425b5b1887264c1/librt-0.15.0-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:ec4b19788f835711a2072f9dbe6b03b3bf32ed1f0fb30cf399bdd59d9f0c33fa", size = 505512, upload-time = "2026-08-07T10:47:45.314Z" }, + { url = "https://files.pythonhosted.org/packages/95/82/01940e40b83c43a546c4a3c896cf34ca272a9690899d55914e4827b3dcce/librt-0.15.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d4c7bacb70930f3d0a56f4ecf1be474a1f0d941b01dd73b756f3c256d42cb879", size = 523073, upload-time = "2026-08-07T10:47:46.66Z" }, + { url = "https://files.pythonhosted.org/packages/88/fa/759c0030f3ee371439eb26de34fc745807caf0abb878af7af4b8b7c3dd3d/librt-0.15.0-cp314-cp314-manylinux_2_34_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:3e79f05e4a08b4d880342673312bbc895b56df7765605796f15902eb5367d3ae", size = 515080, upload-time = "2026-08-07T10:47:48.319Z" }, + { url = "https://files.pythonhosted.org/packages/0b/27/894e072228fcb159703c655da69f8cd10dbed489c36e3df7dd032a2483be/librt-0.15.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:a417149c0cba4d50b61e992e5a15e69eaf96746609b461cc4ed168aeef6b79dd", size = 534164, upload-time = "2026-08-07T10:47:49.875Z" }, + { url = "https://files.pythonhosted.org/packages/98/a3/0078e91c1f36f8815db17827de15650b9a3fe56c55fbf998c854b34e40d3/librt-0.15.0-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:da7a94d6a3411f579d72aa3e3bc5fbca7ed4549f3dbd7e5de3aa567333374285", size = 540616, upload-time = "2026-08-07T10:47:51.408Z" }, + { url = "https://files.pythonhosted.org/packages/86/33/81a29b796dd52a45e9ef7974c7732926e8f10f15b8d2be505665979f896d/librt-0.15.0-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:856f743ae607f2c1380eccb566c0038a9fb3eabf0fc2be2704d76d9f73557239", size = 545890, upload-time = "2026-08-07T10:47:52.818Z" }, + { url = "https://files.pythonhosted.org/packages/05/82/8be1baa1350e5d30cfd70ae79d0a6f4dc5862ef47f7bb2808aabc9bb86e5/librt-0.15.0-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:779a6e7c894737e5983e7790a9c78c4000c30e23c9aada08081bdbea53b0fa60", size = 523287, upload-time = "2026-08-07T10:47:54.165Z" }, + { url = "https://files.pythonhosted.org/packages/c6/4f/d1be6a01a35c20ef734e0e44113f87d4af756a9354a89dcfbe3b4f8af5e1/librt-0.15.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:96bb17dbe8bab3c0954fbebfc69ed395599de75b6bbc35e3270a878e15d4dd65", size = 565868, upload-time = "2026-08-07T10:47:55.566Z" }, + { url = "https://files.pythonhosted.org/packages/67/88/649cfa33f5825927b160610f670bdab012a64d627eddb94fa795ea4292fd/librt-0.15.0-cp314-cp314-pyemscripten_2026_0_wasm32.whl", hash = "sha256:7220697efaa6e5348fc3d18ee7f8563d4bfecd9872b37ffb915bfc1d08840622", size = 81619, upload-time = "2026-08-07T10:47:56.886Z" }, + { url = "https://files.pythonhosted.org/packages/22/31/8e88a8d5e48fc8d1a817787fb6811dfff6499acd6c8683dd83934aa6ede0/librt-0.15.0-cp314-cp314-win32.whl", hash = "sha256:f54598964d357b1c5ab77cf5d92f21e598fe0e23cdbe9618480807f81b4eba15", size = 100138, upload-time = "2026-08-07T10:47:58.093Z" }, + { url = "https://files.pythonhosted.org/packages/80/92/20fd6c4b6a1b1a564b076d55cd3d427d8428217d7638dc25a654cc4791d4/librt-0.15.0-cp314-cp314-win_amd64.whl", hash = "sha256:3ff5893a2c23d886aa9ce786de5ac6ddc74aeeaf90743682b74d920e117d2e28", size = 121258, upload-time = "2026-08-07T10:47:59.564Z" }, + { url = "https://files.pythonhosted.org/packages/fc/28/6af430b44d9ebb897b865a3c363b6dcace51357be2347cc0f8f869656a86/librt-0.15.0-cp314-cp314-win_arm64.whl", hash = "sha256:3722a099730704c9a3d70c879fc0f51daec25fe5f1555672d97bc595abeafb95", size = 106467, upload-time = "2026-08-07T10:48:01.097Z" }, + { url = "https://files.pythonhosted.org/packages/7e/aa/b42bb798942ced219f6d63b27e07f91237887a8d0bd0921666db79a13790/librt-0.15.0-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:38c0c7d4b6fc06c3324b3f9162c8391bfc4fd9dde53afe1033ce7edb48d5a714", size = 159523, upload-time = "2026-08-07T10:48:02.442Z" }, + { url = "https://files.pythonhosted.org/packages/75/03/1b53cd4ef904e73b1d828a5f90143bf94a2967d7cfff0b9ccf93e12aa9b4/librt-0.15.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:8b2fdd7ead3c995c37940a790690660d0ca006c302db26cc51933f6766866fc3", size = 161638, upload-time = "2026-08-07T10:48:03.725Z" }, + { url = "https://files.pythonhosted.org/packages/ac/c4/9f9c9fba097d49e9e694c2b4dc331df31884645ecbc58a93b4b5fc69d2c5/librt-0.15.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2fde98cf1fc4bac144ce23c2c4c017b924ba714509ea9334977b0b27050c837d", size = 701795, upload-time = "2026-08-07T10:48:05.135Z" }, + { url = "https://files.pythonhosted.org/packages/4c/05/0966840bda0380c8ae167b9043c6230202941cc90ea29c48e096964c765e/librt-0.15.0-cp314-cp314t-manylinux2014_i686.manylinux_2_17_i686.manylinux_2_28_i686.whl", hash = "sha256:e3b461183c5fa7681b48560f91515f53a953122fb30c71e07abc67d7ddf58c38", size = 682147, upload-time = "2026-08-07T10:48:06.555Z" }, + { url = "https://files.pythonhosted.org/packages/18/af/1c47ca573c30ea47d195aec26133af522fea1104afaace028d7b32247ea8/librt-0.15.0-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:4bbcc257e3babea20a91715c361b24554ec4e8f51aa578568afc230799fe1a19", size = 696397, upload-time = "2026-08-07T10:48:08.03Z" }, + { url = "https://files.pythonhosted.org/packages/2e/0f/1aed6223d4f9f9d1171a8596ff100ea4c3f7699fea7a4ba657c3e60daa6c/librt-0.15.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b845b8d48088fad0cadc84be4b8fda63203be7e9237b71015b3925443c1f35ab", size = 722542, upload-time = "2026-08-07T10:48:09.569Z" }, + { url = "https://files.pythonhosted.org/packages/c6/22/9e3a929aea456c97d69e6ef3884efea56d4807f97399471cc946baebd8af/librt-0.15.0-cp314-cp314t-manylinux_2_34_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:b30e600e8f337b9bd7f39b86d9fdfedc73cc46e3d0f745931a23a234220bb7e2", size = 729709, upload-time = "2026-08-07T10:48:11.129Z" }, + { url = "https://files.pythonhosted.org/packages/e9/1b/c327ef6018e3a9ca0b8e7c5eddeeb331ba8f9b76c24e126d37d0f6d62faf/librt-0.15.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:64b0c8c35aa4c4ed79896359f3e0b285cbe4e610042106500da4811c322cc108", size = 752891, upload-time = "2026-08-07T10:48:12.558Z" }, + { url = "https://files.pythonhosted.org/packages/d7/d1/d5f1ea02c56930087009e39db9b70660a663e76c730b27b925d786718457/librt-0.15.0-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:0da0d94cb802f32a0524653e7201f2cef72d5f700a5407678f5290483d4fcd08", size = 745301, upload-time = "2026-08-07T10:48:14.55Z" }, + { url = "https://files.pythonhosted.org/packages/d9/3c/5f7c585d15ebb2250c73e7c0ee4e9e47be72c65d520c07ddbcdc62037674/librt-0.15.0-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:4a6369168d371207339b1e50d4532b06a7121586141f82599505a3f315751d47", size = 747921, upload-time = "2026-08-07T10:48:16.453Z" }, + { url = "https://files.pythonhosted.org/packages/7f/52/1443a446486eba966bcbca1696b472e4f210320ec42f490a47f48fbf0fdc/librt-0.15.0-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:c434e072557ade9cbc642d052c89d031efe47d5c9614523619d0d74a02378e81", size = 727561, upload-time = "2026-08-07T10:48:18.089Z" }, + { url = "https://files.pythonhosted.org/packages/79/91/2270a9380f11725cf83ce1925a5e32dd1dde2be9bba597f25c10a38644e7/librt-0.15.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:c7eec6a42018bc1d45763b1c162d3d2bf7c3b9a1b0ed30d3e91dcba390efefcc", size = 774417, upload-time = "2026-08-07T10:48:19.611Z" }, + { url = "https://files.pythonhosted.org/packages/9e/3b/f4b1548d4f5b99186737fe27aec238e9823e8d5d23bf4df007c030689dc5/librt-0.15.0-cp314-cp314t-win32.whl", hash = "sha256:6912fa5e635d74529ac7cdb1bdf6ca3af4453da8d1edbe0110ee1cb4ad407ebf", size = 104381, upload-time = "2026-08-07T10:48:21.048Z" }, + { url = "https://files.pythonhosted.org/packages/80/b6/134afad262def1de04c0843c376d02135f1168af43f22e09a52bd8394727/librt-0.15.0-cp314-cp314t-win_amd64.whl", hash = "sha256:8e11699ed745931c395acd3621b07062e0f840efa6935aad87a64ed0995f0915", size = 127034, upload-time = "2026-08-07T10:48:22.561Z" }, + { url = "https://files.pythonhosted.org/packages/99/5f/1b6846b20572bd699c9e9ec321a5f781845bee477df2aa2a43b28bc40119/librt-0.15.0-cp314-cp314t-win_arm64.whl", hash = "sha256:5d2a91724463bfed4f573cd7a9fdc856d2e230d0c0e5a61416a93481dccd8605", size = 110827, upload-time = "2026-08-07T10:48:23.804Z" }, + { url = "https://files.pythonhosted.org/packages/c6/44/4de9f4ddadb009a55c7758eb5736d62534a7daaf27bd71bc50e64b606b06/librt-0.15.0-cp315-cp315-macosx_10_15_x86_64.whl", hash = "sha256:8443e38dcfcfdbcf5add5118c623efd788d65ac2e25756d6251a54a06a4d0aca", size = 149843, upload-time = "2026-08-07T10:48:25.148Z" }, + { url = "https://files.pythonhosted.org/packages/1f/eb/5d9ab71e30119c44094e0275f38b47dd327aea0f843a080396677029d508/librt-0.15.0-cp315-cp315-macosx_11_0_arm64.whl", hash = "sha256:6d15a29033c57490cfe2069097c6fc4049e4e65ffbb749be7dc453b7c4c68965", size = 154510, upload-time = "2026-08-07T10:48:26.485Z" }, + { url = "https://files.pythonhosted.org/packages/d0/9c/8505d1b8f5e8c19587bd03f7429993b3e9ce5c06819d856bfb11d919374c/librt-0.15.0-cp315-cp315-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d2c05c729b589e734c09578bf5964be48a911765484840d017bbc84f49d4c4ad", size = 497543, upload-time = "2026-08-07T10:48:28.045Z" }, + { url = "https://files.pythonhosted.org/packages/1d/9a/3a8390775cb095765aded027ac9c63e7c8ea74e731498607544c6505de0e/librt-0.15.0-cp315-cp315-manylinux2014_i686.manylinux_2_17_i686.manylinux_2_28_i686.whl", hash = "sha256:fa60887537e1d0cd2d9982269d33a709bf54b195cd2b9364fc0a758022af5bd9", size = 480452, upload-time = "2026-08-07T10:48:29.531Z" }, + { url = "https://files.pythonhosted.org/packages/e7/40/258a4a7117ee915d66de5cd9b8ade65a440993161107ce3a686f1859955c/librt-0.15.0-cp315-cp315-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:d8bc24219b24c0af375718942ab75e3544b2763085f40f965be4326734ae8328", size = 507768, upload-time = "2026-08-07T10:48:31.007Z" }, + { url = "https://files.pythonhosted.org/packages/6b/c6/2f4dd296c97a0b85b98894519b279408ec9dd602d4f692b1ea0e25dee670/librt-0.15.0-cp315-cp315-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:86a21a7bd3fe3a419512ef424cc1c020f6771d0b29cfddff36d1635a855e63f0", size = 525122, upload-time = "2026-08-07T10:48:32.7Z" }, + { url = "https://files.pythonhosted.org/packages/49/dd/29eab42be13b2bf0ea8cb227135a45d44693e30a7e8b92871981ff56b82b/librt-0.15.0-cp315-cp315-manylinux_2_34_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:dbab647e88d90b3167b91efe7091e248653688ed4337e4f90907a722c7361bb9", size = 520371, upload-time = "2026-08-07T10:48:34.294Z" }, + { url = "https://files.pythonhosted.org/packages/91/ed/4bad71adeca8fe208b775c2a35417fa5a2584c8f4791daaf89a89450fea1/librt-0.15.0-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:d8edcf6f550e918dca779c069b9e156385c60b406f99fc7641f32c52f7193659", size = 537258, upload-time = "2026-08-07T10:48:35.88Z" }, + { url = "https://files.pythonhosted.org/packages/4c/63/59dba6143fdcc7240c54458b629f3250000a61b8945890fc9efd451b19c5/librt-0.15.0-cp315-cp315-musllinux_1_2_i686.whl", hash = "sha256:8b62076030baa2d8b1501a46bf0e19c27a489aa90671c55665bff7887f7660b0", size = 527432, upload-time = "2026-08-07T10:48:37.466Z" }, + { url = "https://files.pythonhosted.org/packages/ec/21/21a24c6a2327d8362580efebe77286bf47b0f4062ec5ea41766e609d3c7d/librt-0.15.0-cp315-cp315-musllinux_1_2_ppc64le.whl", hash = "sha256:d00d20d1818e82a07a0ee0aa89a98b17ed7916b92441090b683719cb20a59b6d", size = 548108, upload-time = "2026-08-07T10:48:39.384Z" }, + { url = "https://files.pythonhosted.org/packages/5a/6d/fc68c89a7971418b41f9a873623ff935cb864097544c6a2f8ce491c8ef5d/librt-0.15.0-cp315-cp315-musllinux_1_2_riscv64.whl", hash = "sha256:4e6ee93fc3cf848dcbf0cce2eca73d8e7dcd0cc2b6df3a529d57750b30a4c55c", size = 529681, upload-time = "2026-08-07T10:48:41.392Z" }, + { url = "https://files.pythonhosted.org/packages/65/7e/c2d98766124400d722063a630b0fde38a9fc768705d37eecca15c47dc192/librt-0.15.0-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:32896a0af72508ea979e0acb4e4c04cbeeae04938167950d535c83c45597167d", size = 567736, upload-time = "2026-08-07T10:48:43.124Z" }, + { url = "https://files.pythonhosted.org/packages/55/6c/f8c34a95e3a515c6e1c192b89511e7253c89a7760c6b500d57ffdb8d2dc8/librt-0.15.0-cp315-cp315-pyemscripten_2026_5_wasm32.whl", hash = "sha256:ec3ba415afaf951f6951b1dd16d3c8e4f540065fc382d7e70b823a79567ca374", size = 81673, upload-time = "2026-08-07T10:48:44.645Z" }, + { url = "https://files.pythonhosted.org/packages/c9/9e/e23fa8e78679ec45728188650b39e8ff476c83b691c96f749217df3b1b7c/librt-0.15.0-cp315-cp315-win32.whl", hash = "sha256:d2813ba2503764f0450680c533d13df7cff9b49df1411062eded5f67db4195b9", size = 100081, upload-time = "2026-08-07T10:48:46.171Z" }, + { url = "https://files.pythonhosted.org/packages/e1/dc/3eb4c5e297343f0620a55532cd7c8d764d3001fa2159212dadf480464827/librt-0.15.0-cp315-cp315-win_amd64.whl", hash = "sha256:b87d67e33afaf265262f2a66db578284b88ee2e6fcd224579cb5c15518677ad8", size = 121228, upload-time = "2026-08-07T10:48:47.631Z" }, + { url = "https://files.pythonhosted.org/packages/97/70/43abce19f04e49762f8ec834c8fafee13cc40fd6b94a72a24e534febfcd0/librt-0.15.0-cp315-cp315-win_arm64.whl", hash = "sha256:713bd7df21170b982e729e46870f31d6b437bd1a9b4648cffb529bd3c2ec5c4b", size = 106487, upload-time = "2026-08-07T10:48:49.095Z" }, + { url = "https://files.pythonhosted.org/packages/de/15/83f2deddb9368b8951ec8c9477269b5b9b8bd9bbf15e57402d0f38817dca/librt-0.15.0-cp315-cp315t-macosx_10_15_x86_64.whl", hash = "sha256:3de789c82752730f94782a5ee518baf9c05edf85733aeaf73bb6e518755cdf54", size = 159448, upload-time = "2026-08-07T10:48:50.649Z" }, + { url = "https://files.pythonhosted.org/packages/06/bf/043097353f9b3c73b583d07f6b8e552795463f4bfc8caf85e42eee50c26a/librt-0.15.0-cp315-cp315t-macosx_11_0_arm64.whl", hash = "sha256:e0b5deec9a8664eb722c797241970fd4aa1894d25fda36a1ddac0f7407606bd6", size = 161686, upload-time = "2026-08-07T10:48:52.174Z" }, + { url = "https://files.pythonhosted.org/packages/f4/2a/8ae77f9719d42ce71cd708560a3557b38ac3c17a0383e57f87084de45bbe/librt-0.15.0-cp315-cp315t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5563302a8359bc2295bb7084d1a8ed1519df96afb30eb2aa4e0bff7b54228988", size = 710668, upload-time = "2026-08-07T10:48:53.782Z" }, + { url = "https://files.pythonhosted.org/packages/61/34/c0436ea134deb9a0d6da80a396a2739a81cb31e0418f7227239e23140898/librt-0.15.0-cp315-cp315t-manylinux2014_i686.manylinux_2_17_i686.manylinux_2_28_i686.whl", hash = "sha256:22d6263b9d39d7bbb286fa791945646e3218f1be2d693e36fb630f1d0e59cd13", size = 679396, upload-time = "2026-08-07T10:48:55.645Z" }, + { url = "https://files.pythonhosted.org/packages/4a/9f/001e0d99aa9250d5cd5715a9081291a20656083459f9019cda15255329e1/librt-0.15.0-cp315-cp315t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:39ffd14646190c454f0d86e0d256b33f00a87a26ab410e619773b841d0e41416", size = 704313, upload-time = "2026-08-07T10:48:57.46Z" }, + { url = "https://files.pythonhosted.org/packages/2d/53/b34fa9d0ff00f136f4d58ebb4c411ff634baed1eb412bb602a2bc8dcafcb/librt-0.15.0-cp315-cp315t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c47318cd3a61401452de11282242937e3e057c4fd3dbaf601e269d0928a06c0a", size = 729847, upload-time = "2026-08-07T10:48:59.231Z" }, + { url = "https://files.pythonhosted.org/packages/86/ac/fa4d7a424665040e95baf480a6d523446057684b6758624c85338e8a23b2/librt-0.15.0-cp315-cp315t-manylinux_2_34_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a56a1d4f859a82ca5b99fc4b82c9b027b15e3c455c5cd99e7d0719f27bb20b6c", size = 742736, upload-time = "2026-08-07T10:49:01.151Z" }, + { url = "https://files.pythonhosted.org/packages/8a/f1/e17a9bb5de6fb8c3186ed1a7d68d21618b027ac2d3633e03d3b6109c67ae/librt-0.15.0-cp315-cp315t-musllinux_1_2_aarch64.whl", hash = "sha256:077471b3182db4e17c36ae91555f36a4d2c00080b267f749bcad34a478a9a302", size = 763454, upload-time = "2026-08-07T10:49:03.039Z" }, + { url = "https://files.pythonhosted.org/packages/1d/ec/ecd02cd30935b931b9cdbfed6ab5a099c51b280b4e7baa274da80978ed27/librt-0.15.0-cp315-cp315t-musllinux_1_2_i686.whl", hash = "sha256:411ca4d1b905b860ceba7570dd6717a71dedaddcc4b0f77ece710aa41ee11f8d", size = 743296, upload-time = "2026-08-07T10:49:04.941Z" }, + { url = "https://files.pythonhosted.org/packages/e6/b5/b3c2b8353ce820a4854f78d19321344242f89fa71c975b71132ba9bf242a/librt-0.15.0-cp315-cp315t-musllinux_1_2_ppc64le.whl", hash = "sha256:1256589e0b0adb31751d685a68bce29d73407ddf4ef05d4188f49d5dcf9566d9", size = 756217, upload-time = "2026-08-07T10:49:06.825Z" }, + { url = "https://files.pythonhosted.org/packages/3c/52/6cc22542ba59146b05cca2a656f9ff8bb67e38e63d12c3b0cc183d837bf1/librt-0.15.0-cp315-cp315t-musllinux_1_2_riscv64.whl", hash = "sha256:f42b74a53e5f26a0ba0007411a7455b66c67ce4022a39cc1f56fc4efd65bcbab", size = 741934, upload-time = "2026-08-07T10:49:08.839Z" }, + { url = "https://files.pythonhosted.org/packages/40/32/a04b72b1aa86e3be23b2ecff8c1aad2dcc955bd3956d6d26e7e34267e57a/librt-0.15.0-cp315-cp315t-musllinux_1_2_x86_64.whl", hash = "sha256:291bf73caf78b9e88d6fae9bfd693207ff7d832e2fdbe2cf8e746bc13f5f892b", size = 783763, upload-time = "2026-08-07T10:49:10.661Z" }, + { url = "https://files.pythonhosted.org/packages/6c/f0/89eb11dffbe9279ff37144dec786927314502ae0b114f1449dc78c458aab/librt-0.15.0-cp315-cp315t-win32.whl", hash = "sha256:c16d15ee371643ab48dc8248a3e680ebbeca573a13af2c3dd0c985b142d77162", size = 104313, upload-time = "2026-08-07T10:49:12.305Z" }, + { url = "https://files.pythonhosted.org/packages/6d/4a/1f1978c200f563beda63c36adff2d65bbecb81e365e8e69e572f5f70fbc6/librt-0.15.0-cp315-cp315t-win_amd64.whl", hash = "sha256:dbd605739f228912dc49027cb764456b9757750bdc2b6b7773164db7096c6fd1", size = 126889, upload-time = "2026-08-07T10:49:13.881Z" }, + { url = "https://files.pythonhosted.org/packages/38/a6/800800bfed7b1fb10fc3f3d557785c3854e80d3f7a9800d784b176a1fc2d/librt-0.15.0-cp315-cp315t-win_arm64.whl", hash = "sha256:84d244b00604d17df3fc7736c327892d6bba66181254aa4087be807b6c342bdc", size = 110700, upload-time = "2026-08-07T10:49:15.499Z" }, +] + +[[package]] +name = "mypy" +version = "2.3.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "ast-serialize" }, + { name = "librt", marker = "platform_python_implementation != 'PyPy'" }, + { name = "mypy-extensions" }, + { name = "pathspec" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/82/6a/878cc1097d4035f82bd516658d0c528d2a9955bc7b363afcbd0b07fea11b/mypy-2.3.1.tar.gz", hash = "sha256:47c1b1207258513a9d93495f69c8be9de73916186f0e52703e8c461b7a623419", size = 3992554, upload-time = "2026-08-15T03:03:38.549Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a4/be/c624d4241484f37dc62839e177ab607a9b8b3e96f0866544ca99e8e41d51/mypy-2.3.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:94f04929f1c44c35fb0061e912087edaf504acede963a4a7d00680bd089d8531", size = 13936739, upload-time = "2026-08-15T03:03:26.475Z" }, + { url = "https://files.pythonhosted.org/packages/53/84/e3cf72f90dce5960871c82551c8fba6da05fc1018f79be41c047bd126bdd/mypy-2.3.1-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f5d716048611e85ca9eefb2e1baa5d73ede389b5820ded260ea27c757d667af8", size = 14166460, upload-time = "2026-08-15T03:01:50.565Z" }, + { url = "https://files.pythonhosted.org/packages/4a/ff/6b97d58aa0f79a5ab9b472db1f6d6df1b11a51d74d0c08ab3760d3a613ba/mypy-2.3.1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b091a455111214cb5c9d54a57b9618e9a49f9fe2a42e4e1ac86e9d104ed96ce8", size = 15100476, upload-time = "2026-08-15T03:03:12.079Z" }, + { url = "https://files.pythonhosted.org/packages/da/f0/cbb4b7d2ae3ac635f6b4f2d9b04070b8a92edf50da599d3b39e5ed109001/mypy-2.3.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:df12e20c9efd614738c71b390007ecd0181125afc4ccafca04d78a1d2eed2c01", size = 15347826, upload-time = "2026-08-15T03:03:02.856Z" }, + { url = "https://files.pythonhosted.org/packages/5f/10/91dcdc6f8d43fc08e6a06ab1f9732f3abaaf835ac1b2e67b9dff56910855/mypy-2.3.1-cp311-cp311-win_amd64.whl", hash = "sha256:52eaf3a155f35cf80b40220288c861eb45f14a2340c1f6cbfbdb0feff32879d1", size = 11142615, upload-time = "2026-08-15T03:03:36.316Z" }, + { url = "https://files.pythonhosted.org/packages/3d/8a/28d54535bf4b9aa43b2d8918c2ef660378b9f66b23d78dcee052744ae622/mypy-2.3.1-cp311-cp311-win_arm64.whl", hash = "sha256:9b4eacbee8a69836c06eff6d0dd4e134a07c2b047755b30c08625fe214f322c6", size = 10141145, upload-time = "2026-08-15T03:03:07.406Z" }, + { url = "https://files.pythonhosted.org/packages/85/da/d6effc4f808a842d91edc22535dc9e799d2ff6e91449168b7f47a0771f54/mypy-2.3.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:a32bbbb940af990d3be0b8af321c7b6815bb1b3b48142fe7459b9cc5f58959ff", size = 14047547, upload-time = "2026-08-15T03:02:57.707Z" }, + { url = "https://files.pythonhosted.org/packages/e4/e6/478229701dab76f26485fc8ff5d6f241f393da22447400bbc56f6946aebe/mypy-2.3.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ff715e45b2231a8e85de1d163d1b42791e4d7aab8f5145f85fee1b710b735aff", size = 14216515, upload-time = "2026-08-15T03:01:26.496Z" }, + { url = "https://files.pythonhosted.org/packages/8d/fe/7c42327a3b21e84681f691982cbfe43f334a3685f3b683b72c376476c4fa/mypy-2.3.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:858fc57d3d91fa728e33e7ad71def60fc6272694607b306cd3292db53ae39080", size = 15307789, upload-time = "2026-08-15T03:03:31.62Z" }, + { url = "https://files.pythonhosted.org/packages/59/f4/7e597edbe01b5a56fa958ce541302dcaabfed979966f1dffedbea0ea0fc2/mypy-2.3.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:851833db876e7b650f93719c74b7879a08e338979c96054fdfc3bfd90a486355", size = 15548831, upload-time = "2026-08-15T03:03:15.55Z" }, + { url = "https://files.pythonhosted.org/packages/a3/52/cb31e084bc0314a1e384bdd677a4b80e55af04ccac077545e2238b9d320a/mypy-2.3.1-cp312-cp312-win_amd64.whl", hash = "sha256:4c5095a327483591c94e0c8d3ef9e50d4ab1369b541eae007c1f23bc2a41f6bb", size = 11226359, upload-time = "2026-08-15T03:03:29.002Z" }, + { url = "https://files.pythonhosted.org/packages/7a/47/88fcf6217b43fa2da81a8c2611370af18141536a4f0294bbf98b457d456d/mypy-2.3.1-cp312-cp312-win_arm64.whl", hash = "sha256:bbfe022634a2a195406bd469e888d2eaf193b02ba7e607391cd7640374aaae3b", size = 10214707, upload-time = "2026-08-15T03:02:48.807Z" }, + { url = "https://files.pythonhosted.org/packages/de/cf/862010ee800ca9c2bd0c4c0dacf0f092e5411824a09b8f97ad4be8fe250e/mypy-2.3.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:114dff494000f18bd10d5d95d84b8567b26da60279ecbe838131841df20e635d", size = 13964542, upload-time = "2026-08-15T03:02:21.43Z" }, + { url = "https://files.pythonhosted.org/packages/75/5a/3f3a2107b41e3e92e617e25daaee121413b91e9784bea733131ed4fecc5d/mypy-2.3.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c8637731bb5eee3671eb2c3200827aa3564ed8a9309ecee4d1afe77e6d031bdb", size = 14168922, upload-time = "2026-08-15T03:03:00.351Z" }, + { url = "https://files.pythonhosted.org/packages/8b/41/04dc4fe7e63d7820fa4eff272e95157d30cbea921388f3ab3fe77794cd0b/mypy-2.3.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1c80fbc405ed8020f5ff3802dc18cf060197bcdd3fbdd6a26ef2fd34dfdd5226", size = 15244791, upload-time = "2026-08-15T03:02:31.089Z" }, + { url = "https://files.pythonhosted.org/packages/96/fc/c3053b26b9054949285aa868cb6af8c10e7591541cacd79c5dcc06a1fcf9/mypy-2.3.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:84081f538ce27375045c02e3d7f81bd11d853400621ae245d87ce7b6c420ec74", size = 15501627, upload-time = "2026-08-15T03:03:34.128Z" }, + { url = "https://files.pythonhosted.org/packages/70/4e/d77daab008bbc4e5001374d7928f4a260d28f0e6747af444fc4763f7a310/mypy-2.3.1-cp313-cp313-win_amd64.whl", hash = "sha256:e9144ac16fde007096f9563eb2041b4433c2d705c4218edeb79e7e9d01035ee6", size = 11243961, upload-time = "2026-08-15T03:02:11.952Z" }, + { url = "https://files.pythonhosted.org/packages/f0/f8/7eb68c136e4abd30569fe31ef2bfcb7eceae9952cab80017c04cd09f5d0c/mypy-2.3.1-cp313-cp313-win_arm64.whl", hash = "sha256:77ad9529e67dca28e511f5cd5671436584ce91f6d3bac159a353158187b986ac", size = 10213219, upload-time = "2026-08-15T03:02:26.361Z" }, + { url = "https://files.pythonhosted.org/packages/be/c4/42a49d44aeff804edf1b19acce0b49e8bd1a9c57dee9605dd8d980aa43d7/mypy-2.3.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:192abaedf75da1bc0b1cef104927e70ec49c1ef0031cc4825c7ee10a438ed24d", size = 13986778, upload-time = "2026-08-15T03:01:33.69Z" }, + { url = "https://files.pythonhosted.org/packages/45/13/9331fd2dfed7194d66c5304072894a8be3e51e9deda6863c1eceaa35a43d/mypy-2.3.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:bf678dffd16efcda2c15cbd30e9ecc0081388e29ea23687a88e686ed92638dc3", size = 14188467, upload-time = "2026-08-15T03:02:40.554Z" }, + { url = "https://files.pythonhosted.org/packages/78/f7/f4a34edab45667c5465855dc585a20e87978ffa8aee711445b7239d120c6/mypy-2.3.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8e036f06b41630f4c8a1d48f9ac6aa26acc65f8be089973f5519da643318f03f", size = 15225538, upload-time = "2026-08-15T03:03:09.761Z" }, + { url = "https://files.pythonhosted.org/packages/40/05/534b3590757bd05794f73e07f6666c2a77b8597ffed795c94ce570096aa0/mypy-2.3.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:71af9c8a894e862b58e92abb08e53b05a384a1e5e5d6dc7cda59126211a53d82", size = 15480805, upload-time = "2026-08-15T03:01:41.134Z" }, + { url = "https://files.pythonhosted.org/packages/55/da/bdfba852e2562f599624af5bb7d29e36b0b4f526f2b8bac85efe0dd1803d/mypy-2.3.1-cp314-cp314-pyemscripten_2026_0_wasm32.whl", hash = "sha256:3c80cd23d85368bdd9f37d5231dfd97d35bcbf5bf41af96ef3a9b078ad1957f9", size = 7761712, upload-time = "2026-08-15T03:02:36.008Z" }, + { url = "https://files.pythonhosted.org/packages/98/31/60fc64a74cdba4f2a5d642d32317993e479163e1ac7d91b695e5d15e2264/mypy-2.3.1-cp314-cp314-win_amd64.whl", hash = "sha256:4956f34d145e145562a0a0bf367f642bbc85c04ec2baf47ae015947c3169a85d", size = 11423968, upload-time = "2026-08-15T03:02:06.931Z" }, + { url = "https://files.pythonhosted.org/packages/a9/23/eb5950b24cd26ba3b78f87707a275568d633c77dae8e61c9661be6055ca6/mypy-2.3.1-cp314-cp314-win_arm64.whl", hash = "sha256:cfb12e360242d23d91f5e978d94f58ea66acf5804c4fb6f2f794a20d4cb1b595", size = 10399323, upload-time = "2026-08-15T03:02:33.671Z" }, + { url = "https://files.pythonhosted.org/packages/82/c7/f80f4e46c0b9a00eb5f78a79d49dda8bdf56a5230f7257fb33e76be04da7/mypy-2.3.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:e5f1c50bb05b64e2026b52867e8d21106f01313c744a2c4ecc34c90d12e8d6e2", size = 15121308, upload-time = "2026-08-15T03:01:46.053Z" }, + { url = "https://files.pythonhosted.org/packages/5d/74/9b04f17c7074cc5188f02fb63a2ca1d43fedf479e84fe3091c39061a1d7f/mypy-2.3.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:667196b352f4cf304ded4c10f90cfc179263a1acfb3cdcfa984bdfd340d498bc", size = 15536590, upload-time = "2026-08-15T03:01:35.941Z" }, + { url = "https://files.pythonhosted.org/packages/26/04/c837ef6208e567774e2ed1f863f8ba6ec4817b1b6dd426315e5d559b6ec9/mypy-2.3.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b9c53e395c12cad2c6d4b67d5da7c6057638a132d85c08b73646b18f802a0045", size = 16791074, upload-time = "2026-08-15T03:01:31.073Z" }, + { url = "https://files.pythonhosted.org/packages/37/68/48730230afa45192d5bd429a6a2ff24a6f8dedda90fdf2b221792b54518f/mypy-2.3.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:18162b128c3f9c703cd35f5537446900b0d21a2549aa7a95d21380d2ef643fb0", size = 17069183, upload-time = "2026-08-15T03:02:28.566Z" }, + { url = "https://files.pythonhosted.org/packages/1c/ea/ca23fc9c20eeda09a15c9cbcf50015d0e73f409f6ead059e42aa69a608ff/mypy-2.3.1-cp314-cp314t-win_amd64.whl", hash = "sha256:30c0477d4aab7b7f39c8397dc877f2c96b9fe5588ec379f372c56eb63d599f63", size = 12154679, upload-time = "2026-08-15T03:02:04.809Z" }, + { url = "https://files.pythonhosted.org/packages/3b/67/8d982126034990869466f73b8db80dcb2234a7ac39b4dad093e047a79835/mypy-2.3.1-cp314-cp314t-win_arm64.whl", hash = "sha256:6941ab3619377bc3f32ca02876b07d27f216f5201604b664d3937ea0fdd23bb4", size = 10969159, upload-time = "2026-08-15T03:02:38.152Z" }, + { url = "https://files.pythonhosted.org/packages/ee/f7/41e7f2d8117fbc7a7587286162ffe2f688984b69c46ed63cf5f2e4fc3bae/mypy-2.3.1-cp315-cp315-macosx_11_0_arm64.whl", hash = "sha256:6f041a6de52c9217ca125e78ba0a335cb7fd98a1c0580978e49ab2b126f70b57", size = 13990694, upload-time = "2026-08-15T03:03:21.919Z" }, + { url = "https://files.pythonhosted.org/packages/06/85/8f665811a0c8f3bf6fa1d9acd665ec2d97a2bcc453ae68dcd92340941cd6/mypy-2.3.1-cp315-cp315-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5159ae60f5dbc3a498af5ba8365505808ac8031bc63f9e00304ad545d40bdd9b", size = 14203518, upload-time = "2026-08-15T03:01:48.455Z" }, + { url = "https://files.pythonhosted.org/packages/2d/82/91b866c8546b120bff83b73a439d90d2d63ef3aff113599e6b8e4d566848/mypy-2.3.1-cp315-cp315-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:47a8a7a0a7f6f6e63995c0ac36fa0c07b127413fdc81f0439b7f3dccafd33561", size = 15220224, upload-time = "2026-08-15T03:01:23.577Z" }, + { url = "https://files.pythonhosted.org/packages/c8/78/c226c99208ee40de7c768369fa533f933afa003dfdc606ff021450724e91/mypy-2.3.1-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:2329c0501293d4e1f33bc15d04d6304d65a1cdda967ee93a05c1e681a3923133", size = 15501512, upload-time = "2026-08-15T03:02:09.453Z" }, + { url = "https://files.pythonhosted.org/packages/a9/e7/7cfb3f106c393979f4cc37ad6c0586044d50401e3c35b0c003e4f3ba6bc9/mypy-2.3.1-cp315-cp315-pyemscripten_2026_5_wasm32.whl", hash = "sha256:bb26deed807bdb0457cf3e3f1cd7c4a1cf9d66864eaf1b4a61e06805d4c6b1f9", size = 7761913, upload-time = "2026-08-15T03:01:55.65Z" }, + { url = "https://files.pythonhosted.org/packages/99/3c/52affefa273b97939a1f474ae4a349c8718635c15b941112dfab4291b0c1/mypy-2.3.1-cp315-cp315-win_amd64.whl", hash = "sha256:375d7013876a8233b2d05be185bfa09f689696cd999ce8b1cfe6acac5c80e8a3", size = 11422533, upload-time = "2026-08-15T03:03:24.101Z" }, + { url = "https://files.pythonhosted.org/packages/2a/b7/75643e70c72a5b346d8a9b1543c967ea8824df2ee3fb7ccba652c272b7bb/mypy-2.3.1-cp315-cp315-win_arm64.whl", hash = "sha256:586b3612214cceabb3c0f588c97e7d1e535393f06a60e912e994f6b3ace97523", size = 10397931, upload-time = "2026-08-15T03:02:55.265Z" }, + { url = "https://files.pythonhosted.org/packages/10/ce/53be21f2d4adfcd26f63f1184a13ed797015ab463853f117e2e11e4d726f/mypy-2.3.1-cp315-cp315t-macosx_11_0_arm64.whl", hash = "sha256:ef0c6335cda9d807f8193d8ff6204a72bc909fa9882aacbca14f43cdb7188306", size = 15118669, upload-time = "2026-08-15T03:02:51.479Z" }, + { url = "https://files.pythonhosted.org/packages/62/43/20de757cd42989d291a17fad607742c4c74e875ce5cea00e5a5225020ac1/mypy-2.3.1-cp315-cp315t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e598c8c66401d26b150872154a286e6d484cf2789c3bb28a7556806298423021", size = 15545627, upload-time = "2026-08-15T03:03:05.132Z" }, + { url = "https://files.pythonhosted.org/packages/7e/fc/092bdf77ad280eaf501422f0f3b966012b528076cc13e41a774861c907d1/mypy-2.3.1-cp315-cp315t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:eda22fd4efa9dcd39331d1dede9b5b8b8a7fd69af07592e778433da98610d29e", size = 16764157, upload-time = "2026-08-15T03:02:23.958Z" }, + { url = "https://files.pythonhosted.org/packages/94/5c/c94c4d62d909b07f552d0d9356d7acc943825558e602a64822ffa2231536/mypy-2.3.1-cp315-cp315t-musllinux_1_2_x86_64.whl", hash = "sha256:2a0ba2e57847849fb0d1fcdabb32786d223095ed8bc121dfe322bcdb3d9c46bc", size = 17073258, upload-time = "2026-08-15T03:02:14.573Z" }, + { url = "https://files.pythonhosted.org/packages/c0/f7/511a88b89e478053c02d22039bb8f3ce4183efe8fd7a4f0a5910a8bb0a32/mypy-2.3.1-cp315-cp315t-win_amd64.whl", hash = "sha256:3f7e865dd51f235f60a2dbcd8728a1c095f5ca28f095d48a725b84cd935735c4", size = 12135505, upload-time = "2026-08-15T03:02:16.714Z" }, + { url = "https://files.pythonhosted.org/packages/71/bf/02573b56964ecb0f7c644f915f53c325ae15c3faec521c5adf11599a32df/mypy-2.3.1-cp315-cp315t-win_arm64.whl", hash = "sha256:8ad80807dc3ab8ea978b1b2b6e4a657194ace1d4ef03e0e731aff1abd517da29", size = 10962647, upload-time = "2026-08-15T03:01:43.712Z" }, + { url = "https://files.pythonhosted.org/packages/8e/41/9675c7a1e78edecfba0b79e587a52594c56e189368261dc7b3a7fffb9527/mypy-2.3.1-py3-none-any.whl", hash = "sha256:6ed5c7e3419083268e5c9258bd1c1ef91af44a9e89374dbcaf37b775716e72eb", size = 2754338, upload-time = "2026-08-15T03:02:53.4Z" }, +] + +[[package]] +name = "mypy-extensions" +version = "1.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/a2/6e/371856a3fb9d31ca8dac321cda606860fa4548858c0cc45d9d1d4ca2628b/mypy_extensions-1.1.0.tar.gz", hash = "sha256:52e68efc3284861e772bbcd66823fde5ae21fd2fdb51c62a211403730b916558", size = 6343, upload-time = "2025-04-22T14:54:24.164Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/79/7b/2c79738432f5c924bef5071f933bcc9efd0473bac3b4aa584a6f7c1c8df8/mypy_extensions-1.1.0-py3-none-any.whl", hash = "sha256:1be4cccdb0f2482337c4743e60421de3a356cd97508abadd57d47403e94f5505", size = 4963, upload-time = "2025-04-22T14:54:22.983Z" }, +] + +[[package]] +name = "packaging" +version = "26.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/7d/fa/3944b40b07da9ce895c0e6303a5ab7d53da063554f534556b134a54d6093/packaging-26.3.tar.gz", hash = "sha256:94edc256424af38762eb31306eed28beb9f0efc50a8837492c9d6fd6004aed79", size = 313412, upload-time = "2026-08-04T18:15:28.737Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/63/34/ba1c580383c9eada3711951fef0795c80b829a078d72188184bcab9dd527/packaging-26.3-py3-none-any.whl", hash = "sha256:d7193f7c8e4e93f444fde0262bf90af30e16fa0ad0ad44cb553c87339b23cd1c", size = 129956, upload-time = "2026-08-04T18:15:27.159Z" }, +] + +[[package]] +name = "pathspec" +version = "1.1.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/5a/82/42f767fc1c1143d6fd36efb827202a2d997a375e160a71eb2888a925aac1/pathspec-1.1.1.tar.gz", hash = "sha256:17db5ecd524104a120e173814c90367a96a98d07c45b2e10c2f3919fff91bf5a", size = 135180, upload-time = "2026-04-27T01:46:08.907Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f1/d9/7fb5aa316bc299258e68c73ba3bddbc499654a07f151cba08f6153988714/pathspec-1.1.1-py3-none-any.whl", hash = "sha256:a00ce642f577bf7f473932318056212bc4f8bfdf53128c78bbd5af0b9b20b189", size = 57328, upload-time = "2026-04-27T01:46:07.06Z" }, +] + +[[package]] +name = "pyproject-hooks" +version = "1.2.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/e7/82/28175b2414effca1cdac8dc99f76d660e7a4fb0ceefa4b4ab8f5f6742925/pyproject_hooks-1.2.0.tar.gz", hash = "sha256:1e859bd5c40fae9448642dd871adf459e5e2084186e8d2c2a79a824c970da1f8", size = 19228, upload-time = "2024-09-29T09:24:13.293Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/bd/24/12818598c362d7f300f18e74db45963dbcb85150324092410c8b49405e42/pyproject_hooks-1.2.0-py3-none-any.whl", hash = "sha256:9e5c6bfa8dcc30091c74b0cf803c81fdd29d94f01992a7707bc97babb1141913", size = 10216, upload-time = "2024-09-29T09:24:11.978Z" }, +] + +[[package]] +name = "ruff" +version = "0.16.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/00/8f/d8074b1f25e003164087a8bfe79a0f1a3945135764dbb6aaab04103dcaf9/ruff-0.16.4.tar.gz", hash = "sha256:13171aa9d9af2240ee3504e639de73122c67e74036de5ba2e1d01422cd17e3dc", size = 4899731, upload-time = "2026-08-20T17:43:59.196Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ff/80/779895ef584e089d22f2c6df0d0e99a65ec2df0805f1fffd439415b8c1f0/ruff-0.16.4-py3-none-linux_armv6l.whl", hash = "sha256:df4075f71ddac40b9934af60c3ec8a53047dd5a5fdc43224e6e4e8e9a27cb6f7", size = 10006909, upload-time = "2026-08-20T17:43:16.888Z" }, + { url = "https://files.pythonhosted.org/packages/a9/e6/f553199b5e8927a05cb5c422d921fd0656b29ab976e91c44802107c6b0da/ruff-0.16.4-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:0c95538517af68004306b0fb3214ff2f2af67a65092aee77cd9eb86db6656604", size = 10240201, upload-time = "2026-08-20T17:43:19.337Z" }, + { url = "https://files.pythonhosted.org/packages/1c/70/4a6dc4bb34da4dee35e30f09bbd1bfbdd26f33b62fb9b8df31f08a199cd2/ruff-0.16.4-py3-none-macosx_11_0_arm64.whl", hash = "sha256:963f83df8e69e575b64d67dd447ebbc917db41a14bf38d4593a4183e7aaa8255", size = 9835122, upload-time = "2026-08-20T17:43:21.708Z" }, + { url = "https://files.pythonhosted.org/packages/24/12/c6e22d686372c15bcb7af99831f1a1be96df696491babf4f24e4f942c527/ruff-0.16.4-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:32a5057c7ff3f6e6480a48fccfb3a412a690f48a3d03ac5cf08177d6c2da3ade", size = 9977162, upload-time = "2026-08-20T17:43:24.236Z" }, + { url = "https://files.pythonhosted.org/packages/46/49/72b10ec912f5ab5854992eaf7aa7cd36729b6937d9dc4e0fb41b3bf428ec/ruff-0.16.4-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:b3dce8d9b0c57c265b91885a66a567d8ea1372e8eb4e250fa8e5e3f579e99cff", size = 9829789, upload-time = "2026-08-20T17:43:26.966Z" }, + { url = "https://files.pythonhosted.org/packages/fa/80/0f30e32e7f6ee26edc39075502db9d368d788a44a79b55f763eb4ab03796/ruff-0.16.4-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7dc651db49283c69f8e72c834eec4fe5573e4c646856aebece0ce385dceb2a80", size = 10527949, upload-time = "2026-08-20T17:43:29.384Z" }, + { url = "https://files.pythonhosted.org/packages/52/3d/86e8ad3542169e56cac3859a343afdb9df2ad54d35a59ce1e67baee83421/ruff-0.16.4-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3817b87dbcabc92f13b05019257c5b89b5b4d51b5fb20f56fb5235ceb723cd07", size = 11333695, upload-time = "2026-08-20T17:43:31.872Z" }, + { url = "https://files.pythonhosted.org/packages/d0/16/481c29b380c20a0054a8261066665e1b3488e23636c49d0a43e75975b9bb/ruff-0.16.4-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e9fce1499134b2c8c68e5166f95705a5812062bb93aacc5f9873bb1a27084bc7", size = 10727741, upload-time = "2026-08-20T17:43:34.596Z" }, + { url = "https://files.pythonhosted.org/packages/5e/b6/56bc0b8cf45b54b28b3a5e6381c8945d51b5b18adf659454c32295209a31/ruff-0.16.4-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f2d812e482f5a7e02eee26cd73d2a37ebbdf47d795ea63ba1b89110ae93e9fb3", size = 10286522, upload-time = "2026-08-20T17:43:37.288Z" }, + { url = "https://files.pythonhosted.org/packages/e8/8b/b345b4fb110f2fbe2bd31eabd271e5e8b3b7e4ee6c0e02f2dc6be78db000/ruff-0.16.4-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:6baaf984aa7976edf93d3b627fe2d1d22ee94bbca05fa6f90fc76d73924e3454", size = 10584182, upload-time = "2026-08-20T17:43:39.984Z" }, + { url = "https://files.pythonhosted.org/packages/29/e5/827b34041c35f58774a9681a4213994c164fc987800f4dddabcf451da0bf/ruff-0.16.4-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:bdfcf0b28662eb890372d50f92c283bb94e67e7635ed93c7fd533970acff7b2b", size = 10134195, upload-time = "2026-08-20T17:43:42.351Z" }, + { url = "https://files.pythonhosted.org/packages/0f/10/d0bffcdd6729b87afc82ba0ef377173356a7dc8e972f5179968cf2fdf98c/ruff-0.16.4-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:b66b02cb9b04f537643cadf5768e5f98dc461890d530cb67113d71c8c76e605d", size = 9825821, upload-time = "2026-08-20T17:43:44.532Z" }, + { url = "https://files.pythonhosted.org/packages/f5/32/0db2a863b796ca62d83e92a07a3ccf00921b14db02059347576a2fda3d4b/ruff-0.16.4-py3-none-musllinux_1_2_i686.whl", hash = "sha256:8528bf9a4b291a60bf02ea453511e8ce6215bd2b982ee80405b66b008b6c30a0", size = 10267658, upload-time = "2026-08-20T17:43:46.989Z" }, + { url = "https://files.pythonhosted.org/packages/b2/a0/fbdeb59e48c6261f523e56c8f12e9c08fbe693786595cc7e3959207a9232/ruff-0.16.4-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:fbd85d2875fdd67e833213a651f613bbf25303abf6aa822a5121f4531195678d", size = 10697071, upload-time = "2026-08-20T17:43:49.891Z" }, + { url = "https://files.pythonhosted.org/packages/aa/28/0c6dd865859c6d17bc8ccc34cb72b0e02d6c7eb25e8a1e22b5bea681e2c0/ruff-0.16.4-py3-none-win32.whl", hash = "sha256:312769988007aaeb8e189b443ccdd03c0e6374489e053467be6d96518ebff76e", size = 10021687, upload-time = "2026-08-20T17:43:52.281Z" }, + { url = "https://files.pythonhosted.org/packages/a3/03/e724450f621698117f9aa6dd241c94d0274ae96781378dc86745ae29f0e7/ruff-0.16.4-py3-none-win_amd64.whl", hash = "sha256:05d9d27a18c4bcbefada602480ec9e01e0bc949d432e0ced5df77edac195919c", size = 10567657, upload-time = "2026-08-20T17:43:54.78Z" }, + { url = "https://files.pythonhosted.org/packages/0e/fe/da8b9e1347696bb22120b77280ec5ce25d500ca5cb39d5ad6e5c18de19c1/ruff-0.16.4-py3-none-win_arm64.whl", hash = "sha256:a3a61621c9b6f6a89573e938a080e648f1695baa3f58570a3a707bc51ff65a21", size = 10451579, upload-time = "2026-08-20T17:43:57.135Z" }, +] + +[[package]] +name = "setuptools" +version = "84.0.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/6d/44/f5da03a8ef95d369145c5bb53050e7877c9f3d312e128605fd9504829143/setuptools-84.0.0.tar.gz", hash = "sha256:f4695c21257f0d9b537ec2692c941d02ee143b7cc1276941349a546573b2ef73", size = 1168449, upload-time = "2026-08-08T18:27:58.365Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/95/9c/c510029fc6ef33a6275cd2c5d3cecd6613dfd6aa401d57c54f1c18852ccf/setuptools-84.0.0-py3-none-any.whl", hash = "sha256:51a52592b3b99e102b609654876bd65f19f999935166d1352678931132b0c670", size = 818216, upload-time = "2026-08-08T18:27:56.719Z" }, +] + +[[package]] +name = "typing-extensions" +version = "4.16.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f6/cc/6253133b5bb138fc3306cebfbda2c520f545d36b5be2c7255cc528bb45d6/typing_extensions-4.16.0.tar.gz", hash = "sha256:dc983d19a509c94dba722ee6abd33940f7c05a89e243c47e907eb4db6f1a43e5", size = 113555, upload-time = "2026-07-02T08:40:05.92Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/49/d3/b8441a820a491ddfc024b0b0cf0393375b75ea13866d9c66727e54c2fc80/typing_extensions-4.16.0-py3-none-any.whl", hash = "sha256:481caa481374e813c1b176ada14e97f1f67a4539ce9cfeb3f350d78d6370c2e8", size = 45571, upload-time = "2026-07-02T08:40:04.659Z" }, +] From f41410aa95dc1a169853bea2a16ab8ec41b50e67 Mon Sep 17 00:00:00 2001 From: rica Date: Tue, 25 Aug 2026 17:43:44 +0900 Subject: [PATCH 2/4] [RAG Projection] Add revision-aware derived indexing and rebuilds (#6) Refs #3 --- .github/workflows/ci.yml | 3 +- PYTHON_MODULE_INDEX.md | 65 +- README.md | 135 ++- docs/api.md | 228 ++-- docs/lifecycle.md | 74 +- docs/projection.md | 257 +++++ docs/security-and-privacy.md | 134 ++- src/generic_rag/contracts.py | 245 ++++ src/generic_rag/ports.py | 10 + src/generic_rag/projection.py | 814 +++++++++++++ tests/support/clean_import_probe.py | 33 + tests/support/verify_artifacts.py | 1 + tests/test_package_boundaries.py | 28 + tests/test_ports.py | 32 +- tests/test_projection.py | 1661 +++++++++++++++++++++++++++ tests/test_projection_contracts.py | 545 ++++++++- 16 files changed, 4068 insertions(+), 197 deletions(-) create mode 100644 docs/projection.md create mode 100644 src/generic_rag/projection.py create mode 100644 tests/test_projection.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 329b6c2..90a7532 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -162,7 +162,8 @@ jobs: generic_rag \ generic_rag.errors \ generic_rag.contracts \ - generic_rag.ports + generic_rag.ports \ + generic_rag.projection do ( cd "$rag_probe_dir" diff --git a/PYTHON_MODULE_INDEX.md b/PYTHON_MODULE_INDEX.md index 5a4d342..08092cc 100644 --- a/PYTHON_MODULE_INDEX.md +++ b/PYTHON_MODULE_INDEX.md @@ -4,9 +4,9 @@ - Declared source root: `src` - Packaging source of truth: `pyproject.toml` -- Importable production units: 4 -- Indexed production units: 4 -- Source/index parity: 4/4 +- Importable production units: 5 +- Indexed production units: 5 +- Source/index parity: 5/5 - Package data: `src/generic_rag/py.typed` - Locked verification owner: `.github/workflows/ci.yml` (supporting workflow, not an importable unit) @@ -41,9 +41,9 @@ - Owned state or external resources: none. - Material side effects: none. - Verification: `tests/test_errors.py`, `tests/test_package_boundaries.py`, - `tests/support/clean_import_probe.py`, and the locked CI import and boundary - checks. -- Documentation: `docs/api.md`. + `tests/test_projection.py`, `tests/support/clean_import_probe.py`, and the + locked CI import and boundary checks. +- Documentation: `docs/api.md` and `docs/projection.md`. ## `generic_rag.contracts` @@ -53,9 +53,12 @@ - Supported public imports: `DocumentKey`, `DocumentIdentity`, `Document`, `FragmentIdentity`, `Fragment`, `EmbeddingIdentity`, `EmbeddingVector`, `VectorRecord`, `ProjectionIdentity`, `ProjectionCheckpoint`, - `ProjectionOutcome`, `ProjectionReceipt`, `RetrievalQuery`, - `RetrievalOutcome`, `RetrievalHit`, and `RetrievalResult` from - `generic_rag.contracts`. + `ProjectionOutcome`, `ProjectionReceipt`, `ChunkingPolicy`, + `ProjectionLimits`, `ProjectionRequest`, `ProjectionManifestEntry`, + `ProjectionManifest`, `ProjectionStateAvailability`, + `ProjectionStateSnapshot`, `ProjectionStateStatus`, `ProjectionResult`, + `RetrievalQuery`, `RetrievalOutcome`, `RetrievalHit`, and `RetrievalResult` + from `generic_rag.contracts`. - Re-exports: exactly the names in the module's `__all__`; none from the package root. - Direct internal dependencies: `generic_rag.errors`. @@ -63,11 +66,12 @@ values. - Material side effects: none. - Verification: `tests/test_contract_values.py`, - `tests/test_projection_contracts.py`, `tests/test_retrieval_contracts.py`, - `tests/test_package_boundaries.py`, `tests/support/clean_import_probe.py`, - `tests/support/verify_artifacts.py`, and the locked CI import, boundary, and - artifact checks. -- Documentation: `docs/api.md` and `docs/security-and-privacy.md`. + `tests/test_projection_contracts.py`, `tests/test_projection.py`, + `tests/test_retrieval_contracts.py`, `tests/test_package_boundaries.py`, + `tests/support/clean_import_probe.py`, `tests/support/verify_artifacts.py`, and + the locked CI import, boundary, and artifact checks. +- Documentation: `docs/api.md`, `docs/projection.md`, and + `docs/security-and-privacy.md`. ## `generic_rag.ports` @@ -75,7 +79,8 @@ - Responsibility: define synchronous injected collaborator interfaces and explicit caller-owned borrowing semantics. - Supported public imports: `Borrowed`, `Embedder`, `VectorIndexWriter`, - `VectorIndexReader`, and `LexicalRetriever` from `generic_rag.ports`. + `VectorIndexResetter`, `VectorIndexReader`, and `LexicalRetriever` from + `generic_rag.ports`. - Re-exports: exactly the names in the module's `__all__`; none from the package root. - Direct internal dependencies: `generic_rag.contracts`. @@ -83,7 +88,33 @@ owns, acquires, releases, closes, or shuts down the resource. - Material side effects: none. - Verification: `tests/test_ports.py`, `tests/test_package_boundaries.py`, - `tests/support/clean_import_probe.py`, and the locked CI import and boundary + `tests/test_projection.py`, `tests/support/clean_import_probe.py`, and the + locked CI import and boundary checks. +- Documentation: `docs/api.md`, `docs/lifecycle.md`, `docs/projection.md`, and + `docs/security-and-privacy.md`. + +## `generic_rag.projection` + +- Source: `src/generic_rag/projection.py` +- Responsibility: deterministically plan and synchronously execute bounded, + revision-aware document projection against caller-supplied state. +- Supported public imports: `ProjectionFailureStage`, `ProjectionStateError`, + `ProjectionOperationError`, `project_documents`, and `rebuild_projection` + from `generic_rag.projection`. +- Re-exports: exactly the names in the module's `__all__`; none from the package + root. +- Direct internal dependencies: `generic_rag.contracts`, `generic_rag.errors`, + and `generic_rag.ports`. +- Owned state or external resources: none; planning state is immutable and + local to each call, while every embedder, writer, and resetter remains + caller-owned through `Borrowed`. +- Material side effects: none at import time. At explicit workflow call time it + may invoke the borrowed embedder and vector writer, and full rebuild may + invoke the borrowed corpus resetter; it performs no persistence, network, + retry, acquisition, release, or lifecycle action itself. +- Verification: `tests/test_projection.py`, `tests/test_package_boundaries.py`, + `tests/support/clean_import_probe.py`, `tests/support/verify_artifacts.py`, and + the locked CI test, lint, type, build, clean-install, import, and artifact checks. -- Documentation: `docs/api.md`, `docs/lifecycle.md`, and +- Documentation: `docs/projection.md`, `docs/api.md`, `docs/lifecycle.md`, and `docs/security-and-privacy.md`. diff --git a/README.md b/README.md index 9471b69..1c7bd03 100644 --- a/README.md +++ b/README.md @@ -3,9 +3,10 @@ `generic-rag` is a provider-neutral, runtime-dependency-free foundation for retrieval-augmented generation (RAG). Version 0.1.0 requires Python 3.11 or later and provides immutable contracts, typed error categories, synchronous -collaborator protocols, and explicit caller-owned borrowing. +collaborator protocols, deterministic bounded projection orchestration, and +explicit caller-owned borrowing. -No projection or retrieval algorithm is implemented in 0.1.0. The package has +Retrieval and result composition are not implemented in 0.1.0. The package has no built-in adapter, provider, factory, persistence, network client, configuration system, authentication, citation mechanism, or CLI. @@ -21,78 +22,113 @@ python -m pip install . The installed package has no runtime dependencies. Build and development tools are separate locked dependency groups. -## Use the contracts in application code +## Project approved documents The host application remains responsible for authorization and policy checks. -After approving a source and query, application code can construct generic -values and accept an application-owned provider through a protocol: +After approving source content, construct a complete bounded target, wrap +application-owned adapters in `Borrowed`, and call a projection workflow: ```python from generic_rag.contracts import ( + ChunkingPolicy, Document, DocumentIdentity, DocumentKey, + EmbeddingIdentity, EmbeddingVector, - RetrievalQuery, + ProjectionIdentity, + ProjectionLimits, + ProjectionRequest, + ProjectionStateAvailability, + ProjectionStateSnapshot, ) -from generic_rag.ports import Borrowed, Embedder +from generic_rag.ports import Borrowed +from generic_rag.projection import rebuild_projection -# Construct these values only after application-specific authorization. -approved_document = Document( - identity=DocumentIdentity( - key=DocumentKey(corpus_id="corpus-a", document_id="document-1"), - revision_id="revision-3", +class ExampleEmbedder: + identity = EmbeddingIdentity("example-model", 2) + + def embed(self, texts, /): + return tuple(EmbeddingVector((float(len(text)), 0.0)) for text in texts) + + +class ExampleWriter: + def replace_document(self, document, records, /): + # Replace the complete projection for this stable document key. + return None + + def delete_document(self, document, /): + return None + + +class ExampleResetter: + def reset_corpus(self, corpus_id, /): + # Remove every projected document for this corpus. + return None + + +request = ProjectionRequest( + "corpus-a", + ProjectionIdentity("schema-v1", ExampleEmbedder.identity), + ChunkingPolicy(max_fragment_codepoints=800, overlap_codepoints=80), + ProjectionLimits( + max_documents=100, + max_document_codepoints=100_000, + max_embedding_batch_size=32, + ), + ( + Document( + DocumentIdentity( + DocumentKey("corpus-a", "document-1"), + "revision-3", + ), + "Approved source text", + (("classification", "internal"),), + ), ), - text="Approved source text", - attributes=(("classification", "internal"),), -) -query = RetrievalQuery( - corpus_id=approved_document.identity.key.corpus_id, - text="What does the source say?", - hit_limit=5, - candidate_limit=20, ) +# Bootstrap and recovery are explicit and destructive: reset, then replace. +result = rebuild_projection( + request, + ProjectionStateSnapshot(ProjectionStateAvailability.MISSING, None), + Borrowed(ExampleEmbedder()), + Borrowed(ExampleWriter()), + Borrowed(ExampleResetter()), +) -def application_embed_query( - provider: Embedder, - request: RetrievalQuery, -) -> EmbeddingVector: - with Borrowed(provider) as embedder: - vectors = embedder.embed((request.text,)) - if len(vectors) != 1: - raise ValueError("the provider violated the Embedder contract") - return vectors[0] +# Persist result.manifest in application-owned state only after success. ``` -This is application orchestration, not a package retrieval workflow. Version -0.1.0 defines the boundary that provider implementations and later generic -workflows will use; it does not construct providers or call them on a user's -behalf. +For normal updates, load that manifest into a present +`ProjectionStateSnapshot` and call `project_documents`; it changes only added, +updated, removed, or rechunked documents. Use `rebuild_projection` only when an +explicit corpus-wide reset is intended. See the [projection guide](docs/projection.md) +for the complete lifecycle, state matrix, adapter obligations, and failure +behavior. Public values must be imported from their owning modules: - `generic_rag.contracts` - `generic_rag.errors` - `generic_rag.ports` +- `generic_rag.projection` The package root intentionally has no re-exports: `generic_rag.__all__ == ()`. See the [API reference](docs/api.md) for every supported name and invariant. -## Planned RAG flow +## Application RAG flow The package itself has no concept of a user or agent. A consuming application decides which sources a user may approve, which queries may be submitted, which provider implementations receive data, and whether retrieved fragments are shown to a user or supplied to a downstream tool or agent. -- [Issue #3](https://github.com/Kims-DeveloperGroup/generic-rag/issues/3) is - planned to add generic projection orchestration. Its intended responsibility - is to accept caller-approved documents and explicitly injected collaborators, - derive fragments under a defined chunking policy, embed ordered fragment - text, replace or delete complete document projections, and report truthful - checkpoints and receipts. Its precise API and failure behavior are not part - of 0.1.0. +- Projection accepts caller-approved documents and explicitly injected + collaborators. It derives deterministic fragments under a bounded chunking + policy, embeds ordered fragment text, replaces or deletes complete document + projections, and returns a manifest and truthful receipt for caller-owned + persistence. - [Issue #4](https://github.com/Kims-DeveloperGroup/generic-rag/issues/4) is planned to add retrieval and composition. Its intended responsibility is to use an injected `Embedder` and `VectorIndexReader` for semantic candidates @@ -100,8 +136,11 @@ shown to a user or supplied to a downstream tool or agent. deduplication, fusion, limiting, and outcome behavior. Provider rank will be the input; raw provider scores are not represented or assumed comparable. -The caller/provider ownership model remains explicit throughout this plan. See -[resource lifecycle](docs/lifecycle.md) and +There is no end-user or agent query workflow yet. A consuming application can +project data now, but must wait for or implement a separate reviewed retrieval +layer before supplying retrieved context to users, tools, or agents. The +caller/provider ownership model remains explicit throughout. See [resource +lifecycle](docs/lifecycle.md) and [security and privacy](docs/security-and-privacy.md). ## Compatibility @@ -111,9 +150,10 @@ assume compatibility across minor releases. For this release, direct imports from the documented owning modules are the supported public paths; root-level imports are not. -The distribution includes `py.typed`. The wheel contains exactly the four +The distribution includes `py.typed`. The wheel contains exactly the five importable modules `generic_rag`, `generic_rag.errors`, -`generic_rag.contracts`, and `generic_rag.ports`, plus the typing marker. +`generic_rag.contracts`, `generic_rag.ports`, and `generic_rag.projection`, plus +the typing marker. ## Development verification @@ -140,6 +180,5 @@ uv run --frozen python tests/support/verify_artifacts.py "$rag_dist_dir" CI is configured to run the tests on Python 3.11 and 3.14. On Python 3.11 it also runs lint, format, strict type, compilation, artifact, source-rebuild, -clean-install, and isolated-import checks. A local Python 3.14.2 run currently -contains 53 passing tests; the CI matrix is the authoritative cross-version -result. +clean-install, and isolated-import checks. The CI matrix is the authoritative +cross-version result. diff --git a/docs/api.md b/docs/api.md index 863105e..115c259 100644 --- a/docs/api.md +++ b/docs/api.md @@ -1,10 +1,10 @@ # Public API -Version 0.1.0 exposes immutable values, typed error categories, and synchronous -collaborator protocols. It does not expose projection or retrieval algorithms. -See the [project overview](../README.md), [resource lifecycle](lifecycle.md), -and [security and privacy boundary](security-and-privacy.md) for the surrounding -usage contract. +Version 0.1.0 exposes immutable values, typed error categories, synchronous +collaborator protocols, and deterministic bounded document projection. It does +not expose retrieval or result-composition orchestration. See the [projection +guide](projection.md), [resource lifecycle](lifecycle.md), and [security and +privacy boundary](security-and-privacy.md) for the surrounding usage contract. ## Import boundary @@ -33,6 +33,15 @@ public symbols. Import names from their owning modules instead. - `ProjectionCheckpoint` - `ProjectionOutcome` - `ProjectionReceipt` +- `ChunkingPolicy` +- `ProjectionLimits` +- `ProjectionRequest` +- `ProjectionManifestEntry` +- `ProjectionManifest` +- `ProjectionStateAvailability` +- `ProjectionStateSnapshot` +- `ProjectionStateStatus` +- `ProjectionResult` - `RetrievalQuery` - `RetrievalOutcome` - `RetrievalHit` @@ -43,10 +52,20 @@ public symbols. Import names from their owning modules instead. - `Borrowed` - `Embedder` - `VectorIndexWriter` +- `VectorIndexResetter` - `VectorIndexReader` - `LexicalRetriever` -The package does not support importing public values from the package root. +`generic_rag.projection` exports exactly: + +- `ProjectionFailureStage` +- `ProjectionStateError` +- `ProjectionOperationError` +- `project_documents` +- `rebuild_projection` + +The package does not support importing any of these names from the package +root. ## Shared value rules @@ -72,23 +91,41 @@ and pair order and duplicates are preserved. ## Errors -`GenericRagError` is the base for the three public categories: +The base error relationships used by projection are: ```text GenericRagError ├── ContractValidationError ├── CollaborationError +│ └── ProjectionOperationError └── StateCompatibilityError + └── ProjectionStateError ``` -- `ContractValidationError` reports a violated public value invariant. -- `CollaborationError` is reserved for a workflow that translates a - collaborator operation failure. -- `StateCompatibilityError` is reserved for a workflow that detects derived - state with an incompatible projection identity. +- `ContractValidationError` reports a violated public value invariant or an + invalid top-level workflow input. +- `ProjectionStateError(status: ProjectionStateStatus)` reports state that an + incremental operation cannot use. Its exact `status` field gives the reason; + its message contains no state identifier. +- `ProjectionOperationError(stage, affected_document, receipt)` translates an + ordinary collaborator failure or invalid collaborator return. Its message is + content-free. `affected_document` is a stable key for document-specific + failures and otherwise `None`; `receipt` is truthful `FAILED` or `PARTIAL` + progress, or `None` when there were zero document attempts. -Version 0.1.0 has no projection or retrieval workflow that raises the latter -two categories. `Borrowed` also leaves provider exceptions unchanged. +The closed string enum `ProjectionFailureStage` has exact values: + +| Member | String value | +| --- | --- | +| `EMBEDDER_IDENTITY` | `"embedder_identity"` | +| `EMBEDDING` | `"embedding"` | +| `REPLACEMENT` | `"replacement"` | +| `DELETION` | `"deletion"` | +| `RESET` | `"reset"` | + +An ordinary collaborator exception is chained as the operation error's cause. +An invalid identity, malformed vector result, or non-`None` command result has +no internal cause. `KeyboardInterrupt` and `SystemExit` pass through unchanged. ## Documents and fragments @@ -104,31 +141,65 @@ A fragment range is half-open, `[start, end)`, in Python Unicode code points. It is not measured in bytes or user-perceived grapheme clusters. For example, `"😀"` has one code point while `"e\u0301"` has two. -The package checks that fragment text length equals the range width. Version -0.1.0 does not retain an authoritative `Document` beside a `Fragment`, so it -cannot verify that the text equals the indicated source slice. The caller, or -a future projection workflow, must establish that correspondence. +The contract checks that fragment text length equals the range width. A +standalone fragment cannot prove that its text equals the indicated source +slice; projection establishes that correspondence for fragments it derives. ## Embeddings and vector records | Type | Fields | Construction rules | | --- | --- | --- | | `EmbeddingIdentity` | `model_id: str`, `dimensions: int` | The model ID is nonblank and opaque; dimensions is a positive exact integer. | -| `EmbeddingVector` | `values: tuple[float, ...]` | The tuple is exact and nonempty. Every coordinate is an exact `int` or `float`, excluding `bool`, and must convert to a finite float without overflow. Oversized integers that cannot be represented as finite floats are rejected; accepted coordinates are stored canonically as floats. | +| `EmbeddingVector` | `values: tuple[float, ...]` | The tuple is exact and nonempty. Every coordinate is an exact `int` or `float`, excluding `bool`, and must convert to a finite float without overflow. Accepted coordinates are stored canonically as floats. | | `VectorRecord` | `fragment: Fragment`, `embedding: EmbeddingVector` | Both fields require their exact contract classes. | -A standalone `EmbeddingVector` does not carry an `EmbeddingIdentity`. Version -0.1.0 therefore does not compare the vector length with an identity's declared -`dimensions`; a provider and future orchestration must satisfy that semantic -relationship. +A standalone `EmbeddingVector` does not carry an `EmbeddingIdentity` and does +not itself compare length with declared dimensions. Projection validates the +embedder identity and every returned vector's exact dimensionality and +canonical finite-float representation. -## Projection state +## Projection request and manifest values | Type | Fields | Construction rules | | --- | --- | --- | | `ProjectionIdentity` | `schema_id: str`, `embedding: EmbeddingIdentity` | The schema ID is nonblank and opaque; embedding requires its exact class. | -| `ProjectionCheckpoint` | `corpus_id: str`, `projection: ProjectionIdentity`, `token: str` | Corpus and token are nonblank opaque strings; projection requires its exact class. | -| `ProjectionReceipt` | `corpus_id: str`, `projection: ProjectionIdentity`, `outcome: ProjectionOutcome`, `attempted_documents: int`, `completed_documents: int`, `checkpoint: ProjectionCheckpoint \| None` | Nested values require their exact classes; counts are nonnegative exact integers and completed cannot exceed attempted. | +| `ChunkingPolicy` | `max_fragment_codepoints: int`, `overlap_codepoints: int` | Maximum is positive; overlap is nonnegative and smaller than maximum. | +| `ProjectionLimits` | `max_documents: int`, `max_document_codepoints: int`, `max_embedding_batch_size: int` | All three values are positive exact integers. | +| `ProjectionRequest` | `corpus_id: str`, `projection: ProjectionIdentity`, `chunking: ChunkingPolicy`, `limits: ProjectionLimits`, `documents: tuple[Document, ...]` | Documents must match the corpus, have unique stable keys, and remain within the count and per-document text caps. They are canonicalized by opaque `document_id`. | +| `ProjectionManifestEntry` | `document: DocumentIdentity`, `source_digest: str`, `fragment_count: int` | Digest must be lowercase `sha256:<64hex>` and fragment count is nonnegative. | +| `ProjectionCheckpoint` | `corpus_id: str`, `projection: ProjectionIdentity`, `token: str` | Corpus and token are nonblank; projection requires its exact class. | +| `ProjectionManifest` | `corpus_id: str`, `projection: ProjectionIdentity`, `chunking: ChunkingPolicy`, `entries: tuple[ProjectionManifestEntry, ...]`, `checkpoint: ProjectionCheckpoint` | Entries match the corpus, have unique stable keys, and are canonicalized by `document_id`; checkpoint corpus and projection match the manifest. | + +The package produces manifests; the caller owns their persistence. Source +digests, fragment IDs, and checkpoint tokens are deterministic under explicit +v1 domains described in the [projection guide](projection.md#deterministic-projection-values). + +## Projection state and results + +`ProjectionStateAvailability` is a closed string enum: + +| Member | String value | Manifest rule | +| --- | --- | --- | +| `MISSING` | `"missing"` | Must be `None` | +| `PRESENT` | `"present"` | Must be an exact `ProjectionManifest` | +| `CORRUPT` | `"corrupt"` | Must be `None` | + +`ProjectionStateSnapshot(availability, manifest)` stores that caller-supplied +state. Projection evaluates it to a closed `ProjectionStateStatus`: + +| Member | String value | Meaning | +| --- | --- | --- | +| `MISSING` | `"missing"` | No state is available. | +| `CURRENT` | `"current"` | Valid state exactly matches the complete target. | +| `STALE` | `"stale"` | Valid compatible state requires mutations. | +| `CORRUPT` | `"corrupt"` | State is declared corrupt or fails integrity/consistency checks. | +| `SCHEMA_MISMATCH` | `"schema_mismatch"` | The schema ID differs. | +| `EMBEDDING_MISMATCH` | `"embedding_mismatch"` | The embedding identity differs. | + +`ProjectionResult(status_before, receipt, manifest)` represents only complete +success. Its receipt and manifest have the same corpus, projection, and +checkpoint. An `UNCHANGED` result requires `CURRENT` state and zero attempted +and completed documents. `ProjectionOutcome` is a closed string enum with these exact member values: @@ -139,8 +210,9 @@ relationship. | `PARTIAL` | `"partial"` | | `FAILED` | `"failed"` | -Unknown enum values raise `ContractValidationError`. A receipt can represent -only the following truthful combinations: +`ProjectionReceipt` has fields `corpus_id`, `projection`, `outcome`, +`attempted_documents`, `completed_documents`, and `checkpoint`. It permits only +these truthful combinations: | Outcome | Counts | Checkpoint | | --- | --- | --- | @@ -150,7 +222,43 @@ only the following truthful combinations: | `FAILED` | `attempted_documents > 0` and `completed_documents == 0` | Forbidden | Any supplied checkpoint must have exactly the receipt's `corpus_id` and -`projection`. +`projection`. A successful workflow returns `ProjectionResult`; a failed +workflow exposes a failed or partial receipt only through +`ProjectionOperationError`. + +## Projection workflows + +Both public functions are synchronous and all parameters are positional-only: + +```python +def project_documents( + request: ProjectionRequest, + state: ProjectionStateSnapshot, + embedder: Borrowed[Embedder], + writer: Borrowed[VectorIndexWriter], + /, +) -> ProjectionResult: ... + +def rebuild_projection( + request: ProjectionRequest, + state: ProjectionStateSnapshot, + embedder: Borrowed[Embedder], + writer: Borrowed[VectorIndexWriter], + resetter: Borrowed[VectorIndexResetter], + /, +) -> ProjectionResult: ... +``` + +`project_documents` returns without collaborator effects when state is +`CURRENT`, applies only the canonical delta when state is `STALE`, and raises +`ProjectionStateError` before effects for every other status. + +`rebuild_projection` accepts every state status and always calls the resetter. +For a nonempty target it verifies embedder identity before reset, then replaces +every target document. For an empty target it resets without accessing the +embedder or writer. It is intentionally destructive and supplies no rollback +or retry. See the [projection guide](projection.md) for the full state and +failure matrices. ## Retrieval values @@ -160,60 +268,46 @@ Any supplied checkpoint must have exactly the receipt's `corpus_id` and | `RetrievalHit` | `fragment: Fragment`, `rank: int` | Fragment requires its exact class and rank is a positive exact integer. There is no score field. | | `RetrievalResult` | `query: RetrievalQuery`, `outcome: RetrievalOutcome`, `hits: tuple[RetrievalHit, ...]`, `truncated: bool` | Nested values, the hit tuple, and the boolean require exact types. Hit count cannot exceed `query.hit_limit`. | -`RetrievalOutcome` is a closed string enum with these exact member values: - -| Member | String value | -| --- | --- | -| `COMPLETE` | `"complete"` | -| `PARTIAL` | `"partial"` | -| `UNAVAILABLE` | `"unavailable"` | -| `STALE` | `"stale"` | -| `FAILED` | `"failed"` | - -Unknown enum values raise `ContractValidationError`. Within every result, ranks -must be contiguous starting at one, fragment identities must be unique, and -every fragment's corpus must match the query corpus. Fragment attributes do not -make two otherwise identical fragment identities distinct. - -The outcome matrix is: - -| Outcome | Hits | `truncated` | -| --- | --- | --- | -| `COMPLETE` | Zero through `query.hit_limit` | Either boolean | -| `PARTIAL` | One through `query.hit_limit` | Either boolean | -| `UNAVAILABLE` | None | `False` | -| `STALE` | None | `False` | -| `FAILED` | None | `False` | +`RetrievalOutcome` is a closed string enum with exact values `"complete"`, +`"partial"`, `"unavailable"`, `"stale"`, and `"failed"`. Within every result, +ranks are contiguous from one, fragment identities are unique, and every +fragment belongs to the query corpus. `PARTIAL` requires at least one hit; +`UNAVAILABLE`, `STALE`, and `FAILED` require no hits and `truncated=False`. `truncated=True` is the caller's explicit assertion that otherwise valid work -or results were cut by the query budget. It does not imply that -`len(hits) == query.hit_limit`; a bounded `COMPLETE` result may therefore still -be truncated. Hits and reader ports are score-free. Raw scores from different -providers are neither represented nor promised to be comparable. +or results were cut by the query budget. Hits and reader ports are score-free; +raw provider scores are neither represented nor promised comparable. + +These are value contracts only. Version 0.1.0 has no package retrieval, +composition, citation, user, tool, or agent workflow. ## Collaborator ports The protocols are synchronous, injected, structurally typed, and decorated -with `runtime_checkable`. Runtime protocol checks establish structural presence, -not the behavioral obligations below. Version 0.1.0 provides no implementation, -adapter, factory, provider discovery, or provider-behavior enforcement. +with `runtime_checkable`. Runtime protocol checks establish structural +presence, not the behavioral obligations below. Version 0.1.0 provides no +adapter, factory, or provider discovery. | Port | Exact public operation | Semantic obligation | | --- | --- | --- | | `Embedder` | `identity: EmbeddingIdentity` | Identify the exact model used for produced vectors. | | `Embedder` | `embed(texts: tuple[str, ...], /) -> tuple[EmbeddingVector, ...]` | Return one same-order vector per input text, each with `identity.dimensions` coordinates; empty input returns an empty tuple. | -| `VectorIndexWriter` | `replace_document(document: DocumentIdentity, records: tuple[VectorRecord, ...], /) -> None` | Replace all derived vectors for the document's stable key. Every record carries the supplied full `DocumentIdentity`; an empty record tuple is valid. | +| `VectorIndexWriter` | `replace_document(document: DocumentIdentity, records: tuple[VectorRecord, ...], /) -> None` | Replace all derived vectors for the document's stable key. Every record carries the supplied full identity; an empty record tuple is valid. | | `VectorIndexWriter` | `delete_document(document: DocumentKey, /) -> None` | Delete every derived revision for the stable document key. | +| `VectorIndexResetter` | `reset_corpus(corpus_id: str, /) -> None` | Remove the complete derived vector projection for the corpus. | | `VectorIndexReader` | `search(query: RetrievalQuery, embedding: EmbeddingVector, /) -> tuple[Fragment, ...]` | Return fragments from the requested corpus, in provider rank order, with at most `query.candidate_limit` entries. | | `LexicalRetriever` | `search(query: RetrievalQuery, /) -> tuple[Fragment, ...]` | Return fragments from the requested corpus, in provider rank order, with at most `query.candidate_limit` entries. | +Projection enforces the embedder result rules and requires each writer or +resetter command to return exactly `None`. It cannot enforce external storage, +atomicity, authorization, concurrency, or lifecycle behavior. + `Borrowed[T]` is the companion ownership marker, not a provider port. Its exact behavior is documented in [resource lifecycle](lifecycle.md). -## Planned workflows +## Planned retrieval workflow -Projection orchestration is planned for Issue #3. Retrieval and composition are -planned for Issue #4. Those future workflows are expected to accept protocol- -compatible collaborators explicitly, but their algorithms, APIs, compatibility -checks, exception translation, and outcome mapping are not implemented or -promised by version 0.1.0. +Retrieval and composition remain planned for Issue #4. The existing query, +result, reader, and lexical contracts do not promise an implemented workflow, +fusion algorithm, compatibility check, exception mapping, citation policy, or +user/agent integration. diff --git a/docs/lifecycle.md b/docs/lifecycle.md index c362a41..bac169f 100644 --- a/docs/lifecycle.md +++ b/docs/lifecycle.md @@ -1,21 +1,30 @@ # Resource Lifecycle Version 0.1.0 uses an explicit caller-owned lifecycle. The package defines -collaborator protocols and `Borrowed[T]`; it does not acquire or own provider -resources. See the [API reference](api.md) for the exact port signatures. +collaborator protocols and `Borrowed[T]`; it does not acquire, configure, +discover, persist, synchronize, or release provider resources. Projection calls +borrowed collaborators only during an explicit workflow invocation. See the +[API reference](api.md) for exact port signatures. ## Ownership rule The caller or provider integration owns every lifecycle decision: -1. Acquire and configure the resource. -2. Establish any required synchronization or exclusive access. -3. Pass the resource explicitly to application code or a future generic - workflow. -4. Reset, release, close, or shut down the resource according to the provider's - rules after use. - -`generic-rag` does none of those steps implicitly. +1. Acquire and configure the embedder, vector index, projection-state store, + and any required synchronization. +2. Authorize the complete source set and construct a bounded + `ProjectionRequest`. +3. Wrap the application-owned collaborators in `Borrowed` and call + `project_documents` for a compatible incremental update, or deliberately + call `rebuild_projection` for bootstrap or destructive recovery. +4. On complete success, publish the returned manifest through caller-owned + persistence and synchronization. +5. Release, close, or shut down the real resources according to their provider + contracts. + +`generic-rag` performs none of the acquisition, manifest persistence, +publication, synchronization, or release steps implicitly. It provides no +transaction across collaborators and the manifest store. ## `Borrowed[T]` @@ -31,9 +40,11 @@ has exactly these context semantics: provider health checks. These guarantees still apply if the wrapped provider defines its own context -manager or lifecycle methods. +manager or lifecycle methods. Projection itself translates ordinary method +failures into `ProjectionOperationError`; that workflow behavior does not +change `Borrowed` semantics. -## Successful scope +## Successful borrowed scope The resource that leaves the scope is still the application-owned object: @@ -50,7 +61,7 @@ assert active_provider is provider # Borrowed did not close or replace it. Application code decides when and how to release the real provider afterward. -## Exceptional scope +## Exceptional borrowed scope An exception leaves the borrowed scope unchanged and unsuppressed: @@ -68,15 +79,38 @@ except RuntimeError as caught: assert caught is failure ``` -If a caller needs cleanup after either success or failure, it must arrange that +If a caller needs cleanup after success or failure, it must arrange that cleanup around the borrowed scope according to the provider's contract. -## Future workflows +## Projection call scope + +`project_documents` and `rebuild_projection` receive exact `Borrowed` wrappers. +They enter only those no-op wrappers; they never enter, close, or shut down the +underlying adapter objects. + +The caller must keep each adapter alive for the complete synchronous call. +Providers that require sessions, transactions, locks, or thread affinity must +be prepared before wrapping and must remain valid until the call returns or +raises. + +An incremental call may perform multiple ordered document mutations. A rebuild +performs a corpus reset before its document replacements. The package does not +roll back or retry earlier effects if a later operation fails. The +`ProjectionOperationError.receipt` reports completed document operations but +never supplies a checkpoint for partial work. Keep partially updated state out +of service and recover under application-owned coordination. + +The caller should persist `ProjectionResult.manifest` only after complete +success. The package does not retain it, and a vector index without the matching +published manifest cannot be used safely by the next incremental operation. +See the [projection guide](projection.md) for the state matrix and destructive +rebuild ordering. + +## Retrieval lifecycle is not implemented -Projection and retrieval orchestration are planned for Issues #3 and #4. Their -collaborators are intended to remain explicitly injected and caller-owned. The -precise future APIs are not part of version 0.1.0, and `Borrowed` must not be -read as a promise that a workflow already exists. +The query and reader contracts do not create a package retrieval workflow. +Retrieval and composition remain planned for Issue #4, and no user or agent +query lifecycle is implied by the current projection API. Review the [security and privacy boundary](security-and-privacy.md) before -passing content to a provider implementation. +passing content to any adapter implementation. diff --git a/docs/projection.md b/docs/projection.md new file mode 100644 index 0000000..e407b6a --- /dev/null +++ b/docs/projection.md @@ -0,0 +1,257 @@ +# Projection + +Version 0.1.0 can turn a complete, caller-approved document set into a +deterministic vector projection. It supplies orchestration, contracts, and +failure reporting; the caller supplies and owns the embedder, vector index, +projection-state persistence, authorization policy, and synchronization. + +Projection does not make content retrievable through this package. Retrieval, +result composition, and user or agent integration remain planned for [Issue +#4](https://github.com/Kims-DeveloperGroup/generic-rag/issues/4). + +## Required adapters and state + +Implement application-specific objects that structurally satisfy these public +ports: + +- `Embedder` exposes an `EmbeddingIdentity` and returns one same-order vector + of exactly that dimensionality for every input text. +- `VectorIndexWriter` replaces the complete record set for one stable document + key or deletes every revision of that key. Both commands must return `None`. +- `VectorIndexResetter` removes every projected document for one corpus and + returns `None`. It is required only by `rebuild_projection`. + +The package does not provide adapters or a manifest store. Persist the +successful `ProjectionResult.manifest` in application-owned state, then load it +as a `ProjectionStateSnapshot` for the next operation. The stored manifest and +the vector index are one logical projection: publish them under application- +controlled synchronization so another operation cannot observe an unintended +combination. + +All workflow parameters are positional-only. Each collaborator must be wrapped +in the exact `Borrowed` class; borrowing never acquires, closes, resets, or +otherwise owns the wrapped resource. + +## Bootstrap, then update incrementally + +The first projection has no valid present manifest, so bootstrap it through the +explicitly destructive rebuild path: + +```python +from generic_rag.contracts import ( + ChunkingPolicy, + Document, + DocumentIdentity, + DocumentKey, + EmbeddingIdentity, + EmbeddingVector, + ProjectionIdentity, + ProjectionLimits, + ProjectionRequest, + ProjectionStateAvailability, + ProjectionStateSnapshot, +) +from generic_rag.ports import Borrowed +from generic_rag.projection import project_documents, rebuild_projection + + +class ApplicationEmbedder: + identity = EmbeddingIdentity("embedding-model-v1", 2) + + def embed(self, texts, /): + return tuple(EmbeddingVector((float(len(text)), 0.0)) for text in texts) + + +class ApplicationVectorIndex: + def replace_document(self, document, records, /): + # Replace all derived records for document.key in one adapter operation. + return None + + def delete_document(self, document, /): + # Delete every projected revision of this stable document key. + return None + + def reset_corpus(self, corpus_id, /): + # Delete the complete vector projection for this corpus. + return None + + +def target(revision_id, text): + return ProjectionRequest( + "approved-corpus", + ProjectionIdentity("schema-v1", ApplicationEmbedder.identity), + ChunkingPolicy(800, 80), + ProjectionLimits(100, 100_000, 32), + ( + Document( + DocumentIdentity( + DocumentKey("approved-corpus", "document-1"), + revision_id, + ), + text, + (("source", "application-authorized"),), + ), + ), + ) + + +embedder = ApplicationEmbedder() +index = ApplicationVectorIndex() +initial_request = target("revision-1", "First approved source text") + +initial_result = rebuild_projection( + initial_request, + ProjectionStateSnapshot(ProjectionStateAvailability.MISSING, None), + Borrowed(embedder), + Borrowed(index), + Borrowed(index), +) +# Persist initial_result.manifest only after the call succeeds. + +updated_request = target("revision-2", "Updated approved source text") +updated_result = project_documents( + updated_request, + ProjectionStateSnapshot( + ProjectionStateAvailability.PRESENT, + initial_result.manifest, + ), + Borrowed(embedder), + Borrowed(index), +) +# Atomically publish updated_result.manifest as the new application-owned state. +``` + +The example uses one object for the writer and resetter ports, but separate +objects are equally valid. Production adapters must implement the stated +complete replacement, deletion, reset, persistence, and synchronization +semantics; the example deliberately omits storage. + +## Incremental lifecycle + +`project_documents(request, state, embedder, writer)` applies a complete target +to compatible state: + +1. It validates the exact request, snapshot, and `Borrowed` wrapper types and + prepares the complete target before calling a collaborator. +2. It evaluates the supplied manifest against the target. `current` returns an + `UNCHANGED` result with zero attempts and no collaborator calls. Only + `stale` proceeds; every other status raises `ProjectionStateError` before + effects. +3. It sorts mutations by opaque `document_id`. Added documents, new revisions, + and documents affected by a chunking change are replaced; absent target + documents are deleted; unchanged entries are skipped. A source change under + an unchanged full document identity is corrupt state, not an incremental + replacement. +4. If any replacement is needed, it validates the embedder identity once, + embeds nonempty ordered batches no larger than + `max_embedding_batch_size`, and calls the writer once per document. A + delete-only update does not access the embedder. +5. After every mutation succeeds, it returns `COMPLETED` with the complete + target manifest and checkpoint. The caller may then publish that manifest. + +An empty document produces no fragments and is still replaced with an explicit +empty record tuple. An empty target incrementally deletes every document in the +previous valid manifest. + +## Destructive rebuild lifecycle + +`rebuild_projection(request, state, embedder, writer, resetter)` is the explicit +bootstrap and recovery operation. It accepts every state status, including +`current`, but always resets the requested corpus and recreates the complete +target: + +1. It validates inputs and computes `status_before` without effects. +2. For a nonempty target, it validates the embedder identity once *before* the + reset. A mismatch or invalid identity therefore leaves reset and writer + untouched. +3. It calls `reset_corpus`, then embeds and replaces every target document in + canonical `document_id` order. An empty target calls only the resetter. +4. Complete success returns a `COMPLETED` result, even when the target has zero + documents. + +Reset is intentionally destructive. The package provides no transaction, +rollback, retry, or two-phase publication across the resetter, writer, and +caller-owned manifest store. If reset succeeds and a later replacement fails, +the vector index can contain an incomplete rebuild and no successful +checkpoint is issued. Keep the prior manifest out of service and run an +application-controlled recovery, normally another full rebuild. + +## State matrix + +`ProjectionStateAvailability` describes what the caller could load. +`ProjectionStateStatus` is the workflow's evaluation of that snapshot against +the requested complete target. + +| Supplied state | Evaluated status | `project_documents` | `rebuild_projection` | +| --- | --- | --- | --- | +| `MISSING` with no manifest | `MISSING` | Raises before effects | Resets and builds target | +| `CORRUPT` with no manifest | `CORRUPT` | Raises before effects | Resets and builds target | +| Valid present manifest exactly matching target | `CURRENT` | Returns `UNCHANGED`; no effects | Resets and rebuilds target | +| Valid present manifest with compatible target differences | `STALE` | Applies incremental mutations | Resets and rebuilds target | +| Present manifest with invalid checkpoint, wrong corpus, or same-revision source/count inconsistency | `CORRUPT` | Raises before effects | Resets and rebuilds target | +| Present manifest with another schema ID | `SCHEMA_MISMATCH` | Raises before effects | Resets and rebuilds target | +| Present manifest with another embedding identity | `EMBEDDING_MISMATCH` | Raises before effects | Resets and rebuilds target | + +A revision change is a normal stale update. For the same full document +identity, changing the source digest—or the fragment count under unchanged +chunking—is treated as corrupt state rather than an unannounced rewrite. + +## Deterministic projection values + +Documents and manifest entries are canonicalized by opaque `document_id`. +Fragments use half-open Python Unicode code-point ranges and copy the source's +ordered attributes. Each fragment contains at most +`max_fragment_codepoints`; consecutive fragments overlap by +`overlap_codepoints`. Attribute order and duplicates remain significant. + +Source digests, fragment IDs, and checkpoint tokens use lowercase +`sha256:<64hex>` values. The current algorithms serialize tagged fields as +UTF-8 with `surrogatepass`, prefix each encoded field with its unsigned +eight-byte big-endian length, and hash them under these versioned domains: + +| Value | v1 domain | Bound inputs | +| --- | --- | --- | +| Source digest | `generic-rag:projection-source:v1` | Exact document text and ordered attributes | +| Fragment ID | `generic-rag:fragment-id:v1` | Corpus, document, revision, and fragment range | +| Checkpoint token | `generic-rag:projection-checkpoint:v1` | Corpus, projection and embedding identities, chunking policy, and ordered manifest entries | + +These values are reproducible for the same inputs and current v1 algorithm, +including across clean processes. The v1 domain names do not promise that a +future package version will retain the same algorithm or accept an old +manifest. Consumers that persist projection state should pin and review the +package version and use explicit rebuild for an incompatible upgrade. + +Hashes are deterministic comparison and identity values, not encryption, +authorization, or a proof of source ownership. See [security and +privacy](security-and-privacy.md). + +## Failures and receipts + +Invalid public values raise `ContractValidationError`. Incremental state that +is not `current` or `stale` raises `ProjectionStateError`, whose `status` gives +the evaluated reason. Both cases are detected before collaborator effects. + +An ordinary collaborator exception, or an invalid collaborator return, raises +`ProjectionOperationError` with: + +- `stage`: `EMBEDDER_IDENTITY`, `EMBEDDING`, `REPLACEMENT`, `DELETION`, or + `RESET`; +- `affected_document`: the stable key for a document-specific failure, else + `None`; and +- `receipt`: `FAILED` when no planned document completed, `PARTIAL` after one + or more but not all planned documents completed, or `None` when there were + zero document attempts. + +Failure receipts never contain a checkpoint. `attempted_documents` is the +total mutation or rebuild-document count; `completed_documents` counts only +fully completed document operations. The original ordinary exception is +chained as the cause. A structurally invalid identity, vector result, or +non-`None` writer/resetter return has no internal cause. `KeyboardInterrupt` +and `SystemExit` are neither translated nor retried. + +The public error messages do not include document or vector content. Adapter +exception messages remain reachable through exception chaining, so adapters +and application logging must avoid disclosing sensitive values. + +See the [API reference](api.md) for exact signatures and value invariants and +[resource lifecycle](lifecycle.md) for ownership details. diff --git a/docs/security-and-privacy.md b/docs/security-and-privacy.md index 7147817..91abad7 100644 --- a/docs/security-and-privacy.md +++ b/docs/security-and-privacy.md @@ -1,64 +1,114 @@ # Security and Privacy -Version 0.1.0 defines in-process values and collaborator boundaries. By itself, -the package performs no persistence, network transmission, provider discovery, -credential loading, telemetry, or background work. Contract objects do retain -caller-supplied values in process memory. +Version 0.1.0 defines in-process values, collaborator boundaries, and an +explicit projection workflow. The package itself performs no persistence, +network setup, provider discovery, credential loading, telemetry, or background +work. A projection call does pass derived fragment text and metadata to the +caller-supplied embedder and vector writer, whose effects are outside the +package. ## Caller responsibility -Before constructing a `Document`, `Fragment`, or `RetrievalQuery`, the caller -must perform its own authorization and policy checks. The caller also controls: +Before constructing a `ProjectionRequest`, the caller must authorize every +source, revision, attribute, and intended destination. The package does not +authenticate an authoritative source or decide whether a user, tool, or agent +may project it. -- which source content and metadata enter the contracts; -- which provider implementations receive document text, fragment text, - embeddings, attributes, identifiers, or queries; +The caller also controls: + +- corpus and tenant isolation; +- which adapter implementations receive document text, fragment text, + embeddings, attributes, identifiers, or future queries; - provider account, region, transport, and credential configuration; -- retention, replacement, deletion, backup, and recovery behavior; -- logging, tracing, metrics, redaction, and incident response; and -- whether retrieved fragments are displayed, persisted, or supplied to another - tool or agent. +- vector-index and manifest-store access control, retention, replacement, + deletion, backup, and recovery; +- synchronization between vector mutations and manifest publication; +- logging, tracing, metrics, exception rendering, redaction, and incident + response; and +- whether future retrieved fragments are displayed, persisted, or supplied to + another tool or agent. -Do not place credentials or other secrets in attributes, opaque identifiers, or -checkpoint tokens. These fields deliberately preserve caller input and do not -apply redaction, escaping, access control, or tenant isolation. +Do not place credentials or other secrets in attributes, opaque identifiers, +or checkpoint tokens. These fields preserve caller input and do not apply +redaction, escaping, authorization, or tenant isolation. -## Provider effects +## Adapter effects -The protocols describe operations that an external implementation may perform. -Calling an embedder, vector index, or lexical retriever can store or transmit -data according to that implementation. Version 0.1.0 supplies no such provider -and does not call one on the caller's behalf. +Calling `project_documents` can invoke the supplied embedder and writer. +Calling `rebuild_projection` can additionally reset all projected data for the +requested corpus. Those adapters may persist or transmit data according to +their implementations. Review their transport, storage, subprocess, network, +credential, and deletion behavior before use. -`Borrowed` does not reduce this responsibility. It only marks the wrapped -resource as caller-owned and does not acquire, close, reset, authenticate, or -synchronize it. See [resource lifecycle](lifecycle.md). +`Borrowed` only marks resources as caller-owned. It does not acquire, close, +authenticate, synchronize, sandbox, or reduce the privileges of an adapter. +The package supplies no transaction or rollback across the vector index and +caller-owned manifest store. See [resource lifecycle](lifecycle.md). -## Sensitive derived data +## Sensitive source and derived data Treat all of the following as potentially sensitive: -- document and query text; +- document and future query text; - ordered attributes and opaque identities; - fragments and their source ranges; -- embeddings and vector records; and -- projection checkpoint tokens. - -Fragments and embeddings are derived data but may reveal information from the -source. Deleting an authoritative source does not automatically delete copies -or derived values held by an application or provider. +- embeddings and vector records; +- source digests, fragment IDs, manifests, and checkpoint tokens; and +- adapter exceptions and logs. + +Fragments and embeddings may reveal source information. Deterministic IDs and +digests may allow equality correlation or guessing attacks against predictable +content. Their `sha256:` representation provides neither encryption nor access +control and should not be used as proof of source ownership. + +Deleting an authoritative source does not automatically delete copies, +backups, logs, embeddings, or derived records held by an application or +provider. Incremental deletion and corpus reset cover only the behavior promised +by the supplied vector-index adapter. + +## Limits and resource policy + +`ProjectionLimits` rejects a request that exceeds its configured document count +or per-document code-point cap and bounds each embedding batch. +`ChunkingPolicy` bounds fragment size and overlap. These checks prevent one +accepted request from exceeding caller-selected values; they are not global +quotas, rate limits, memory isolation, provider billing controls, timeouts, or +admission control. + +Choose limits from trusted application policy rather than untrusted request +parameters. Account for the fact that a small fragment size and large permitted +document set can still produce many fragments and provider operations. Supply +external cancellation, concurrency, cost, and capacity controls where needed. + +## Errors and logging + +`ProjectionStateError` and the direct message of +`ProjectionOperationError` are content-free. Ordinary adapter exceptions are +preserved as chained causes, so rendering the full exception chain may expose +an adapter's message or fields. Adapter implementations must avoid placing +document text, fragments, embeddings, credentials, or sensitive identifiers in +exceptions and logs. Applications should apply redaction before exporting +traces or error reports. + +The package does not retry collaborator operations. A failure can leave earlier +document mutations in place, and a rebuild failure can occur after the corpus +was reset. Do not publish a failed or partial receipt as a completed checkpoint; +isolate the affected projection and recover under caller-owned policy. ## Trust boundary -The package does not own, authenticate, or prove an authoritative source or -revision. A fragment range checks only a half-open code-point width against the -fragment text length; it does not verify the text against a source document. -Ranges and attributes are not a citation or provenance-verification mechanism. +A fragment range is a half-open Python code-point range. Standalone contracts +check its width but do not prove that text came from the indicated source. +Projection derives its own fragment text from the supplied document, but the +package still cannot prove that the supplied document or revision was +authoritative or authorized. Version 0.1.0 provides no built-in encryption, authentication, authorization, -ACL, content filter, persistence, network security, citation validation, or -vendor guarantee. A consuming application must select and assess those controls -for its environment. - -The complete public value and provider boundaries are listed in the -[API reference](api.md). +ACL, content filter, persistence security, network security, citation +validation, secret management, vendor guarantee, retrieval workflow, or +user/agent policy. A consuming application must select and assess those +controls for its environment. + +The complete public value and collaborator boundaries are listed in the [API +reference](api.md), and deterministic projection behavior is documented in the +[projection guide](projection.md). diff --git a/src/generic_rag/contracts.py b/src/generic_rag/contracts.py index a8cfb66..978cbc5 100644 --- a/src/generic_rag/contracts.py +++ b/src/generic_rag/contracts.py @@ -2,6 +2,7 @@ from __future__ import annotations +import re from dataclasses import dataclass from enum import StrEnum from math import isfinite @@ -21,12 +22,23 @@ "ProjectionCheckpoint", "ProjectionOutcome", "ProjectionReceipt", + "ChunkingPolicy", + "ProjectionLimits", + "ProjectionRequest", + "ProjectionManifestEntry", + "ProjectionManifest", + "ProjectionStateAvailability", + "ProjectionStateSnapshot", + "ProjectionStateStatus", + "ProjectionResult", "RetrievalQuery", "RetrievalOutcome", "RetrievalHit", "RetrievalResult", ) +_SOURCE_DIGEST = re.compile(r"sha256:[0-9a-f]{64}") + def _require_exact_type(name: str, value: object, expected: type[object]) -> None: if type(value) is not expected: @@ -310,6 +322,239 @@ def __post_init__(self) -> None: ) +@dataclass(frozen=True, slots=True) +class ChunkingPolicy: + """Code-point chunk size and overlap used by one projection.""" + + max_fragment_codepoints: int + overlap_codepoints: int + + def __post_init__(self) -> None: + maximum = _require_positive_integer( + "max_fragment_codepoints", + self.max_fragment_codepoints, + ) + overlap = _require_nonnegative_integer( + "overlap_codepoints", + self.overlap_codepoints, + ) + if overlap >= maximum: + raise ContractValidationError( + "overlap_codepoints must be smaller than max_fragment_codepoints" + ) + + +@dataclass(frozen=True, slots=True) +class ProjectionLimits: + """Independent document, text, and embedding-batch projection bounds.""" + + max_documents: int + max_document_codepoints: int + max_embedding_batch_size: int + + def __post_init__(self) -> None: + _require_positive_integer("max_documents", self.max_documents) + _require_positive_integer( + "max_document_codepoints", + self.max_document_codepoints, + ) + _require_positive_integer( + "max_embedding_batch_size", + self.max_embedding_batch_size, + ) + + +@dataclass(frozen=True, slots=True) +class ProjectionRequest: + """A complete bounded canonical target projection for one corpus.""" + + corpus_id: str + projection: ProjectionIdentity + chunking: ChunkingPolicy + limits: ProjectionLimits + documents: tuple[Document, ...] + + def __post_init__(self) -> None: + _require_nonblank_string("corpus_id", self.corpus_id) + _require_exact_type("projection", self.projection, ProjectionIdentity) + _require_exact_type("chunking", self.chunking, ChunkingPolicy) + _require_exact_type("limits", self.limits, ProjectionLimits) + _require_exact_type("documents", self.documents, tuple) + if len(self.documents) > self.limits.max_documents: + raise ContractValidationError("documents must not exceed max_documents") + + keys: set[DocumentKey] = set() + canonical: list[Document] = [] + for index, document in enumerate(self.documents): + _require_exact_type(f"documents[{index}]", document, Document) + if document.identity.key.corpus_id != self.corpus_id: + raise ContractValidationError( + "every document corpus_id must match the request corpus_id" + ) + if len(document.text) > self.limits.max_document_codepoints: + raise ContractValidationError( + "document text must not exceed max_document_codepoints" + ) + if document.identity.key in keys: + raise ContractValidationError( + "documents must have unique stable document keys" + ) + keys.add(document.identity.key) + canonical.append(document) + canonical.sort(key=lambda document: document.identity.key.document_id) + object.__setattr__(self, "documents", tuple(canonical)) + + +@dataclass(frozen=True, slots=True) +class ProjectionManifestEntry: + """One projected document revision, source digest, and fragment count.""" + + document: DocumentIdentity + source_digest: str + fragment_count: int + + def __post_init__(self) -> None: + _require_exact_type("document", self.document, DocumentIdentity) + _require_exact_type("source_digest", self.source_digest, str) + if _SOURCE_DIGEST.fullmatch(self.source_digest) is None: + raise ContractValidationError( + "source_digest must be lowercase sha256:<64hex>" + ) + _require_nonnegative_integer("fragment_count", self.fragment_count) + + +@dataclass(frozen=True, slots=True) +class ProjectionManifest: + """A complete canonical successful projection checkpoint manifest.""" + + corpus_id: str + projection: ProjectionIdentity + chunking: ChunkingPolicy + entries: tuple[ProjectionManifestEntry, ...] + checkpoint: ProjectionCheckpoint + + def __post_init__(self) -> None: + _require_nonblank_string("corpus_id", self.corpus_id) + _require_exact_type("projection", self.projection, ProjectionIdentity) + _require_exact_type("chunking", self.chunking, ChunkingPolicy) + _require_exact_type("entries", self.entries, tuple) + _require_exact_type("checkpoint", self.checkpoint, ProjectionCheckpoint) + if self.checkpoint.corpus_id != self.corpus_id: + raise ContractValidationError( + "checkpoint corpus_id must match the manifest corpus_id" + ) + if self.checkpoint.projection != self.projection: + raise ContractValidationError( + "checkpoint projection must match the manifest projection" + ) + + keys: set[DocumentKey] = set() + canonical: list[ProjectionManifestEntry] = [] + for index, entry in enumerate(self.entries): + _require_exact_type(f"entries[{index}]", entry, ProjectionManifestEntry) + if entry.document.key.corpus_id != self.corpus_id: + raise ContractValidationError( + "every manifest entry corpus_id must match the manifest corpus_id" + ) + if entry.document.key in keys: + raise ContractValidationError( + "manifest entries must have unique stable document keys" + ) + keys.add(entry.document.key) + canonical.append(entry) + canonical.sort(key=lambda entry: entry.document.key.document_id) + object.__setattr__(self, "entries", tuple(canonical)) + + +class ProjectionStateAvailability(StrEnum): + """Structural availability of a caller-supplied projection snapshot.""" + + MISSING = "missing" + PRESENT = "present" + CORRUPT = "corrupt" + + @classmethod + def _missing_(cls, value: object) -> None: + raise ContractValidationError(f"{cls.__name__} value is not a defined member") + + +@dataclass(frozen=True, slots=True) +class ProjectionStateSnapshot: + """Caller-supplied projection state without package-owned persistence.""" + + availability: ProjectionStateAvailability + manifest: ProjectionManifest | None + + def __post_init__(self) -> None: + _require_exact_type( + "availability", + self.availability, + ProjectionStateAvailability, + ) + if self.availability is ProjectionStateAvailability.PRESENT: + _require_exact_type("manifest", self.manifest, ProjectionManifest) + elif self.manifest is not None: + raise ContractValidationError( + "missing and corrupt state snapshots must not contain a manifest" + ) + + +class ProjectionStateStatus(StrEnum): + """Compatibility of supplied state with one complete projection target.""" + + MISSING = "missing" + CURRENT = "current" + STALE = "stale" + CORRUPT = "corrupt" + SCHEMA_MISMATCH = "schema_mismatch" + EMBEDDING_MISMATCH = "embedding_mismatch" + + @classmethod + def _missing_(cls, value: object) -> None: + raise ContractValidationError(f"{cls.__name__} value is not a defined member") + + +@dataclass(frozen=True, slots=True) +class ProjectionResult: + """One completely successful projection result and authoritative manifest.""" + + status_before: ProjectionStateStatus + receipt: ProjectionReceipt + manifest: ProjectionManifest + + def __post_init__(self) -> None: + _require_exact_type("status_before", self.status_before, ProjectionStateStatus) + _require_exact_type("receipt", self.receipt, ProjectionReceipt) + _require_exact_type("manifest", self.manifest, ProjectionManifest) + if self.receipt.outcome not in ( + ProjectionOutcome.COMPLETED, + ProjectionOutcome.UNCHANGED, + ): + raise ContractValidationError( + "projection results require a completed or unchanged receipt" + ) + if self.receipt.corpus_id != self.manifest.corpus_id: + raise ContractValidationError( + "receipt corpus_id must match the result manifest" + ) + if self.receipt.projection != self.manifest.projection: + raise ContractValidationError( + "receipt projection must match the result manifest" + ) + if self.receipt.checkpoint != self.manifest.checkpoint: + raise ContractValidationError( + "receipt checkpoint must equal the result manifest checkpoint" + ) + if self.receipt.outcome is ProjectionOutcome.UNCHANGED and ( + self.status_before is not ProjectionStateStatus.CURRENT + or self.receipt.attempted_documents != 0 + or self.receipt.completed_documents != 0 + ): + raise ContractValidationError( + "unchanged results require current state and zero document attempts" + ) + + @dataclass(frozen=True, slots=True) class RetrievalQuery: """A bounded retrieval request for one opaque corpus.""" diff --git a/src/generic_rag/ports.py b/src/generic_rag/ports.py index fdc931b..0eb7105 100644 --- a/src/generic_rag/ports.py +++ b/src/generic_rag/ports.py @@ -20,6 +20,7 @@ "Borrowed", "Embedder", "VectorIndexWriter", + "VectorIndexResetter", "VectorIndexReader", "LexicalRetriever", ) @@ -81,6 +82,15 @@ def delete_document(self, document: DocumentKey, /) -> None: ... +@runtime_checkable +class VectorIndexResetter(Protocol): + """Synchronously removes every projected document for one corpus.""" + + def reset_corpus(self, corpus_id: str, /) -> None: + """Remove the complete derived vector projection for the corpus.""" + ... + + @runtime_checkable class VectorIndexReader(Protocol): """Returns best-first vector candidates without exposing raw scores.""" diff --git a/src/generic_rag/projection.py b/src/generic_rag/projection.py new file mode 100644 index 0000000..a8ac690 --- /dev/null +++ b/src/generic_rag/projection.py @@ -0,0 +1,814 @@ +"""Deterministic bounded document projection orchestration.""" + +from __future__ import annotations + +import hashlib +from collections.abc import Callable +from dataclasses import dataclass +from enum import StrEnum +from math import isfinite +from typing import NoReturn, cast + +from .contracts import ( + ChunkingPolicy, + Document, + DocumentIdentity, + DocumentKey, + EmbeddingIdentity, + EmbeddingVector, + Fragment, + FragmentIdentity, + ProjectionCheckpoint, + ProjectionIdentity, + ProjectionLimits, + ProjectionManifest, + ProjectionManifestEntry, + ProjectionOutcome, + ProjectionReceipt, + ProjectionRequest, + ProjectionResult, + ProjectionStateAvailability, + ProjectionStateSnapshot, + ProjectionStateStatus, + VectorRecord, +) +from .errors import ( + CollaborationError, + ContractValidationError, + StateCompatibilityError, +) +from .ports import Borrowed, Embedder, VectorIndexResetter, VectorIndexWriter + +__all__ = ( + "ProjectionFailureStage", + "ProjectionStateError", + "ProjectionOperationError", + "project_documents", + "rebuild_projection", +) + +_SOURCE_DIGEST_DOMAIN = "generic-rag:projection-source:v1" +_FRAGMENT_ID_DOMAIN = "generic-rag:fragment-id:v1" +_CHECKPOINT_DOMAIN = "generic-rag:projection-checkpoint:v1" + + +class ProjectionFailureStage(StrEnum): + """The collaborator stage at which a projection operation failed.""" + + EMBEDDER_IDENTITY = "embedder_identity" + EMBEDDING = "embedding" + REPLACEMENT = "replacement" + DELETION = "deletion" + RESET = "reset" + + @classmethod + def _missing_(cls, value: object) -> None: + raise ContractValidationError(f"{cls.__name__} value is not a defined member") + + +class ProjectionStateError(StateCompatibilityError): + """Raised before effects when incremental projection cannot use state.""" + + status: ProjectionStateStatus + + def __init__(self, status: ProjectionStateStatus) -> None: + _require_exact_type("status", status, ProjectionStateStatus) + self.status = status + super().__init__("projection state is not compatible with incremental update") + + +class ProjectionOperationError(CollaborationError): + """A content-free translation of one collaborator operation failure.""" + + stage: ProjectionFailureStage + affected_document: DocumentKey | None + receipt: ProjectionReceipt | None + + def __init__( + self, + stage: ProjectionFailureStage, + affected_document: DocumentKey | None, + receipt: ProjectionReceipt | None, + ) -> None: + _require_exact_type("stage", stage, ProjectionFailureStage) + if affected_document is not None: + _validate_document_key(affected_document) + if receipt is not None: + _require_exact_type("receipt", receipt, ProjectionReceipt) + if receipt.outcome not in ( + ProjectionOutcome.FAILED, + ProjectionOutcome.PARTIAL, + ): + raise ContractValidationError( + "operation error receipt must be failed or partial" + ) + self.stage = stage + self.affected_document = affected_document + self.receipt = receipt + super().__init__(f"projection collaborator failed during {stage.value}") + + +@dataclass(frozen=True, slots=True) +class _PreparedDocument: + source: Document + fragments: tuple[Fragment, ...] + entry: ProjectionManifestEntry + + +@dataclass(frozen=True, slots=True) +class _PreparedTarget: + request: ProjectionRequest + documents: tuple[_PreparedDocument, ...] + manifest: ProjectionManifest + + +@dataclass(frozen=True, slots=True) +class _Mutation: + key: DocumentKey + replacement: _PreparedDocument | None + + +def _require_exact_type(name: str, value: object, expected: type[object]) -> None: + if type(value) is not expected: + raise ContractValidationError( + f"{name} must be exactly {expected.__name__}, not {type(value).__name__}" + ) + + +def _validate_document_key(value: object) -> DocumentKey: + _require_exact_type("document key", value, DocumentKey) + assert isinstance(value, DocumentKey) + return DocumentKey(value.corpus_id, value.document_id) + + +def _validate_document_identity(value: object) -> DocumentIdentity: + _require_exact_type("document identity", value, DocumentIdentity) + assert isinstance(value, DocumentIdentity) + return DocumentIdentity( + _validate_document_key(value.key), + value.revision_id, + ) + + +def _validate_document(value: object) -> Document: + _require_exact_type("document", value, Document) + assert isinstance(value, Document) + return Document( + _validate_document_identity(value.identity), + value.text, + value.attributes, + ) + + +def _validate_embedding_identity(value: object) -> EmbeddingIdentity: + _require_exact_type("embedding identity", value, EmbeddingIdentity) + assert isinstance(value, EmbeddingIdentity) + return EmbeddingIdentity(value.model_id, value.dimensions) + + +def _validate_projection_identity(value: object) -> ProjectionIdentity: + _require_exact_type("projection identity", value, ProjectionIdentity) + assert isinstance(value, ProjectionIdentity) + return ProjectionIdentity( + value.schema_id, + _validate_embedding_identity(value.embedding), + ) + + +def _validate_chunking(value: object) -> ChunkingPolicy: + _require_exact_type("chunking", value, ChunkingPolicy) + assert isinstance(value, ChunkingPolicy) + return ChunkingPolicy( + value.max_fragment_codepoints, + value.overlap_codepoints, + ) + + +def _validate_limits(value: object) -> ProjectionLimits: + _require_exact_type("limits", value, ProjectionLimits) + assert isinstance(value, ProjectionLimits) + return ProjectionLimits( + value.max_documents, + value.max_document_codepoints, + value.max_embedding_batch_size, + ) + + +def _validate_request(value: object) -> ProjectionRequest: + _require_exact_type("request", value, ProjectionRequest) + assert isinstance(value, ProjectionRequest) + _require_exact_type("request documents", value.documents, tuple) + documents = tuple(_validate_document(document) for document in value.documents) + canonical = ProjectionRequest( + value.corpus_id, + _validate_projection_identity(value.projection), + _validate_chunking(value.chunking), + _validate_limits(value.limits), + documents, + ) + if documents != canonical.documents: + raise ContractValidationError("request documents must be in canonical order") + return canonical + + +def _validate_manifest_entry(value: object) -> ProjectionManifestEntry: + _require_exact_type("manifest entry", value, ProjectionManifestEntry) + assert isinstance(value, ProjectionManifestEntry) + return ProjectionManifestEntry( + _validate_document_identity(value.document), + value.source_digest, + value.fragment_count, + ) + + +def _validate_checkpoint(value: object) -> ProjectionCheckpoint: + _require_exact_type("checkpoint", value, ProjectionCheckpoint) + assert isinstance(value, ProjectionCheckpoint) + return ProjectionCheckpoint( + value.corpus_id, + _validate_projection_identity(value.projection), + value.token, + ) + + +def _validate_manifest(value: object) -> ProjectionManifest: + _require_exact_type("manifest", value, ProjectionManifest) + assert isinstance(value, ProjectionManifest) + _require_exact_type("manifest entries", value.entries, tuple) + entries = tuple(_validate_manifest_entry(entry) for entry in value.entries) + canonical = ProjectionManifest( + value.corpus_id, + _validate_projection_identity(value.projection), + _validate_chunking(value.chunking), + entries, + _validate_checkpoint(value.checkpoint), + ) + if entries != canonical.entries: + raise ContractValidationError("manifest entries must be in canonical order") + return canonical + + +def _validate_state(value: object) -> ProjectionStateSnapshot: + _require_exact_type("state", value, ProjectionStateSnapshot) + assert isinstance(value, ProjectionStateSnapshot) + _require_exact_type( + "state availability", + value.availability, + ProjectionStateAvailability, + ) + manifest = None if value.manifest is None else _validate_manifest(value.manifest) + return ProjectionStateSnapshot(value.availability, manifest) + + +def _validate_borrowed(name: str, value: object) -> None: + _require_exact_type(name, value, Borrowed) + + +def _sha256_fields(fields: tuple[str, ...]) -> str: + digest = hashlib.sha256() + for field in fields: + encoded = field.encode("utf-8", "surrogatepass") + digest.update(len(encoded).to_bytes(8, "big", signed=False)) + digest.update(encoded) + return f"sha256:{digest.hexdigest()}" + + +def _source_digest(document: Document) -> str: + fields = [ + _SOURCE_DIGEST_DOMAIN, + "text", + document.text, + "attributes_count", + str(len(document.attributes)), + ] + for key, value in document.attributes: + fields.extend(("attribute_key", key, "attribute_value", value)) + return _sha256_fields(tuple(fields)) + + +def _fragment_id(document: DocumentIdentity, start: int, end: int) -> str: + return _sha256_fields( + ( + _FRAGMENT_ID_DOMAIN, + "corpus_id", + document.key.corpus_id, + "document_id", + document.key.document_id, + "revision_id", + document.revision_id, + "start", + str(start), + "end", + str(end), + ) + ) + + +def _checkpoint_token( + corpus_id: str, + projection: ProjectionIdentity, + chunking: ChunkingPolicy, + entries: tuple[ProjectionManifestEntry, ...], +) -> str: + fields = [ + _CHECKPOINT_DOMAIN, + "corpus_id", + corpus_id, + "schema_id", + projection.schema_id, + "embedding_model_id", + projection.embedding.model_id, + "embedding_dimensions", + str(projection.embedding.dimensions), + "max_fragment_codepoints", + str(chunking.max_fragment_codepoints), + "overlap_codepoints", + str(chunking.overlap_codepoints), + "entry_count", + str(len(entries)), + ] + for entry in entries: + fields.extend( + ( + "document_id", + entry.document.key.document_id, + "revision_id", + entry.document.revision_id, + "source_digest", + entry.source_digest, + "fragment_count", + str(entry.fragment_count), + ) + ) + return _sha256_fields(tuple(fields)) + + +def _fragments(document: Document, chunking: ChunkingPolicy) -> tuple[Fragment, ...]: + fragments: list[Fragment] = [] + start = 0 + text_length = len(document.text) + while start < text_length: + end = min(start + chunking.max_fragment_codepoints, text_length) + identity = FragmentIdentity( + document.identity, + _fragment_id(document.identity, start, end), + start, + end, + ) + fragments.append( + Fragment( + identity, + document.text[start:end], + document.attributes, + ) + ) + if end == text_length: + break + start = end - chunking.overlap_codepoints + return tuple(fragments) + + +def _prepare_target(request: ProjectionRequest) -> _PreparedTarget: + prepared: list[_PreparedDocument] = [] + for document in request.documents: + fragments = _fragments(document, request.chunking) + entry = ProjectionManifestEntry( + document.identity, + _source_digest(document), + len(fragments), + ) + prepared.append(_PreparedDocument(document, fragments, entry)) + entries = tuple(item.entry for item in prepared) + checkpoint = ProjectionCheckpoint( + request.corpus_id, + request.projection, + _checkpoint_token( + request.corpus_id, + request.projection, + request.chunking, + entries, + ), + ) + manifest = ProjectionManifest( + request.corpus_id, + request.projection, + request.chunking, + entries, + checkpoint, + ) + return _PreparedTarget(request, tuple(prepared), manifest) + + +def _has_valid_checkpoint(manifest: ProjectionManifest) -> bool: + expected = _checkpoint_token( + manifest.corpus_id, + manifest.projection, + manifest.chunking, + manifest.entries, + ) + return manifest.checkpoint.token == expected + + +def _state_status( + state: ProjectionStateSnapshot, + target: _PreparedTarget, +) -> ProjectionStateStatus: + if state.availability is ProjectionStateAvailability.MISSING: + return ProjectionStateStatus.MISSING + if state.availability is ProjectionStateAvailability.CORRUPT: + return ProjectionStateStatus.CORRUPT + + manifest = state.manifest + assert manifest is not None + if not _has_valid_checkpoint(manifest): + return ProjectionStateStatus.CORRUPT + if manifest.corpus_id != target.request.corpus_id: + return ProjectionStateStatus.CORRUPT + if manifest.projection.schema_id != target.request.projection.schema_id: + return ProjectionStateStatus.SCHEMA_MISMATCH + if manifest.projection.embedding != target.request.projection.embedding: + return ProjectionStateStatus.EMBEDDING_MISMATCH + + target_by_key = {item.entry.document.key: item.entry for item in target.documents} + for previous in manifest.entries: + current = target_by_key.get(previous.document.key) + if current is None or previous.document != current.document: + continue + if previous.source_digest != current.source_digest: + return ProjectionStateStatus.CORRUPT + if ( + manifest.chunking == target.request.chunking + and previous.fragment_count != current.fragment_count + ): + return ProjectionStateStatus.CORRUPT + + if ( + manifest.chunking == target.request.chunking + and manifest.entries == target.manifest.entries + ): + return ProjectionStateStatus.CURRENT + return ProjectionStateStatus.STALE + + +def _incremental_plan( + manifest: ProjectionManifest, + target: _PreparedTarget, +) -> tuple[_Mutation, ...]: + previous_by_key = {entry.document.key: entry for entry in manifest.entries} + target_by_key = {item.entry.document.key: item for item in target.documents} + mutations: list[_Mutation] = [] + + for key, item in target_by_key.items(): + previous = previous_by_key.get(key) + if manifest.chunking != target.request.chunking or previous != item.entry: + mutations.append(_Mutation(key, item)) + for key in previous_by_key.keys() - target_by_key.keys(): + mutations.append(_Mutation(key, None)) + mutations.sort(key=lambda mutation: mutation.key.document_id) + return tuple(mutations) + + +def _rebuild_plan(target: _PreparedTarget) -> tuple[_Mutation, ...]: + return tuple(_Mutation(item.entry.document.key, item) for item in target.documents) + + +def _failure_receipt( + target: _PreparedTarget, + attempted: int, + completed: int, +) -> ProjectionReceipt | None: + if attempted == 0: + return None + outcome = ProjectionOutcome.FAILED if completed == 0 else ProjectionOutcome.PARTIAL + return ProjectionReceipt( + target.request.corpus_id, + target.request.projection, + outcome, + attempted, + completed, + None, + ) + + +def _raise_operation_error( + stage: ProjectionFailureStage, + affected_document: DocumentKey | None, + receipt: ProjectionReceipt | None, + cause: Exception | None = None, +) -> NoReturn: + error = ProjectionOperationError(stage, affected_document, receipt) + if cause is None: + raise error + raise error from cause + + +def _require_embedder_identity( + embedder: Embedder, + expected: EmbeddingIdentity, + receipt: ProjectionReceipt | None, +) -> None: + try: + identity = embedder.identity + except Exception as exc: + _raise_operation_error( + ProjectionFailureStage.EMBEDDER_IDENTITY, + None, + receipt, + exc, + ) + try: + validated = _validate_embedding_identity(identity) + except ContractValidationError: + _raise_operation_error( + ProjectionFailureStage.EMBEDDER_IDENTITY, + None, + receipt, + ) + if validated != expected: + _raise_operation_error( + ProjectionFailureStage.EMBEDDER_IDENTITY, + None, + receipt, + ) + + +def _validate_embeddings( + value: object, + expected_count: int, + expected_dimensions: int, +) -> tuple[EmbeddingVector, ...]: + if type(value) is not tuple or len(value) != expected_count: + raise ContractValidationError( + "embedder output must be an exact same-count tuple" + ) + assert isinstance(value, tuple) + for vector in value: + _require_exact_type("embedding", vector, EmbeddingVector) + assert isinstance(vector, EmbeddingVector) + if ( + type(vector.values) is not tuple + or len(vector.values) != expected_dimensions + ): + raise ContractValidationError( + "embedding vector must have the requested dimensions" + ) + if any( + type(coordinate) is not float or not isfinite(coordinate) + for coordinate in vector.values + ): + raise ContractValidationError( + "embedding vector coordinates must be canonical finite floats" + ) + return value + + +def _records_for_document( + embedder: Embedder, + item: _PreparedDocument, + batch_size: int, + target: _PreparedTarget, + attempted: int, + completed: int, +) -> tuple[VectorRecord, ...]: + records: list[VectorRecord] = [] + for start in range(0, len(item.fragments), batch_size): + fragments = item.fragments[start : start + batch_size] + texts = tuple(fragment.text for fragment in fragments) + try: + output = embedder.embed(texts) + except Exception as exc: + _raise_operation_error( + ProjectionFailureStage.EMBEDDING, + item.entry.document.key, + _failure_receipt(target, attempted, completed), + exc, + ) + try: + vectors = _validate_embeddings( + output, + len(fragments), + target.request.projection.embedding.dimensions, + ) + except ContractValidationError: + _raise_operation_error( + ProjectionFailureStage.EMBEDDING, + item.entry.document.key, + _failure_receipt(target, attempted, completed), + ) + records.extend( + VectorRecord(fragment, vector) + for fragment, vector in zip(fragments, vectors, strict=True) + ) + return tuple(records) + + +def _execute_prevalidated_mutations( + target: _PreparedTarget, + mutations: tuple[_Mutation, ...], + embedder: Embedder | None, + writer_scope: Borrowed[VectorIndexWriter], +) -> None: + if not mutations: + return + attempted = len(mutations) + completed = 0 + with writer_scope as writer: + for mutation in mutations: + if mutation.replacement is None: + try: + delete_document = cast( + Callable[[DocumentKey], object], + writer.delete_document, + ) + command_result = delete_document(mutation.key) + except Exception as exc: + _raise_operation_error( + ProjectionFailureStage.DELETION, + mutation.key, + _failure_receipt(target, attempted, completed), + exc, + ) + if command_result is not None: + _raise_operation_error( + ProjectionFailureStage.DELETION, + mutation.key, + _failure_receipt(target, attempted, completed), + ) + else: + assert embedder is not None + records = _records_for_document( + embedder, + mutation.replacement, + target.request.limits.max_embedding_batch_size, + target, + attempted, + completed, + ) + try: + replace_document = cast( + Callable[ + [DocumentIdentity, tuple[VectorRecord, ...]], + object, + ], + writer.replace_document, + ) + command_result = replace_document( + mutation.replacement.entry.document, + records, + ) + except Exception as exc: + _raise_operation_error( + ProjectionFailureStage.REPLACEMENT, + mutation.key, + _failure_receipt(target, attempted, completed), + exc, + ) + if command_result is not None: + _raise_operation_error( + ProjectionFailureStage.REPLACEMENT, + mutation.key, + _failure_receipt(target, attempted, completed), + ) + completed += 1 + + +def _execute_mutations( + target: _PreparedTarget, + mutations: tuple[_Mutation, ...], + embedder_scope: Borrowed[Embedder], + writer_scope: Borrowed[VectorIndexWriter], +) -> None: + needs_embedder = any(mutation.replacement is not None for mutation in mutations) + if not needs_embedder: + _execute_prevalidated_mutations(target, mutations, None, writer_scope) + return + + with embedder_scope as embedder: + _require_embedder_identity( + embedder, + target.request.projection.embedding, + _failure_receipt(target, len(mutations), 0), + ) + _execute_prevalidated_mutations(target, mutations, embedder, writer_scope) + + +def _reset_projection( + target: _PreparedTarget, + attempted: int, + resetter_scope: Borrowed[VectorIndexResetter], +) -> None: + try: + with resetter_scope as resetter: + reset_corpus = cast( + Callable[[str], object], + resetter.reset_corpus, + ) + command_result = reset_corpus(target.request.corpus_id) + except Exception as exc: + _raise_operation_error( + ProjectionFailureStage.RESET, + None, + _failure_receipt(target, attempted, 0), + exc, + ) + if command_result is not None: + _raise_operation_error( + ProjectionFailureStage.RESET, + None, + _failure_receipt(target, attempted, 0), + ) + + +def _successful_result( + target: _PreparedTarget, + status_before: ProjectionStateStatus, + outcome: ProjectionOutcome, + attempted: int, +) -> ProjectionResult: + receipt = ProjectionReceipt( + target.request.corpus_id, + target.request.projection, + outcome, + attempted, + attempted, + target.manifest.checkpoint, + ) + return ProjectionResult(status_before, receipt, target.manifest) + + +def project_documents( + request: ProjectionRequest, + state: ProjectionStateSnapshot, + embedder: Borrowed[Embedder], + writer: Borrowed[VectorIndexWriter], + /, +) -> ProjectionResult: + """Project a compatible present state to the complete target. + + Incompatible state raises ``ProjectionStateError`` before collaborator effects. + """ + + canonical_request = _validate_request(request) + canonical_state = _validate_state(state) + _validate_borrowed("embedder", embedder) + _validate_borrowed("writer", writer) + target = _prepare_target(canonical_request) + status = _state_status(canonical_state, target) + if status is ProjectionStateStatus.CURRENT: + return _successful_result(target, status, ProjectionOutcome.UNCHANGED, 0) + if status is not ProjectionStateStatus.STALE: + raise ProjectionStateError(status) + + manifest = canonical_state.manifest + assert manifest is not None + mutations = _incremental_plan(manifest, target) + _execute_mutations(target, mutations, embedder, writer) + return _successful_result( + target, + status, + ProjectionOutcome.COMPLETED, + len(mutations), + ) + + +def rebuild_projection( + request: ProjectionRequest, + state: ProjectionStateSnapshot, + embedder: Borrowed[Embedder], + writer: Borrowed[VectorIndexWriter], + resetter: Borrowed[VectorIndexResetter], + /, +) -> ProjectionResult: + """Reset one corpus and write its complete canonical target projection.""" + + canonical_request = _validate_request(request) + canonical_state = _validate_state(state) + _validate_borrowed("embedder", embedder) + _validate_borrowed("writer", writer) + _validate_borrowed("resetter", resetter) + target = _prepare_target(canonical_request) + status = _state_status(canonical_state, target) + mutations = _rebuild_plan(target) + + if mutations: + with embedder as embedder_resource: + _require_embedder_identity( + embedder_resource, + target.request.projection.embedding, + _failure_receipt(target, len(mutations), 0), + ) + _reset_projection(target, len(mutations), resetter) + _execute_prevalidated_mutations( + target, + mutations, + embedder_resource, + writer, + ) + else: + _reset_projection(target, 0, resetter) + return _successful_result( + target, + status, + ProjectionOutcome.COMPLETED, + len(mutations), + ) diff --git a/tests/support/clean_import_probe.py b/tests/support/clean_import_probe.py index d0db065..e61cdaa 100644 --- a/tests/support/clean_import_probe.py +++ b/tests/support/clean_import_probe.py @@ -161,6 +161,32 @@ def _assert_forbidden_paths_absent(paths: list[str]) -> None: ) +def _bootstrap_source_root(raw_source_root: str | None) -> Path | None: + if raw_source_root is None: + return None + source_root = Path(raw_source_root).resolve(strict=True) + if not source_root.is_dir(): + raise AssertionError(f"source root is not a directory: {source_root}") + sys.path.insert(0, os.fspath(source_root)) + return source_root + + +def _assert_import_origin(imported: ModuleType, source_root: Path | None) -> None: + if source_root is None: + return + raw_origin = getattr(imported, "__file__", None) + if not isinstance(raw_origin, str): + raise AssertionError(f"source import has no file origin: {imported.__name__}") + origin = Path(raw_origin).resolve(strict=True) + expected_package = source_root / _PACKAGE_ROOT + try: + origin.relative_to(expected_package) + except ValueError: + raise AssertionError( + f"{imported.__name__} came from {origin}, not {expected_package}" + ) from None + + def _parse_arguments() -> argparse.Namespace: parser = argparse.ArgumentParser() parser.add_argument("module") @@ -170,6 +196,10 @@ def _parse_arguments() -> argparse.Namespace: default=[], help="Fail when this path or one of its children occurs on sys.path.", ) + parser.add_argument( + "--source-root", + help="Explicit resolved src root for an isolated source-tree probe.", + ) return parser.parse_args() @@ -177,14 +207,17 @@ def main() -> int: arguments = _parse_arguments() module_name = cast(str, arguments.module) forbidden_paths = cast(list[str], arguments.forbid_path) + raw_source_root = cast(str | None, arguments.source_root) if module_name != _PACKAGE_ROOT and not module_name.startswith(f"{_PACKAGE_ROOT}."): raise ValueError(f"probe is restricted to {_PACKAGE_ROOT} modules") _assert_forbidden_paths_absent(forbidden_paths) + source_root = _bootstrap_source_root(raw_source_root) before = set(sys.modules) imported = _import_with_guards(module_name) after = set(sys.modules) _assert_no_external_imports(before, after) + _assert_import_origin(imported, source_root) if imported.__name__ != module_name: raise AssertionError(f"requested {module_name}, imported {imported.__name__}") diff --git a/tests/support/verify_artifacts.py b/tests/support/verify_artifacts.py index bf78892..ba6ea5f 100644 --- a/tests/support/verify_artifacts.py +++ b/tests/support/verify_artifacts.py @@ -16,6 +16,7 @@ "generic_rag/contracts.py", "generic_rag/errors.py", "generic_rag/ports.py", + "generic_rag/projection.py", } _PACKAGE_DATA = {"generic_rag/py.typed"} diff --git a/tests/test_package_boundaries.py b/tests/test_package_boundaries.py index eedc054..a5b4eef 100644 --- a/tests/test_package_boundaries.py +++ b/tests/test_package_boundaries.py @@ -16,6 +16,7 @@ import generic_rag.contracts as contracts import generic_rag.errors as errors import generic_rag.ports as ports +import generic_rag.projection as projection _PROJECT_ROOT = Path(__file__).resolve().parents[1] _SOURCE_ROOT = _PROJECT_ROOT / "src" @@ -26,12 +27,18 @@ "generic_rag.contracts": "src/generic_rag/contracts.py", "generic_rag.errors": "src/generic_rag/errors.py", "generic_rag.ports": "src/generic_rag/ports.py", + "generic_rag.projection": "src/generic_rag/projection.py", } _EXPECTED_DEPENDENCIES = { "generic_rag": set(), "generic_rag.contracts": {"generic_rag.errors"}, "generic_rag.errors": set(), "generic_rag.ports": {"generic_rag.contracts"}, + "generic_rag.projection": { + "generic_rag.contracts", + "generic_rag.errors", + "generic_rag.ports", + }, } _EXPECTED_EXPORTS = { "generic_rag": (), @@ -54,6 +61,15 @@ "ProjectionCheckpoint", "ProjectionOutcome", "ProjectionReceipt", + "ChunkingPolicy", + "ProjectionLimits", + "ProjectionRequest", + "ProjectionManifestEntry", + "ProjectionManifest", + "ProjectionStateAvailability", + "ProjectionStateSnapshot", + "ProjectionStateStatus", + "ProjectionResult", "RetrievalQuery", "RetrievalOutcome", "RetrievalHit", @@ -63,9 +79,17 @@ "Borrowed", "Embedder", "VectorIndexWriter", + "VectorIndexResetter", "VectorIndexReader", "LexicalRetriever", ), + "generic_rag.projection": ( + "ProjectionFailureStage", + "ProjectionStateError", + "ProjectionOperationError", + "project_documents", + "rebuild_projection", + ), } @@ -150,6 +174,7 @@ def test_supported_exports_are_exact_and_owned(self) -> None: "generic_rag.errors": errors, "generic_rag.contracts": contracts, "generic_rag.ports": ports, + "generic_rag.projection": projection, } for module_name, expected_exports in _EXPECTED_EXPORTS.items(): @@ -164,6 +189,7 @@ def test_supported_exports_are_exact_and_owned(self) -> None: *_EXPECTED_EXPORTS["generic_rag.errors"], *_EXPECTED_EXPORTS["generic_rag.contracts"], *_EXPECTED_EXPORTS["generic_rag.ports"], + *_EXPECTED_EXPORTS["generic_rag.projection"], ): with self.subTest(root_reexport=name): self.assertFalse(hasattr(generic_rag, name)) @@ -339,6 +365,8 @@ def test_each_module_imports_in_an_isolated_side_effect_guarded_process( "-B", str(_CLEAN_IMPORT_PROBE), module_name, + "--source-root", + str(_SOURCE_ROOT), ), cwd=working_directory, env=environment, diff --git a/tests/test_ports.py b/tests/test_ports.py index d3bc6f3..4033810 100644 --- a/tests/test_ports.py +++ b/tests/test_ports.py @@ -4,9 +4,10 @@ import inspect import unittest +from collections.abc import Callable from dataclasses import FrozenInstanceError, fields from types import TracebackType -from typing import get_type_hints +from typing import cast, get_type_hints import generic_rag.ports as ports from generic_rag.contracts import ( @@ -24,6 +25,7 @@ Embedder, LexicalRetriever, VectorIndexReader, + VectorIndexResetter, VectorIndexWriter, ) @@ -107,6 +109,14 @@ def delete_document(self, document: DocumentKey, /) -> None: self.deletions.append(document) +class _FakeVectorResetter: + def __init__(self) -> None: + self.corpora: list[str] = [] + + def reset_corpus(self, corpus_id: str, /) -> None: + self.corpora.append(corpus_id) + + class _FakeVectorReader: def __init__(self, ranked: tuple[Fragment, ...]) -> None: self.ranked = ranked @@ -189,6 +199,7 @@ def test_exports_are_exact_and_owned_by_the_module(self) -> None: "Borrowed", "Embedder", "VectorIndexWriter", + "VectorIndexResetter", "VectorIndexReader", "LexicalRetriever", ) @@ -201,12 +212,14 @@ def test_exports_are_exact_and_owned_by_the_module(self) -> None: def test_protocols_are_runtime_checkable_structural_shapes(self) -> None: self.assertIsInstance(_FakeEmbedder(), Embedder) self.assertIsInstance(_FakeVectorWriter(), VectorIndexWriter) + self.assertIsInstance(_FakeVectorResetter(), VectorIndexResetter) self.assertIsInstance(_FakeVectorReader(()), VectorIndexReader) self.assertIsInstance(_FakeLexicalRetriever(()), LexicalRetriever) missing = _MissingMethods() self.assertNotIsInstance(missing, Embedder) self.assertNotIsInstance(missing, VectorIndexWriter) + self.assertNotIsInstance(missing, VectorIndexResetter) self.assertNotIsInstance(missing, VectorIndexReader) self.assertNotIsInstance(missing, LexicalRetriever) @@ -235,6 +248,7 @@ def test_protocol_methods_are_synchronous_and_positional_only(self) -> None: ("self", "document", "records"), ), (VectorIndexWriter.delete_document, ("self", "document")), + (VectorIndexResetter.reset_corpus, ("self", "corpus_id")), ( VectorIndexReader.search, ("self", "query", "embedding"), @@ -279,6 +293,13 @@ def test_protocol_annotations_keep_scores_and_lifecycle_out(self) -> None: "return": type(None), }, ) + self.assertEqual( + get_type_hints(VectorIndexResetter.reset_corpus), + { + "corpus_id": str, + "return": type(None), + }, + ) self.assertEqual( get_type_hints(VectorIndexReader.search), { @@ -298,6 +319,7 @@ def test_protocol_annotations_keep_scores_and_lifecycle_out(self) -> None: for protocol in ( Embedder, VectorIndexWriter, + VectorIndexResetter, VectorIndexReader, LexicalRetriever, ): @@ -340,6 +362,14 @@ def test_writer_witness_replaces_complete_sets_and_accepts_empty(self) -> None: with self.assertRaises(AssertionError): writer.replace_document(wrong_document, records) + def test_resetter_witness_removes_one_exact_corpus_and_returns_none(self) -> None: + resetter = _FakeVectorResetter() + + result = cast(Callable[[str], object], resetter.reset_corpus)(" Corpus/../A ") + + self.assertIsNone(result) + self.assertEqual(resetter.corpora, [" Corpus/../A "]) + def test_reader_witnesses_preserve_rank_and_enforce_candidate_bound(self) -> None: ranked = ( _fragment("first"), diff --git a/tests/test_projection.py b/tests/test_projection.py new file mode 100644 index 0000000..700f2df --- /dev/null +++ b/tests/test_projection.py @@ -0,0 +1,1661 @@ +"""Deterministic projection orchestration, failure, and lifecycle tests.""" + +from __future__ import annotations + +import hashlib +import inspect +import json +import os +import subprocess +import sys +import textwrap +import unittest +from collections.abc import Callable +from pathlib import Path +from typing import cast + +import generic_rag.projection as projection_module +from generic_rag.contracts import ( + ChunkingPolicy, + Document, + DocumentIdentity, + DocumentKey, + EmbeddingIdentity, + EmbeddingVector, + ProjectionCheckpoint, + ProjectionIdentity, + ProjectionLimits, + ProjectionManifest, + ProjectionManifestEntry, + ProjectionOutcome, + ProjectionReceipt, + ProjectionRequest, + ProjectionStateAvailability, + ProjectionStateSnapshot, + ProjectionStateStatus, + VectorRecord, +) +from generic_rag.errors import ( + CollaborationError, + ContractValidationError, + StateCompatibilityError, +) +from generic_rag.ports import ( + Borrowed, + Embedder, + VectorIndexResetter, + VectorIndexWriter, +) +from generic_rag.projection import ( + ProjectionFailureStage, + ProjectionOperationError, + ProjectionStateError, + project_documents, + rebuild_projection, +) + +_SOURCE_ROOT = Path(__file__).resolve().parents[1] / "src" + + +def _document( + document_id: str, + *, + revision_id: str = "revision-1", + text: str = "abcdefgh", + corpus_id: str = "corpus", + attributes: tuple[tuple[str, str], ...] = (), +) -> Document: + return Document( + DocumentIdentity(DocumentKey(corpus_id, document_id), revision_id), + text, + attributes, + ) + + +def _identity( + *, + schema_id: str = "schema-v1", + model_id: str = "model-v1", + dimensions: int = 2, +) -> ProjectionIdentity: + return ProjectionIdentity( + schema_id, + EmbeddingIdentity(model_id, dimensions), + ) + + +def _request( + documents: tuple[Document, ...], + *, + projection: ProjectionIdentity | None = None, + chunking: ChunkingPolicy | None = None, + batch_size: int = 2, + corpus_id: str = "corpus", +) -> ProjectionRequest: + return ProjectionRequest( + corpus_id, + projection or _identity(), + chunking or ChunkingPolicy(4, 1), + ProjectionLimits(max(1, len(documents)), 100, batch_size), + documents, + ) + + +def _hash_fields(fields_to_hash: tuple[str, ...]) -> str: + """Independent length-delimited SHA-256 oracle from the public contract.""" + + digest = hashlib.sha256() + for value in fields_to_hash: + encoded = value.encode("utf-8", "surrogatepass") + digest.update(len(encoded).to_bytes(8, "big", signed=False)) + digest.update(encoded) + return f"sha256:{digest.hexdigest()}" + + +def _source_digest(document: Document) -> str: + source_fields = [ + "generic-rag:projection-source:v1", + "text", + document.text, + "attributes_count", + str(len(document.attributes)), + ] + for key, value in document.attributes: + source_fields.extend(("attribute_key", key, "attribute_value", value)) + return _hash_fields(tuple(source_fields)) + + +def _fragment_id(document: DocumentIdentity, start: int, end: int) -> str: + return _hash_fields( + ( + "generic-rag:fragment-id:v1", + "corpus_id", + document.key.corpus_id, + "document_id", + document.key.document_id, + "revision_id", + document.revision_id, + "start", + str(start), + "end", + str(end), + ) + ) + + +def _expected_ranges( + text: str, chunking: ChunkingPolicy +) -> tuple[tuple[int, int], ...]: + ranges: list[tuple[int, int]] = [] + start = 0 + while start < len(text): + end = min(start + chunking.max_fragment_codepoints, len(text)) + ranges.append((start, end)) + if end == len(text): + break + start = end - chunking.overlap_codepoints + return tuple(ranges) + + +def _checkpoint_token( + request: ProjectionRequest, + entries: tuple[ProjectionManifestEntry, ...], +) -> str: + token_fields = [ + "generic-rag:projection-checkpoint:v1", + "corpus_id", + request.corpus_id, + "schema_id", + request.projection.schema_id, + "embedding_model_id", + request.projection.embedding.model_id, + "embedding_dimensions", + str(request.projection.embedding.dimensions), + "max_fragment_codepoints", + str(request.chunking.max_fragment_codepoints), + "overlap_codepoints", + str(request.chunking.overlap_codepoints), + "entry_count", + str(len(entries)), + ] + for entry in entries: + token_fields.extend( + ( + "document_id", + entry.document.key.document_id, + "revision_id", + entry.document.revision_id, + "source_digest", + entry.source_digest, + "fragment_count", + str(entry.fragment_count), + ) + ) + return _hash_fields(tuple(token_fields)) + + +def _manifest(request: ProjectionRequest) -> ProjectionManifest: + entries = tuple( + ProjectionManifestEntry( + document.identity, + _source_digest(document), + len(_expected_ranges(document.text, request.chunking)), + ) + for document in request.documents + ) + checkpoint = ProjectionCheckpoint( + request.corpus_id, + request.projection, + _checkpoint_token(request, entries), + ) + return ProjectionManifest( + request.corpus_id, + request.projection, + request.chunking, + entries, + checkpoint, + ) + + +def _present(request: ProjectionRequest) -> ProjectionStateSnapshot: + return ProjectionStateSnapshot( + ProjectionStateAvailability.PRESENT, + _manifest(request), + ) + + +class _FakeEmbedder: + def __init__( + self, + expected_identity: EmbeddingIdentity, + events: list[str], + *, + identity_value: object | None = None, + identity_failure: BaseException | None = None, + output: Callable[[tuple[str, ...]], object] | None = None, + embed_failure_at: int | None = None, + embed_failure: BaseException | None = None, + ) -> None: + self.expected_identity = expected_identity + self.events = events + self.identity_value = ( + expected_identity if identity_value is None else identity_value + ) + self.identity_failure = identity_failure + self.output = output + self.embed_failure_at = embed_failure_at + self.embed_failure = embed_failure or RuntimeError("embedding failed") + self.identity_calls = 0 + self.embed_calls: list[tuple[str, ...]] = [] + self.lifecycle_calls: list[str] = [] + + @property + def identity(self) -> EmbeddingIdentity: + self.identity_calls += 1 + self.events.append("identity") + if self.identity_failure is not None: + raise self.identity_failure + return cast(EmbeddingIdentity, self.identity_value) + + def embed(self, texts: tuple[str, ...], /) -> tuple[EmbeddingVector, ...]: + call_index = len(self.embed_calls) + self.embed_calls.append(texts) + self.events.append("embed:" + "|".join(texts)) + if self.embed_failure_at == call_index: + raise self.embed_failure + if self.output is not None: + return cast(tuple[EmbeddingVector, ...], self.output(texts)) + return tuple( + EmbeddingVector((float(len(text)), float(index))) + for index, text in enumerate(texts) + ) + + def __enter__(self) -> _FakeEmbedder: + self.lifecycle_calls.append("enter") + return self + + def __exit__(self, *arguments: object) -> bool: + del arguments + self.lifecycle_calls.append("exit") + return True + + def close(self) -> None: + self.lifecycle_calls.append("close") + + def shutdown(self) -> None: + self.lifecycle_calls.append("shutdown") + + +class _FakeWriter: + def __init__( + self, + events: list[str], + *, + replace_failure_at: int | None = None, + replace_failure: BaseException | None = None, + replace_result: object = None, + delete_failure_at: int | None = None, + delete_failure: BaseException | None = None, + delete_result: object = None, + ) -> None: + self.events = events + self.replace_failure_at = replace_failure_at + self.replace_failure = replace_failure or RuntimeError("replacement failed") + self.replace_result = replace_result + self.delete_failure_at = delete_failure_at + self.delete_failure = delete_failure or RuntimeError("deletion failed") + self.delete_result = delete_result + self.replacements: list[tuple[DocumentIdentity, tuple[VectorRecord, ...]]] = [] + self.deletions: list[DocumentKey] = [] + self.lifecycle_calls: list[str] = [] + + def replace_document( + self, + document: DocumentIdentity, + records: tuple[VectorRecord, ...], + /, + ) -> None: + call_index = len(self.replacements) + self.replacements.append((document, records)) + self.events.append(f"replace:{document.key.document_id}") + if self.replace_failure_at == call_index: + raise self.replace_failure + return cast(None, self.replace_result) + + def delete_document(self, document: DocumentKey, /) -> None: + call_index = len(self.deletions) + self.deletions.append(document) + self.events.append(f"delete:{document.document_id}") + if self.delete_failure_at == call_index: + raise self.delete_failure + return cast(None, self.delete_result) + + def __enter__(self) -> _FakeWriter: + self.lifecycle_calls.append("enter") + return self + + def __exit__(self, *arguments: object) -> bool: + del arguments + self.lifecycle_calls.append("exit") + return True + + def close(self) -> None: + self.lifecycle_calls.append("close") + + def shutdown(self) -> None: + self.lifecycle_calls.append("shutdown") + + +class _FakeResetter: + def __init__( + self, + events: list[str], + *, + failure: BaseException | None = None, + result: object = None, + ) -> None: + self.events = events + self.failure = failure + self.result = result + self.corpora: list[str] = [] + self.lifecycle_calls: list[str] = [] + + def reset_corpus(self, corpus_id: str, /) -> None: + self.corpora.append(corpus_id) + self.events.append(f"reset:{corpus_id}") + if self.failure is not None: + raise self.failure + return cast(None, self.result) + + def __enter__(self) -> _FakeResetter: + self.lifecycle_calls.append("enter") + return self + + def __exit__(self, *arguments: object) -> bool: + del arguments + self.lifecycle_calls.append("exit") + return True + + def close(self) -> None: + self.lifecycle_calls.append("close") + + def shutdown(self) -> None: + self.lifecycle_calls.append("shutdown") + + +def _borrow_embedder(embedder: _FakeEmbedder) -> Borrowed[Embedder]: + return Borrowed(cast(Embedder, embedder)) + + +def _borrow_writer(writer: _FakeWriter) -> Borrowed[VectorIndexWriter]: + return Borrowed(cast(VectorIndexWriter, writer)) + + +def _borrow_resetter(resetter: _FakeResetter) -> Borrowed[VectorIndexResetter]: + return Borrowed(cast(VectorIndexResetter, resetter)) + + +def _constant_output(value: object) -> Callable[[tuple[str, ...]], object]: + def output(texts: tuple[str, ...]) -> object: + del texts + return value + + return output + + +def _collaborators( + request: ProjectionRequest, +) -> tuple[_FakeEmbedder, _FakeWriter, _FakeResetter, list[str]]: + events: list[str] = [] + return ( + _FakeEmbedder(request.projection.embedding, events), + _FakeWriter(events), + _FakeResetter(events), + events, + ) + + +class ProjectionPublicApiTests(unittest.TestCase): + def test_exports_are_exact_and_owned_by_projection_module(self) -> None: + expected = ( + "ProjectionFailureStage", + "ProjectionStateError", + "ProjectionOperationError", + "project_documents", + "rebuild_projection", + ) + + self.assertEqual(projection_module.__all__, expected) + for name in expected: + self.assertEqual( + getattr(projection_module, name).__module__, + "generic_rag.projection", + ) + + def test_public_workflows_are_synchronous_and_positional_only(self) -> None: + expected_parameters = { + project_documents: ("request", "state", "embedder", "writer"), + rebuild_projection: ( + "request", + "state", + "embedder", + "writer", + "resetter", + ), + } + for function, expected in expected_parameters.items(): + with self.subTest(function=function.__name__): + self.assertFalse(inspect.iscoroutinefunction(function)) + parameters = tuple( + inspect.signature( + cast(Callable[..., object], function) + ).parameters.values() + ) + self.assertEqual(tuple(item.name for item in parameters), expected) + self.assertTrue( + all( + item.kind is inspect.Parameter.POSITIONAL_ONLY + for item in parameters + ) + ) + + def test_failure_stage_is_an_exact_closed_string_enum(self) -> None: + self.assertEqual( + tuple((member.name, member.value) for member in ProjectionFailureStage), + ( + ("EMBEDDER_IDENTITY", "embedder_identity"), + ("EMBEDDING", "embedding"), + ("REPLACEMENT", "replacement"), + ("DELETION", "deletion"), + ("RESET", "reset"), + ), + ) + for value in ("EMBEDDING", "unknown", "", None, 1, object()): + with self.subTest(value=value): + with self.assertRaises(ContractValidationError): + ProjectionFailureStage(cast(str, value)) + + def test_state_error_is_typed_content_free_and_requires_exact_status(self) -> None: + error = ProjectionStateError(ProjectionStateStatus.SCHEMA_MISMATCH) + + self.assertIsInstance(error, StateCompatibilityError) + self.assertIs(error.status, ProjectionStateStatus.SCHEMA_MISMATCH) + self.assertNotIn("schema", str(error).lower()) + with self.assertRaises(ContractValidationError): + ProjectionStateError(cast(ProjectionStateStatus, "schema_mismatch")) + + def test_operation_error_fields_and_receipt_are_exact_and_truthful(self) -> None: + request = _request((_document("alpha"),)) + receipt = ProjectionReceipt( + request.corpus_id, + request.projection, + ProjectionOutcome.FAILED, + 1, + 0, + None, + ) + key = request.documents[0].identity.key + error = ProjectionOperationError( + ProjectionFailureStage.REPLACEMENT, + key, + receipt, + ) + + self.assertIsInstance(error, CollaborationError) + self.assertEqual( + ProjectionOperationError.__annotations__, + { + "stage": "ProjectionFailureStage", + "affected_document": "DocumentKey | None", + "receipt": "ProjectionReceipt | None", + }, + ) + self.assertIs(error.stage, ProjectionFailureStage.REPLACEMENT) + self.assertEqual(error.affected_document, key) + self.assertIs(error.receipt, receipt) + for stage, affected, value_receipt in ( + ("replacement", key, receipt), + (ProjectionFailureStage.REPLACEMENT, object(), receipt), + ( + ProjectionFailureStage.REPLACEMENT, + key, + ProjectionReceipt( + request.corpus_id, + request.projection, + ProjectionOutcome.COMPLETED, + 1, + 1, + _manifest(request).checkpoint, + ), + ), + (ProjectionFailureStage.REPLACEMENT, key, object()), + ): + with self.subTest(stage=stage, affected=affected): + with self.assertRaises(ContractValidationError): + ProjectionOperationError( + cast(ProjectionFailureStage, stage), + cast(DocumentKey, affected), + cast(ProjectionReceipt, value_receipt), + ) + + def test_invalid_top_level_inputs_are_rejected_before_any_effect(self) -> None: + request = _request((_document("alpha"),)) + state = _present(_request((), projection=request.projection)) + embedder, writer, resetter, events = _collaborators(request) + invalid_calls: tuple[Callable[[], object], ...] = ( + lambda: project_documents( + cast(ProjectionRequest, object()), + state, + _borrow_embedder(embedder), + _borrow_writer(writer), + ), + lambda: project_documents( + request, + cast(ProjectionStateSnapshot, object()), + _borrow_embedder(embedder), + _borrow_writer(writer), + ), + lambda: project_documents( + request, + state, + cast(Borrowed[Embedder], object()), + _borrow_writer(writer), + ), + lambda: project_documents( + request, + state, + _borrow_embedder(embedder), + cast(Borrowed[VectorIndexWriter], object()), + ), + lambda: rebuild_projection( + request, + state, + _borrow_embedder(embedder), + _borrow_writer(writer), + cast(Borrowed[VectorIndexResetter], object()), + ), + ) + + for call in invalid_calls: + with self.subTest(call=call): + with self.assertRaises(ContractValidationError): + call() + self.assertEqual(events, []) + + +class ProjectionDeterminismTests(unittest.TestCase): + def test_unicode_codepoint_chunks_overlap_and_preserve_source_attributes( + self, + ) -> None: + document = _document( + "unicode", + text="A😀e\u0301한Z", + attributes=(("", ""), ("tag", "one"), ("tag", "one")), + ) + request = _request( + (document,), + chunking=ChunkingPolicy(3, 1), + batch_size=8, + ) + previous = _request( + (), projection=request.projection, chunking=request.chunking + ) + embedder, writer, _, _ = _collaborators(request) + + result = project_documents( + request, + _present(previous), + _borrow_embedder(embedder), + _borrow_writer(writer), + ) + + records = writer.replacements[0][1] + self.assertEqual( + tuple( + (record.fragment.identity.start, record.fragment.identity.end) + for record in records + ), + ((0, 3), (2, 5), (4, 6)), + ) + self.assertEqual( + tuple(record.fragment.text for record in records), + ("A😀e", "e\u0301한", "한Z"), + ) + self.assertTrue( + all(record.fragment.attributes == document.attributes for record in records) + ) + self.assertTrue( + all( + record.fragment.identity.document == document.identity + for record in records + ) + ) + self.assertEqual(result.manifest.entries[0].fragment_count, 3) + + def test_fragment_ids_source_digest_and_checkpoint_match_independent_oracle( + self, + ) -> None: + document = _document( + "doc/../A", + revision_id=" rev e\u0301 ", + text="abcdef", + attributes=(("k", "v"), ("k", "v"), ("", "")), + ) + request = _request((document,), chunking=ChunkingPolicy(4, 1)) + embedder, writer, resetter, _ = _collaborators(request) + + result = rebuild_projection( + request, + ProjectionStateSnapshot(ProjectionStateAvailability.MISSING, None), + _borrow_embedder(embedder), + _borrow_writer(writer), + _borrow_resetter(resetter), + ) + + expected_manifest = _manifest(request) + self.assertEqual(result.manifest, expected_manifest) + self.assertEqual( + tuple( + record.fragment.identity.fragment_id + for record in writer.replacements[0][1] + ), + tuple( + _fragment_id(document.identity, start, end) + for start, end in _expected_ranges(document.text, request.chunking) + ), + ) + self.assertEqual( + result.manifest.entries[0].source_digest, + _source_digest(document), + ) + self.assertEqual( + result.manifest.checkpoint.token, + _checkpoint_token(request, expected_manifest.entries), + ) + + def test_source_digest_distinguishes_attribute_order_duplicates_and_text( + self, + ) -> None: + base = _document("doc", text="same", attributes=(("a", "1"), ("b", "2"))) + variants = ( + _document("doc", text="same", attributes=(("b", "2"), ("a", "1"))), + _document("doc", text="same", attributes=(("a", "1"), ("a", "1"))), + _document("doc", text="same!", attributes=(("a", "1"), ("b", "2"))), + ) + + self.assertEqual(len({_source_digest(base), *map(_source_digest, variants)}), 4) + + def test_embedding_batches_are_bounded_ordered_and_never_empty(self) -> None: + document = _document("doc", text="abcdefghijklmn") + request = _request( + (document,), + chunking=ChunkingPolicy(3, 0), + batch_size=2, + ) + previous = _request( + (), projection=request.projection, chunking=request.chunking + ) + embedder, writer, _, _ = _collaborators(request) + + project_documents( + request, + _present(previous), + _borrow_embedder(embedder), + _borrow_writer(writer), + ) + + self.assertEqual( + embedder.embed_calls, + [("abc", "def"), ("ghi", "jkl"), ("mn",)], + ) + self.assertTrue(all(batch for batch in embedder.embed_calls)) + self.assertEqual( + tuple(record.fragment.text for record in writer.replacements[0][1]), + ("abc", "def", "ghi", "jkl", "mn"), + ) + + def test_empty_document_is_replaced_by_an_explicit_empty_record_tuple(self) -> None: + document = _document("doc", revision_id="revision-2", text="") + request = _request((document,)) + previous = _request( + (_document("doc", revision_id="revision-1", text="old"),), + projection=request.projection, + chunking=request.chunking, + ) + embedder, writer, _, _ = _collaborators(request) + + result = project_documents( + request, + _present(previous), + _borrow_embedder(embedder), + _borrow_writer(writer), + ) + + self.assertEqual(embedder.embed_calls, []) + self.assertEqual(writer.replacements, [(document.identity, ())]) + self.assertEqual(result.manifest.entries[0].fragment_count, 0) + + def test_document_and_mutation_order_is_canonical_by_opaque_document_id( + self, + ) -> None: + request = _request( + ( + _document("c", revision_id="new"), + _document("a", revision_id="new"), + ) + ) + previous = _request( + ( + _document("d", revision_id="old"), + _document("b", revision_id="old"), + ), + projection=request.projection, + chunking=request.chunking, + ) + embedder, writer, _, events = _collaborators(request) + + project_documents( + request, + _present(previous), + _borrow_embedder(embedder), + _borrow_writer(writer), + ) + + mutation_events = [ + event for event in events if event.startswith(("replace:", "delete:")) + ] + self.assertEqual( + mutation_events, + ["replace:a", "delete:b", "replace:c", "delete:d"], + ) + + def test_fixed_projection_is_identical_in_clean_processes(self) -> None: + script = textwrap.dedent( + f""" + import json + import sys + sys.path.insert(0, {os.fspath(_SOURCE_ROOT)!r}) + from generic_rag.contracts import * + from generic_rag.ports import Borrowed + from generic_rag.projection import rebuild_projection + + class Embedder: + identity = EmbeddingIdentity('model-v1', 2) + def embed(self, texts, /): + return tuple( + EmbeddingVector((len(text), index)) + for index, text in enumerate(texts) + ) + class Writer: + def __init__(self): self.records = () + def replace_document(self, document, records, /): self.records = records + def delete_document(self, document, /): return None + class Resetter: + def reset_corpus(self, corpus_id, /): return None + + document = Document( + DocumentIdentity(DocumentKey('corpus', 'doc/../A'), ' rev e\\u0301 '), + 'abcdef', + (('k', 'v'), ('k', 'v'), ('', '')), + ) + request = ProjectionRequest( + 'corpus', + ProjectionIdentity('schema-v1', EmbeddingIdentity('model-v1', 2)), + ChunkingPolicy(4, 1), + ProjectionLimits(1, 100, 2), + (document,), + ) + writer = Writer() + result = rebuild_projection( + request, + ProjectionStateSnapshot(ProjectionStateAvailability.MISSING, None), + Borrowed(Embedder()), + Borrowed(writer), + Borrowed(Resetter()), + ) + print(json.dumps({{ + 'source': result.manifest.entries[0].source_digest, + 'checkpoint': result.manifest.checkpoint.token, + 'fragments': [ + record.fragment.identity.fragment_id + for record in writer.records + ], + }}, sort_keys=True)) + """ + ) + outputs: list[dict[str, object]] = [] + for _ in range(2): + completed = subprocess.run( + (sys.executable, "-I", "-B", "-c", script), + check=False, + capture_output=True, + text=True, + timeout=10, + ) + self.assertEqual(completed.returncode, 0, completed.stderr) + outputs.append(cast(dict[str, object], json.loads(completed.stdout))) + + request = _request( + ( + _document( + "doc/../A", + revision_id=" rev e\u0301 ", + text="abcdef", + attributes=(("k", "v"), ("k", "v"), ("", "")), + ), + ), + chunking=ChunkingPolicy(4, 1), + ) + expected = _manifest(request) + self.assertEqual(outputs[0], outputs[1]) + self.assertEqual(outputs[0]["source"], expected.entries[0].source_digest) + self.assertEqual(outputs[0]["checkpoint"], expected.checkpoint.token) + self.assertEqual( + outputs[0]["fragments"], + [ + _fragment_id(request.documents[0].identity, start, end) + for start, end in _expected_ranges( + request.documents[0].text, + request.chunking, + ) + ], + ) + + +class IncrementalProjectionTests(unittest.TestCase): + def test_current_state_is_unchanged_without_touching_collaborators(self) -> None: + request = _request((_document("alpha"), _document("beta"))) + embedder, writer, _, events = _collaborators(request) + embedder.identity_failure = AssertionError("identity must not be read") + writer.replace_failure_at = 0 + + result = project_documents( + request, + _present(request), + _borrow_embedder(embedder), + _borrow_writer(writer), + ) + + self.assertIs(result.status_before, ProjectionStateStatus.CURRENT) + self.assertIs(result.receipt.outcome, ProjectionOutcome.UNCHANGED) + self.assertEqual(result.receipt.attempted_documents, 0) + self.assertEqual(result.receipt.completed_documents, 0) + self.assertEqual(result.manifest, _manifest(request)) + self.assertEqual(events, []) + + def test_incompatible_state_statuses_fail_before_collaborator_effects(self) -> None: + request = _request((_document("alpha"),)) + schema_request = _request( + request.documents, + projection=_identity(schema_id="other-schema"), + ) + embedding_request = _request( + request.documents, + projection=_identity(model_id="other-model"), + ) + corpus_request = _request( + (_document("alpha", corpus_id="other"),), + projection=request.projection, + corpus_id="other", + ) + cases = ( + ( + ProjectionStateSnapshot(ProjectionStateAvailability.MISSING, None), + ProjectionStateStatus.MISSING, + ), + ( + ProjectionStateSnapshot(ProjectionStateAvailability.CORRUPT, None), + ProjectionStateStatus.CORRUPT, + ), + (_present(schema_request), ProjectionStateStatus.SCHEMA_MISMATCH), + (_present(embedding_request), ProjectionStateStatus.EMBEDDING_MISMATCH), + (_present(corpus_request), ProjectionStateStatus.CORRUPT), + ) + + for state, expected_status in cases: + embedder, writer, _, events = _collaborators(request) + with self.subTest(status=expected_status): + with self.assertRaises(ProjectionStateError) as raised: + project_documents( + request, + state, + _borrow_embedder(embedder), + _borrow_writer(writer), + ) + self.assertIs(raised.exception.status, expected_status) + self.assertEqual(events, []) + + def test_invalid_checkpoint_is_corrupt_before_any_effect(self) -> None: + request = _request((_document("alpha"),)) + valid = _manifest(request) + corrupt = ProjectionManifest( + valid.corpus_id, + valid.projection, + valid.chunking, + valid.entries, + ProjectionCheckpoint(valid.corpus_id, valid.projection, "wrong-token"), + ) + embedder, writer, _, events = _collaborators(request) + + with self.assertRaises(ProjectionStateError) as raised: + project_documents( + request, + ProjectionStateSnapshot( + ProjectionStateAvailability.PRESENT, + corrupt, + ), + _borrow_embedder(embedder), + _borrow_writer(writer), + ) + + self.assertIs(raised.exception.status, ProjectionStateStatus.CORRUPT) + self.assertEqual(events, []) + + def test_same_revision_digest_or_fragment_count_drift_is_corrupt(self) -> None: + request = _request((_document("alpha"),)) + valid = _manifest(request) + original = valid.entries[0] + changed_entries = ( + ( + ProjectionManifestEntry( + original.document, + "sha256:" + "f" * 64, + original.fragment_count, + ), + ), + ( + ProjectionManifestEntry( + original.document, + original.source_digest, + original.fragment_count + 1, + ), + ), + ) + + for entries in changed_entries: + previous_request = _request( + request.documents, + projection=request.projection, + chunking=request.chunking, + ) + manifest = ProjectionManifest( + request.corpus_id, + request.projection, + request.chunking, + entries, + ProjectionCheckpoint( + request.corpus_id, + request.projection, + _checkpoint_token(previous_request, entries), + ), + ) + embedder, writer, _, events = _collaborators(request) + with self.subTest(entries=entries): + with self.assertRaises(ProjectionStateError) as raised: + project_documents( + request, + ProjectionStateSnapshot( + ProjectionStateAvailability.PRESENT, + manifest, + ), + _borrow_embedder(embedder), + _borrow_writer(writer), + ) + self.assertIs( + raised.exception.status, + ProjectionStateStatus.CORRUPT, + ) + self.assertEqual(events, []) + + def test_revision_and_chunking_changes_are_stale_and_replaced(self) -> None: + target = _request( + (_document("alpha", revision_id="revision-2", text="new text"),), + chunking=ChunkingPolicy(4, 1), + ) + previous_requests = ( + _request( + (_document("alpha", revision_id="revision-1", text="new text"),), + projection=target.projection, + chunking=target.chunking, + ), + _request( + target.documents, + projection=target.projection, + chunking=ChunkingPolicy(5, 0), + ), + ) + + for previous in previous_requests: + embedder, writer, _, _ = _collaborators(target) + with self.subTest(previous=previous): + result = project_documents( + target, + _present(previous), + _borrow_embedder(embedder), + _borrow_writer(writer), + ) + self.assertIs(result.status_before, ProjectionStateStatus.STALE) + self.assertEqual( + [identity for identity, _ in writer.replacements], + [target.documents[0].identity], + ) + + def test_same_revision_source_change_is_corrupt_not_stale(self) -> None: + target = _request( + (_document("alpha", revision_id="revision-2", text="new text"),) + ) + previous = _request( + (_document("alpha", revision_id="revision-2", text="old text"),), + projection=target.projection, + chunking=target.chunking, + ) + embedder, writer, _, events = _collaborators(target) + + with self.assertRaises(ProjectionStateError) as raised: + project_documents( + target, + _present(previous), + _borrow_embedder(embedder), + _borrow_writer(writer), + ) + + self.assertIs(raised.exception.status, ProjectionStateStatus.CORRUPT) + self.assertEqual(events, []) + + def test_delete_only_plan_does_not_access_embedder(self) -> None: + target = _request(()) + previous = _request( + (_document("zeta"), _document("alpha")), + projection=target.projection, + chunking=target.chunking, + ) + embedder, writer, _, events = _collaborators(target) + embedder.identity_failure = AssertionError("delete-only must not embed") + + result = project_documents( + target, + _present(previous), + _borrow_embedder(embedder), + _borrow_writer(writer), + ) + + self.assertEqual( + writer.deletions, + [DocumentKey("corpus", "alpha"), DocumentKey("corpus", "zeta")], + ) + self.assertEqual(embedder.identity_calls, 0) + self.assertFalse(any(event.startswith("embed:") for event in events)) + self.assertEqual(result.receipt.attempted_documents, 2) + self.assertEqual(result.receipt.completed_documents, 2) + + def test_success_receipt_counts_only_changed_and_removed_documents(self) -> None: + unchanged = _document("same", revision_id="r1", text="same") + target = _request( + ( + unchanged, + _document("added", revision_id="r1", text="added"), + _document("changed", revision_id="r2", text="new"), + ) + ) + previous = _request( + ( + unchanged, + _document("changed", revision_id="r1", text="old"), + _document("removed", revision_id="r1", text="gone"), + ), + projection=target.projection, + chunking=target.chunking, + ) + embedder, writer, _, _ = _collaborators(target) + + result = project_documents( + target, + _present(previous), + _borrow_embedder(embedder), + _borrow_writer(writer), + ) + + self.assertIs(result.receipt.outcome, ProjectionOutcome.COMPLETED) + self.assertEqual(result.receipt.attempted_documents, 3) + self.assertEqual(result.receipt.completed_documents, 3) + self.assertEqual(result.receipt.checkpoint, result.manifest.checkpoint) + self.assertNotIn(unchanged.identity, [item[0] for item in writer.replacements]) + + +class RebuildProjectionTests(unittest.TestCase): + def test_nonempty_rebuild_validates_identity_once_before_reset_and_reuses_it( + self, + ) -> None: + request = _request((_document("alpha"), _document("beta"))) + embedder, writer, resetter, events = _collaborators(request) + + result = rebuild_projection( + request, + ProjectionStateSnapshot(ProjectionStateAvailability.MISSING, None), + _borrow_embedder(embedder), + _borrow_writer(writer), + _borrow_resetter(resetter), + ) + + self.assertEqual(embedder.identity_calls, 1) + self.assertEqual(resetter.corpora, ["corpus"]) + self.assertEqual( + [identity.key.document_id for identity, _ in writer.replacements], + ["alpha", "beta"], + ) + self.assertEqual(events[0:2], ["identity", "reset:corpus"]) + self.assertEqual(sum(event == "identity" for event in events), 1) + self.assertIs(result.status_before, ProjectionStateStatus.MISSING) + self.assertEqual(result.receipt.attempted_documents, 2) + + def test_wrong_embedder_identity_fails_before_reset_without_a_cause(self) -> None: + request = _request((_document("alpha"),)) + events: list[str] = [] + embedder = _FakeEmbedder( + request.projection.embedding, + events, + identity_value=EmbeddingIdentity("other-model", 2), + ) + writer = _FakeWriter(events) + resetter = _FakeResetter(events) + + with self.assertRaises(ProjectionOperationError) as raised: + rebuild_projection( + request, + ProjectionStateSnapshot(ProjectionStateAvailability.MISSING, None), + _borrow_embedder(embedder), + _borrow_writer(writer), + _borrow_resetter(resetter), + ) + + self.assertIs( + raised.exception.stage, + ProjectionFailureStage.EMBEDDER_IDENTITY, + ) + self.assertIsNone(raised.exception.affected_document) + self.assertIsNone(raised.exception.__cause__) + self.assertIsNotNone(raised.exception.receipt) + assert raised.exception.receipt is not None + self.assertIs(raised.exception.receipt.outcome, ProjectionOutcome.FAILED) + self.assertEqual(resetter.corpora, []) + self.assertEqual(writer.replacements, []) + + def test_identity_exception_is_preserved_as_cause_before_reset(self) -> None: + request = _request((_document("alpha"),)) + failure = RuntimeError("identity provider failed") + events: list[str] = [] + embedder = _FakeEmbedder( + request.projection.embedding, + events, + identity_failure=failure, + ) + writer = _FakeWriter(events) + resetter = _FakeResetter(events) + + with self.assertRaises(ProjectionOperationError) as raised: + rebuild_projection( + request, + ProjectionStateSnapshot(ProjectionStateAvailability.MISSING, None), + _borrow_embedder(embedder), + _borrow_writer(writer), + _borrow_resetter(resetter), + ) + + self.assertIs(raised.exception.__cause__, failure) + self.assertEqual(resetter.corpora, []) + + def test_empty_rebuild_resets_only_without_embedder_or_writer_access(self) -> None: + request = _request(()) + embedder, writer, resetter, events = _collaborators(request) + embedder.identity_failure = AssertionError("empty rebuild must not embed") + writer.replace_failure_at = 0 + + result = rebuild_projection( + request, + ProjectionStateSnapshot(ProjectionStateAvailability.CORRUPT, None), + _borrow_embedder(embedder), + _borrow_writer(writer), + _borrow_resetter(resetter), + ) + + self.assertEqual(events, ["reset:corpus"]) + self.assertEqual(embedder.identity_calls, 0) + self.assertEqual(writer.replacements, []) + self.assertIs(result.status_before, ProjectionStateStatus.CORRUPT) + self.assertEqual(result.receipt.attempted_documents, 0) + self.assertEqual(result.receipt.completed_documents, 0) + self.assertIs(result.receipt.outcome, ProjectionOutcome.COMPLETED) + + def test_rebuild_accepts_every_state_status_and_reports_it_truthfully(self) -> None: + request = _request((_document("alpha"),)) + other_schema = _request( + request.documents, + projection=_identity(schema_id="other"), + ) + other_embedding = _request( + request.documents, + projection=_identity(model_id="other"), + ) + stale = _request( + (_document("alpha", revision_id="old"),), + projection=request.projection, + ) + states = ( + ( + ProjectionStateSnapshot(ProjectionStateAvailability.MISSING, None), + ProjectionStateStatus.MISSING, + ), + ( + ProjectionStateSnapshot(ProjectionStateAvailability.CORRUPT, None), + ProjectionStateStatus.CORRUPT, + ), + (_present(request), ProjectionStateStatus.CURRENT), + (_present(stale), ProjectionStateStatus.STALE), + (_present(other_schema), ProjectionStateStatus.SCHEMA_MISMATCH), + (_present(other_embedding), ProjectionStateStatus.EMBEDDING_MISMATCH), + ) + + for state, status in states: + embedder, writer, resetter, _ = _collaborators(request) + with self.subTest(status=status): + result = rebuild_projection( + request, + state, + _borrow_embedder(embedder), + _borrow_writer(writer), + _borrow_resetter(resetter), + ) + self.assertIs(result.status_before, status) + self.assertEqual(resetter.corpora, ["corpus"]) + self.assertIs(result.receipt.outcome, ProjectionOutcome.COMPLETED) + + def test_reset_exception_has_failed_receipt_and_preserves_cause(self) -> None: + request = _request((_document("alpha"), _document("beta"))) + failure = RuntimeError("reset failed") + events: list[str] = [] + embedder = _FakeEmbedder(request.projection.embedding, events) + writer = _FakeWriter(events) + resetter = _FakeResetter(events, failure=failure) + + with self.assertRaises(ProjectionOperationError) as raised: + rebuild_projection( + request, + ProjectionStateSnapshot(ProjectionStateAvailability.MISSING, None), + _borrow_embedder(embedder), + _borrow_writer(writer), + _borrow_resetter(resetter), + ) + + error = raised.exception + self.assertIs(error.stage, ProjectionFailureStage.RESET) + self.assertIsNone(error.affected_document) + self.assertIs(error.__cause__, failure) + self.assertIsNotNone(error.receipt) + assert error.receipt is not None + self.assertIs(error.receipt.outcome, ProjectionOutcome.FAILED) + self.assertEqual(error.receipt.attempted_documents, 2) + self.assertEqual(error.receipt.completed_documents, 0) + self.assertIsNone(error.receipt.checkpoint) + self.assertEqual(writer.replacements, []) + + def test_non_none_reset_result_is_failure_without_cause(self) -> None: + for documents, expected_receipt in ( + ((_document("alpha"),), True), + ((), False), + ): + request = _request(documents) + events: list[str] = [] + embedder = _FakeEmbedder(request.projection.embedding, events) + writer = _FakeWriter(events) + resetter = _FakeResetter(events, result=False) + with self.subTest(documents=documents): + with self.assertRaises(ProjectionOperationError) as raised: + rebuild_projection( + request, + ProjectionStateSnapshot( + ProjectionStateAvailability.MISSING, + None, + ), + _borrow_embedder(embedder), + _borrow_writer(writer), + _borrow_resetter(resetter), + ) + self.assertIs(raised.exception.stage, ProjectionFailureStage.RESET) + self.assertIsNone(raised.exception.__cause__) + self.assertEqual( + raised.exception.receipt is not None, + expected_receipt, + ) + + +class ProjectionFailureTests(unittest.TestCase): + def _incremental_target( + self, + documents: tuple[Document, ...], + *, + batch_size: int = 2, + ) -> tuple[ProjectionRequest, ProjectionStateSnapshot]: + target = _request(documents, batch_size=batch_size) + previous = _request( + (), + projection=target.projection, + chunking=target.chunking, + batch_size=batch_size, + ) + return target, _present(previous) + + def test_embedding_exception_is_called_once_and_preserved_as_cause(self) -> None: + request, state = self._incremental_target((_document("alpha", text="x"),)) + failure = RuntimeError("provider failed") + events: list[str] = [] + embedder = _FakeEmbedder( + request.projection.embedding, + events, + embed_failure_at=0, + embed_failure=failure, + ) + writer = _FakeWriter(events) + + with self.assertRaises(ProjectionOperationError) as raised: + project_documents( + request, + state, + _borrow_embedder(embedder), + _borrow_writer(writer), + ) + + error = raised.exception + self.assertIs(error.stage, ProjectionFailureStage.EMBEDDING) + self.assertEqual(error.affected_document, DocumentKey("corpus", "alpha")) + self.assertIs(error.__cause__, failure) + self.assertEqual(embedder.embed_calls, [("x",)]) + self.assertEqual(writer.replacements, []) + self.assertIsNotNone(error.receipt) + assert error.receipt is not None + self.assertIs(error.receipt.outcome, ProjectionOutcome.FAILED) + self.assertEqual(error.receipt.completed_documents, 0) + self.assertIsNone(error.receipt.checkpoint) + + def test_malformed_embedding_outputs_fail_without_internal_cause(self) -> None: + request, state = self._incremental_target((_document("alpha", text="x"),)) + nonfinite = EmbeddingVector((1.0, 2.0)) + object.__setattr__(nonfinite, "values", (float("nan"), 2.0)) + noncanonical = EmbeddingVector((1.0, 2.0)) + object.__setattr__(noncanonical, "values", (1, 2.0)) + outputs: tuple[object, ...] = ( + [EmbeddingVector((1.0, 2.0))], + (), + (object(),), + (EmbeddingVector((1.0,)),), + (EmbeddingVector((1.0, 2.0, 3.0)),), + (nonfinite,), + (noncanonical,), + ) + + for output in outputs: + events: list[str] = [] + embedder = _FakeEmbedder( + request.projection.embedding, + events, + output=_constant_output(output), + ) + writer = _FakeWriter(events) + with self.subTest(output=output): + with self.assertRaises(ProjectionOperationError) as raised: + project_documents( + request, + state, + _borrow_embedder(embedder), + _borrow_writer(writer), + ) + self.assertIs( + raised.exception.stage, + ProjectionFailureStage.EMBEDDING, + ) + self.assertIsNone(raised.exception.__cause__) + self.assertEqual(len(embedder.embed_calls), 1) + self.assertEqual(writer.replacements, []) + + def test_second_document_embedding_failure_reports_partial_completion(self) -> None: + request, state = self._incremental_target( + (_document("alpha"), _document("beta")), + batch_size=2, + ) + events: list[str] = [] + failure = RuntimeError("second document failed") + embedder = _FakeEmbedder( + request.projection.embedding, + events, + embed_failure_at=2, + embed_failure=failure, + ) + writer = _FakeWriter(events) + + with self.assertRaises(ProjectionOperationError) as raised: + project_documents( + request, + state, + _borrow_embedder(embedder), + _borrow_writer(writer), + ) + + error = raised.exception + self.assertEqual(error.affected_document, DocumentKey("corpus", "beta")) + self.assertIsNotNone(error.receipt) + assert error.receipt is not None + self.assertIs(error.receipt.outcome, ProjectionOutcome.PARTIAL) + self.assertEqual(error.receipt.attempted_documents, 2) + self.assertEqual(error.receipt.completed_documents, 1) + self.assertEqual( + [identity.key.document_id for identity, _ in writer.replacements], + ["alpha"], + ) + self.assertIs(error.__cause__, failure) + + def test_replacement_exception_has_matching_failed_receipt_and_cause(self) -> None: + request, state = self._incremental_target((_document("alpha", text="x"),)) + events: list[str] = [] + failure = RuntimeError("writer failed") + embedder = _FakeEmbedder(request.projection.embedding, events) + writer = _FakeWriter( + events, + replace_failure_at=0, + replace_failure=failure, + ) + + with self.assertRaises(ProjectionOperationError) as raised: + project_documents( + request, + state, + _borrow_embedder(embedder), + _borrow_writer(writer), + ) + + error = raised.exception + self.assertIs(error.stage, ProjectionFailureStage.REPLACEMENT) + self.assertEqual(error.affected_document, DocumentKey("corpus", "alpha")) + self.assertIs(error.__cause__, failure) + self.assertIsNotNone(error.receipt) + assert error.receipt is not None + self.assertIs(error.receipt.outcome, ProjectionOutcome.FAILED) + self.assertEqual(len(writer.replacements), 1) + + def test_non_none_replacement_result_is_failure_without_cause(self) -> None: + request, state = self._incremental_target((_document("alpha", text="x"),)) + events: list[str] = [] + embedder = _FakeEmbedder(request.projection.embedding, events) + writer = _FakeWriter(events, replace_result=False) + + with self.assertRaises(ProjectionOperationError) as raised: + project_documents( + request, + state, + _borrow_embedder(embedder), + _borrow_writer(writer), + ) + + self.assertIs(raised.exception.stage, ProjectionFailureStage.REPLACEMENT) + self.assertIsNone(raised.exception.__cause__) + self.assertIsNotNone(raised.exception.receipt) + assert raised.exception.receipt is not None + self.assertEqual(raised.exception.receipt.completed_documents, 0) + + def test_deletion_exception_after_replacement_reports_partial_receipt(self) -> None: + target = _request((_document("alpha", text="x"),)) + previous = _request( + (_document("beta", text="x"),), + projection=target.projection, + chunking=target.chunking, + ) + events: list[str] = [] + failure = RuntimeError("delete failed") + embedder = _FakeEmbedder(target.projection.embedding, events) + writer = _FakeWriter( + events, + delete_failure_at=0, + delete_failure=failure, + ) + + with self.assertRaises(ProjectionOperationError) as raised: + project_documents( + target, + _present(previous), + _borrow_embedder(embedder), + _borrow_writer(writer), + ) + + error = raised.exception + self.assertIs(error.stage, ProjectionFailureStage.DELETION) + self.assertEqual(error.affected_document, DocumentKey("corpus", "beta")) + self.assertIs(error.__cause__, failure) + self.assertIsNotNone(error.receipt) + assert error.receipt is not None + self.assertIs(error.receipt.outcome, ProjectionOutcome.PARTIAL) + self.assertEqual(error.receipt.attempted_documents, 2) + self.assertEqual(error.receipt.completed_documents, 1) + self.assertIsNone(error.receipt.checkpoint) + self.assertEqual( + [event for event in events if event.startswith(("replace:", "delete:"))], + ["replace:alpha", "delete:beta"], + ) + + def test_non_none_deletion_result_is_failed_without_cause(self) -> None: + target = _request(()) + previous = _request( + (_document("alpha", text="x"),), + projection=target.projection, + chunking=target.chunking, + ) + events: list[str] = [] + embedder = _FakeEmbedder(target.projection.embedding, events) + writer = _FakeWriter(events, delete_result=0) + + with self.assertRaises(ProjectionOperationError) as raised: + project_documents( + target, + _present(previous), + _borrow_embedder(embedder), + _borrow_writer(writer), + ) + + self.assertIs(raised.exception.stage, ProjectionFailureStage.DELETION) + self.assertIsNone(raised.exception.__cause__) + self.assertIsNotNone(raised.exception.receipt) + assert raised.exception.receipt is not None + self.assertIs(raised.exception.receipt.outcome, ProjectionOutcome.FAILED) + + def test_rebuild_replacement_failure_never_claims_a_checkpoint(self) -> None: + request = _request((_document("alpha"), _document("beta"))) + events: list[str] = [] + embedder = _FakeEmbedder(request.projection.embedding, events) + writer = _FakeWriter(events, replace_failure_at=1) + resetter = _FakeResetter(events) + + with self.assertRaises(ProjectionOperationError) as raised: + rebuild_projection( + request, + ProjectionStateSnapshot(ProjectionStateAvailability.CORRUPT, None), + _borrow_embedder(embedder), + _borrow_writer(writer), + _borrow_resetter(resetter), + ) + + error = raised.exception + self.assertIs(error.stage, ProjectionFailureStage.REPLACEMENT) + self.assertEqual(error.affected_document, DocumentKey("corpus", "beta")) + self.assertIsNotNone(error.receipt) + assert error.receipt is not None + self.assertIs(error.receipt.outcome, ProjectionOutcome.PARTIAL) + self.assertEqual(error.receipt.completed_documents, 1) + self.assertIsNone(error.receipt.checkpoint) + self.assertEqual(resetter.corpora, ["corpus"]) + + def test_control_flow_exceptions_are_never_translated_or_retried(self) -> None: + for failure in (KeyboardInterrupt("interrupt"), SystemExit("exit")): + request, state = self._incremental_target((_document("alpha", text="x"),)) + events: list[str] = [] + embedder = _FakeEmbedder( + request.projection.embedding, + events, + embed_failure_at=0, + embed_failure=failure, + ) + writer = _FakeWriter(events) + with self.subTest(failure=type(failure).__name__): + with self.assertRaises(type(failure)) as raised: + project_documents( + request, + state, + _borrow_embedder(embedder), + _borrow_writer(writer), + ) + self.assertIs(raised.exception, failure) + self.assertEqual(len(embedder.embed_calls), 1) + self.assertEqual(writer.replacements, []) + + def test_reset_control_flow_exception_is_not_translated_or_retried(self) -> None: + request = _request((_document("alpha"),)) + failure = KeyboardInterrupt("reset interrupted") + events: list[str] = [] + embedder = _FakeEmbedder(request.projection.embedding, events) + writer = _FakeWriter(events) + resetter = _FakeResetter(events, failure=failure) + + with self.assertRaises(KeyboardInterrupt) as raised: + rebuild_projection( + request, + ProjectionStateSnapshot(ProjectionStateAvailability.MISSING, None), + _borrow_embedder(embedder), + _borrow_writer(writer), + _borrow_resetter(resetter), + ) + + self.assertIs(raised.exception, failure) + self.assertEqual(resetter.corpora, ["corpus"]) + self.assertEqual(writer.replacements, []) + + def test_workflows_never_enter_close_shutdown_or_suppress_resources(self) -> None: + request, state = self._incremental_target((_document("alpha", text="x"),)) + embedder, writer, resetter, _ = _collaborators(request) + + project_documents( + request, + state, + _borrow_embedder(embedder), + _borrow_writer(writer), + ) + rebuild_projection( + request, + ProjectionStateSnapshot(ProjectionStateAvailability.MISSING, None), + _borrow_embedder(embedder), + _borrow_writer(writer), + _borrow_resetter(resetter), + ) + + self.assertEqual(embedder.lifecycle_calls, []) + self.assertEqual(writer.lifecycle_calls, []) + self.assertEqual(resetter.lifecycle_calls, []) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_projection_contracts.py b/tests/test_projection_contracts.py index 7f59932..e4c2cb1 100644 --- a/tests/test_projection_contracts.py +++ b/tests/test_projection_contracts.py @@ -1,4 +1,4 @@ -"""Contract tests for projection identities, checkpoints, and receipts.""" +"""Contract tests for immutable projection values and truthful state.""" from __future__ import annotations @@ -7,11 +7,23 @@ from typing import cast from generic_rag.contracts import ( + ChunkingPolicy, + Document, + DocumentIdentity, + DocumentKey, EmbeddingIdentity, ProjectionCheckpoint, ProjectionIdentity, + ProjectionLimits, + ProjectionManifest, + ProjectionManifestEntry, ProjectionOutcome, ProjectionReceipt, + ProjectionRequest, + ProjectionResult, + ProjectionStateAvailability, + ProjectionStateSnapshot, + ProjectionStateStatus, ) from generic_rag.errors import ContractValidationError @@ -49,6 +61,60 @@ def _checkpoint( ) +def _document( + document_id: str = "document", + *, + corpus_id: str = "corpus", + revision_id: str = "revision", + text: str = "text", +) -> Document: + return Document( + DocumentIdentity(DocumentKey(corpus_id, document_id), revision_id), + text, + ) + + +def _entry( + document_id: str = "document", + *, + corpus_id: str = "corpus", + revision_id: str = "revision", + digest_digit: str = "0", + fragment_count: int = 1, +) -> ProjectionManifestEntry: + return ProjectionManifestEntry( + _document( + document_id, + corpus_id=corpus_id, + revision_id=revision_id, + ).identity, + f"sha256:{digest_digit * 64}", + fragment_count, + ) + + +def _manifest( + *, + corpus_id: str = "corpus", + projection: ProjectionIdentity | None = None, + chunking: ChunkingPolicy | None = None, + entries: tuple[ProjectionManifestEntry, ...] | None = None, + token: str = "checkpoint", +) -> ProjectionManifest: + resolved_projection = projection or _projection() + return ProjectionManifest( + corpus_id, + resolved_projection, + chunking or ChunkingPolicy(4, 1), + (_entry(corpus_id=corpus_id),) if entries is None else entries, + _checkpoint( + corpus_id=corpus_id, + projection=resolved_projection, + token=token, + ), + ) + + class ProjectionContractTests(unittest.TestCase): def test_projection_fields_are_exact_frozen_and_slotted(self) -> None: self.assertEqual( @@ -372,5 +438,482 @@ def test_receipt_rejects_inexact_nested_and_discrete_types(self) -> None: ) +class ProjectionWorkflowContractTests(unittest.TestCase): + def test_new_projection_values_have_exact_frozen_slotted_fields(self) -> None: + expected_fields = { + ChunkingPolicy: ("max_fragment_codepoints", "overlap_codepoints"), + ProjectionLimits: ( + "max_documents", + "max_document_codepoints", + "max_embedding_batch_size", + ), + ProjectionRequest: ( + "corpus_id", + "projection", + "chunking", + "limits", + "documents", + ), + ProjectionManifestEntry: ( + "document", + "source_digest", + "fragment_count", + ), + ProjectionManifest: ( + "corpus_id", + "projection", + "chunking", + "entries", + "checkpoint", + ), + ProjectionStateSnapshot: ("availability", "manifest"), + ProjectionResult: ("status_before", "receipt", "manifest"), + } + projection = _projection() + manifest = _manifest(projection=projection) + instances = ( + ChunkingPolicy(4, 1), + ProjectionLimits(2, 20, 2), + ProjectionRequest( + "corpus", + projection, + ChunkingPolicy(4, 1), + ProjectionLimits(2, 20, 2), + (_document(),), + ), + manifest.entries[0], + manifest, + ProjectionStateSnapshot(ProjectionStateAvailability.PRESENT, manifest), + ProjectionResult( + ProjectionStateStatus.STALE, + ProjectionReceipt( + "corpus", + projection, + ProjectionOutcome.COMPLETED, + 1, + 1, + manifest.checkpoint, + ), + manifest, + ), + ) + + for instance in instances: + with self.subTest(value=type(instance).__name__): + self.assertEqual( + tuple(field.name for field in fields(type(instance))), + expected_fields[type(instance)], + ) + self.assertFalse(hasattr(instance, "__dict__")) + with self.assertRaises(FrozenInstanceError): + setattr(instance, expected_fields[type(instance)][0], object()) + self.assertIsInstance(hash(instance), int) + + def test_chunking_policy_uses_positive_codepoint_size_and_bounded_overlap( + self, + ) -> None: + self.assertEqual(ChunkingPolicy(4, 0), ChunkingPolicy(4, 0)) + self.assertEqual(ChunkingPolicy(4, 3).overlap_codepoints, 3) + + for maximum, overlap in ( + (0, 0), + (-1, 0), + (4, -1), + (4, 4), + (4, 5), + (True, 0), + (4, True), + (4.0, 0), + (4, 1.0), + (_IntegerSubclass(4), 0), + ): + with self.subTest(maximum=maximum, overlap=overlap): + with self.assertRaises(ContractValidationError): + ChunkingPolicy(cast(int, maximum), cast(int, overlap)) + + def test_projection_limits_require_three_positive_exact_integers(self) -> None: + self.assertEqual(ProjectionLimits(1, 1, 1), ProjectionLimits(1, 1, 1)) + + for field_index in range(3): + for invalid in (0, -1, True, 1.0, _IntegerSubclass(1)): + values: list[object] = [2, 20, 3] + values[field_index] = invalid + with self.subTest(field=field_index, value=invalid): + with self.assertRaises(ContractValidationError): + ProjectionLimits( + cast(int, values[0]), + cast(int, values[1]), + cast(int, values[2]), + ) + + def test_request_preserves_opaque_values_and_canonicalizes_document_order( + self, + ) -> None: + documents = ( + _document("zeta", text=""), + _document("alpha", revision_id=" revision/../2 ", text="e\u0301"), + ) + request = ProjectionRequest( + "corpus", + _projection(), + ChunkingPolicy(2, 1), + ProjectionLimits(2, 2, 1), + documents, + ) + + self.assertEqual( + tuple(document.identity.key.document_id for document in request.documents), + ("alpha", "zeta"), + ) + self.assertEqual(request.documents[0].identity.revision_id, " revision/../2 ") + self.assertEqual(request.documents[0].text, "e\u0301") + self.assertEqual( + ProjectionRequest( + "corpus", + _projection(), + ChunkingPolicy(2, 1), + ProjectionLimits(2, 2, 1), + (), + ).documents, + (), + ) + + def test_request_rejects_inexact_fields_and_document_bound_violations( + self, + ) -> None: + projection = _projection() + chunking = ChunkingPolicy(4, 1) + limits = ProjectionLimits(1, 4, 2) + valid = _document(text="four") + invalid_requests = ( + ("", projection, chunking, limits, (valid,)), + (_StringSubclass("corpus"), projection, chunking, limits, (valid,)), + ("corpus", object(), chunking, limits, (valid,)), + ("corpus", projection, object(), limits, (valid,)), + ("corpus", projection, chunking, object(), (valid,)), + ("corpus", projection, chunking, limits, [valid]), + ("corpus", projection, chunking, limits, (object(),)), + ("corpus", projection, chunking, limits, (valid, _document("two"))), + ("corpus", projection, chunking, limits, (_document(text="12345"),)), + ( + "corpus", + projection, + chunking, + limits, + (_document(corpus_id="other"),), + ), + ( + "corpus", + projection, + chunking, + ProjectionLimits(2, 4, 2), + (_document(revision_id="r1"), _document(revision_id="r2")), + ), + ) + for values in invalid_requests: + with self.subTest(values=values): + with self.assertRaises(ContractValidationError): + ProjectionRequest( + values[0], + cast(ProjectionIdentity, values[1]), + cast(ChunkingPolicy, values[2]), + cast(ProjectionLimits, values[3]), + cast(tuple[Document, ...], values[4]), + ) + + def test_manifest_entry_requires_lowercase_sha256_and_nonnegative_count( + self, + ) -> None: + entry = _entry(fragment_count=0) + self.assertEqual(entry.source_digest, "sha256:" + "0" * 64) + self.assertEqual(entry.fragment_count, 0) + + for digest in ( + "", + "0" * 64, + "sha256:" + "A" * 64, + "sha256:" + "0" * 63, + "sha256:" + "0" * 65, + _StringSubclass("sha256:" + "0" * 64), + ): + with self.subTest(digest=digest): + with self.assertRaises(ContractValidationError): + ProjectionManifestEntry( + _document().identity, + digest, + 1, + ) + for count in (-1, True, 1.0, _IntegerSubclass(1)): + with self.subTest(count=count): + with self.assertRaises(ContractValidationError): + ProjectionManifestEntry( + _document().identity, + "sha256:" + "0" * 64, + cast(int, count), + ) + with self.assertRaises(ContractValidationError): + ProjectionManifestEntry( + cast(DocumentIdentity, object()), + "sha256:" + "0" * 64, + 1, + ) + + def test_manifest_canonicalizes_entries_and_preserves_checkpoint(self) -> None: + projection = _projection() + checkpoint = _checkpoint(projection=projection, token=" token/../x ") + zeta = _entry("zeta", digest_digit="1") + alpha = _entry("alpha", digest_digit="2") + manifest = ProjectionManifest( + "corpus", + projection, + ChunkingPolicy(4, 1), + (zeta, alpha), + checkpoint, + ) + + self.assertEqual(manifest.entries, (alpha, zeta)) + self.assertIs(manifest.checkpoint, checkpoint) + self.assertEqual(manifest.checkpoint.token, " token/../x ") + + def test_manifest_rejects_mismatched_inexact_or_duplicate_content(self) -> None: + projection = _projection() + chunking = ChunkingPolicy(4, 1) + checkpoint = _checkpoint(projection=projection) + entry = _entry() + invalid_values = ( + ("", projection, chunking, (entry,), checkpoint), + ("corpus", object(), chunking, (entry,), checkpoint), + ("corpus", projection, object(), (entry,), checkpoint), + ("corpus", projection, chunking, [entry], checkpoint), + ("corpus", projection, chunking, (object(),), checkpoint), + ("corpus", projection, chunking, (entry,), object()), + ( + "corpus", + projection, + chunking, + (_entry(corpus_id="other"),), + checkpoint, + ), + ("corpus", projection, chunking, (entry, entry), checkpoint), + ( + "corpus", + projection, + chunking, + (entry,), + _checkpoint(corpus_id="other", projection=projection), + ), + ( + "corpus", + projection, + chunking, + (entry,), + _checkpoint(projection=_projection(schema_id="other")), + ), + ) + for values in invalid_values: + with self.subTest(values=values): + with self.assertRaises(ContractValidationError): + ProjectionManifest( + values[0], + cast(ProjectionIdentity, values[1]), + cast(ChunkingPolicy, values[2]), + cast(tuple[ProjectionManifestEntry, ...], values[3]), + cast(ProjectionCheckpoint, values[4]), + ) + + def test_state_availability_is_an_exact_closed_string_enum(self) -> None: + self.assertEqual( + tuple( + (member.name, member.value) for member in ProjectionStateAvailability + ), + (("MISSING", "missing"), ("PRESENT", "present"), ("CORRUPT", "corrupt")), + ) + for invalid in ("MISSING", "", "unknown", None, 1, object()): + with self.subTest(value=invalid): + with self.assertRaises(ContractValidationError): + ProjectionStateAvailability(cast(str, invalid)) + + def test_state_snapshot_requires_manifest_only_when_present(self) -> None: + manifest = _manifest() + self.assertIs( + ProjectionStateSnapshot( + ProjectionStateAvailability.PRESENT, + manifest, + ).manifest, + manifest, + ) + for availability in ( + ProjectionStateAvailability.MISSING, + ProjectionStateAvailability.CORRUPT, + ): + self.assertIsNone(ProjectionStateSnapshot(availability, None).manifest) + with self.assertRaises(ContractValidationError): + ProjectionStateSnapshot(availability, manifest) + with self.assertRaises(ContractValidationError): + ProjectionStateSnapshot(ProjectionStateAvailability.PRESENT, None) + with self.assertRaises(ContractValidationError): + ProjectionStateSnapshot( + cast(ProjectionStateAvailability, "present"), + manifest, + ) + + def test_state_status_is_an_exact_closed_string_enum(self) -> None: + self.assertEqual( + tuple((member.name, member.value) for member in ProjectionStateStatus), + ( + ("MISSING", "missing"), + ("CURRENT", "current"), + ("STALE", "stale"), + ("CORRUPT", "corrupt"), + ("SCHEMA_MISMATCH", "schema_mismatch"), + ("EMBEDDING_MISMATCH", "embedding_mismatch"), + ), + ) + for invalid in ("CURRENT", "", "unknown", None, 1, object()): + with self.subTest(value=invalid): + with self.assertRaises(ContractValidationError): + ProjectionStateStatus(cast(str, invalid)) + + def test_result_accepts_matching_completed_and_truthful_unchanged_values( + self, + ) -> None: + manifest = _manifest() + completed = ProjectionResult( + ProjectionStateStatus.STALE, + ProjectionReceipt( + "corpus", + manifest.projection, + ProjectionOutcome.COMPLETED, + 2, + 2, + manifest.checkpoint, + ), + manifest, + ) + unchanged = ProjectionResult( + ProjectionStateStatus.CURRENT, + ProjectionReceipt( + "corpus", + manifest.projection, + ProjectionOutcome.UNCHANGED, + 0, + 0, + manifest.checkpoint, + ), + manifest, + ) + + self.assertEqual(completed.status_before, ProjectionStateStatus.STALE) + self.assertEqual(unchanged.receipt.attempted_documents, 0) + + def test_result_rejects_inexact_mismatched_or_untruthful_values(self) -> None: + manifest = _manifest() + completed = ProjectionReceipt( + "corpus", + manifest.projection, + ProjectionOutcome.COMPLETED, + 1, + 1, + manifest.checkpoint, + ) + invalid_values = ( + ("stale", completed, manifest), + (ProjectionStateStatus.STALE, object(), manifest), + (ProjectionStateStatus.STALE, completed, object()), + ( + ProjectionStateStatus.STALE, + ProjectionReceipt( + "other", + manifest.projection, + ProjectionOutcome.COMPLETED, + 1, + 1, + _checkpoint(corpus_id="other", projection=manifest.projection), + ), + manifest, + ), + ( + ProjectionStateStatus.STALE, + ProjectionReceipt( + "corpus", + manifest.projection, + ProjectionOutcome.PARTIAL, + 2, + 1, + None, + ), + manifest, + ), + ( + ProjectionStateStatus.STALE, + ProjectionReceipt( + "corpus", + manifest.projection, + ProjectionOutcome.FAILED, + 1, + 0, + None, + ), + manifest, + ), + ( + ProjectionStateStatus.STALE, + ProjectionReceipt( + "corpus", + _projection(schema_id="other"), + ProjectionOutcome.COMPLETED, + 1, + 1, + _checkpoint(projection=_projection(schema_id="other")), + ), + manifest, + ), + ( + ProjectionStateStatus.STALE, + ProjectionReceipt( + "corpus", + manifest.projection, + ProjectionOutcome.COMPLETED, + 1, + 1, + _checkpoint(projection=manifest.projection, token="other"), + ), + manifest, + ), + ( + ProjectionStateStatus.STALE, + ProjectionReceipt( + "corpus", + manifest.projection, + ProjectionOutcome.UNCHANGED, + 0, + 0, + manifest.checkpoint, + ), + manifest, + ), + ( + ProjectionStateStatus.CURRENT, + ProjectionReceipt( + "corpus", + manifest.projection, + ProjectionOutcome.UNCHANGED, + 1, + 1, + manifest.checkpoint, + ), + manifest, + ), + ) + for status, receipt, value_manifest in invalid_values: + with self.subTest(status=status, receipt=receipt): + with self.assertRaises(ContractValidationError): + ProjectionResult( + cast(ProjectionStateStatus, status), + cast(ProjectionReceipt, receipt), + cast(ProjectionManifest, value_manifest), + ) + + if __name__ == "__main__": unittest.main() From 0d5861ef57a11fe11b9c75082a85e7eea1469038 Mon Sep 17 00:00:00 2001 From: rica Date: Tue, 25 Aug 2026 19:38:25 +0900 Subject: [PATCH 3/4] [RAG Retrieval] Add deterministic semantic and hybrid retrieval (#8) Implements and documents the final generic-rag retrieval leaf. Closes #4. --- PYTHON_MODULE_INDEX.md | 77 +- README.md | 56 +- docs/api.md | 124 ++- docs/lifecycle.md | 63 +- docs/projection.md | 21 +- docs/retrieval.md | 324 ++++++ docs/security-and-privacy.md | 74 +- src/generic_rag/contracts.py | 11 + src/generic_rag/projection.py | 108 +- src/generic_rag/projection_integrity.py | 267 +++++ src/generic_rag/retrieval.py | 517 +++++++++ tests/support/verify_artifacts.py | 2 + tests/test_package_boundaries.py | 32 + tests/test_projection_integrity.py | 464 ++++++++ tests/test_retrieval.py | 1312 +++++++++++++++++++++++ tests/test_retrieval_contracts.py | 21 + 16 files changed, 3299 insertions(+), 174 deletions(-) create mode 100644 docs/retrieval.md create mode 100644 src/generic_rag/projection_integrity.py create mode 100644 src/generic_rag/retrieval.py create mode 100644 tests/test_projection_integrity.py create mode 100644 tests/test_retrieval.py diff --git a/PYTHON_MODULE_INDEX.md b/PYTHON_MODULE_INDEX.md index 08092cc..d2d0c18 100644 --- a/PYTHON_MODULE_INDEX.md +++ b/PYTHON_MODULE_INDEX.md @@ -4,9 +4,9 @@ - Declared source root: `src` - Packaging source of truth: `pyproject.toml` -- Importable production units: 5 -- Indexed production units: 5 -- Source/index parity: 5/5 +- Importable production units: 7 +- Indexed production units: 7 +- Source/index parity: 7/7 - Package data: `src/generic_rag/py.typed` - Locked verification owner: `.github/workflows/ci.yml` (supporting workflow, not an importable unit) @@ -57,8 +57,8 @@ `ProjectionLimits`, `ProjectionRequest`, `ProjectionManifestEntry`, `ProjectionManifest`, `ProjectionStateAvailability`, `ProjectionStateSnapshot`, `ProjectionStateStatus`, `ProjectionResult`, - `RetrievalQuery`, `RetrievalOutcome`, `RetrievalHit`, and `RetrievalResult` - from `generic_rag.contracts`. + `RetrievalLimits`, `RetrievalQuery`, `RetrievalOutcome`, `RetrievalHit`, and + `RetrievalResult` from `generic_rag.contracts`. - Re-exports: exactly the names in the module's `__all__`; none from the package root. - Direct internal dependencies: `generic_rag.errors`. @@ -70,7 +70,7 @@ `tests/test_retrieval_contracts.py`, `tests/test_package_boundaries.py`, `tests/support/clean_import_probe.py`, `tests/support/verify_artifacts.py`, and the locked CI import, boundary, and artifact checks. -- Documentation: `docs/api.md`, `docs/projection.md`, and +- Documentation: `docs/api.md`, `docs/projection.md`, `docs/retrieval.md`, and `docs/security-and-privacy.md`. ## `generic_rag.ports` @@ -90,8 +90,8 @@ - Verification: `tests/test_ports.py`, `tests/test_package_boundaries.py`, `tests/test_projection.py`, `tests/support/clean_import_probe.py`, and the locked CI import and boundary checks. -- Documentation: `docs/api.md`, `docs/lifecycle.md`, `docs/projection.md`, and - `docs/security-and-privacy.md`. +- Documentation: `docs/api.md`, `docs/lifecycle.md`, `docs/projection.md`, + `docs/retrieval.md`, and `docs/security-and-privacy.md`. ## `generic_rag.projection` @@ -104,7 +104,10 @@ - Re-exports: exactly the names in the module's `__all__`; none from the package root. - Direct internal dependencies: `generic_rag.contracts`, `generic_rag.errors`, - and `generic_rag.ports`. + `generic_rag.ports`, and `generic_rag.projection_integrity`. +- Integrity ownership: delegates deterministic source-digest, fragment-ID, and + checkpoint-token derivation and checkpoint validation to + `generic_rag.projection_integrity`. - Owned state or external resources: none; planning state is immutable and local to each call, while every embedder, writer, and resetter remains caller-owned through `Borrowed`. @@ -118,3 +121,59 @@ checks. - Documentation: `docs/projection.md`, `docs/api.md`, `docs/lifecycle.md`, and `docs/security-and-privacy.md`. + +## `generic_rag.projection_integrity` + +- Source: `src/generic_rag/projection_integrity.py` +- Responsibility: own deterministic v1 derivation and validation algorithms + for projection integrity values. +- Supported public imports: `derive_source_digest`, `derive_fragment_id`, + `derive_projection_checkpoint_token`, and + `has_valid_projection_checkpoint` from `generic_rag.projection_integrity`. +- Re-exports: exactly the names in the module's `__all__`; none from the package + root. +- Direct internal dependencies: `generic_rag.contracts` and + `generic_rag.errors`. +- Owned state or external resources: none; each call derives or checks an + immutable value from exact caller-supplied contracts. +- Material side effects: none at import or call time. +- Verification: `tests/test_projection_integrity.py`, + `tests/test_projection.py`, `tests/test_retrieval.py`, + `tests/test_package_boundaries.py`, `tests/support/clean_import_probe.py`, + `tests/support/verify_artifacts.py`, and the locked CI test, lint, strict + type, compile, build, clean-install, import, and artifact checks. +- Documentation: `docs/api.md`, `docs/projection.md`, `docs/retrieval.md`, and + `docs/security-and-privacy.md`. + +## `generic_rag.retrieval` + +- Source: `src/generic_rag/retrieval.py` +- Responsibility: deterministically compose bounded semantic and hybrid + retrieval results from published caller-owned projection state. +- Supported public imports: `retrieve_semantic` and `retrieve_hybrid` from + `generic_rag.retrieval`. +- Re-exports: exactly the names in the module's `__all__`; none from the package + root. +- Direct internal dependencies: `generic_rag.contracts`, `generic_rag.errors`, + `generic_rag.ports`, and `generic_rag.projection_integrity`. +- Owned state or external resources: none; ranking state is local to each call, + while every embedder, vector reader, and lexical retriever remains + caller-owned through `Borrowed`. +- Material side effects: none at import time. At explicit workflow call time it + may read one borrowed embedder identity, embed one query, search one borrowed + vector reader, and for hybrid retrieval search one borrowed lexical + retriever; it performs no persistence, network, retry, acquisition, release, + logging, authorization, citation, or lifecycle action itself. +- Retrieval semantics: validates and deduplicates exact fragment identities, + filters candidates against the full current published document revision, + and preserves semantic provider order or fuses original semantic and lexical + ranks deterministically without comparing raw scores. Returned fragments + remain non-authoritative; this module does not authorize a query or source, + validate authoritative source text, or create citations. +- Verification: `tests/test_retrieval.py`, + `tests/test_retrieval_contracts.py`, `tests/test_ports.py`, + `tests/test_package_boundaries.py`, `tests/support/clean_import_probe.py`, + `tests/support/verify_artifacts.py`, and the locked CI test, lint, strict + type, compile, build, clean-install, import, and artifact checks. +- Documentation: `docs/retrieval.md`, `docs/api.md`, `docs/lifecycle.md`, + `docs/security-and-privacy.md`, and `README.md`. diff --git a/README.md b/README.md index 1c7bd03..ec47813 100644 --- a/README.md +++ b/README.md @@ -3,12 +3,13 @@ `generic-rag` is a provider-neutral, runtime-dependency-free foundation for retrieval-augmented generation (RAG). Version 0.1.0 requires Python 3.11 or later and provides immutable contracts, typed error categories, synchronous -collaborator protocols, deterministic bounded projection orchestration, and -explicit caller-owned borrowing. +collaborator protocols, deterministic bounded projection orchestration, +semantic retrieval, rank-based hybrid retrieval, and explicit caller-owned +borrowing. -Retrieval and result composition are not implemented in 0.1.0. The package has -no built-in adapter, provider, factory, persistence, network client, -configuration system, authentication, citation mechanism, or CLI. +The package has no built-in adapter, provider, factory, persistence, network +client, configuration system, authentication, authorization, citation +mechanism, or CLI. ## Install from a checkout @@ -107,12 +108,23 @@ explicit corpus-wide reset is intended. See the [projection guide](docs/projecti for the complete lifecycle, state matrix, adapter obligations, and failure behavior. +Load the corresponding published manifest before retrieval, reauthorize each +query and source in the host, and call `retrieve_semantic` or +`retrieve_hybrid` with borrowed provider implementations. Returned fragment +text is non-authoritative: resolve each identity against the still-authorized +source revision and create host-owned citations before showing results to a +user or injecting bounded context into an agent. See the [retrieval +guide](docs/retrieval.md) for an executable independent-consumer example, +deterministic fusion behavior, outcome handling, and the complete host flow. + Public values must be imported from their owning modules: - `generic_rag.contracts` - `generic_rag.errors` - `generic_rag.ports` - `generic_rag.projection` +- `generic_rag.projection_integrity` +- `generic_rag.retrieval` The package root intentionally has no re-exports: `generic_rag.__all__ == ()`. See the [API reference](docs/api.md) for every supported name and invariant. @@ -129,19 +141,20 @@ shown to a user or supplied to a downstream tool or agent. policy, embeds ordered fragment text, replaces or deletes complete document projections, and returns a manifest and truthful receipt for caller-owned persistence. -- [Issue #4](https://github.com/Kims-DeveloperGroup/generic-rag/issues/4) is - planned to add retrieval and composition. Its intended responsibility is to - use an injected `Embedder` and `VectorIndexReader` for semantic candidates - and an injected `LexicalRetriever` for lexical candidates, then define - deduplication, fusion, limiting, and outcome behavior. Provider rank will be - the input; raw provider scores are not represented or assumed comparable. - -There is no end-user or agent query workflow yet. A consuming application can -project data now, but must wait for or implement a separate reviewed retrieval -layer before supplying retrieved context to users, tools, or agents. The -caller/provider ownership model remains explicit throughout. See [resource -lifecycle](docs/lifecycle.md) and -[security and privacy](docs/security-and-privacy.md). +- Retrieval accepts the matching caller-loaded published state and injected + semantic or lexical providers. It validates and filters current-revision + candidates, deduplicates exact identities, and returns bounded score-free + hits. Hybrid retrieval fuses provider ranks deterministically; it does not + compare raw provider scores. +- The host reauthorizes every query, resolves each returned fragment identity + against authoritative source data, verifies the exact source slice, and + creates citations. It may then show cited results to a user or inject bounded + cited context into an agent; the agent must retain those citations. + +The package does not decide provider selection, authentication, authorization, +prompt or tool policy, retry, display, logging, or resource lifecycle. See the +[retrieval guide](docs/retrieval.md), [resource lifecycle](docs/lifecycle.md), +and [security and privacy](docs/security-and-privacy.md). ## Compatibility @@ -150,10 +163,11 @@ assume compatibility across minor releases. For this release, direct imports from the documented owning modules are the supported public paths; root-level imports are not. -The distribution includes `py.typed`. The wheel contains exactly the five +The distribution includes `py.typed`. The wheel contains exactly the seven importable modules `generic_rag`, `generic_rag.errors`, -`generic_rag.contracts`, `generic_rag.ports`, and `generic_rag.projection`, plus -the typing marker. +`generic_rag.contracts`, `generic_rag.ports`, `generic_rag.projection`, +`generic_rag.projection_integrity`, and `generic_rag.retrieval`, plus the typing +marker. ## Development verification diff --git a/docs/api.md b/docs/api.md index 115c259..47d3c7c 100644 --- a/docs/api.md +++ b/docs/api.md @@ -1,10 +1,11 @@ # Public API Version 0.1.0 exposes immutable values, typed error categories, synchronous -collaborator protocols, and deterministic bounded document projection. It does -not expose retrieval or result-composition orchestration. See the [projection -guide](projection.md), [resource lifecycle](lifecycle.md), and [security and -privacy boundary](security-and-privacy.md) for the surrounding usage contract. +collaborator protocols, deterministic bounded document projection, and +score-free semantic and hybrid retrieval. See the [projection guide](projection.md), +[retrieval guide](retrieval.md), [resource lifecycle](lifecycle.md), and +[security and privacy boundary](security-and-privacy.md) for the surrounding +usage contract. ## Import boundary @@ -42,6 +43,7 @@ public symbols. Import names from their owning modules instead. - `ProjectionStateSnapshot` - `ProjectionStateStatus` - `ProjectionResult` +- `RetrievalLimits` - `RetrievalQuery` - `RetrievalOutcome` - `RetrievalHit` @@ -64,6 +66,18 @@ public symbols. Import names from their owning modules instead. - `project_documents` - `rebuild_projection` +`generic_rag.projection_integrity` exports exactly: + +- `derive_source_digest` +- `derive_fragment_id` +- `derive_projection_checkpoint_token` +- `has_valid_projection_checkpoint` + +`generic_rag.retrieval` exports exactly: + +- `retrieve_semantic` +- `retrieve_hybrid` + The package does not support importing any of these names from the package root. @@ -127,6 +141,15 @@ An ordinary collaborator exception is chained as the operation error's cause. An invalid identity, malformed vector result, or non-`None` command result has no internal cause. `KeyboardInterrupt` and `SystemExit` pass through unchanged. +Retrieval defines no workflow-specific public exception. Invalid top-level +workflow inputs, including a query that exceeds `RetrievalLimits`, raise +`ContractValidationError` before collaborator effects. After a collaborator +boundary is entered, ordinary `Exception` failures and malformed collaborator +returns contribute a content-free failure state. A failed branch contributes no +fragment or exception text; a `PARTIAL` result can contain independently +validated hits from another branch. No exception cause crosses the result +boundary. `BaseException` subclasses pass through unchanged. + ## Documents and fragments | Type | Fields | Construction rules | @@ -174,6 +197,38 @@ The package produces manifests; the caller owns their persistence. Source digests, fragment IDs, and checkpoint tokens are deterministic under explicit v1 domains described in the [projection guide](projection.md#deterministic-projection-values). +The public integrity functions are synchronous and positional-only: + +```python +def derive_source_digest(document: Document, /) -> str: ... + +def derive_fragment_id( + document: DocumentIdentity, + start: int, + end: int, + /, +) -> str: ... + +def derive_projection_checkpoint_token( + corpus_id: str, + projection: ProjectionIdentity, + chunking: ChunkingPolicy, + entries: tuple[ProjectionManifestEntry, ...], + /, +) -> str: ... + +def has_valid_projection_checkpoint( + manifest: ProjectionManifest, + /, +) -> bool: ... +``` + +They reproduce the same v1 integrity values used by projection and retrieval. +They validate exact public contract shapes and canonical manifest-entry order; +invalid inputs raise `ContractValidationError`. Checkpoint validation returns +whether the supplied token equals the derived token. It does not inspect a +provider index or establish authorization. + ## Projection state and results `ProjectionStateAvailability` is a closed string enum: @@ -264,6 +319,7 @@ failure matrices. | Type | Fields | Construction rules | | --- | --- | --- | +| `RetrievalLimits` | `max_query_codepoints: int` | The query-text cap is a positive exact integer. | | `RetrievalQuery` | `corpus_id: str`, `text: str`, `hit_limit: int`, `candidate_limit: int` | Corpus and text are nonblank exact strings; limits are positive exact integers and `hit_limit <= candidate_limit`. Values are preserved exactly. | | `RetrievalHit` | `fragment: Fragment`, `rank: int` | Fragment requires its exact class and rank is a positive exact integer. There is no score field. | | `RetrievalResult` | `query: RetrievalQuery`, `outcome: RetrievalOutcome`, `hits: tuple[RetrievalHit, ...]`, `truncated: bool` | Nested values, the hit tuple, and the boolean require exact types. Hit count cannot exceed `query.hit_limit`. | @@ -274,12 +330,12 @@ ranks are contiguous from one, fragment identities are unique, and every fragment belongs to the query corpus. `PARTIAL` requires at least one hit; `UNAVAILABLE`, `STALE`, and `FAILED` require no hits and `truncated=False`. -`truncated=True` is the caller's explicit assertion that otherwise valid work -or results were cut by the query budget. Hits and reader ports are score-free; -raw provider scores are neither represented nor promised comparable. - -These are value contracts only. Version 0.1.0 has no package retrieval, -composition, citation, user, tool, or agent workflow. +The workflows set `truncated=True` exactly when validated, unique, +current-revision candidates exceed `query.hit_limit`. Hits and reader ports are +score-free; raw provider scores are neither represented nor promised +comparable. When constructing a `RetrievalResult` directly, callers remain +responsible for supplying a truthful `truncated` value because the value +contract cannot reconstruct discarded candidates. ## Collaborator ports @@ -302,12 +358,50 @@ Projection enforces the embedder result rules and requires each writer or resetter command to return exactly `None`. It cannot enforce external storage, atomicity, authorization, concurrency, or lifecycle behavior. +Retrieval checks its embedder identity and output, provider tuple types and +candidate bounds, fragment integrity, corpus and published revisions, and +cross-provider identity consistency. The caller still owns provider selection, +authorization, persistence, concurrency, retries, and authoritative source +validation. + `Borrowed[T]` is the companion ownership marker, not a provider port. Its exact behavior is documented in [resource lifecycle](lifecycle.md). -## Planned retrieval workflow +## Retrieval workflows + +Both public functions are synchronous and all parameters are positional-only: + +```python +def retrieve_semantic( + query: RetrievalQuery, + state: ProjectionStateSnapshot, + limits: RetrievalLimits, + embedder: Borrowed[Embedder], + vector_reader: Borrowed[VectorIndexReader], + /, +) -> RetrievalResult: ... + +def retrieve_hybrid( + query: RetrievalQuery, + state: ProjectionStateSnapshot, + limits: RetrievalLimits, + embedder: Borrowed[Embedder], + vector_reader: Borrowed[VectorIndexReader], + lexical_retriever: Borrowed[LexicalRetriever], + /, +) -> RetrievalResult: ... +``` -Retrieval and composition remain planned for Issue #4. The existing query, -result, reader, and lexical contracts do not promise an implemented workflow, -fusion algorithm, compatibility check, exception mapping, citation policy, or -user/agent integration. +Semantic retrieval embeds the query once, validates at most +`candidate_limit` vector candidates, preserves provider order through +current-revision filtering, and returns at most `hit_limit` hits. Hybrid +retrieval also obtains at most `candidate_limit` lexical candidates, preserves +each provider's original ranks, and fuses exact identities using deterministic +reciprocal rank fusion with offset 60. It uses provider ranks rather than raw +scores and applies opaque identity ordering to ties. + +The workflows return `complete`, `partial`, `unavailable`, `stale`, or `failed` +according to published-state and collaborator results. They do not authorize, +cite, persist, log, retry, or manage collaborator resources. See the +[retrieval guide](retrieval.md) for candidate validation, exact outcome +handling, deterministic fusion, and the required user and agent host flow. diff --git a/docs/lifecycle.md b/docs/lifecycle.md index bac169f..167d06a 100644 --- a/docs/lifecycle.md +++ b/docs/lifecycle.md @@ -3,15 +3,16 @@ Version 0.1.0 uses an explicit caller-owned lifecycle. The package defines collaborator protocols and `Borrowed[T]`; it does not acquire, configure, discover, persist, synchronize, or release provider resources. Projection calls -borrowed collaborators only during an explicit workflow invocation. See the -[API reference](api.md) for exact port signatures. +and retrieval calls use borrowed collaborators only during an explicit +workflow invocation. See the [API reference](api.md) for exact port signatures. ## Ownership rule The caller or provider integration owns every lifecycle decision: -1. Acquire and configure the embedder, vector index, projection-state store, - and any required synchronization. +1. Select, acquire, and configure the embedder, vector index, lexical + retriever, projection-state store, credentials, network clients, and any + required synchronization. 2. Authorize the complete source set and construct a bounded `ProjectionRequest`. 3. Wrap the application-owned collaborators in `Borrowed` and call @@ -26,6 +27,12 @@ The caller or provider integration owns every lifecycle decision: publication, synchronization, or release steps implicitly. It provides no transaction across collaborators and the manifest store. +For retrieval, the caller also loads the manifest paired with the index, +reauthorizes the requester and source set for each query, applies retry and +timeout policy around package calls, resolves returned identities against +authoritative source revisions, creates citations, and decides whether cited +results may be shown to a user or supplied to an agent. + ## `Borrowed[T]` `Borrowed` is a frozen, slotted context manager that retains a reference. It @@ -40,9 +47,11 @@ has exactly these context semantics: provider health checks. These guarantees still apply if the wrapped provider defines its own context -manager or lifecycle methods. Projection itself translates ordinary method -failures into `ProjectionOperationError`; that workflow behavior does not -change `Borrowed` semantics. +manager or lifecycle methods. Projection translates ordinary method failures +into `ProjectionOperationError`; retrieval represents an ordinary method +failure as a content-free failed branch and may return independent validated +hits from another branch. Those workflow behaviors do not change `Borrowed` +semantics. ## Successful borrowed scope @@ -106,11 +115,41 @@ published manifest cannot be used safely by the next incremental operation. See the [projection guide](projection.md) for the state matrix and destructive rebuild ordering. -## Retrieval lifecycle is not implemented - -The query and reader contracts do not create a package retrieval workflow. -Retrieval and composition remain planned for Issue #4, and no user or agent -query lifecycle is implied by the current projection API. +## Retrieval call scope + +`retrieve_semantic` and `retrieve_hybrid` receive exact `Borrowed` wrappers and +a caller-loaded `ProjectionStateSnapshot`. As with projection, they enter only +the no-op wrappers and never enter, close, or shut down the underlying provider +objects. Keep every provider alive for the complete synchronous call. + +Semantic retrieval reads the embedder identity, embeds one query, and then +searches the vector reader. Hybrid retrieval completes that semantic branch +and then calls the lexical retriever; an ordinary semantic failure does not +prevent the lexical call. The package does not retry. Ordinary provider +exceptions and malformed returns produce a failed branch: the overall result +is `failed` without hits or `partial` when the other branch supplies validated +hits. An identity or revision mismatch similarly produces `stale` without hits +or `partial` alongside another branch's hits. `BaseException` subclasses +propagate through the no-op borrowed scope. + +The host owns the complete operating lifecycle around those calls: + +- persist and publish each manifest with the provider index it describes; +- prevent projection, retrieval, rebuild, disable, and purge operations from + observing incompatible manifest/index combinations; +- select and acquire providers, credentials, and network resources and apply + provider-specific retry, timeout, and fallback policy; +- reauthorize each query and authoritative source, then revalidate returned + fragment identities and create citations; +- interpret `complete`, `partial`, `unavailable`, `stale`, and `failed` under + application policy and record any required audit events; and +- coordinate in-flight calls before disabling, rebuilding, purging, or shutting + down caller-owned state and resources. + +The package provides no daemon, background worker, registry, persistence, +dynamic loading, network discovery, lock, transaction, purge command, or +shutdown hook. See the [retrieval guide](retrieval.md) for the exact query flow +and outcome meanings. Review the [security and privacy boundary](security-and-privacy.md) before passing content to any adapter implementation. diff --git a/docs/projection.md b/docs/projection.md index e407b6a..933721b 100644 --- a/docs/projection.md +++ b/docs/projection.md @@ -5,9 +5,11 @@ deterministic vector projection. It supplies orchestration, contracts, and failure reporting; the caller supplies and owns the embedder, vector index, projection-state persistence, authorization policy, and synchronization. -Projection does not make content retrievable through this package. Retrieval, -result composition, and user or agent integration remain planned for [Issue -#4](https://github.com/Kims-DeveloperGroup/generic-rag/issues/4). +Projection establishes the published manifest and derived vector index that +retrieval validates. The host must publish and load those as one matching +logical state, then reauthorize queries and authoritative sources. See the +[retrieval guide](retrieval.md) for semantic and hybrid use; neither workflow +implements authentication, authorization, citations, or user/agent policy. ## Required adapters and state @@ -221,6 +223,14 @@ future package version will retain the same algorithm or accept an old manifest. Consumers that persist projection state should pin and review the package version and use explicit rebuild for an incompatible upgrade. +The positional-only public helpers `derive_source_digest`, +`derive_fragment_id`, `derive_projection_checkpoint_token`, and +`has_valid_projection_checkpoint` live in +`generic_rag.projection_integrity`. Projection and retrieval call the same +helpers; hosts may use them to construct or validate published state. They +validate contract structure but do not inspect provider storage. See the [API +reference](api.md) for their exact signatures. + Hashes are deterministic comparison and identity values, not encryption, authorization, or a proof of source ownership. See [security and privacy](security-and-privacy.md). @@ -253,5 +263,6 @@ The public error messages do not include document or vector content. Adapter exception messages remain reachable through exception chaining, so adapters and application logging must avoid disclosing sensitive values. -See the [API reference](api.md) for exact signatures and value invariants and -[resource lifecycle](lifecycle.md) for ownership details. +See the [API reference](api.md) for exact signatures and value invariants, +[resource lifecycle](lifecycle.md) for ownership details, and the [retrieval +guide](retrieval.md) for consuming a successfully published projection. diff --git a/docs/retrieval.md b/docs/retrieval.md new file mode 100644 index 0000000..3e25f21 --- /dev/null +++ b/docs/retrieval.md @@ -0,0 +1,324 @@ +# Retrieval + +`generic-rag` provides synchronous semantic and hybrid retrieval over a +host-published projection. The package composes caller-supplied collaborators; +it does not select, load, configure, retry, or close a provider. + +The package root intentionally exports nothing. Import retrieval contracts from +`generic_rag.contracts`, borrowed collaborator protocols from +`generic_rag.ports`, integrity helpers from +`generic_rag.projection_integrity`, and workflows from +`generic_rag.retrieval`. + +## Independent consumer example + +This complete example constructs a valid published state, injects local fake +providers, invokes both workflows, and resolves returned fragment identities +against a still-authorized authoritative source before creating citations. + +```python +from generic_rag.contracts import ( + ChunkingPolicy, + Document, + DocumentIdentity, + DocumentKey, + EmbeddingIdentity, + EmbeddingVector, + Fragment, + FragmentIdentity, + ProjectionCheckpoint, + ProjectionIdentity, + ProjectionManifest, + ProjectionManifestEntry, + ProjectionStateAvailability, + ProjectionStateSnapshot, + RetrievalLimits, + RetrievalOutcome, + RetrievalQuery, +) +from generic_rag.ports import Borrowed +from generic_rag.projection_integrity import ( + derive_fragment_id, + derive_projection_checkpoint_token, + derive_source_digest, +) +from generic_rag.retrieval import retrieve_hybrid, retrieve_semantic + +corpus_id = "corpus-a" +source_document = Document( + identity=DocumentIdentity( + key=DocumentKey(corpus_id=corpus_id, document_id="document-a"), + revision_id="revision-1", + ), + text="Approved source text.", + attributes=(("classification", "public"),), +) +chunking = ChunkingPolicy(max_fragment_codepoints=128, overlap_codepoints=0) +embedding_identity = EmbeddingIdentity(model_id="example-embedding", dimensions=2) +projection_identity = ProjectionIdentity( + schema_id="schema-v1", + embedding=embedding_identity, +) +entries = ( + ProjectionManifestEntry( + document=source_document.identity, + source_digest=derive_source_digest(source_document), + fragment_count=1, + ), +) +checkpoint_token = derive_projection_checkpoint_token( + corpus_id, + projection_identity, + chunking, + entries, +) +published_manifest = ProjectionManifest( + corpus_id=corpus_id, + projection=projection_identity, + chunking=chunking, + entries=entries, + checkpoint=ProjectionCheckpoint( + corpus_id=corpus_id, + projection=projection_identity, + token=checkpoint_token, + ), +) +published_state = ProjectionStateSnapshot( + availability=ProjectionStateAvailability.PRESENT, + manifest=published_manifest, +) + +start, end = 0, len(source_document.text) +fragment = Fragment( + identity=FragmentIdentity( + document=source_document.identity, + fragment_id=derive_fragment_id(source_document.identity, start, end), + start=start, + end=end, + ), + text=source_document.text[start:end], + attributes=source_document.attributes, +) + + +class ExampleEmbedder: + @property + def identity(self): + return embedding_identity + + def embed(self, texts, /): + assert texts == ("Where is the approved source?",) + return (EmbeddingVector(values=(1.0, 0.0)),) + + +class ExampleVectorReader: + def search(self, query, embedding, /): + assert embedding == EmbeddingVector(values=(1.0, 0.0)) + return (fragment,) + + +class ExampleLexicalRetriever: + def search(self, query, /): + return (fragment,) + + +query = RetrievalQuery( + corpus_id=corpus_id, + text="Where is the approved source?", + hit_limit=2, + candidate_limit=4, +) +limits = RetrievalLimits(max_query_codepoints=200) + +# The host reauthorizes the corpus and documents for this query before calling +# retrieval. The published state must describe the projection being searched. +authorized_revisions = {source_document.identity: source_document} + +semantic_result = retrieve_semantic( + query, + published_state, + limits, + Borrowed(ExampleEmbedder()), + Borrowed(ExampleVectorReader()), +) +hybrid_result = retrieve_hybrid( + query, + published_state, + limits, + Borrowed(ExampleEmbedder()), + Borrowed(ExampleVectorReader()), + Borrowed(ExampleLexicalRetriever()), +) + + +def host_citations(result): + if result.outcome not in ( + RetrievalOutcome.COMPLETE, + RetrievalOutcome.PARTIAL, + ): + return () + + citations = [] + for hit in result.hits: + identity = hit.fragment.identity + authoritative = authorized_revisions.get(identity.document) + if authoritative is None: + continue + authoritative_text = authoritative.text[identity.start : identity.end] + if authoritative_text != hit.fragment.text: + continue + citations.append( + ( + identity.document.key.document_id, + identity.document.revision_id, + identity.start, + identity.end, + ) + ) + return tuple(citations) + + +assert semantic_result.outcome is RetrievalOutcome.COMPLETE +assert hybrid_result.outcome is RetrievalOutcome.COMPLETE +assert host_citations(semantic_result) == (("document-a", "revision-1", 0, 21),) +assert host_citations(hybrid_result) == (("document-a", "revision-1", 0, 21),) +``` + +`authorized_revisions` represents a host authorization decision made for this +query; it is not package state. A real host also verifies that the requesting +user or agent may use the corpus before invoking retrieval. + +## Inputs and limits + +Both workflows require: + +- a `RetrievalQuery` with a nonblank corpus and query, a positive `hit_limit`, + and a positive `candidate_limit` where `hit_limit <= candidate_limit`; +- the matching `ProjectionStateSnapshot` loaded by the host; +- a `RetrievalLimits` whose positive `max_query_codepoints` bounds the query + before any collaborator call; and +- caller-owned collaborators wrapped in `Borrowed`. + +The limits are caller-selected work budgets, not tenant quotas, rate limits, +authorization rules, or guarantees about a provider's resource use. Each +provider may return at most `candidate_limit` candidates. The package validates +that bound but does not configure the provider or restrict work hidden behind +its interface. + +A missing state produces `unavailable`. A corrupt state or invalid checkpoint +produces `failed`. A state for another corpus produces `unavailable`. A valid +published state with no entries produces an empty `complete` result without +calling a provider. + +Every provider return must be an exact tuple with no more than +`candidate_limit` entries. Each entry must be an exact `Fragment` for the query +corpus, stay within the manifest's fragment-width bound, and carry the +deterministic fragment ID for its document identity and range. Retrieval looks +up each stable `DocumentKey` in the manifest and keeps it only when the full +published `DocumentIdentity`, including revision, matches. Missing and old +revisions are filtered as stale. These checks still cannot prove that candidate +text equals its claimed authoritative source slice. + +## Semantic retrieval + +`retrieve_semantic` performs these steps in order: + +1. Validate the top-level query, state, limits, and borrowed handles without + collaborator effects. +2. Check that the borrowed embedder identity exactly matches the manifest's + embedding identity. A mismatch produces `stale` without embedding or vector + search. +3. Embed the exact one-element tuple `(query.text,)` and require exactly one + finite vector with the published dimensions. +4. Ask the vector reader for up to `candidate_limit` fragments. +5. Validate, deduplicate, and filter the candidates against current manifest + revisions, then return at most `hit_limit` hits in provider order. + +Provider ranks used during validation and fusion are the candidates' original +one-based positions in the provider tuple. Filtering or exact-identity +deduplication does not close gaps in those source ranks. Semantic retrieval +preserves the remaining provider order, while public `RetrievalHit.rank` values +are final contiguous ranks from one. The first identical occurrence wins. +Conflicting payloads for one exact fragment identity fail that provider branch. + +## Hybrid retrieval + +`retrieve_hybrid` executes the semantic branch and then the lexical branch. +Semantic failure or staleness does not prevent the lexical call. Each provider +returns at most `candidate_limit` candidates, and its original ranks are +preserved. + +Validated current-revision candidates are fused by reciprocal rank fusion: + +```text +fused score = sum(1 / (60 + provider rank)) +``` + +Raw provider scores are neither accepted nor returned. Exact fragment +identities are deduplicated across providers. Identical payloads contribute +both ranks; conflicting payloads produce a content-free `failed` result. Hits +sort by descending fused score, then by this opaque identity tuple: + +```text +(corpus_id, document_id, revision_id, fragment_id, start, end) +``` + +That final comparison is deterministic raw string/integer ordering, not text +normalization or semantic relevance. `truncated` is true only when the number +of validated, unique, current-revision fragments exceeds `hit_limit`. + +## Outcomes + +| Outcome | Meaning and host response | +| --- | --- | +| `complete` | All required branches completed. Hits may be empty. Resolve every returned identity against current authorized source data before use. | +| `partial` | At least one validated hit is available, but a branch was stale or failed. Revalidate and cite usable hits; apply host policy before showing or injecting them. | +| `unavailable` | State is missing or the loaded manifest belongs to another corpus. Do not treat this as an empty authoritative answer. | +| `stale` | No hits are usable and a provider identity or candidate revision is stale. Reconcile or rebuild the projection. | +| `failed` | No hits are usable because state, collaborator output, or an ordinary collaborator call failed. Apply host retry, fallback, and audit policy. | + +If stale and failed conditions coexist without hits, `failed` takes precedence. +Ordinary `Exception` failures at collaborator boundaries contribute only a +content-free failure state: the failed branch contributes no fragment or +exception text. A terminal `failed` result is empty; a `partial` result contains +only independently validated hits from another branch. `BaseException` +subclasses such as cancellation signals propagate and must be handled by the +host runtime. + +## User and agent utilization + +The required host flow is: + +1. Authorize documents before projection and publish the resulting manifest + and provider index under host-controlled persistence and concurrency. +2. Reauthorize the user or agent, corpus, and documents for every query. +3. Load the published manifest that matches the provider index being searched. +4. Invoke semantic or hybrid retrieval with caller-owned collaborators. +5. Resolve every returned `FragmentIdentity` against the still-authorized, + authoritative source revision. Verify the exact source slice still equals + the returned text, then create a host citation. +6. Show cited results to the authorized user, or inject only bounded cited + context into an agent. Agents must retain the host citations in any derived + answer or artifact. + +Fragment text is derived, non-authoritative data. A collaborator candidate can +prove neither that its text equals the authoritative source slice nor that the +requester remains authorized. Matching deterministic identities and the +published revision is necessary but insufficient; host revalidation is +mandatory. + +The package never authenticates or authorizes users, creates citations, logs +queries or results, loads providers, opens network connections, owns credentials +or provider resources, or decides prompt, tool, display, retry, or fallback +policy. See [Lifecycle and ownership](lifecycle.md) and +[Security and privacy](security-and-privacy.md) for host responsibilities. + +## Operational ownership + +The host owns provider selection and acquisition, network and credential +handling, retry and timeout policy, manifest and index persistence, concurrency +control, audit, disable/rebuild/purge procedures, and orderly shutdown. +`Borrowed` is only a no-op ownership marker: retrieval neither enters nor exits +the underlying collaborator. See [Projection](projection.md) for how to create +the published state and [API reference](api.md) for exact imports and +signatures. diff --git a/docs/security-and-privacy.md b/docs/security-and-privacy.md index 91abad7..179e188 100644 --- a/docs/security-and-privacy.md +++ b/docs/security-and-privacy.md @@ -1,11 +1,12 @@ # Security and Privacy -Version 0.1.0 defines in-process values, collaborator boundaries, and an -explicit projection workflow. The package itself performs no persistence, -network setup, provider discovery, credential loading, telemetry, or background -work. A projection call does pass derived fragment text and metadata to the -caller-supplied embedder and vector writer, whose effects are outside the -package. +Version 0.1.0 defines in-process values, collaborator boundaries, and explicit +projection and retrieval workflows. The package itself performs no +persistence, network setup, provider discovery, credential loading, telemetry, +or background work. Projection passes derived fragment text and metadata to +caller-supplied providers. Retrieval passes query data to caller-supplied +providers and accepts fragment candidates from them. Those collaborator +effects are outside the package. ## Caller responsibility @@ -14,19 +15,26 @@ source, revision, attribute, and intended destination. The package does not authenticate an authoritative source or decide whether a user, tool, or agent may project it. +Before every retrieval, the caller must reauthorize the requester, corpus, and +source revisions, then load the published manifest paired with the index being +searched. After retrieval, it must resolve each fragment identity against +still-authorized authoritative data, verify the exact source slice, and create +a citation before display or agent use. An agent must retain those host +citations in derived output. + The caller also controls: - corpus and tenant isolation; - which adapter implementations receive document text, fragment text, - embeddings, attributes, identifiers, or future queries; + embeddings, attributes, identifiers, or queries; - provider account, region, transport, and credential configuration; - vector-index and manifest-store access control, retention, replacement, deletion, backup, and recovery; - synchronization between vector mutations and manifest publication; - logging, tracing, metrics, exception rendering, redaction, and incident response; and -- whether future retrieved fragments are displayed, persisted, or supplied to - another tool or agent. +- whether retrieved fragments are displayed, persisted, or supplied to another + tool or agent, and the prompt/tool policy applied to them. Do not place credentials or other secrets in attributes, opaque identifiers, or checkpoint tokens. These fields preserve caller input and do not apply @@ -40,6 +48,13 @@ requested corpus. Those adapters may persist or transmit data according to their implementations. Review their transport, storage, subprocess, network, credential, and deletion behavior before use. +Calling `retrieve_semantic` passes the exact query text to the embedder, then +passes the query and derived query embedding to the vector reader. Calling +`retrieve_hybrid` also passes the query to the lexical retriever. A provider may +transmit, retain, correlate, or log those values according to its +implementation. The package has no hidden network or runtime dependency, but +injected collaborators can have both. + `Borrowed` only marks resources as caller-owned. It does not acquire, close, authenticate, synchronize, sandbox, or reduce the privileges of an adapter. The package supplies no transaction or rollback across the vector index and @@ -49,18 +64,24 @@ caller-owned manifest store. See [resource lifecycle](lifecycle.md). Treat all of the following as potentially sensitive: -- document and future query text; +- document and query text; - ordered attributes and opaque identities; - fragments and their source ranges; - embeddings and vector records; -- source digests, fragment IDs, manifests, and checkpoint tokens; and -- adapter exceptions and logs. +- source digests, fragment IDs, manifests, and checkpoint tokens; +- retrieval ranks and outcomes; and +- adapter exceptions and host logs. Fragments and embeddings may reveal source information. Deterministic IDs and digests may allow equality correlation or guessing attacks against predictable content. Their `sha256:` representation provides neither encryption nor access control and should not be used as proof of source ownership. +Retrieval results expose no raw provider score, but their ranks, identities, +fragment text, truncation flag, and outcome can still reveal content, +correlation, or availability information. Do not treat the score-free boundary +as anonymization. + Deleting an authoritative source does not automatically delete copies, backups, logs, embeddings, or derived records held by an application or provider. Incremental deletion and corpus reset cover only the behavior promised @@ -75,6 +96,12 @@ accepted request from exceeding caller-selected values; they are not global quotas, rate limits, memory isolation, provider billing controls, timeouts, or admission control. +`RetrievalLimits.max_query_codepoints` bounds query text before collaborator +effects. `candidate_limit` bounds the tuple accepted from each provider, and +`hit_limit` bounds returned hits. These are likewise caller-selected work +budgets, not quotas, authorization, cost controls, or proof that a provider did +only bounded internal work. + Choose limits from trusted application policy rather than untrusted request parameters. Account for the fact that a small fragment size and large permitted document set can still produce many fragments and provider operations. Supply @@ -95,6 +122,18 @@ document mutations in place, and a rebuild failure can occur after the corpus was reset. Do not publish a failed or partial receipt as a completed checkpoint; isolate the affected projection and recover under caller-owned policy. +Retrieval maps ordinary collaborator exceptions and malformed collaborator +returns to a failed provider branch, producing `failed` when no validated hit +remains or `partial` when another branch supplies one. A `failed` result is +content-free, and a `partial` result contains only independently validated +hits; provider exception text and rejected candidates do not cross the result +boundary. Stale provider identity or revision data is reported separately as +`stale`, or as `partial` when another branch supplies a validated hit. The host +decides whether to retry, fall back, audit, or suppress a result. +`BaseException` subclasses propagate unchanged, so the host runtime must handle +its own cancellation and shutdown signals. The package does not log queries, +fragments, provider failures, or outcomes. + ## Trust boundary A fragment range is a half-open Python code-point range. Standalone contracts @@ -103,12 +142,19 @@ Projection derives its own fragment text from the supplied document, but the package still cannot prove that the supplied document or revision was authoritative or authorized. +Retrieval candidates are collaborator-supplied derived data. Deterministic +fragment identity and a current manifest revision do not prove that candidate +text equals the authoritative source slice or that the requester remains +authorized. Host reauthorization, source resolution, exact-slice validation, +and citation creation are mandatory before use. + Version 0.1.0 provides no built-in encryption, authentication, authorization, ACL, content filter, persistence security, network security, citation -validation, secret management, vendor guarantee, retrieval workflow, or +creation or validation, secret management, vendor guarantee, logging, or user/agent policy. A consuming application must select and assess those controls for its environment. The complete public value and collaborator boundaries are listed in the [API reference](api.md), and deterministic projection behavior is documented in the -[projection guide](projection.md). +[projection guide](projection.md). Retrieval behavior and the required host +flow are documented in the [retrieval guide](retrieval.md). diff --git a/src/generic_rag/contracts.py b/src/generic_rag/contracts.py index 978cbc5..077349f 100644 --- a/src/generic_rag/contracts.py +++ b/src/generic_rag/contracts.py @@ -31,6 +31,7 @@ "ProjectionStateSnapshot", "ProjectionStateStatus", "ProjectionResult", + "RetrievalLimits", "RetrievalQuery", "RetrievalOutcome", "RetrievalHit", @@ -555,6 +556,16 @@ def __post_init__(self) -> None: ) +@dataclass(frozen=True, slots=True) +class RetrievalLimits: + """Independent bound on caller-supplied retrieval query text.""" + + max_query_codepoints: int + + def __post_init__(self) -> None: + _require_positive_integer("max_query_codepoints", self.max_query_codepoints) + + @dataclass(frozen=True, slots=True) class RetrievalQuery: """A bounded retrieval request for one opaque corpus.""" diff --git a/src/generic_rag/projection.py b/src/generic_rag/projection.py index a8ac690..019d79c 100644 --- a/src/generic_rag/projection.py +++ b/src/generic_rag/projection.py @@ -2,7 +2,6 @@ from __future__ import annotations -import hashlib from collections.abc import Callable from dataclasses import dataclass from enum import StrEnum @@ -38,6 +37,12 @@ StateCompatibilityError, ) from .ports import Borrowed, Embedder, VectorIndexResetter, VectorIndexWriter +from .projection_integrity import ( + derive_fragment_id, + derive_projection_checkpoint_token, + derive_source_digest, + has_valid_projection_checkpoint, +) __all__ = ( "ProjectionFailureStage", @@ -47,10 +52,6 @@ "rebuild_projection", ) -_SOURCE_DIGEST_DOMAIN = "generic-rag:projection-source:v1" -_FRAGMENT_ID_DOMAIN = "generic-rag:fragment-id:v1" -_CHECKPOINT_DOMAIN = "generic-rag:projection-checkpoint:v1" - class ProjectionFailureStage(StrEnum): """The collaborator stage at which a projection operation failed.""" @@ -264,85 +265,6 @@ def _validate_borrowed(name: str, value: object) -> None: _require_exact_type(name, value, Borrowed) -def _sha256_fields(fields: tuple[str, ...]) -> str: - digest = hashlib.sha256() - for field in fields: - encoded = field.encode("utf-8", "surrogatepass") - digest.update(len(encoded).to_bytes(8, "big", signed=False)) - digest.update(encoded) - return f"sha256:{digest.hexdigest()}" - - -def _source_digest(document: Document) -> str: - fields = [ - _SOURCE_DIGEST_DOMAIN, - "text", - document.text, - "attributes_count", - str(len(document.attributes)), - ] - for key, value in document.attributes: - fields.extend(("attribute_key", key, "attribute_value", value)) - return _sha256_fields(tuple(fields)) - - -def _fragment_id(document: DocumentIdentity, start: int, end: int) -> str: - return _sha256_fields( - ( - _FRAGMENT_ID_DOMAIN, - "corpus_id", - document.key.corpus_id, - "document_id", - document.key.document_id, - "revision_id", - document.revision_id, - "start", - str(start), - "end", - str(end), - ) - ) - - -def _checkpoint_token( - corpus_id: str, - projection: ProjectionIdentity, - chunking: ChunkingPolicy, - entries: tuple[ProjectionManifestEntry, ...], -) -> str: - fields = [ - _CHECKPOINT_DOMAIN, - "corpus_id", - corpus_id, - "schema_id", - projection.schema_id, - "embedding_model_id", - projection.embedding.model_id, - "embedding_dimensions", - str(projection.embedding.dimensions), - "max_fragment_codepoints", - str(chunking.max_fragment_codepoints), - "overlap_codepoints", - str(chunking.overlap_codepoints), - "entry_count", - str(len(entries)), - ] - for entry in entries: - fields.extend( - ( - "document_id", - entry.document.key.document_id, - "revision_id", - entry.document.revision_id, - "source_digest", - entry.source_digest, - "fragment_count", - str(entry.fragment_count), - ) - ) - return _sha256_fields(tuple(fields)) - - def _fragments(document: Document, chunking: ChunkingPolicy) -> tuple[Fragment, ...]: fragments: list[Fragment] = [] start = 0 @@ -351,7 +273,7 @@ def _fragments(document: Document, chunking: ChunkingPolicy) -> tuple[Fragment, end = min(start + chunking.max_fragment_codepoints, text_length) identity = FragmentIdentity( document.identity, - _fragment_id(document.identity, start, end), + derive_fragment_id(document.identity, start, end), start, end, ) @@ -374,7 +296,7 @@ def _prepare_target(request: ProjectionRequest) -> _PreparedTarget: fragments = _fragments(document, request.chunking) entry = ProjectionManifestEntry( document.identity, - _source_digest(document), + derive_source_digest(document), len(fragments), ) prepared.append(_PreparedDocument(document, fragments, entry)) @@ -382,7 +304,7 @@ def _prepare_target(request: ProjectionRequest) -> _PreparedTarget: checkpoint = ProjectionCheckpoint( request.corpus_id, request.projection, - _checkpoint_token( + derive_projection_checkpoint_token( request.corpus_id, request.projection, request.chunking, @@ -399,16 +321,6 @@ def _prepare_target(request: ProjectionRequest) -> _PreparedTarget: return _PreparedTarget(request, tuple(prepared), manifest) -def _has_valid_checkpoint(manifest: ProjectionManifest) -> bool: - expected = _checkpoint_token( - manifest.corpus_id, - manifest.projection, - manifest.chunking, - manifest.entries, - ) - return manifest.checkpoint.token == expected - - def _state_status( state: ProjectionStateSnapshot, target: _PreparedTarget, @@ -420,7 +332,7 @@ def _state_status( manifest = state.manifest assert manifest is not None - if not _has_valid_checkpoint(manifest): + if not has_valid_projection_checkpoint(manifest): return ProjectionStateStatus.CORRUPT if manifest.corpus_id != target.request.corpus_id: return ProjectionStateStatus.CORRUPT diff --git a/src/generic_rag/projection_integrity.py b/src/generic_rag/projection_integrity.py new file mode 100644 index 0000000..f03d30b --- /dev/null +++ b/src/generic_rag/projection_integrity.py @@ -0,0 +1,267 @@ +"""Deterministic integrity values shared by projection and retrieval.""" + +from __future__ import annotations + +import hashlib + +from .contracts import ( + ChunkingPolicy, + Document, + DocumentIdentity, + DocumentKey, + EmbeddingIdentity, + ProjectionCheckpoint, + ProjectionIdentity, + ProjectionManifest, + ProjectionManifestEntry, +) +from .errors import ContractValidationError + +__all__ = ( + "derive_source_digest", + "derive_fragment_id", + "derive_projection_checkpoint_token", + "has_valid_projection_checkpoint", +) + +_SOURCE_DIGEST_DOMAIN = "generic-rag:projection-source:v1" +_FRAGMENT_ID_DOMAIN = "generic-rag:fragment-id:v1" +_CHECKPOINT_DOMAIN = "generic-rag:projection-checkpoint:v1" + + +def _require_exact_type(name: str, value: object, expected: type[object]) -> None: + if type(value) is not expected: + raise ContractValidationError( + f"{name} must be exactly {expected.__name__}, not {type(value).__name__}" + ) + + +def _require_nonblank_string(name: str, value: object) -> str: + _require_exact_type(name, value, str) + assert isinstance(value, str) + if not value or value.isspace(): + raise ContractValidationError(f"{name} must not be empty or whitespace-only") + return value + + +def _require_nonnegative_integer(name: str, value: object) -> int: + _require_exact_type(name, value, int) + assert isinstance(value, int) + if value < 0: + raise ContractValidationError(f"{name} must be nonnegative") + return value + + +def _validate_document_key(value: object) -> DocumentKey: + _require_exact_type("document key", value, DocumentKey) + assert isinstance(value, DocumentKey) + return DocumentKey(value.corpus_id, value.document_id) + + +def _validate_document_identity(value: object) -> DocumentIdentity: + _require_exact_type("document identity", value, DocumentIdentity) + assert isinstance(value, DocumentIdentity) + return DocumentIdentity( + _validate_document_key(value.key), + value.revision_id, + ) + + +def _validate_document(value: object) -> Document: + _require_exact_type("document", value, Document) + assert isinstance(value, Document) + return Document( + _validate_document_identity(value.identity), + value.text, + value.attributes, + ) + + +def _validate_embedding_identity(value: object) -> EmbeddingIdentity: + _require_exact_type("embedding identity", value, EmbeddingIdentity) + assert isinstance(value, EmbeddingIdentity) + return EmbeddingIdentity(value.model_id, value.dimensions) + + +def _validate_projection_identity(value: object) -> ProjectionIdentity: + _require_exact_type("projection identity", value, ProjectionIdentity) + assert isinstance(value, ProjectionIdentity) + return ProjectionIdentity( + value.schema_id, + _validate_embedding_identity(value.embedding), + ) + + +def _validate_chunking(value: object) -> ChunkingPolicy: + _require_exact_type("chunking", value, ChunkingPolicy) + assert isinstance(value, ChunkingPolicy) + return ChunkingPolicy( + value.max_fragment_codepoints, + value.overlap_codepoints, + ) + + +def _validate_manifest_entry(value: object) -> ProjectionManifestEntry: + _require_exact_type("manifest entry", value, ProjectionManifestEntry) + assert isinstance(value, ProjectionManifestEntry) + return ProjectionManifestEntry( + _validate_document_identity(value.document), + value.source_digest, + value.fragment_count, + ) + + +def _validate_checkpoint(value: object) -> ProjectionCheckpoint: + _require_exact_type("checkpoint", value, ProjectionCheckpoint) + assert isinstance(value, ProjectionCheckpoint) + return ProjectionCheckpoint( + value.corpus_id, + _validate_projection_identity(value.projection), + value.token, + ) + + +def _validate_manifest(value: object) -> ProjectionManifest: + _require_exact_type("manifest", value, ProjectionManifest) + assert isinstance(value, ProjectionManifest) + _require_exact_type("manifest entries", value.entries, tuple) + entries = tuple(_validate_manifest_entry(entry) for entry in value.entries) + canonical = ProjectionManifest( + value.corpus_id, + _validate_projection_identity(value.projection), + _validate_chunking(value.chunking), + entries, + _validate_checkpoint(value.checkpoint), + ) + if entries != canonical.entries: + raise ContractValidationError("manifest entries must be in canonical order") + return canonical + + +def _sha256_fields(fields: tuple[str, ...]) -> str: + digest = hashlib.sha256() + for field in fields: + encoded = field.encode("utf-8", "surrogatepass") + digest.update(len(encoded).to_bytes(8, "big", signed=False)) + digest.update(encoded) + return f"sha256:{digest.hexdigest()}" + + +def derive_source_digest(document: Document, /) -> str: + """Derive the stable v1 digest of exact source text and attributes.""" + + canonical = _validate_document(document) + fields = [ + _SOURCE_DIGEST_DOMAIN, + "text", + canonical.text, + "attributes_count", + str(len(canonical.attributes)), + ] + for key, value in canonical.attributes: + fields.extend(("attribute_key", key, "attribute_value", value)) + return _sha256_fields(tuple(fields)) + + +def derive_fragment_id( + document: DocumentIdentity, + start: int, + end: int, + /, +) -> str: + """Derive the stable v1 identity for one valid half-open source range.""" + + canonical = _validate_document_identity(document) + canonical_start = _require_nonnegative_integer("start", start) + canonical_end = _require_nonnegative_integer("end", end) + if canonical_start >= canonical_end: + raise ContractValidationError("fragment range must satisfy start < end") + return _sha256_fields( + ( + _FRAGMENT_ID_DOMAIN, + "corpus_id", + canonical.key.corpus_id, + "document_id", + canonical.key.document_id, + "revision_id", + canonical.revision_id, + "start", + str(canonical_start), + "end", + str(canonical_end), + ) + ) + + +def derive_projection_checkpoint_token( + corpus_id: str, + projection: ProjectionIdentity, + chunking: ChunkingPolicy, + entries: tuple[ProjectionManifestEntry, ...], + /, +) -> str: + """Derive the stable v1 token for one canonical projection manifest.""" + + canonical_corpus_id = _require_nonblank_string("corpus_id", corpus_id) + canonical_projection = _validate_projection_identity(projection) + canonical_chunking = _validate_chunking(chunking) + _require_exact_type("entries", entries, tuple) + canonical_entries = tuple(_validate_manifest_entry(entry) for entry in entries) + ordering_witness = ProjectionManifest( + canonical_corpus_id, + canonical_projection, + canonical_chunking, + canonical_entries, + ProjectionCheckpoint( + canonical_corpus_id, + canonical_projection, + "integrity-validation", + ), + ) + if canonical_entries != ordering_witness.entries: + raise ContractValidationError("manifest entries must be in canonical order") + + fields = [ + _CHECKPOINT_DOMAIN, + "corpus_id", + canonical_corpus_id, + "schema_id", + canonical_projection.schema_id, + "embedding_model_id", + canonical_projection.embedding.model_id, + "embedding_dimensions", + str(canonical_projection.embedding.dimensions), + "max_fragment_codepoints", + str(canonical_chunking.max_fragment_codepoints), + "overlap_codepoints", + str(canonical_chunking.overlap_codepoints), + "entry_count", + str(len(canonical_entries)), + ] + for entry in canonical_entries: + fields.extend( + ( + "document_id", + entry.document.key.document_id, + "revision_id", + entry.document.revision_id, + "source_digest", + entry.source_digest, + "fragment_count", + str(entry.fragment_count), + ) + ) + return _sha256_fields(tuple(fields)) + + +def has_valid_projection_checkpoint(manifest: ProjectionManifest, /) -> bool: + """Return whether a structurally valid manifest has its exact v1 token.""" + + canonical = _validate_manifest(manifest) + expected = derive_projection_checkpoint_token( + canonical.corpus_id, + canonical.projection, + canonical.chunking, + canonical.entries, + ) + return canonical.checkpoint.token == expected diff --git a/src/generic_rag/retrieval.py b/src/generic_rag/retrieval.py new file mode 100644 index 0000000..6341631 --- /dev/null +++ b/src/generic_rag/retrieval.py @@ -0,0 +1,517 @@ +"""Bounded semantic retrieval and deterministic rank-based hybrid fusion.""" + +from __future__ import annotations + +from dataclasses import dataclass +from fractions import Fraction +from math import isfinite + +from .contracts import ( + ChunkingPolicy, + DocumentIdentity, + DocumentKey, + EmbeddingIdentity, + EmbeddingVector, + Fragment, + FragmentIdentity, + ProjectionCheckpoint, + ProjectionIdentity, + ProjectionManifest, + ProjectionManifestEntry, + ProjectionStateAvailability, + ProjectionStateSnapshot, + RetrievalHit, + RetrievalLimits, + RetrievalOutcome, + RetrievalQuery, + RetrievalResult, +) +from .errors import ContractValidationError +from .ports import Borrowed, Embedder, LexicalRetriever, VectorIndexReader +from .projection_integrity import ( + derive_fragment_id, + has_valid_projection_checkpoint, +) + +__all__ = ("retrieve_semantic", "retrieve_hybrid") + +_RRF_OFFSET = 60 + + +@dataclass(frozen=True, slots=True) +class _RankedFragment: + fragment: Fragment + source_rank: int + + +@dataclass(frozen=True, slots=True) +class _BranchResult: + all_candidates: tuple[_RankedFragment, ...] + current_candidates: tuple[_RankedFragment, ...] + stale: bool + failed: bool + + +def _require_exact_type(name: str, value: object, expected: type[object]) -> None: + if type(value) is not expected: + raise ContractValidationError( + f"{name} must be exactly {expected.__name__}, not {type(value).__name__}" + ) + + +def _validate_document_key(value: object) -> DocumentKey: + _require_exact_type("document key", value, DocumentKey) + assert isinstance(value, DocumentKey) + return DocumentKey(value.corpus_id, value.document_id) + + +def _validate_document_identity(value: object) -> DocumentIdentity: + _require_exact_type("document identity", value, DocumentIdentity) + assert isinstance(value, DocumentIdentity) + return DocumentIdentity( + _validate_document_key(value.key), + value.revision_id, + ) + + +def _validate_embedding_identity(value: object) -> EmbeddingIdentity: + _require_exact_type("embedding identity", value, EmbeddingIdentity) + assert isinstance(value, EmbeddingIdentity) + return EmbeddingIdentity(value.model_id, value.dimensions) + + +def _validate_projection_identity(value: object) -> ProjectionIdentity: + _require_exact_type("projection identity", value, ProjectionIdentity) + assert isinstance(value, ProjectionIdentity) + return ProjectionIdentity( + value.schema_id, + _validate_embedding_identity(value.embedding), + ) + + +def _validate_chunking(value: object) -> ChunkingPolicy: + _require_exact_type("chunking", value, ChunkingPolicy) + assert isinstance(value, ChunkingPolicy) + return ChunkingPolicy( + value.max_fragment_codepoints, + value.overlap_codepoints, + ) + + +def _validate_manifest_entry(value: object) -> ProjectionManifestEntry: + _require_exact_type("manifest entry", value, ProjectionManifestEntry) + assert isinstance(value, ProjectionManifestEntry) + return ProjectionManifestEntry( + _validate_document_identity(value.document), + value.source_digest, + value.fragment_count, + ) + + +def _validate_checkpoint(value: object) -> ProjectionCheckpoint: + _require_exact_type("checkpoint", value, ProjectionCheckpoint) + assert isinstance(value, ProjectionCheckpoint) + return ProjectionCheckpoint( + value.corpus_id, + _validate_projection_identity(value.projection), + value.token, + ) + + +def _validate_manifest(value: object) -> ProjectionManifest: + _require_exact_type("manifest", value, ProjectionManifest) + assert isinstance(value, ProjectionManifest) + _require_exact_type("manifest entries", value.entries, tuple) + entries = tuple(_validate_manifest_entry(entry) for entry in value.entries) + canonical = ProjectionManifest( + value.corpus_id, + _validate_projection_identity(value.projection), + _validate_chunking(value.chunking), + entries, + _validate_checkpoint(value.checkpoint), + ) + if entries != canonical.entries: + raise ContractValidationError("manifest entries must be in canonical order") + return canonical + + +def _validate_state(value: object) -> ProjectionStateSnapshot: + _require_exact_type("state", value, ProjectionStateSnapshot) + assert isinstance(value, ProjectionStateSnapshot) + _require_exact_type( + "state availability", + value.availability, + ProjectionStateAvailability, + ) + manifest = None if value.manifest is None else _validate_manifest(value.manifest) + return ProjectionStateSnapshot(value.availability, manifest) + + +def _validate_query(value: object) -> RetrievalQuery: + _require_exact_type("query", value, RetrievalQuery) + assert isinstance(value, RetrievalQuery) + return RetrievalQuery( + value.corpus_id, + value.text, + value.hit_limit, + value.candidate_limit, + ) + + +def _validate_limits(value: object) -> RetrievalLimits: + _require_exact_type("limits", value, RetrievalLimits) + assert isinstance(value, RetrievalLimits) + return RetrievalLimits(value.max_query_codepoints) + + +def _validate_borrowed(name: str, value: object) -> None: + _require_exact_type(name, value, Borrowed) + + +def _validate_embedding( + value: object, + expected_dimensions: int, +) -> EmbeddingVector: + _require_exact_type("embedding", value, EmbeddingVector) + assert isinstance(value, EmbeddingVector) + _require_exact_type("embedding values", value.values, tuple) + if len(value.values) != expected_dimensions: + raise ContractValidationError( + "embedding vector must have the requested dimensions" + ) + if any( + type(coordinate) is not float or not isfinite(coordinate) + for coordinate in value.values + ): + raise ContractValidationError( + "embedding vector coordinates must be canonical finite floats" + ) + return EmbeddingVector(value.values) + + +def _validate_fragment( + value: object, + query: RetrievalQuery, + chunking: ChunkingPolicy, +) -> Fragment: + _require_exact_type("candidate", value, Fragment) + assert isinstance(value, Fragment) + identity_value = value.identity + _require_exact_type("candidate identity", identity_value, FragmentIdentity) + assert isinstance(identity_value, FragmentIdentity) + identity = FragmentIdentity( + _validate_document_identity(identity_value.document), + identity_value.fragment_id, + identity_value.start, + identity_value.end, + ) + canonical = Fragment(identity, value.text, value.attributes) + if canonical.identity.document.key.corpus_id != query.corpus_id: + raise ContractValidationError("candidate corpus_id must match the query") + if len(canonical.text) > chunking.max_fragment_codepoints: + raise ContractValidationError( + "candidate width must not exceed the published chunk bound" + ) + if canonical.identity.fragment_id != derive_fragment_id( + canonical.identity.document, + canonical.identity.start, + canonical.identity.end, + ): + raise ContractValidationError( + "candidate fragment_id must match its deterministic identity" + ) + return canonical + + +def _validate_provider_candidates( + value: object, + query: RetrievalQuery, + manifest: ProjectionManifest, +) -> _BranchResult: + if type(value) is not tuple: + raise ContractValidationError("retriever output must be exactly tuple") + assert isinstance(value, tuple) + if len(value) > query.candidate_limit: + raise ContractValidationError( + "retriever output must not exceed query.candidate_limit" + ) + + manifest_by_key = {entry.document.key: entry for entry in manifest.entries} + first_by_identity: dict[FragmentIdentity, _RankedFragment] = {} + ordered: list[_RankedFragment] = [] + for source_rank, candidate in enumerate(value, start=1): + fragment = _validate_fragment(candidate, query, manifest.chunking) + previous = first_by_identity.get(fragment.identity) + if previous is not None: + if previous.fragment != fragment: + raise ContractValidationError( + "one provider returned conflicting candidate payloads" + ) + continue + ranked = _RankedFragment(fragment, source_rank) + first_by_identity[fragment.identity] = ranked + ordered.append(ranked) + + current: list[_RankedFragment] = [] + stale = False + for candidate in ordered: + published = manifest_by_key.get(candidate.fragment.identity.document.key) + if ( + published is None + or candidate.fragment.identity.document != published.document + ): + stale = True + continue + current.append(candidate) + return _BranchResult(tuple(ordered), tuple(current), stale, False) + + +def _failed_branch() -> _BranchResult: + return _BranchResult((), (), False, True) + + +def _stale_branch() -> _BranchResult: + return _BranchResult((), (), True, False) + + +def _semantic_branch( + query: RetrievalQuery, + manifest: ProjectionManifest, + embedder: Borrowed[Embedder], + vector_reader: Borrowed[VectorIndexReader], +) -> _BranchResult: + try: + with embedder as embedder_resource: + identity_value = embedder_resource.identity + identity = _validate_embedding_identity(identity_value) + if identity != manifest.projection.embedding: + return _stale_branch() + embedding_values = embedder_resource.embed((query.text,)) + if type(embedding_values) is not tuple or len(embedding_values) != 1: + raise ContractValidationError( + "embedder output must be an exact one-vector tuple" + ) + embedding = _validate_embedding( + embedding_values[0], + identity.dimensions, + ) + except Exception: + return _failed_branch() + + try: + with vector_reader as reader_resource: + candidates = reader_resource.search(query, embedding) + return _validate_provider_candidates(candidates, query, manifest) + except Exception: + return _failed_branch() + + +def _lexical_branch( + query: RetrievalQuery, + manifest: ProjectionManifest, + lexical_retriever: Borrowed[LexicalRetriever], +) -> _BranchResult: + try: + with lexical_retriever as lexical_resource: + candidates = lexical_resource.search(query) + return _validate_provider_candidates(candidates, query, manifest) + except Exception: + return _failed_branch() + + +def _terminal_result( + query: RetrievalQuery, + outcome: RetrievalOutcome, +) -> RetrievalResult: + return RetrievalResult(query, outcome, (), False) + + +def _state_manifest_or_result( + query: RetrievalQuery, + state: ProjectionStateSnapshot, +) -> ProjectionManifest | RetrievalResult: + if state.availability is ProjectionStateAvailability.MISSING: + return _terminal_result(query, RetrievalOutcome.UNAVAILABLE) + if state.availability is ProjectionStateAvailability.CORRUPT: + return _terminal_result(query, RetrievalOutcome.FAILED) + + manifest = state.manifest + assert manifest is not None + if manifest.corpus_id != query.corpus_id: + return _terminal_result(query, RetrievalOutcome.UNAVAILABLE) + if not has_valid_projection_checkpoint(manifest): + return _terminal_result(query, RetrievalOutcome.FAILED) + if not manifest.entries: + return _terminal_result(query, RetrievalOutcome.COMPLETE) + return manifest + + +def _outcome_for_hits( + has_hits: bool, + *, + stale: bool, + failed: bool, +) -> RetrievalOutcome: + if has_hits: + if stale or failed: + return RetrievalOutcome.PARTIAL + return RetrievalOutcome.COMPLETE + if failed: + return RetrievalOutcome.FAILED + if stale: + return RetrievalOutcome.STALE + return RetrievalOutcome.COMPLETE + + +def _ranked_result( + query: RetrievalQuery, + fragments: tuple[Fragment, ...], + *, + stale: bool, + failed: bool, +) -> RetrievalResult: + truncated = len(fragments) > query.hit_limit + selected = fragments[: query.hit_limit] + hits = tuple( + RetrievalHit(fragment, rank) for rank, fragment in enumerate(selected, start=1) + ) + outcome = _outcome_for_hits(bool(hits), stale=stale, failed=failed) + return RetrievalResult(query, outcome, hits, truncated) + + +def _identity_order(fragment: Fragment) -> tuple[str, str, str, str, int, int]: + identity = fragment.identity + return ( + identity.document.key.corpus_id, + identity.document.key.document_id, + identity.document.revision_id, + identity.fragment_id, + identity.start, + identity.end, + ) + + +def _has_cross_provider_conflict( + semantic: _BranchResult, + lexical: _BranchResult, +) -> bool: + semantic_by_identity = { + candidate.fragment.identity: candidate.fragment + for candidate in semantic.all_candidates + } + for candidate in lexical.all_candidates: + previous = semantic_by_identity.get(candidate.fragment.identity) + if previous is not None and previous != candidate.fragment: + return True + return False + + +def _fuse_candidates( + semantic: _BranchResult, + lexical: _BranchResult, +) -> tuple[Fragment, ...]: + fragments: dict[FragmentIdentity, Fragment] = {} + scores: dict[FragmentIdentity, Fraction] = {} + for branch in (semantic, lexical): + for candidate in branch.current_candidates: + identity = candidate.fragment.identity + fragments.setdefault(identity, candidate.fragment) + scores[identity] = scores.get(identity, Fraction()) + Fraction( + 1, + _RRF_OFFSET + candidate.source_rank, + ) + ranked_identities = sorted( + scores, + key=lambda identity: ( + -scores[identity], + _identity_order(fragments[identity]), + ), + ) + return tuple(fragments[identity] for identity in ranked_identities) + + +def retrieve_semantic( + query: RetrievalQuery, + state: ProjectionStateSnapshot, + limits: RetrievalLimits, + embedder: Borrowed[Embedder], + vector_reader: Borrowed[VectorIndexReader], + /, +) -> RetrievalResult: + """Retrieve bounded current fragments from one borrowed semantic index.""" + + canonical_query = _validate_query(query) + canonical_state = _validate_state(state) + canonical_limits = _validate_limits(limits) + _validate_borrowed("embedder", embedder) + _validate_borrowed("vector_reader", vector_reader) + if len(canonical_query.text) > canonical_limits.max_query_codepoints: + raise ContractValidationError( + "query text must not exceed limits.max_query_codepoints" + ) + + state_result = _state_manifest_or_result(canonical_query, canonical_state) + if type(state_result) is RetrievalResult: + return state_result + assert isinstance(state_result, ProjectionManifest) + branch = _semantic_branch( + canonical_query, + state_result, + embedder, + vector_reader, + ) + fragments = tuple(candidate.fragment for candidate in branch.current_candidates) + return _ranked_result( + canonical_query, + fragments, + stale=branch.stale, + failed=branch.failed, + ) + + +def retrieve_hybrid( + query: RetrievalQuery, + state: ProjectionStateSnapshot, + limits: RetrievalLimits, + embedder: Borrowed[Embedder], + vector_reader: Borrowed[VectorIndexReader], + lexical_retriever: Borrowed[LexicalRetriever], + /, +) -> RetrievalResult: + """Fuse bounded semantic and lexical ranks without comparing raw scores.""" + + canonical_query = _validate_query(query) + canonical_state = _validate_state(state) + canonical_limits = _validate_limits(limits) + _validate_borrowed("embedder", embedder) + _validate_borrowed("vector_reader", vector_reader) + _validate_borrowed("lexical_retriever", lexical_retriever) + if len(canonical_query.text) > canonical_limits.max_query_codepoints: + raise ContractValidationError( + "query text must not exceed limits.max_query_codepoints" + ) + + state_result = _state_manifest_or_result(canonical_query, canonical_state) + if type(state_result) is RetrievalResult: + return state_result + assert isinstance(state_result, ProjectionManifest) + semantic = _semantic_branch( + canonical_query, + state_result, + embedder, + vector_reader, + ) + lexical = _lexical_branch( + canonical_query, + state_result, + lexical_retriever, + ) + if _has_cross_provider_conflict(semantic, lexical): + return _terminal_result(canonical_query, RetrievalOutcome.FAILED) + fragments = _fuse_candidates(semantic, lexical) + return _ranked_result( + canonical_query, + fragments, + stale=semantic.stale or lexical.stale, + failed=semantic.failed or lexical.failed, + ) diff --git a/tests/support/verify_artifacts.py b/tests/support/verify_artifacts.py index ba6ea5f..667753d 100644 --- a/tests/support/verify_artifacts.py +++ b/tests/support/verify_artifacts.py @@ -17,6 +17,8 @@ "generic_rag/errors.py", "generic_rag/ports.py", "generic_rag/projection.py", + "generic_rag/projection_integrity.py", + "generic_rag/retrieval.py", } _PACKAGE_DATA = {"generic_rag/py.typed"} diff --git a/tests/test_package_boundaries.py b/tests/test_package_boundaries.py index a5b4eef..5ac439d 100644 --- a/tests/test_package_boundaries.py +++ b/tests/test_package_boundaries.py @@ -17,6 +17,8 @@ import generic_rag.errors as errors import generic_rag.ports as ports import generic_rag.projection as projection +import generic_rag.projection_integrity as projection_integrity +import generic_rag.retrieval as retrieval _PROJECT_ROOT = Path(__file__).resolve().parents[1] _SOURCE_ROOT = _PROJECT_ROOT / "src" @@ -28,6 +30,8 @@ "generic_rag.errors": "src/generic_rag/errors.py", "generic_rag.ports": "src/generic_rag/ports.py", "generic_rag.projection": "src/generic_rag/projection.py", + "generic_rag.projection_integrity": ("src/generic_rag/projection_integrity.py"), + "generic_rag.retrieval": "src/generic_rag/retrieval.py", } _EXPECTED_DEPENDENCIES = { "generic_rag": set(), @@ -38,6 +42,17 @@ "generic_rag.contracts", "generic_rag.errors", "generic_rag.ports", + "generic_rag.projection_integrity", + }, + "generic_rag.projection_integrity": { + "generic_rag.contracts", + "generic_rag.errors", + }, + "generic_rag.retrieval": { + "generic_rag.contracts", + "generic_rag.errors", + "generic_rag.ports", + "generic_rag.projection_integrity", }, } _EXPECTED_EXPORTS = { @@ -70,6 +85,7 @@ "ProjectionStateSnapshot", "ProjectionStateStatus", "ProjectionResult", + "RetrievalLimits", "RetrievalQuery", "RetrievalOutcome", "RetrievalHit", @@ -90,6 +106,16 @@ "project_documents", "rebuild_projection", ), + "generic_rag.projection_integrity": ( + "derive_source_digest", + "derive_fragment_id", + "derive_projection_checkpoint_token", + "has_valid_projection_checkpoint", + ), + "generic_rag.retrieval": ( + "retrieve_semantic", + "retrieve_hybrid", + ), } @@ -166,6 +192,7 @@ def test_production_module_inventory_is_exact(self) -> None: for path in _production_sources() } + self.assertEqual(len(actual), 7) self.assertEqual(actual, _EXPECTED_SOURCES) def test_supported_exports_are_exact_and_owned(self) -> None: @@ -175,6 +202,8 @@ def test_supported_exports_are_exact_and_owned(self) -> None: "generic_rag.contracts": contracts, "generic_rag.ports": ports, "generic_rag.projection": projection, + "generic_rag.projection_integrity": projection_integrity, + "generic_rag.retrieval": retrieval, } for module_name, expected_exports in _EXPECTED_EXPORTS.items(): @@ -190,6 +219,8 @@ def test_supported_exports_are_exact_and_owned(self) -> None: *_EXPECTED_EXPORTS["generic_rag.contracts"], *_EXPECTED_EXPORTS["generic_rag.ports"], *_EXPECTED_EXPORTS["generic_rag.projection"], + *_EXPECTED_EXPORTS["generic_rag.projection_integrity"], + *_EXPECTED_EXPORTS["generic_rag.retrieval"], ): with self.subTest(root_reexport=name): self.assertFalse(hasattr(generic_rag, name)) @@ -210,6 +241,7 @@ def test_internal_dependency_graph_is_exact_and_acyclic(self) -> None: for module_name, relative_path in _EXPECTED_SOURCES.items() } + self.assertEqual(sum(map(len, actual.values())), 12) self.assertEqual(actual, _EXPECTED_DEPENDENCIES) root_tree = _syntax_tree(_PROJECT_ROOT / _EXPECTED_SOURCES["generic_rag"]) root_imports = [ diff --git a/tests/test_projection_integrity.py b/tests/test_projection_integrity.py new file mode 100644 index 0000000..5e3c67a --- /dev/null +++ b/tests/test_projection_integrity.py @@ -0,0 +1,464 @@ +"""Independent v1 goldens and validation for projection integrity values.""" + +from __future__ import annotations + +import hashlib +import inspect +import unittest +from collections.abc import Callable +from typing import cast + +import generic_rag.projection_integrity as integrity +from generic_rag.contracts import ( + ChunkingPolicy, + Document, + DocumentIdentity, + DocumentKey, + EmbeddingIdentity, + EmbeddingVector, + ProjectionCheckpoint, + ProjectionIdentity, + ProjectionLimits, + ProjectionManifest, + ProjectionManifestEntry, + ProjectionRequest, + ProjectionStateAvailability, + ProjectionStateSnapshot, + VectorRecord, +) +from generic_rag.errors import ContractValidationError +from generic_rag.ports import Borrowed, Embedder, VectorIndexResetter, VectorIndexWriter +from generic_rag.projection import rebuild_projection +from generic_rag.projection_integrity import ( + derive_fragment_id, + derive_projection_checkpoint_token, + derive_source_digest, + has_valid_projection_checkpoint, +) + +_CORPUS_ID = " Corpus/../e\u0301 " +_SOURCE_GOLDEN = ( + "sha256:19390f5d911704efab45a06821296d2309454cfb8bdca63799d1c451afa1a9ea" +) +_FRAGMENT_GOLDEN = ( + "sha256:49d28767699855dbf6778396488946303a62dc8394c46adcdd22bef653950293" +) +_CHECKPOINT_GOLDEN = ( + "sha256:f62fca95057be15ab4e4bab8eebde94b7f92eb87773ae2d82d849664c80ba59f" +) + + +class _TupleSubclass(tuple[object, ...]): + pass + + +def _golden_document() -> Document: + return Document( + DocumentIdentity( + DocumentKey(_CORPUS_ID, "doc/\0😀"), + "rev/한", + ), + "A\0😀e\u0301\n", + (("k", "v"), ("k", "v"), ("", "한")), + ) + + +def _projection() -> ProjectionIdentity: + return ProjectionIdentity( + "schema/😀", + EmbeddingIdentity("model/e\u0301", 3), + ) + + +def _checkpoint_entries() -> tuple[ProjectionManifestEntry, ...]: + return ( + ProjectionManifestEntry( + DocumentIdentity(DocumentKey(_CORPUS_ID, "alpha/e\u0301"), "r1😀"), + _SOURCE_GOLDEN, + 2, + ), + ProjectionManifestEntry( + DocumentIdentity(DocumentKey(_CORPUS_ID, "한"), "r2\0"), + "sha256:" + "b" * 64, + 0, + ), + ) + + +def _manifest(token: str = _CHECKPOINT_GOLDEN) -> ProjectionManifest: + projection = _projection() + return ProjectionManifest( + _CORPUS_ID, + projection, + ChunkingPolicy(4, 1), + _checkpoint_entries(), + ProjectionCheckpoint(_CORPUS_ID, projection, token), + ) + + +def _independent_hash(fields: tuple[str, ...]) -> str: + digest = hashlib.sha256() + for field in fields: + encoded = field.encode("utf-8", "surrogatepass") + digest.update(len(encoded).to_bytes(8, "big", signed=False)) + digest.update(encoded) + return f"sha256:{digest.hexdigest()}" + + +class _ProjectionEmbedder: + identity = _projection().embedding + + def embed(self, texts: tuple[str, ...], /) -> tuple[EmbeddingVector, ...]: + return tuple(EmbeddingVector((float(len(text)), 0.0, 1.0)) for text in texts) + + +class _ProjectionWriter: + def __init__(self) -> None: + self.records: tuple[VectorRecord, ...] = () + + def replace_document( + self, + document: DocumentIdentity, + records: tuple[VectorRecord, ...], + /, + ) -> None: + del document + self.records = records + + def delete_document(self, document: DocumentKey, /) -> None: + del document + + +class _ProjectionResetter: + def reset_corpus(self, corpus_id: str, /) -> None: + del corpus_id + + +class ProjectionIntegrityPublicTests(unittest.TestCase): + def test_exports_are_exact_owned_synchronous_and_positional_only(self) -> None: + expected = ( + "derive_source_digest", + "derive_fragment_id", + "derive_projection_checkpoint_token", + "has_valid_projection_checkpoint", + ) + expected_parameters = { + derive_source_digest: ("document",), + derive_fragment_id: ("document", "start", "end"), + derive_projection_checkpoint_token: ( + "corpus_id", + "projection", + "chunking", + "entries", + ), + has_valid_projection_checkpoint: ("manifest",), + } + + self.assertEqual(integrity.__all__, expected) + for function, parameter_names in expected_parameters.items(): + with self.subTest(function=function.__name__): + self.assertEqual( + function.__module__, "generic_rag.projection_integrity" + ) + self.assertFalse(inspect.iscoroutinefunction(function)) + parameters = tuple( + inspect.signature( + cast(Callable[..., object], function) + ).parameters.values() + ) + self.assertEqual( + tuple(parameter.name for parameter in parameters), + parameter_names, + ) + self.assertTrue( + all( + parameter.kind is inspect.Parameter.POSITIONAL_ONLY + for parameter in parameters + ) + ) + + def test_fixed_unicode_nul_and_duplicate_attribute_v1_goldens(self) -> None: + document = _golden_document() + entries = _checkpoint_entries() + + source_fields = [ + "generic-rag:projection-source:v1", + "text", + document.text, + "attributes_count", + str(len(document.attributes)), + ] + for key, value in document.attributes: + source_fields.extend(("attribute_key", key, "attribute_value", value)) + self.assertEqual(_independent_hash(tuple(source_fields)), _SOURCE_GOLDEN) + self.assertEqual(derive_source_digest(document), _SOURCE_GOLDEN) + + fragment_fields = ( + "generic-rag:fragment-id:v1", + "corpus_id", + document.identity.key.corpus_id, + "document_id", + document.identity.key.document_id, + "revision_id", + document.identity.revision_id, + "start", + "1", + "end", + "5", + ) + self.assertEqual(_independent_hash(fragment_fields), _FRAGMENT_GOLDEN) + self.assertEqual(derive_fragment_id(document.identity, 1, 5), _FRAGMENT_GOLDEN) + + checkpoint_fields = [ + "generic-rag:projection-checkpoint:v1", + "corpus_id", + _CORPUS_ID, + "schema_id", + _projection().schema_id, + "embedding_model_id", + _projection().embedding.model_id, + "embedding_dimensions", + "3", + "max_fragment_codepoints", + "4", + "overlap_codepoints", + "1", + "entry_count", + "2", + ] + for entry in entries: + checkpoint_fields.extend( + ( + "document_id", + entry.document.key.document_id, + "revision_id", + entry.document.revision_id, + "source_digest", + entry.source_digest, + "fragment_count", + str(entry.fragment_count), + ) + ) + self.assertEqual( + _independent_hash(tuple(checkpoint_fields)), _CHECKPOINT_GOLDEN + ) + self.assertEqual( + derive_projection_checkpoint_token( + _CORPUS_ID, + _projection(), + ChunkingPolicy(4, 1), + entries, + ), + _CHECKPOINT_GOLDEN, + ) + + def test_checkpoint_validation_is_exact_and_does_not_mutate_manifest(self) -> None: + manifest = _manifest() + before = repr(manifest) + + self.assertTrue(has_valid_projection_checkpoint(manifest)) + self.assertFalse(has_valid_projection_checkpoint(_manifest("wrong-token"))) + self.assertEqual(repr(manifest), before) + + def test_projection_workflow_preserves_the_extracted_v1_algorithms(self) -> None: + document = _golden_document() + projection = _projection() + chunking = ChunkingPolicy(4, 1) + request = ProjectionRequest( + _CORPUS_ID, + projection, + chunking, + ProjectionLimits(1, len(document.text), 4), + (document,), + ) + writer = _ProjectionWriter() + + result = rebuild_projection( + request, + ProjectionStateSnapshot(ProjectionStateAvailability.MISSING, None), + Borrowed(cast(Embedder, _ProjectionEmbedder())), + Borrowed(cast(VectorIndexWriter, writer)), + Borrowed(cast(VectorIndexResetter, _ProjectionResetter())), + ) + + entry = result.manifest.entries[0] + self.assertEqual(entry.source_digest, derive_source_digest(document)) + self.assertEqual( + tuple(record.fragment.identity.fragment_id for record in writer.records), + tuple( + derive_fragment_id(document.identity, start, end) + for start, end in ((0, 4), (3, 6)) + ), + ) + self.assertEqual( + result.manifest.checkpoint.token, + derive_projection_checkpoint_token( + request.corpus_id, + request.projection, + request.chunking, + result.manifest.entries, + ), + ) + + +class ProjectionIntegrityValidationTests(unittest.TestCase): + def assert_contract_failure(self, operation: Callable[[], object]) -> None: + with self.assertRaises(ContractValidationError) as raised: + operation() + self.assertIs(type(raised.exception), ContractValidationError) + + def test_top_level_values_require_exact_public_types(self) -> None: + operations: tuple[Callable[[], object], ...] = ( + lambda: derive_source_digest(cast(Document, object())), + lambda: derive_fragment_id( + cast(DocumentIdentity, object()), + 0, + 1, + ), + lambda: derive_projection_checkpoint_token( + cast(str, object()), + _projection(), + ChunkingPolicy(4, 1), + (), + ), + lambda: derive_projection_checkpoint_token( + _CORPUS_ID, + cast(ProjectionIdentity, object()), + ChunkingPolicy(4, 1), + (), + ), + lambda: derive_projection_checkpoint_token( + _CORPUS_ID, + _projection(), + cast(ChunkingPolicy, object()), + (), + ), + lambda: derive_projection_checkpoint_token( + _CORPUS_ID, + _projection(), + ChunkingPolicy(4, 1), + cast(tuple[ProjectionManifestEntry, ...], []), + ), + lambda: has_valid_projection_checkpoint(cast(ProjectionManifest, object())), + ) + + for operation in operations: + with self.subTest(operation=operation): + self.assert_contract_failure(operation) + + def test_corrupted_frozen_nested_values_fail_as_contract_errors(self) -> None: + malformed_documents = (_golden_document(), _golden_document()) + object.__setattr__(malformed_documents[0], "identity", object()) + object.__setattr__(malformed_documents[1], "attributes", [("k", "v")]) + + malformed_identity = _golden_document().identity + object.__setattr__(malformed_identity, "key", object()) + + malformed_projection = _projection() + object.__setattr__(malformed_projection.embedding, "dimensions", True) + + malformed_chunking = ChunkingPolicy(4, 1) + object.__setattr__(malformed_chunking, "overlap_codepoints", 4) + + malformed_entry = _checkpoint_entries()[0] + object.__setattr__(malformed_entry, "fragment_count", -1) + + operations: tuple[Callable[[], object], ...] = ( + *( + lambda value=value: derive_source_digest(value) + for value in malformed_documents + ), + lambda: derive_fragment_id(malformed_identity, 0, 1), + lambda: derive_projection_checkpoint_token( + _CORPUS_ID, + malformed_projection, + ChunkingPolicy(4, 1), + (), + ), + lambda: derive_projection_checkpoint_token( + _CORPUS_ID, + _projection(), + malformed_chunking, + (), + ), + lambda: derive_projection_checkpoint_token( + _CORPUS_ID, + _projection(), + ChunkingPolicy(4, 1), + (malformed_entry,), + ), + ) + + for operation in operations: + with self.subTest(operation=operation): + self.assert_contract_failure(operation) + + def test_fragment_ranges_are_nonnegative_ordered_exact_integers(self) -> None: + document = _golden_document().identity + for start, end in ( + (-1, 1), + (0, 0), + (2, 1), + (True, 1), + (0, False), + (0.0, 1), + (0, 1.0), + ): + with self.subTest(start=start, end=end): + self.assert_contract_failure( + lambda: derive_fragment_id( + document, + cast(int, start), + cast(int, end), + ) + ) + + def test_checkpoint_entries_must_be_an_exact_canonical_tuple(self) -> None: + entries = _checkpoint_entries() + wrong_corpus = ProjectionManifestEntry( + DocumentIdentity(DocumentKey("other", "document"), "revision"), + _SOURCE_GOLDEN, + 1, + ) + duplicate = ProjectionManifestEntry( + entries[0].document, + "sha256:" + "c" * 64, + 1, + ) + malformed_cases: tuple[object, ...] = ( + list(entries), + _TupleSubclass(entries), + tuple(reversed(entries)), + (entries[0], duplicate), + (wrong_corpus,), + (object(),), + ) + + for malformed in malformed_cases: + with self.subTest(container=type(malformed).__name__, value=malformed): + self.assert_contract_failure( + lambda: derive_projection_checkpoint_token( + _CORPUS_ID, + _projection(), + ChunkingPolicy(4, 1), + cast(tuple[ProjectionManifestEntry, ...], malformed), + ) + ) + + def test_checkpoint_validation_rejects_corrupted_manifest_shapes(self) -> None: + malformed_manifests = (_manifest(), _manifest(), _manifest()) + object.__setattr__( + malformed_manifests[0], "entries", list(_checkpoint_entries()) + ) + object.__setattr__(malformed_manifests[1], "checkpoint", object()) + object.__setattr__(malformed_manifests[2].projection, "schema_id", 1) + + for malformed in malformed_manifests: + with self.subTest(manifest=malformed): + self.assert_contract_failure( + lambda: has_valid_projection_checkpoint(malformed) + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_retrieval.py b/tests/test_retrieval.py new file mode 100644 index 0000000..5f66da1 --- /dev/null +++ b/tests/test_retrieval.py @@ -0,0 +1,1312 @@ +"""Behavioral tests for bounded semantic and deterministic hybrid retrieval.""" + +from __future__ import annotations + +import inspect +import itertools +import json +import os +import subprocess +import sys +import textwrap +import unittest +from collections.abc import Callable +from fractions import Fraction +from pathlib import Path +from types import TracebackType +from typing import cast + +import generic_rag.retrieval as retrieval_module +from generic_rag.contracts import ( + ChunkingPolicy, + DocumentIdentity, + DocumentKey, + EmbeddingIdentity, + EmbeddingVector, + Fragment, + FragmentIdentity, + ProjectionCheckpoint, + ProjectionIdentity, + ProjectionManifest, + ProjectionManifestEntry, + ProjectionStateAvailability, + ProjectionStateSnapshot, + RetrievalHit, + RetrievalLimits, + RetrievalOutcome, + RetrievalQuery, + RetrievalResult, +) +from generic_rag.errors import ContractValidationError +from generic_rag.ports import Borrowed, Embedder, LexicalRetriever, VectorIndexReader +from generic_rag.projection_integrity import ( + derive_fragment_id, + derive_projection_checkpoint_token, +) +from generic_rag.retrieval import retrieve_hybrid, retrieve_semantic + +_SOURCE_ROOT = Path(__file__).resolve().parents[1] / "src" +_DEFAULT = object() + + +class _TupleSubclass(tuple[object, ...]): + pass + + +class _ControlFlow(BaseException): + pass + + +def _projection( + *, + model_id: str = "model-v1", + dimensions: int = 2, +) -> ProjectionIdentity: + return ProjectionIdentity( + "schema-v1", + EmbeddingIdentity(model_id, dimensions), + ) + + +def _document( + document_id: str, + *, + corpus_id: str = "corpus", + revision_id: str = "revision-1", +) -> DocumentIdentity: + return DocumentIdentity(DocumentKey(corpus_id, document_id), revision_id) + + +def _manifest( + documents: tuple[DocumentIdentity, ...], + *, + corpus_id: str = "corpus", + projection: ProjectionIdentity | None = None, + chunking: ChunkingPolicy | None = None, + token: str | None = None, +) -> ProjectionManifest: + selected_projection = projection or _projection() + selected_chunking = chunking or ChunkingPolicy(4, 1) + entries = tuple( + sorted( + ( + ProjectionManifestEntry( + document, + "sha256:" + f"{index:064x}", + 1, + ) + for index, document in enumerate(documents, start=1) + ), + key=lambda entry: entry.document.key.document_id, + ) + ) + checkpoint_token = token or derive_projection_checkpoint_token( + corpus_id, + selected_projection, + selected_chunking, + entries, + ) + return ProjectionManifest( + corpus_id, + selected_projection, + selected_chunking, + entries, + ProjectionCheckpoint(corpus_id, selected_projection, checkpoint_token), + ) + + +def _present(manifest: ProjectionManifest) -> ProjectionStateSnapshot: + return ProjectionStateSnapshot(ProjectionStateAvailability.PRESENT, manifest) + + +def _fragment( + document: DocumentIdentity, + *, + text: str = "x", + start: int = 0, + fragment_id: str | None = None, + attributes: tuple[tuple[str, str], ...] = (), +) -> Fragment: + end = start + len(text) + identity = FragmentIdentity( + document, + fragment_id or derive_fragment_id(document, start, end), + start, + end, + ) + return Fragment(identity, text, attributes) + + +def _query( + *, + corpus_id: str = "corpus", + text: str = "query", + hit_limit: int = 4, + candidate_limit: int = 8, +) -> RetrievalQuery: + return RetrievalQuery(corpus_id, text, hit_limit, candidate_limit) + + +def _limits(max_query_codepoints: int = 100) -> RetrievalLimits: + return RetrievalLimits(max_query_codepoints) + + +class _LifecycleResource: + def __init__(self, events: list[str]) -> None: + self.events = events + self.lifecycle_calls: list[str] = [] + + def __enter__(self) -> _LifecycleResource: + self.lifecycle_calls.append("enter") + return self + + def __exit__( + self, + exc_type: type[BaseException] | None, + exc_value: BaseException | None, + traceback: TracebackType | None, + ) -> bool: + del exc_type, exc_value, traceback + self.lifecycle_calls.append("exit") + return True + + def close(self) -> None: + self.lifecycle_calls.append("close") + + def shutdown(self) -> None: + self.lifecycle_calls.append("shutdown") + + +class _FakeEmbedder(_LifecycleResource): + def __init__( + self, + expected_identity: EmbeddingIdentity, + events: list[str], + *, + identity_value: object = _DEFAULT, + output: object = _DEFAULT, + identity_failure: BaseException | None = None, + embed_failure: BaseException | None = None, + ) -> None: + super().__init__(events) + self.expected_identity = expected_identity + self.identity_value = identity_value + self.output = output + self.identity_failure = identity_failure + self.embed_failure = embed_failure + self.identity_calls = 0 + self.embed_calls: list[tuple[str, ...]] = [] + + @property + def identity(self) -> EmbeddingIdentity: + self.identity_calls += 1 + self.events.append("identity") + if self.identity_failure is not None: + raise self.identity_failure + if self.identity_value is _DEFAULT: + return self.expected_identity + return cast(EmbeddingIdentity, self.identity_value) + + def embed(self, texts: tuple[str, ...], /) -> tuple[EmbeddingVector, ...]: + self.embed_calls.append(texts) + self.events.append("embed:" + "|".join(texts)) + if self.embed_failure is not None: + raise self.embed_failure + if self.output is _DEFAULT: + return (EmbeddingVector((1.0, 2.0)),) + return cast(tuple[EmbeddingVector, ...], self.output) + + +class _FakeVectorReader(_LifecycleResource): + def __init__( + self, + events: list[str], + output: object = (), + *, + failure: BaseException | None = None, + ) -> None: + super().__init__(events) + self.output = output + self.failure = failure + self.calls: list[tuple[RetrievalQuery, EmbeddingVector]] = [] + + def search( + self, + query: RetrievalQuery, + embedding: EmbeddingVector, + /, + ) -> tuple[Fragment, ...]: + self.calls.append((query, embedding)) + self.events.append("vector") + if self.failure is not None: + raise self.failure + return cast(tuple[Fragment, ...], self.output) + + +class _FakeLexicalRetriever(_LifecycleResource): + def __init__( + self, + events: list[str], + output: object = (), + *, + failure: BaseException | None = None, + ) -> None: + super().__init__(events) + self.output = output + self.failure = failure + self.calls: list[RetrievalQuery] = [] + + def search(self, query: RetrievalQuery, /) -> tuple[Fragment, ...]: + self.calls.append(query) + self.events.append("lexical") + if self.failure is not None: + raise self.failure + return cast(tuple[Fragment, ...], self.output) + + +def _borrow_embedder(value: _FakeEmbedder) -> Borrowed[Embedder]: + return Borrowed(cast(Embedder, value)) + + +def _borrow_vector(value: _FakeVectorReader) -> Borrowed[VectorIndexReader]: + return Borrowed(cast(VectorIndexReader, value)) + + +def _borrow_lexical(value: _FakeLexicalRetriever) -> Borrowed[LexicalRetriever]: + return Borrowed(cast(LexicalRetriever, value)) + + +def _hit_fragments(result: RetrievalResult) -> tuple[Fragment, ...]: + return tuple(hit.fragment for hit in result.hits) + + +def _identity_order(fragment: Fragment) -> tuple[str, str, str, str, int, int]: + identity = fragment.identity + return ( + identity.document.key.corpus_id, + identity.document.key.document_id, + identity.document.revision_id, + identity.fragment_id, + identity.start, + identity.end, + ) + + +def _independent_fusion( + semantic: tuple[Fragment, ...], + lexical: tuple[Fragment, ...], +) -> tuple[Fragment, ...]: + fragments: dict[FragmentIdentity, Fragment] = {} + scores: dict[FragmentIdentity, Fraction] = {} + for candidates in (semantic, lexical): + seen: set[FragmentIdentity] = set() + for rank, fragment in enumerate(candidates, start=1): + if fragment.identity in seen: + continue + seen.add(fragment.identity) + fragments.setdefault(fragment.identity, fragment) + scores[fragment.identity] = scores.get( + fragment.identity, + Fraction(), + ) + Fraction(1, 60 + rank) + identities = sorted( + scores, + key=lambda identity: ( + -scores[identity], + _identity_order(fragments[identity]), + ), + ) + return tuple(fragments[identity] for identity in identities) + + +class RetrievalPublicAndInputTests(unittest.TestCase): + def test_exports_and_function_shapes_are_exact_and_owned(self) -> None: + expected = ("retrieve_semantic", "retrieve_hybrid") + signatures = { + retrieve_semantic: ( + "query", + "state", + "limits", + "embedder", + "vector_reader", + ), + retrieve_hybrid: ( + "query", + "state", + "limits", + "embedder", + "vector_reader", + "lexical_retriever", + ), + } + + self.assertEqual(retrieval_module.__all__, expected) + for function, names in signatures.items(): + with self.subTest(function=function.__name__): + self.assertEqual(function.__module__, "generic_rag.retrieval") + self.assertFalse(inspect.iscoroutinefunction(function)) + parameters = tuple( + inspect.signature( + cast(Callable[..., object], function) + ).parameters.values() + ) + self.assertEqual(tuple(item.name for item in parameters), names) + self.assertTrue( + all( + item.kind is inspect.Parameter.POSITIONAL_ONLY + for item in parameters + ) + ) + + def test_invalid_top_level_inputs_fail_before_any_collaborator_effect(self) -> None: + document = _document("document") + state = _present(_manifest((document,))) + query = _query(hit_limit=1, candidate_limit=1) + events: list[str] = [] + embedder = _FakeEmbedder(_projection().embedding, events) + vector = _FakeVectorReader(events, (_fragment(document),)) + operations: tuple[Callable[[], object], ...] = ( + lambda: retrieve_semantic( + cast(RetrievalQuery, object()), + state, + _limits(), + _borrow_embedder(embedder), + _borrow_vector(vector), + ), + lambda: retrieve_semantic( + query, + cast(ProjectionStateSnapshot, object()), + _limits(), + _borrow_embedder(embedder), + _borrow_vector(vector), + ), + lambda: retrieve_semantic( + query, + state, + cast(RetrievalLimits, object()), + _borrow_embedder(embedder), + _borrow_vector(vector), + ), + lambda: retrieve_semantic( + query, + state, + _limits(), + cast(Borrowed[Embedder], object()), + _borrow_vector(vector), + ), + lambda: retrieve_semantic( + query, + state, + _limits(), + _borrow_embedder(embedder), + cast(Borrowed[VectorIndexReader], object()), + ), + lambda: retrieve_hybrid( + query, + state, + _limits(), + _borrow_embedder(embedder), + _borrow_vector(vector), + cast(Borrowed[LexicalRetriever], object()), + ), + ) + + for operation in operations: + with self.subTest(operation=operation): + with self.assertRaises(ContractValidationError): + operation() + self.assertEqual(events, []) + + def test_corrupted_frozen_query_and_state_fail_before_effects(self) -> None: + document = _document("document") + query = _query(hit_limit=1, candidate_limit=1) + object.__setattr__(query, "candidate_limit", True) + state = _present(_manifest((document,))) + malformed_state = _present(_manifest((document,))) + assert malformed_state.manifest is not None + object.__setattr__(malformed_state.manifest, "entries", []) + + for malformed_query, malformed_snapshot in ( + (query, state), + (_query(hit_limit=1, candidate_limit=1), malformed_state), + ): + events: list[str] = [] + embedder = _FakeEmbedder(_projection().embedding, events) + vector = _FakeVectorReader(events) + with self.subTest(query=malformed_query, state=malformed_snapshot): + with self.assertRaises(ContractValidationError): + retrieve_semantic( + malformed_query, + malformed_snapshot, + _limits(), + _borrow_embedder(embedder), + _borrow_vector(vector), + ) + self.assertEqual(events, []) + + def test_query_codepoint_cap_is_independent_and_checked_before_effects( + self, + ) -> None: + document = _document("document") + fragment = _fragment(document) + state = _present(_manifest((document,))) + exact_text = "😀e\u0301\0" + self.assertEqual(len(exact_text), 4) + + events: list[str] = [] + embedder = _FakeEmbedder(_projection().embedding, events) + vector = _FakeVectorReader(events, (fragment,)) + result = retrieve_semantic( + _query(text=exact_text, hit_limit=1, candidate_limit=1), + state, + RetrievalLimits(4), + _borrow_embedder(embedder), + _borrow_vector(vector), + ) + self.assertIs(result.outcome, RetrievalOutcome.COMPLETE) + self.assertEqual(embedder.embed_calls, [(exact_text,)]) + + blocked_events: list[str] = [] + blocked_embedder = _FakeEmbedder(_projection().embedding, blocked_events) + blocked_vector = _FakeVectorReader(blocked_events, (fragment,)) + with self.assertRaises(ContractValidationError): + retrieve_semantic( + _query(text=exact_text + "Z", hit_limit=1, candidate_limit=1), + state, + RetrievalLimits(4), + _borrow_embedder(blocked_embedder), + _borrow_vector(blocked_vector), + ) + self.assertEqual(blocked_events, []) + + +class RetrievalStateAndEffectTests(unittest.TestCase): + def test_terminal_state_outcomes_never_touch_collaborators(self) -> None: + document = _document("document") + valid = _manifest((document,)) + invalid_checkpoint = ProjectionManifest( + valid.corpus_id, + valid.projection, + valid.chunking, + valid.entries, + ProjectionCheckpoint(valid.corpus_id, valid.projection, "wrong-token"), + ) + other_document = _document("document", corpus_id="other") + cases = ( + ( + ProjectionStateSnapshot(ProjectionStateAvailability.MISSING, None), + RetrievalOutcome.UNAVAILABLE, + ), + ( + ProjectionStateSnapshot(ProjectionStateAvailability.CORRUPT, None), + RetrievalOutcome.FAILED, + ), + ( + _present(_manifest((other_document,), corpus_id="other")), + RetrievalOutcome.UNAVAILABLE, + ), + (_present(invalid_checkpoint), RetrievalOutcome.FAILED), + (_present(_manifest(())), RetrievalOutcome.COMPLETE), + ) + + for state, expected in cases: + events: list[str] = [] + embedder = _FakeEmbedder( + _projection().embedding, + events, + identity_failure=AssertionError("must not run"), + ) + vector = _FakeVectorReader( + events, + failure=AssertionError("must not run"), + ) + lexical = _FakeLexicalRetriever( + events, + failure=AssertionError("must not run"), + ) + with self.subTest(outcome=expected): + result = retrieve_hybrid( + _query(), + state, + _limits(), + _borrow_embedder(embedder), + _borrow_vector(vector), + _borrow_lexical(lexical), + ) + self.assertIs(result.outcome, expected) + self.assertEqual(result.hits, ()) + self.assertFalse(result.truncated) + self.assertEqual(events, []) + + def test_semantic_and_hybrid_effect_order_and_call_bounds_are_exact(self) -> None: + semantic_document = _document("semantic") + lexical_document = _document("lexical") + state = _present(_manifest((semantic_document, lexical_document))) + query = _query(hit_limit=2, candidate_limit=2) + + semantic_events: list[str] = [] + semantic_embedder = _FakeEmbedder(_projection().embedding, semantic_events) + semantic_vector = _FakeVectorReader( + semantic_events, + (_fragment(semantic_document),), + ) + semantic_result = retrieve_semantic( + query, + state, + _limits(), + _borrow_embedder(semantic_embedder), + _borrow_vector(semantic_vector), + ) + self.assertEqual(semantic_events, ["identity", "embed:query", "vector"]) + self.assertEqual(semantic_embedder.identity_calls, 1) + self.assertEqual(semantic_embedder.embed_calls, [(query.text,)]) + self.assertEqual(len(semantic_vector.calls), 1) + self.assertEqual(semantic_vector.calls[0][0], query) + self.assertEqual(semantic_vector.calls[0][1], EmbeddingVector((1.0, 2.0))) + self.assertIs(semantic_result.outcome, RetrievalOutcome.COMPLETE) + + hybrid_events: list[str] = [] + hybrid_embedder = _FakeEmbedder(_projection().embedding, hybrid_events) + hybrid_vector = _FakeVectorReader( + hybrid_events, + (_fragment(semantic_document),), + ) + hybrid_lexical = _FakeLexicalRetriever( + hybrid_events, + (_fragment(lexical_document),), + ) + hybrid_result = retrieve_hybrid( + query, + state, + _limits(), + _borrow_embedder(hybrid_embedder), + _borrow_vector(hybrid_vector), + _borrow_lexical(hybrid_lexical), + ) + self.assertEqual( + hybrid_events, + ["identity", "embed:query", "vector", "lexical"], + ) + self.assertEqual(hybrid_embedder.identity_calls, 1) + self.assertEqual(hybrid_embedder.embed_calls, [(query.text,)]) + self.assertEqual(len(hybrid_vector.calls), 1) + self.assertEqual(hybrid_lexical.calls, [query]) + self.assertIs(hybrid_result.outcome, RetrievalOutcome.COMPLETE) + self.assertEqual(tuple(hit.rank for hit in hybrid_result.hits), (1, 2)) + + +class RetrievalProviderValidationTests(unittest.TestCase): + def test_embedder_identity_and_vector_outputs_are_strictly_validated(self) -> None: + document = _document("document") + state = _present(_manifest((document,))) + malformed_identity = EmbeddingIdentity("model-v1", 2) + object.__setattr__(malformed_identity, "dimensions", True) + wrong_dimensions = EmbeddingVector((1.0,)) + noncanonical_number = EmbeddingVector((1.0, 2.0)) + object.__setattr__(noncanonical_number, "values", (1, 2.0)) + nonfinite = EmbeddingVector((1.0, 2.0)) + object.__setattr__(nonfinite, "values", (float("nan"), 2.0)) + malformed_container = EmbeddingVector((1.0, 2.0)) + object.__setattr__(malformed_container, "values", [1.0, 2.0]) + cases = ( + ("identity type", object(), _DEFAULT), + ("identity shape", malformed_identity, _DEFAULT), + ("output list", _DEFAULT, [EmbeddingVector((1.0, 2.0))]), + ("output empty", _DEFAULT, ()), + ( + "output count", + _DEFAULT, + (EmbeddingVector((1.0, 2.0)), EmbeddingVector((3.0, 4.0))), + ), + ("vector type", _DEFAULT, (object(),)), + ("dimensions", _DEFAULT, (wrong_dimensions,)), + ("coordinate type", _DEFAULT, (noncanonical_number,)), + ("nonfinite", _DEFAULT, (nonfinite,)), + ("values container", _DEFAULT, (malformed_container,)), + ) + + for label, identity_value, output in cases: + events: list[str] = [] + embedder = _FakeEmbedder( + _projection().embedding, + events, + identity_value=identity_value, + output=output, + ) + vector = _FakeVectorReader(events, (_fragment(document),)) + with self.subTest(case=label): + result = retrieve_semantic( + _query(hit_limit=1, candidate_limit=1), + state, + _limits(), + _borrow_embedder(embedder), + _borrow_vector(vector), + ) + self.assertIs(result.outcome, RetrievalOutcome.FAILED) + self.assertEqual(result.hits, ()) + self.assertFalse(result.truncated) + self.assertLessEqual(embedder.identity_calls, 1) + self.assertLessEqual(len(embedder.embed_calls), 1) + self.assertEqual(vector.calls, []) + + def test_vector_and_lexical_candidate_outputs_share_strict_validation(self) -> None: + documents = tuple(_document(name) for name in ("a", "b", "c")) + state = _present(_manifest(documents)) + valid = tuple(_fragment(document) for document in documents) + wrong_corpus = _fragment(_document("wrong", corpus_id="other")) + too_wide = _fragment(documents[0], text="abcde") + wrong_id = _fragment(documents[0], fragment_id="wrong-fragment-id") + malformed_outputs: tuple[tuple[str, object], ...] = ( + ("list", list(valid[:1])), + ("tuple subclass", _TupleSubclass(valid[:1])), + ("candidate count", valid), + ("candidate type", (object(),)), + ("wrong corpus", (wrong_corpus,)), + ("fragment width", (too_wide,)), + ("fragment id", (wrong_id,)), + ) + query = _query(hit_limit=2, candidate_limit=2) + + for pathway in ("vector", "lexical"): + for label, malformed in malformed_outputs: + events: list[str] = [] + embedder = _FakeEmbedder(_projection().embedding, events) + vector = _FakeVectorReader( + events, + malformed if pathway == "vector" else (), + ) + lexical = _FakeLexicalRetriever( + events, + malformed if pathway == "lexical" else (), + ) + with self.subTest(pathway=pathway, case=label): + if pathway == "vector": + result = retrieve_semantic( + query, + state, + _limits(), + _borrow_embedder(embedder), + _borrow_vector(vector), + ) + else: + result = retrieve_hybrid( + query, + state, + _limits(), + _borrow_embedder(embedder), + _borrow_vector(vector), + _borrow_lexical(lexical), + ) + self.assertIs(result.outcome, RetrievalOutcome.FAILED) + self.assertEqual(result.hits, ()) + self.assertFalse(result.truncated) + if pathway == "vector": + self.assertEqual(len(vector.calls), 1) + self.assertEqual(lexical.calls, []) + else: + self.assertEqual(len(vector.calls), 1) + self.assertEqual(len(lexical.calls), 1) + + def test_revision_and_document_membership_staleness_preserve_current_hits( + self, + ) -> None: + published_a = _document("a", revision_id="current") + published_c = _document("c", revision_id="current") + stale_revision = _fragment(_document("a", revision_id="old")) + absent_document = _fragment(_document("b", revision_id="current")) + current = _fragment(published_c) + state = _present(_manifest((published_a, published_c))) + events: list[str] = [] + embedder = _FakeEmbedder(_projection().embedding, events) + vector = _FakeVectorReader( + events, + (stale_revision, absent_document, current), + ) + + result = retrieve_semantic( + _query(hit_limit=2, candidate_limit=3), + state, + _limits(), + _borrow_embedder(embedder), + _borrow_vector(vector), + ) + + self.assertIs(result.outcome, RetrievalOutcome.PARTIAL) + self.assertEqual(_hit_fragments(result), (current,)) + self.assertEqual(result.hits[0], RetrievalHit(current, 1)) + self.assertFalse(result.truncated) + + def test_embedder_identity_mismatch_is_stale_without_embedding_or_vector_read( + self, + ) -> None: + document = _document("document") + state = _present(_manifest((document,))) + events: list[str] = [] + embedder = _FakeEmbedder( + _projection().embedding, + events, + identity_value=EmbeddingIdentity("other-model", 2), + ) + vector = _FakeVectorReader(events, (_fragment(document),)) + + result = retrieve_semantic( + _query(hit_limit=1, candidate_limit=1), + state, + _limits(), + _borrow_embedder(embedder), + _borrow_vector(vector), + ) + + self.assertIs(result.outcome, RetrievalOutcome.STALE) + self.assertEqual(events, ["identity"]) + self.assertEqual(embedder.embed_calls, []) + self.assertEqual(vector.calls, []) + + +class RetrievalDeduplicationAndFusionTests(unittest.TestCase): + def test_identical_provider_duplicates_are_deduplicated_without_truncation( + self, + ) -> None: + documents = (_document("a"), _document("b")) + first, second = tuple(_fragment(document) for document in documents) + events: list[str] = [] + result = retrieve_semantic( + _query(hit_limit=2, candidate_limit=3), + _present(_manifest(documents)), + _limits(), + _borrow_embedder(_FakeEmbedder(_projection().embedding, events)), + _borrow_vector(_FakeVectorReader(events, (first, first, second))), + ) + + self.assertIs(result.outcome, RetrievalOutcome.COMPLETE) + self.assertEqual(_hit_fragments(result), (first, second)) + self.assertFalse(result.truncated) + + def test_within_and_cross_provider_payload_conflicts_fail_content_free( + self, + ) -> None: + document = _document("document") + original = _fragment(document, attributes=(("variant", "one"),)) + conflict = _fragment(document, attributes=(("sentinel-secret", "two"),)) + state = _present(_manifest((document,))) + + events: list[str] = [] + within = retrieve_semantic( + _query(hit_limit=2, candidate_limit=2), + state, + _limits(), + _borrow_embedder(_FakeEmbedder(_projection().embedding, events)), + _borrow_vector(_FakeVectorReader(events, (original, conflict))), + ) + self.assertIs(within.outcome, RetrievalOutcome.FAILED) + self.assertEqual(within.hits, ()) + self.assertNotIn("sentinel-secret", repr(within)) + + cross_events: list[str] = [] + cross = retrieve_hybrid( + _query(hit_limit=1, candidate_limit=1), + state, + _limits(), + _borrow_embedder(_FakeEmbedder(_projection().embedding, cross_events)), + _borrow_vector(_FakeVectorReader(cross_events, (original,))), + _borrow_lexical(_FakeLexicalRetriever(cross_events, (conflict,))), + ) + self.assertIs(cross.outcome, RetrievalOutcome.FAILED) + self.assertEqual(cross.hits, ()) + self.assertFalse(cross.truncated) + self.assertNotIn("sentinel-secret", repr(cross)) + + def test_exact_cross_provider_duplicate_is_one_fused_hit(self) -> None: + document = _document("document") + fragment = _fragment(document) + events: list[str] = [] + result = retrieve_hybrid( + _query(hit_limit=1, candidate_limit=1), + _present(_manifest((document,))), + _limits(), + _borrow_embedder(_FakeEmbedder(_projection().embedding, events)), + _borrow_vector(_FakeVectorReader(events, (fragment,))), + _borrow_lexical(_FakeLexicalRetriever(events, (fragment,))), + ) + + self.assertIs(result.outcome, RetrievalOutcome.COMPLETE) + self.assertEqual(_hit_fragments(result), (fragment,)) + self.assertFalse(result.truncated) + + def test_duplicate_gaps_retain_original_provider_ranks(self) -> None: + documents = tuple(_document(name) for name in ("a", "b", "c", "d")) + by_id = { + document.key.document_id: _fragment(document) for document in documents + } + semantic = (by_id["d"], by_id["d"], by_id["b"]) + lexical = (by_id["a"], by_id["c"]) + events: list[str] = [] + result = retrieve_hybrid( + _query(hit_limit=4, candidate_limit=4), + _present(_manifest(documents)), + _limits(), + _borrow_embedder(_FakeEmbedder(_projection().embedding, events)), + _borrow_vector(_FakeVectorReader(events, semantic)), + _borrow_lexical(_FakeLexicalRetriever(events, lexical)), + ) + + self.assertEqual(_hit_fragments(result), _independent_fusion(semantic, lexical)) + document_ids = tuple( + hit.fragment.identity.document.key.document_id for hit in result.hits + ) + self.assertLess(document_ids.index("c"), document_ids.index("b")) + + def test_rrf_uses_exact_fraction_offset_sixty_witness(self) -> None: + semantic_ids = ( + "A", + "s02", + "s03", + "s04", + "B", + "s06", + "s07", + "s08", + "s09", + "s10", + "s11", + "s12", + "s13", + ) + lexical_ids = ( + "l01", + "l02", + "l03", + "l04", + "l05", + "l06", + "l07", + "B", + "l09", + "l10", + "l11", + "l12", + "A", + ) + all_ids = tuple(sorted(set((*semantic_ids, *lexical_ids)))) + documents = tuple(_document(document_id) for document_id in all_ids) + fragments = { + document.key.document_id: _fragment(document) for document in documents + } + semantic = tuple(fragments[document_id] for document_id in semantic_ids) + lexical = tuple(fragments[document_id] for document_id in lexical_ids) + events: list[str] = [] + result = retrieve_hybrid( + _query(hit_limit=13, candidate_limit=13), + _present(_manifest(documents)), + _limits(), + _borrow_embedder(_FakeEmbedder(_projection().embedding, events)), + _borrow_vector(_FakeVectorReader(events, semantic)), + _borrow_lexical(_FakeLexicalRetriever(events, lexical)), + ) + + expected = _independent_fusion(semantic, lexical) + self.assertEqual(_hit_fragments(result), expected[:13]) + self.assertEqual( + tuple( + hit.fragment.identity.document.key.document_id + for hit in result.hits[:2] + ), + ("A", "B"), + ) + self.assertGreater( + Fraction(1, 60 + 1) + Fraction(1, 60 + 13), + Fraction(1, 60 + 5) + Fraction(1, 60 + 8), + ) + self.assertLess( + Fraction(1, 61 + 1) + Fraction(1, 61 + 13), + Fraction(1, 61 + 5) + Fraction(1, 61 + 8), + ) + self.assertTrue(result.truncated) + + def test_unicode_ties_and_branch_permutation_use_identity_order(self) -> None: + decomposed = _document("e\u0301") + composed = _document("é") + first = _fragment(decomposed) + second = _fragment(composed) + state = _present(_manifest((decomposed, composed))) + + outputs: list[tuple[Fragment, ...]] = [] + for semantic, lexical in (((first,), (second,)), ((second,), (first,))): + events: list[str] = [] + result = retrieve_hybrid( + _query(hit_limit=2, candidate_limit=2), + state, + _limits(), + _borrow_embedder(_FakeEmbedder(_projection().embedding, events)), + _borrow_vector(_FakeVectorReader(events, semantic)), + _borrow_lexical(_FakeLexicalRetriever(events, lexical)), + ) + outputs.append(_hit_fragments(result)) + + self.assertEqual(outputs, [(first, second), (first, second)]) + + def test_all_small_provider_permutations_match_fraction_oracle(self) -> None: + documents = tuple(_document(name) for name in ("a", "b", "c")) + fragments = tuple(_fragment(document) for document in documents) + state = _present(_manifest(documents)) + + for semantic in itertools.permutations(fragments): + for lexical in itertools.permutations(fragments): + events: list[str] = [] + with self.subTest( + semantic=tuple( + item.identity.document.key.document_id for item in semantic + ), + lexical=tuple( + item.identity.document.key.document_id for item in lexical + ), + ): + result = retrieve_hybrid( + _query(hit_limit=3, candidate_limit=3), + state, + _limits(), + _borrow_embedder( + _FakeEmbedder(_projection().embedding, events) + ), + _borrow_vector(_FakeVectorReader(events, semantic)), + _borrow_lexical(_FakeLexicalRetriever(events, lexical)), + ) + self.assertEqual( + _hit_fragments(result), + _independent_fusion(semantic, lexical), + ) + + def test_truncation_reflects_post_validation_unique_current_candidates( + self, + ) -> None: + documents = tuple(_document(name) for name in ("a", "b", "c")) + current = tuple(_fragment(document) for document in documents) + + events: list[str] = [] + truncated = retrieve_semantic( + _query(hit_limit=2, candidate_limit=3), + _present(_manifest(documents)), + _limits(), + _borrow_embedder(_FakeEmbedder(_projection().embedding, events)), + _borrow_vector(_FakeVectorReader(events, current)), + ) + self.assertEqual(len(truncated.hits), 2) + self.assertTrue(truncated.truncated) + + stale = _fragment(_document("b", revision_id="old")) + filtered_events: list[str] = [] + filtered = retrieve_semantic( + _query(hit_limit=1, candidate_limit=3), + _present(_manifest(documents[:2])), + _limits(), + _borrow_embedder(_FakeEmbedder(_projection().embedding, filtered_events)), + _borrow_vector( + _FakeVectorReader(filtered_events, (current[0], current[0], stale)) + ), + ) + self.assertIs(filtered.outcome, RetrievalOutcome.PARTIAL) + self.assertEqual(_hit_fragments(filtered), (current[0],)) + self.assertFalse(filtered.truncated) + + +class RetrievalFailureAndLifecycleTests(unittest.TestCase): + def assert_no_lifecycle(self, *resources: _LifecycleResource) -> None: + for resource in resources: + self.assertEqual(resource.lifecycle_calls, []) + + def test_failure_and_stale_precedence_is_truthful(self) -> None: + document = _document("document") + fragment = _fragment(document) + state = _present(_manifest((document,))) + wrong_identity = EmbeddingIdentity("other-model", 2) + cases = ( + (wrong_identity, None, (), RetrievalOutcome.STALE), + ( + wrong_identity, + RuntimeError("lexical sentinel"), + (), + RetrievalOutcome.FAILED, + ), + (wrong_identity, None, (fragment,), RetrievalOutcome.PARTIAL), + ( + RuntimeError("identity sentinel"), + None, + (fragment,), + RetrievalOutcome.PARTIAL, + ), + (_DEFAULT, None, (), RetrievalOutcome.COMPLETE), + ) + + for semantic_state, lexical_failure, lexical_output, expected in cases: + events: list[str] = [] + embedder = _FakeEmbedder( + _projection().embedding, + events, + identity_value=( + semantic_state + if isinstance(semantic_state, EmbeddingIdentity) + else _DEFAULT + ), + identity_failure=( + semantic_state + if isinstance(semantic_state, BaseException) + else None + ), + ) + vector = _FakeVectorReader(events) + lexical = _FakeLexicalRetriever( + events, + lexical_output, + failure=lexical_failure, + ) + with self.subTest(expected=expected, semantic=semantic_state): + result = retrieve_hybrid( + _query(hit_limit=1, candidate_limit=1), + state, + _limits(), + _borrow_embedder(embedder), + _borrow_vector(vector), + _borrow_lexical(lexical), + ) + self.assertIs(result.outcome, expected) + self.assertEqual( + bool(result.hits), expected is RetrievalOutcome.PARTIAL + ) + self.assertNotIn("sentinel", repr(result)) + + def test_ordinary_exceptions_are_content_free_and_never_retried(self) -> None: + document = _document("document") + state = _present(_manifest((document,))) + stages = ("identity", "embedding", "vector", "lexical") + expected_calls = { + "identity": (1, 0, 0, 0), + "embedding": (1, 1, 0, 0), + "vector": (1, 1, 1, 0), + "lexical": (1, 1, 1, 1), + } + + for stage in stages: + failure = RuntimeError(f"secret-{stage}-sentinel") + events: list[str] = [] + embedder = _FakeEmbedder( + _projection().embedding, + events, + identity_failure=failure if stage == "identity" else None, + embed_failure=failure if stage == "embedding" else None, + ) + vector = _FakeVectorReader( + events, + failure=failure if stage == "vector" else None, + ) + lexical = _FakeLexicalRetriever( + events, + failure=failure if stage == "lexical" else None, + ) + with self.subTest(stage=stage): + if stage == "lexical": + result = retrieve_hybrid( + _query(hit_limit=1, candidate_limit=1), + state, + _limits(), + _borrow_embedder(embedder), + _borrow_vector(vector), + _borrow_lexical(lexical), + ) + else: + result = retrieve_semantic( + _query(hit_limit=1, candidate_limit=1), + state, + _limits(), + _borrow_embedder(embedder), + _borrow_vector(vector), + ) + self.assertIs(result.outcome, RetrievalOutcome.FAILED) + self.assertEqual(result.hits, ()) + self.assertFalse(result.truncated) + self.assertNotIn(str(failure), repr(result)) + self.assertEqual( + ( + embedder.identity_calls, + len(embedder.embed_calls), + len(vector.calls), + len(lexical.calls), + ), + expected_calls[stage], + ) + self.assert_no_lifecycle(embedder, vector, lexical) + + def test_base_exceptions_propagate_exactly_once_without_lifecycle_actions( + self, + ) -> None: + document = _document("document") + state = _present(_manifest((document,))) + expected_calls = { + "identity": (1, 0, 0, 0), + "embedding": (1, 1, 0, 0), + "vector": (1, 1, 1, 0), + "lexical": (1, 1, 1, 1), + } + for stage in ("identity", "embedding", "vector", "lexical"): + failure = _ControlFlow(f"control-{stage}") + events: list[str] = [] + embedder = _FakeEmbedder( + _projection().embedding, + events, + identity_failure=failure if stage == "identity" else None, + embed_failure=failure if stage == "embedding" else None, + ) + vector = _FakeVectorReader( + events, + failure=failure if stage == "vector" else None, + ) + lexical = _FakeLexicalRetriever( + events, + failure=failure if stage == "lexical" else None, + ) + with self.subTest(stage=stage): + with self.assertRaises(_ControlFlow) as raised: + if stage == "lexical": + retrieve_hybrid( + _query(hit_limit=1, candidate_limit=1), + state, + _limits(), + _borrow_embedder(embedder), + _borrow_vector(vector), + _borrow_lexical(lexical), + ) + else: + retrieve_semantic( + _query(hit_limit=1, candidate_limit=1), + state, + _limits(), + _borrow_embedder(embedder), + _borrow_vector(vector), + ) + self.assertIs(raised.exception, failure) + self.assertEqual( + ( + embedder.identity_calls, + len(embedder.embed_calls), + len(vector.calls), + len(lexical.calls), + ), + expected_calls[stage], + ) + self.assert_no_lifecycle(embedder, vector, lexical) + + def test_success_never_enters_closes_or_shuts_down_borrowed_resources(self) -> None: + document = _document("document") + fragment = _fragment(document) + events: list[str] = [] + embedder = _FakeEmbedder(_projection().embedding, events) + vector = _FakeVectorReader(events, (fragment,)) + lexical = _FakeLexicalRetriever(events, (fragment,)) + + result = retrieve_hybrid( + _query(hit_limit=1, candidate_limit=1), + _present(_manifest((document,))), + _limits(), + _borrow_embedder(embedder), + _borrow_vector(vector), + _borrow_lexical(lexical), + ) + + self.assertIs(result.outcome, RetrievalOutcome.COMPLETE) + self.assert_no_lifecycle(embedder, vector, lexical) + + +class RetrievalProcessDeterminismTests(unittest.TestCase): + def test_hash_seed_does_not_change_hybrid_rank_order(self) -> None: + script = textwrap.dedent( + f""" + import json + import sys + sys.path.insert(0, {os.fspath(_SOURCE_ROOT)!r}) + from generic_rag.contracts import ( + ChunkingPolicy, DocumentIdentity, DocumentKey, + EmbeddingIdentity, EmbeddingVector, Fragment, FragmentIdentity, + ProjectionCheckpoint, ProjectionIdentity, ProjectionManifest, + ProjectionManifestEntry, ProjectionStateAvailability, + ProjectionStateSnapshot, RetrievalLimits, RetrievalQuery, + ) + from generic_rag.ports import Borrowed + from generic_rag.projection_integrity import ( + derive_fragment_id, derive_projection_checkpoint_token, + ) + from generic_rag.retrieval import retrieve_hybrid + + projection = ProjectionIdentity( + 'schema-v1', EmbeddingIdentity('model-v1', 2) + ) + chunking = ChunkingPolicy(4, 1) + documents = tuple( + DocumentIdentity(DocumentKey('corpus', name), 'revision-1') + for name in ('e\\u0301', 'é', '한') + ) + entries = tuple( + ProjectionManifestEntry( + document, 'sha256:' + format(index, '064x'), 1 + ) + for index, document in enumerate(documents, start=1) + ) + token = derive_projection_checkpoint_token( + 'corpus', projection, chunking, entries + ) + manifest = ProjectionManifest( + 'corpus', projection, chunking, entries, + ProjectionCheckpoint('corpus', projection, token), + ) + fragments = tuple( + Fragment( + FragmentIdentity( + document, derive_fragment_id(document, 0, 1), 0, 1 + ), + 'x', + ) + for document in documents + ) + + class E: + identity = projection.embedding + def embed(self, texts, /): + return (EmbeddingVector((1.0, 2.0)),) + class V: + def search(self, query, embedding, /): + return (fragments[2], fragments[0], fragments[1]) + class L: + def search(self, query, /): + return (fragments[1], fragments[0], fragments[2]) + + result = retrieve_hybrid( + RetrievalQuery('corpus', 'query', 3, 3), + ProjectionStateSnapshot(ProjectionStateAvailability.PRESENT, manifest), + RetrievalLimits(100), Borrowed(E()), Borrowed(V()), Borrowed(L()), + ) + print(json.dumps([ + hit.fragment.identity.document.key.document_id + for hit in result.hits + ], ensure_ascii=False)) + """ + ) + outputs: list[list[str]] = [] + for seed in ("1", "17", "987654"): + environment = dict(os.environ) + environment["PYTHONHASHSEED"] = seed + completed = subprocess.run( + (sys.executable, "-B", "-c", script), + env=environment, + check=False, + capture_output=True, + text=True, + timeout=10, + ) + self.assertEqual(completed.returncode, 0, completed.stderr) + outputs.append(cast(list[str], json.loads(completed.stdout))) + + self.assertEqual(outputs[0], outputs[1]) + self.assertEqual(outputs[1], outputs[2]) + self.assertEqual(outputs[0], ["é", "한", "e\u0301"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_retrieval_contracts.py b/tests/test_retrieval_contracts.py index 1798385..2fe397c 100644 --- a/tests/test_retrieval_contracts.py +++ b/tests/test_retrieval_contracts.py @@ -13,6 +13,7 @@ Fragment, FragmentIdentity, RetrievalHit, + RetrievalLimits, RetrievalOutcome, RetrievalQuery, RetrievalResult, @@ -71,6 +72,10 @@ def _query( class RetrievalContractTests(unittest.TestCase): def test_retrieval_fields_are_exact_frozen_and_slotted(self) -> None: + self.assertEqual( + tuple(field.name for field in fields(RetrievalLimits)), + ("max_query_codepoints",), + ) self.assertEqual( tuple(field.name for field in fields(RetrievalQuery)), ("corpus_id", "text", "hit_limit", "candidate_limit"), @@ -84,6 +89,7 @@ def test_retrieval_fields_are_exact_frozen_and_slotted(self) -> None: ("query", "outcome", "hits", "truncated"), ) + limits = RetrievalLimits(100) query = _query() hit = RetrievalHit(_fragment("fragment"), 1) result = RetrievalResult( @@ -93,6 +99,7 @@ def test_retrieval_fields_are_exact_frozen_and_slotted(self) -> None: False, ) for instance, field_name in ( + (limits, "max_query_codepoints"), (query, "text"), (hit, "rank"), (result, "truncated"), @@ -103,6 +110,20 @@ def test_retrieval_fields_are_exact_frozen_and_slotted(self) -> None: setattr(instance, field_name, object()) self.assertIsInstance(hash(instance), int) + def test_retrieval_limits_are_independent_positive_exact_integers(self) -> None: + limits = RetrievalLimits(max_query_codepoints=7) + + self.assertEqual(limits.max_query_codepoints, 7) + self.assertNotIn("hit_limit", inspect.signature(RetrievalLimits).parameters) + self.assertNotIn( + "candidate_limit", + inspect.signature(RetrievalLimits).parameters, + ) + for invalid in (0, -1, True, 1.0, _IntegerSubclass(1)): + with self.subTest(value=invalid): + with self.assertRaises(ContractValidationError): + RetrievalLimits(cast(int, invalid)) + def test_retrieval_outcomes_are_exact_closed_string_enums(self) -> None: self.assertEqual( tuple((member.name, member.value) for member in RetrievalOutcome), From d66a7454e97a8b08ba8f82f4ab303e43ea78c77a Mon Sep 17 00:00:00 2001 From: rica-v3 Date: Tue, 25 Aug 2026 19:53:54 +0900 Subject: [PATCH 4/4] Probe every installed RAG module in CI --- .github/workflows/ci.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 90a7532..31defe3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -163,7 +163,9 @@ jobs: generic_rag.errors \ generic_rag.contracts \ generic_rag.ports \ - generic_rag.projection + generic_rag.projection \ + generic_rag.projection_integrity \ + generic_rag.retrieval do ( cd "$rag_probe_dir"