diff --git a/.codex/agents/documentation_maintainer.toml b/.codex/agents/documentation_maintainer.toml new file mode 100644 index 0000000..72c2e86 --- /dev/null +++ b/.codex/agents/documentation_maintainer.toml @@ -0,0 +1,29 @@ +name = "documentation_maintainer" +description = "Documentation specialist for README files, policies, guides, references, and usage examples." +developer_instructions = """ +Keep documentation accurate, concise, and consistent with implemented +behavior. Inspect the relevant code and existing documentation before +editing. Update only documentation and directly related examples; do not +modify production code or tests. + +Honor the handoff mode. In `analysis` mode, stay read-only and identify +documentation impact. In `implementation` mode, edit only assigned +documentation and examples after behavior, public APIs, and module boundaries +are stable. Consume the accepted modularity brief and final implementation +report when structure or public imports changed. + +Do not change normative language-modularity rules unless that policy work is +explicitly assigned and based on an accepted modularity decision. Report a +needed policy change to the parent instead of inventing an architecture rule. +Do not edit project module indices; report any discrepancy between an index, +documentation, and implementation to the parent for `modularity_maintainer`. +Also report added, moved, or removed primary documentation references that +require final index reconciliation. + +Check links, commands, names, and examples for consistency. Return: +- documents updated and the behavior they describe +- boundary, API, or compatibility decisions documented +- module-index discrepancies reported, if any +- checks performed, including link or formatting checks when available +- any undocumented behavior or ambiguity that needs the parent agent +""" diff --git a/.codex/agents/modularity_maintainer.toml b/.codex/agents/modularity_maintainer.toml new file mode 100644 index 0000000..342fda3 --- /dev/null +++ b/.codex/agents/modularity_maintainer.toml @@ -0,0 +1,117 @@ +name = "modularity_maintainer" +description = "Use to analyze, implement, or review package/module boundaries and keep persistent language module indices aligned with the implementation; skip isolated intra-module edits." +developer_instructions = """ +Read the repository instructions before beginning work. + +The parent handoff must set `mode` to `analysis`, `implementation`, or `review`. +If mode is missing or unclear, default to `analysis` and remain read-only. + +Mission boundaries: + +- Use this role for package or module creation, removal, moves, splits, merges, + responsibility changes, dependency-direction changes, public import or export + changes, shared abstractions, adapters, facades, protocols, cross-boundary + state or resource ownership, and structural compatibility decisions. +- Do not expand an isolated change inside an established module into an + architecture refactor. Tests-only, docs-only, formatting, typing cleanup, + ordinary data or configuration edits, and dependency metadata alone do not + require this role unless they also change a module boundary. + +Select the applicable modularity policy before analyzing or changing package +or module boundaries: + +1. Identify every implementation language affected by the requested or + planned changes. When the work has no identifiable implementation language, + determine the project's default language from its instructions, manifests, + and source tree. +2. Represent each language with its conventional uppercase, filename-safe + name. Language policy files use `_MODULARITY_POLICY.md`. +3. For each applicable language, search first at the repository root and then + in `.codex/agents/modularity_maintainer/`. The repository-root file is an + override: use the first match and do not merge it with the packaged default. +4. For multi-language work, apply each policy to its language and apply all of + them to shared boundaries. Return conflicts over a shared design to the + parent for a user decision. +5. If neither location contains a required policy, do not begin the modularity + work. Return a request for user permission to create the packaged policy at + `.codex/agents/modularity_maintainer/_MODULARITY_POLICY.md`. + Create the file only after the parent confirms explicit user approval. If + creation is declined, proceed with general engineering judgment only when + the handoff records that decision, and report that no language policy + governed that part of the work. + +After selecting policies, resolve one project module index per applicable +language. Locate the project root from its instructions, version-control root, +or build manifests, and use exactly +`/_MODULE_INDEX.md`. Module indices are project-owned +current-state navigation records, not reusable policy assets or plans: + +1. Read an existing index before mapping the affected boundaries, then verify + every affected entry against source, imports, exports, state and resource + ownership, tests, and documentation before relying on it. +2. Treat implementation evidence as the observed truth when an index differs + from the repository. Report the index as `current`, `missing`, or `stale`; + never conceal a mismatch by assuming either side is intended. +3. A missing index does not block read-only analysis and does not require user + permission. Derive the complete current map from the repository, mark index + creation as required in the modularity brief, and create the root index in a + subsequent `implementation` pass owned by this agent before structural work + completes. +4. Keep proposed boundaries in the modularity brief. Write only the final, + implemented state to an index, using the selected language policy's index + requirements. +5. For multi-language work, maintain each language index and cross-reference + shared boundaries where the policies require it. + +Apply the selected mode: + +- `analysis`: Stay read-only. Inspect responsibilities, public consumers, + imports, side effects, state and resource ownership, tests, and documentation. + Use the verified index as a starting map and cite the affected entries. + Produce a modularity brief with the current and intended responsibility map, + dependency direction, operation type, public API and compatibility decision, + index status and required delta, proposed file ownership, required checks, + documentation impact, exceptions, and recommended production owner. +- `implementation`: Require an accepted modularity brief and exclusive file or + symbol ownership. If either is missing, return a blocker without editing. + Implement only the approved architecture-dominant structural change and its + directly required consumers and exports. Preserve established behavior and + compatibility unless the brief explicitly authorizes a migration. Do not edit + files assigned to another writer or add unrelated behavior. After the final + implementation is observable, reconcile every affected module index to that + state before reporting completion. An index-only implementation pass may own + only the selected index files and must use an accepted brief plus the + completed production diff or a repository-derived bootstrap inventory; it + must not change source code. When an index is missing, create it at the + required project-root path without requesting separate user approval. The + sole exception to the brief requirement is an explicitly approved handoff + whose complete scope is creating a missing language policy; it may edit only + the approved policy path and must return to `analysis` mode before structural + work. +- `review`: Stay read-only. Compare the completed production diff with the + accepted modularity brief, selected policies, and affected module indices. + A missing or stale affected entry is a contract violation. Report only + contract deviations and policy violations, ranked by impact, with file + references and exact policy sections. Do not apply fixes or raise style-only + findings. + +For every mode, follow any separate repository dependency or packaging policy +when triggered. Stop and return control to the parent for a missing or +conflicting policy, an unapproved breaking API decision, a policy exception, +scope expansion, or overlapping write ownership. The parent coordinates other +specialists; do not delegate to or direct peer agents. Recommend the next +specialist and give the parent a concise artifact to forward. + +Always return: +- mode, languages, policy files, and module index files selected +- index status, entries referenced or changed, and consistency evidence +- files inspected and files changed, if any +- public API and compatibility impact +- checks run and their results +- missing policies, conflicts, exceptions, unresolved coupling, and blockers +- recommended next owner and follow-up + +In `analysis`, include the modularity brief. In `implementation`, include the +brief decisions implemented and every deviation. In `review`, include findings +or an explicit statement that no contract or policy violations were found. +""" diff --git a/.codex/agents/modularity_maintainer/PYTHON_MODULARITY_POLICY.md b/.codex/agents/modularity_maintainer/PYTHON_MODULARITY_POLICY.md new file mode 100644 index 0000000..f325fc5 --- /dev/null +++ b/.codex/agents/modularity_maintainer/PYTHON_MODULARITY_POLICY.md @@ -0,0 +1,348 @@ +# Python Modularity Policy + +## Authority and Scope + +This policy governs Python package and module boundaries in any project that +selects it. It applies when work adds, removes, moves, splits, or merges an +importable unit; changes responsibility, dependency direction, public imports +or exports, shared abstractions, or state and resource ownership; or makes a +structural compatibility decision. It also applies to behavior-preserving +refactors with those effects. + +It does not require a modularity workflow for every Python edit. A localized +change within an established module responsibility remains outside this policy +when imports, exports, ownership, dependency direction, and public contracts do +not change. + +The terms **MUST**, **MUST NOT**, **SHOULD**, **SHOULD NOT**, and **MAY** are +normative: + +- **MUST** and **MUST NOT** define requirements. An exception requires explicit + authorization and the record described in + [Policy Exceptions](#policy-exceptions). +- **SHOULD** and **SHOULD NOT** define the normal design. Deviations require a + concrete reason in the change description. +- **MAY** identifies an allowed option, not a default requirement. + +This policy governs internal structure. Apply any project dependency, +packaging, security, or compatibility policies separately when a change crosses +those concerns. Repository instructions take precedence when they impose a +stricter compatible requirement. + +## Terms + +- A **package** is an importable namespace that owns a cohesive capability or + feature and may contain modules or subpackages. +- A **module** is one importable source file with a cohesive responsibility. +- A **feature boundary** contains behavior, data, and interfaces that change + for the same product reason. +- A **public interface** is any documented or exported import path, callable, + type, constant, data shape, exception, or observable behavior used outside + its owning module. +- A **private implementation** is an implementation detail not promised to + consumers. A leading underscore communicates intent but does not by itself + make external use safe. +- **Independent** means understandable, importable, testable, and reusable + without initializing or depending on unrelated features. It does not mean + dependency-free. +- **Cohesion** is the degree to which a unit's contents serve the same + responsibility. **Coupling** is the knowledge one unit requires about + another unit's implementation or lifecycle. + +## Python Module Index + +The project inventory is `/PYTHON_MODULE_INDEX.md`. It is a +project-owned descriptive navigation record for the current Python +implementation; this packaged policy remains the normative source for boundary +rules. Proposed or partially implemented structures belong in a modularity +brief, not in the index. When the index is missing, the modularity maintainer +MUST inventory the implementation and create it at the project root in its own +write-enabled pass; separate user approval is not required. + +### Coverage and entry format + +- The index MUST name every production importable package and module beneath + each declared Python source root exactly once. Cache directories, generated + bytecode, and test modules are excluded unless they are intentionally shipped + as part of the installed package. +- Each entry MUST record the import path, source path, one current + responsibility, supported public imports or entry points, direct internal + dependencies, owned state or external resources, material side effects, and + primary verification or documentation references. +- Package entries MUST identify supported re-exports. Determine the supported + surface from package-level exports such as `__all__`, documented import paths, + and entry points in the active build or packaging manifests. A direct import + used only by tests does not by itself make a symbol public. +- Dependencies MUST describe runtime direction between project modules. Type- + only imports MAY be noted when they materially constrain a boundary, but + standard-library and third-party import lists SHOULD NOT be duplicated. +- An entry MUST describe observed behavior only. It MUST NOT present a proposed + move, split, export, dependency, or ownership transfer as already implemented. + +### Freshness and reconciliation + +- Modularity analysis MUST read the index as a starting map and verify all + affected entries against the source files, package exports, imports, tests, + and documentation before relying on them. +- Adding, deleting, renaming, moving, splitting, or merging a module requires a + matching index change. So does a material change to responsibility, supported + imports, direct dependency direction, state or resource ownership, entry + points, or side effects. +- The index MUST be reconciled after the final implementation is observable and + within the same complete change. Removed units and obsolete paths MUST be + deleted rather than retained as historical records. +- When implementation and index disagree, the implementation is the observed + fact for diagnosis, but the change is incomplete. Either bring the code back + to the accepted boundary or update the index to the accepted implemented + state; do not silently choose a new design. +- Review MUST verify source-path parity for the whole declared source root and + semantic parity for every affected entry. A missing index, an unlisted module, + a deleted-but-listed module, or stale responsibility, export, dependency, or + ownership information is a policy violation. +- Index-only maintenance MUST NOT change production behavior. It requires a + repository-derived inventory and exclusive ownership of the index file. + +## Responsibility and Ownership + +### Required boundaries + +- Every package and module MUST have one concise responsibility that can be + stated without joining unrelated concerns with “and.” +- Code that changes for the same feature reason SHOULD remain together. Code + that changes for unrelated reasons SHOULD live behind separate boundaries. +- Each stateful resource, cache, registry, or lifecycle MUST have one clear + owner. Other modules MUST interact with it through that owner's public + interface. +- Feature-specific code MUST remain within its feature unless it satisfies the + shared-code criteria below. +- A module MUST NOT become a dumping ground such as an unbounded `utils`, + `helpers`, `common`, or `misc` module. A shared module needs a specific domain + name and responsibility. + +### When to split + +A package or module SHOULD be split when at least one of these conditions is +demonstrated: + +- it owns responsibilities that change independently; +- consumers need distinct subsets of its interface; +- testing one responsibility requires unrelated setup; +- optional infrastructure is pulled into consumers that do not use it; +- dependency direction becomes unclear or cyclic; +- separate ownership or lifecycle boundaries are being hidden. + +Line count alone MUST NOT trigger a split. The proposed units still need clear +responsibilities and interfaces. + +### When to merge + +Packages or modules SHOULD be merged when they have no meaningful independent +responsibility, always change together, expose forwarding-only interfaces, or +create indirection without reducing coupling. A merge MUST preserve intentional +public imports or include an authorized migration. + +## Dependency Direction + +- Dependencies MUST be explicit in imports, parameters, constructors, or + declared interfaces. Behavior MUST NOT depend on import order or unrelated + initialization having occurred first. +- Dependencies MUST point from orchestration and adapters toward stable domain + contracts, not from reusable domain logic toward entry points or concrete + infrastructure. +- Circular runtime imports between modules or packages are prohibited. +- `typing.TYPE_CHECKING`, local imports, string annotations, or deferred imports + MUST NOT be used merely to conceal an architectural cycle. They MAY break a + type-only import cycle when the runtime dependency direction is already + valid and documented. +- A module MUST NOT reach into another module's underscored names or mutate its + internal state. Cross-boundary use goes through the owner's public interface. +- A lower-level module MUST NOT import a higher-level workflow solely to call + back into it. Use an explicit callback, protocol, or data contract owned by + the lower-level boundary when inversion is required. +- Optional integrations SHOULD be isolated behind adapters so importing core + behavior does not require optional infrastructure. +- Dynamic imports MAY be used only for an intentional extension point or to + satisfy a documented platform constraint. They MUST validate the imported + interface and surface actionable failures. +- Production code MUST NOT modify `sys.path` to cross package boundaries. +- Wildcard imports are prohibited. + +## Public Interfaces and Compatibility + +- Public interfaces MUST be intentional, minimal, documented at the point of + ownership, and covered by contract-focused tests. +- Package `__init__.py` exports and module `__all__` declarations MUST match the + supported public surface. Adding a re-export is an API decision, not a + convenience-only cleanup. +- Consumers SHOULD import from the owning public path. They MUST NOT depend on + a peer's private file layout when a supported package-level path exists. +- Function and method signatures, accepted value domains, return shapes, + raised public exceptions, and externally visible side effects are part of + compatibility unless explicitly documented otherwise. +- Moving a public symbol MUST preserve its established import path with a + forwarding export when compatibility is required. The forwarding export + MUST have a removal plan if it is temporary. +- A breaking interface change requires explicit authorization, updated + consumers, migration notes, compatibility tests where practical, and removal + of stale exports in the same complete change. +- Facades MUST add a stable abstraction boundary. A facade that only duplicates + every underlying symbol without hiding volatility SHOULD NOT be introduced. + +## State, Side Effects, and Resource Lifecycles + +- Importing a reusable module MUST NOT perform network calls, filesystem + writes, provider authentication, subprocess execution, thread creation, or + application startup. +- Import-time reads or registration MAY occur only when they are deterministic, + local, required by the module's stated responsibility, and tested. +- Mutable module-level state SHOULD be avoided. When it is necessary, the + owning module MUST define initialization, mutation, synchronization, reset, + and shutdown behavior. +- External resources MUST have an explicit owner and lifecycle. Acquisition and + release SHOULD use context managers or an equally visible lifecycle API. +- Domain computation SHOULD remain deterministic and side-effect-light. + Filesystem, process, provider, clock, and environment access SHOULD be kept in + adapters and passed into reusable logic through narrow interfaces. +- Tests MUST be able to replace external collaborators without initializing the + entire application. + +## Reuse and Shared Code + +- Shared code MUST represent a stable domain or infrastructure concept, not + merely identical syntax. +- Extraction normally requires at least two real consumers with compatible + semantics. An approved near-term consumer MAY justify earlier extraction + when its contract is already known. +- Shared interfaces MUST be owned by the side that defines the abstraction, + not by an arbitrary consumer or a generic utility package. +- A reusable module SHOULD accept collaborators and data through explicit + inputs instead of importing application singletons. +- Feature flags or mode parameters MUST NOT accumulate unrelated behaviors in + one implementation. Split strategies or adapters when modes have distinct + dependencies, invariants, or lifecycles. +- Limited local duplication MAY be preferable to a premature abstraction. Any + later extraction must reconcile semantic differences rather than hide them. + +## Python Source and Import Conventions + +- Production packages and modules MUST remain under the source roots declared + by the project's build configuration, packaging manifests, or repository + instructions. Introducing or moving a source root is a packaging and + modularity decision that requires explicit scope. +- Importable package directories MUST contain `__init__.py` unless an explicit + namespace-package design is authorized and documented. +- Imports within a package MUST follow the project's established absolute or + explicit-relative style consistently. A change MUST NOT mix styles without a + documented interoperability reason. +- Imports SHOULD be grouped as standard library, third-party, and local, with + unused imports removed. +- Modules MUST be directly importable in a clean interpreter after installation + of declared dependencies. They MUST NOT rely on the current working directory + or test-only path manipulation. +- Public callables, classes, and data structures SHOULD use type annotations. + Boundary types MUST describe accepted optionality and collection shapes + accurately; `Any` requires a boundary-specific reason. +- Protocols or abstract base classes SHOULD be introduced only when multiple + implementations, substitution in tests, or a real dependency inversion + requires them. They MUST NOT duplicate a concrete class without reducing + coupling. +- Package data MUST be declared through the packaging configuration rather than + discovered from an assumed checkout layout. +- The active build or packaging manifest MUST be treated as the source of truth + for package discovery, entry points, package data, and dependency metadata. + Any separate project dependency or packaging policy also applies. + +## Readability and Documentation + +- Names MUST communicate domain responsibility. Generic names are acceptable + only inside a narrowly named owning module where their meaning is clear. +- Public modules, classes, and non-obvious functions SHOULD document purpose, + inputs, outputs, side effects, raised exceptions, and lifecycle constraints. +- Comments SHOULD explain why a boundary or invariant exists, not narrate + syntax. +- Functions and classes SHOULD remain focused enough that their invariants and + collaborators are visible without tracing unrelated workflows. +- File size, function length, or class count MAY be used as investigation + signals but MUST NOT be enforced as standalone architectural thresholds. +- Dead forwarding layers, obsolete compatibility aliases, and unused exports + MUST be removed when their approved compatibility period ends. + +## Testing Requirements + +- New or changed modules MUST have focused tests for their public behavior and + important failure modes. +- Tests for a public contract SHOULD import through its supported public path. + Direct-module tests MAY cover private algorithms but do not establish those + names as public API. +- A new or moved module MUST be tested for direct import in a clean process when + import-time behavior or dependency availability is material. +- Boundary tests MUST cover collaborator failures, invalid data, and resource + cleanup where those behaviors cross modules. +- Refactors MUST retain behavior tests before relying on new implementation + tests. Public symbol moves require coverage for preserved import paths or the + authorized migration. +- Test helpers MUST NOT become a production dependency. Production modules MUST + NOT import from `tests`. +- Verification MUST check for circular imports, stale import paths, unintended + exports, and the relevant focused and broader test suites. Use existing tools + before adding a new dependency solely for boundary checking. + +## Required Change Workflow + +Before editing: + +1. Read the root Python module index, or complete the required bootstrap when + it is missing. Check source-path parity and verify every affected entry + against the implementation. +2. State the current and intended responsibility of every affected package or + module. +3. Inventory public import paths, consumers, imports, side effects, state + ownership, tests, and documentation. +4. Draw or describe the intended dependency direction and identify any cycle, + private reach-through, or compatibility constraint. +5. Decide whether the work is a split, merge, move, extraction, adapter, or + interface change and explain why that operation improves the boundary. + +During implementation: + +1. Make the smallest complete structural change. +2. Ensure consumers, exports, type annotations, tests, and documentation are + updated within the same integrated change. Repository specialists MAY + complete those portions sequentially under the main agent; this requirement + does not authorize concurrent writers or overlapping file ownership. +3. Reconcile affected module-index entries after the implemented structure is + stable. When another agent owns production code, perform index maintenance + as a sequential, index-only pass. +4. Preserve public behavior and import paths unless a breaking change is + explicitly authorized. +5. Avoid unrelated cleanup and new abstractions outside the demonstrated + boundary need. + +Before completion: + +1. Import each affected module through its supported path. +2. Check for cycles, stale paths, private cross-boundary access, and unintended + import-time side effects. +3. Run focused tests followed by the relevant broader suite. +4. Review the diff for unrelated movement, compatibility changes, and missing + documentation. +5. Compare discovered importable units with the module index and recheck every + affected entry for responsibility, exports, dependencies, ownership, and + side effects. +6. Report the resulting responsibility and dependency direction, API impact, + module-index consistency, checks, and any remaining coupling. + +## Policy Exceptions + +An authorized exception MUST be recorded with: + +- the exact rule being waived; +- the technical reason normal compliance is not currently viable; +- the affected files and consumers; +- compatibility, maintenance, and testing risks; +- containment measures and verification performed; and +- a removal condition or explicit decision that the exception is permanent. + +An undocumented exception is a policy violation. Existing noncompliance does +not authorize new noncompliance; avoid expanding it and report it when it is +material to the requested change. diff --git a/.codex/agents/regression_diagnostician.toml b/.codex/agents/regression_diagnostician.toml new file mode 100644 index 0000000..629b645 --- /dev/null +++ b/.codex/agents/regression_diagnostician.toml @@ -0,0 +1,23 @@ +name = "regression_diagnostician" +description = "Read-only diagnostician for reproducing failures and tracing regressions to their cause." +sandbox_mode = "read-only" +developer_instructions = """ +Stay read-only. Reproduce the reported failure when possible, inspect the +relevant execution path, compare expected and actual behavior, and isolate the +smallest likely cause. + +Classify the likely cause as contained within an established module boundary +or crossing a package or module boundary. Flag dependency cycles, private +reach-through, misplaced responsibility, public-contract drift, and state or +resource ownership problems for modularity analysis. Diagnose the cause; do +not design or apply the structural fix. + +Do not apply a fix. Return: +- reproduction steps and observed evidence +- affected files, symbols, or configuration +- likely root cause and confidence +- `boundary_impact = contained | cross_boundary | uncertain` with rationale +- architecture constraints remediation must preserve +- focused remediation recommendation +- checks that should confirm the fix +""" diff --git a/.codex/agents/requirements_analyst.toml b/.codex/agents/requirements_analyst.toml new file mode 100644 index 0000000..b0f2fdf --- /dev/null +++ b/.codex/agents/requirements_analyst.toml @@ -0,0 +1,28 @@ +name = "requirements_analyst" +description = "Read-only analyst for clarifying repository-change requirements, acceptance criteria, and whether module-boundary analysis is required." +sandbox_mode = "read-only" +developer_instructions = """ +Stay in analysis mode and do not modify files. + +Inspect the repository and trace the relevant behavior. Turn the request into +explicit requirements, constraints, affected areas, risks, and acceptance +criteria. Identify ambiguities that would materially change the result. + +Classify the modularity gate as `required`, `not_required`, or `uncertain`. +Base the classification on whether the work changes package or module +responsibilities, dependency direction, public imports or exports, shared +abstractions, or state and resource ownership. Do not design the final module +boundary; provide evidence for the parent and modularity specialist. When the +gate is `required` or `uncertain`, identify the expected per-language module +index paths and the entries likely affected without treating the index as more +authoritative than the implementation. + +Return a concise handoff with: +- objective and assumptions +- relevant files and existing behavior +- proposed scope and acceptance criteria +- `modularity_gate = required | not_required | uncertain`, with rationale, + affected languages, and likely boundary scope +- expected module-index paths, current availability, and likely affected entries +- risks, open questions, and recommended next action +""" diff --git a/.codex/agents/runtime_implementer.toml b/.codex/agents/runtime_implementer.toml new file mode 100644 index 0000000..e8172cb --- /dev/null +++ b/.codex/agents/runtime_implementer.toml @@ -0,0 +1,37 @@ +name = "runtime_implementer" +description = "Implementation specialist for approved production code, configuration, and integration changes." +developer_instructions = """ +Implement only the approved, in-scope production change. Inspect the existing +code and conventions before editing, and preserve unrelated user changes. + +Require `mode = implementation`. If the mode is missing or different, remain +read-only and return the mismatch to the parent agent. + +The handoff must state `modularity_gate = required | not_required`. Do not +write when the value is missing or `uncertain`; return it to the parent for +resolution. When the gate is required, acknowledge and follow the accepted +modularity brief, including its public API, dependency direction, compatibility +constraints, and file ownership. If a required brief is missing, or +implementation reveals unapproved package or module restructuring, stop before +making that structural change and return the decision to the parent agent. + +Do not edit project module indices; `modularity_maintainer` owns their semantic +accuracy and updates them in a sequential pass. When a modularity brief applies, +report the exact implemented boundary delta needed to reconcile affected index +entries, including paths, responsibilities, exports, direct dependencies, and +state or resource ownership. + +Keep the change focused and complete. Avoid speculative abstractions and +unrequested dependency or architecture changes. Do not edit files assigned to +another writer. Do not rewrite documentation or tests except where a small, +directly required update is part of the assigned ownership; report remaining +work for the appropriate specialist. + +Run focused checks when practical and report: +- files changed and behavior implemented +- modularity brief followed and any deviation or newly discovered trigger +- implemented boundary delta for module-index reconciliation, when applicable +- public API and compatibility result +- checks run and their results +- assumptions, limitations, or follow-up work +""" diff --git a/.codex/agents/test_engineer.toml b/.codex/agents/test_engineer.toml new file mode 100644 index 0000000..cfa4de3 --- /dev/null +++ b/.codex/agents/test_engineer.toml @@ -0,0 +1,26 @@ +name = "test_engineer" +description = "Test specialist for designing, updating, and running coverage for production changes." +developer_instructions = """ +Focus on verification of the requested behavior and boundaries. + +Honor the handoff mode. In `analysis` mode, stay read-only and return a test +plan. In `implementation` mode, edit only assigned test files. Inspect existing +test conventions first, identify missing cases, and run the narrowest relevant +checks before broader checks. + +When a modularity brief applies, map tests to its boundary and compatibility +requirements. Cover supported import and export paths, dependency-cycle or +clean-import risks, compatibility shims, collaborator failures, and resource +cleanup as relevant. Do not modify production code to make a test pass; +classify and report production defects to the parent agent. Do not edit module +indices; report added, moved, or removed primary verification references that +require final index reconciliation. + +Return: +- scenarios covered and remaining gaps +- acceptance criteria and modularity requirements mapped to tests +- module-index verification-reference changes, when applicable +- files changed, if any +- exact checks run and results +- failures classified as test defects or production defects, with evidence +""" diff --git a/.codex/repository-engineering-kit.lock.json b/.codex/repository-engineering-kit.lock.json new file mode 100644 index 0000000..7d00367 --- /dev/null +++ b/.codex/repository-engineering-kit.lock.json @@ -0,0 +1,112 @@ +{ + "schemaVersion": 1, + "plugin": "repository-engineering", + "installedPluginVersion": "0.2.0", + "pluginSelector": "repository-engineering@repository-engineering-kit", + "installationSource": { + "kind": "remote", + "marketplace": "repository-engineering-kit", + "marketplaceManifest": ".agents/plugins/marketplace.json", + "marketplaceManifestSha256": "6e0f44e7cf797eff7cdc73c06b25271f33c49915a1bc37dc0c554df06537b7db", + "pluginSource": "plugins/repository-engineering", + "marketplaceSource": "https://github.com/Kims-DeveloperGroup/AI-engineering-kit.git", + "immutableRef": "2cab4e6a2d0d7f693c4a65a9021622410bf23966" + }, + "installedDistributionManifest": { + "path": "skills/repository-engineering-bootstrap/assets/distribution-manifest.json", + "sha256": "71696a46bf80bfd733e52cbc9baca62eabfe584df4dc0c61bbf3ac5806b9813c" + }, + "profiles": [ + "core", + "github", + "python" + ], + "files": [ + { + "asset": "skills/repository-engineering-bootstrap/assets/core/.codex/agents/documentation_maintainer.toml", + "target": ".codex/agents/documentation_maintainer.toml", + "ownership": "managed", + "installedAssetSha256": "dcfc1cea1af8f4cb5728f633d69f9f863ad071feabe21935715e0277d22501a0", + "installedTargetSha256": "dcfc1cea1af8f4cb5728f633d69f9f863ad071feabe21935715e0277d22501a0" + }, + { + "asset": "skills/repository-engineering-bootstrap/assets/core/.codex/agents/modularity_maintainer.toml", + "target": ".codex/agents/modularity_maintainer.toml", + "ownership": "managed", + "installedAssetSha256": "91b41393715b94b23330850dd135e5fa7b634583709fc7db60e746c13701f481", + "installedTargetSha256": "91b41393715b94b23330850dd135e5fa7b634583709fc7db60e746c13701f481" + }, + { + "asset": "skills/repository-engineering-bootstrap/assets/profiles/python/.codex/agents/modularity_maintainer/PYTHON_MODULARITY_POLICY.md", + "target": ".codex/agents/modularity_maintainer/PYTHON_MODULARITY_POLICY.md", + "ownership": "managed", + "installedAssetSha256": "400aeb7d64cc47bed7c551935540507a6282438501613fdfe2c84b668d239578", + "installedTargetSha256": "400aeb7d64cc47bed7c551935540507a6282438501613fdfe2c84b668d239578" + }, + { + "asset": "skills/repository-engineering-bootstrap/assets/core/.codex/agents/regression_diagnostician.toml", + "target": ".codex/agents/regression_diagnostician.toml", + "ownership": "managed", + "installedAssetSha256": "4b677f02df462ed29ac8838ddfcdeed14758142d4f4284b97bfa467b912510f5", + "installedTargetSha256": "4b677f02df462ed29ac8838ddfcdeed14758142d4f4284b97bfa467b912510f5" + }, + { + "asset": "skills/repository-engineering-bootstrap/assets/core/.codex/agents/requirements_analyst.toml", + "target": ".codex/agents/requirements_analyst.toml", + "ownership": "managed", + "installedAssetSha256": "dd4f6b901ad3d738bdc9a684a15ac8450205107f430119e609722bbf7b167d6f", + "installedTargetSha256": "dd4f6b901ad3d738bdc9a684a15ac8450205107f430119e609722bbf7b167d6f" + }, + { + "asset": "skills/repository-engineering-bootstrap/assets/core/.codex/agents/runtime_implementer.toml", + "target": ".codex/agents/runtime_implementer.toml", + "ownership": "managed", + "installedAssetSha256": "9b5160702a3e15c4753da5180d369a10313e32efd6803921e44856dfb1b76d1f", + "installedTargetSha256": "9b5160702a3e15c4753da5180d369a10313e32efd6803921e44856dfb1b76d1f" + }, + { + "asset": "skills/repository-engineering-bootstrap/assets/core/.codex/agents/test_engineer.toml", + "target": ".codex/agents/test_engineer.toml", + "ownership": "managed", + "installedAssetSha256": "ed58c94c7ae7ebebe4c4617afc2383d83c58734337597b8378015cd72fe585fc", + "installedTargetSha256": "ed58c94c7ae7ebebe4c4617afc2383d83c58734337597b8378015cd72fe585fc" + }, + { + "asset": "skills/repository-engineering-bootstrap/assets/core/AGENTS.managed.md", + "target": "AGENTS.md", + "ownership": "templated", + "installedAssetSha256": "0943a60be7dd1ecada4f2ddfda25b8d6a37193bc6bef24801de5fe661cb391a3", + "installedTargetSha256": "1cd27bc57f7027184e44c8b9b98d8e8b8b7b47e991c3945c405816f3c1aaaebe", + "startMarker": "", + "endMarker": "" + }, + { + "asset": "skills/repository-engineering-bootstrap/assets/profiles/github/AGENT_TASK_RUNBOOK.md", + "target": "AGENT_TASK_RUNBOOK.md", + "ownership": "managed", + "installedAssetSha256": "77f4d24b814cfced855f2b558bb9fe3c5b0544551fdf2695697a9992aebcfe4d", + "installedTargetSha256": "77f4d24b814cfced855f2b558bb9fe3c5b0544551fdf2695697a9992aebcfe4d" + }, + { + "asset": "skills/repository-engineering-bootstrap/assets/core/COMMIT_POLICY.md", + "target": "COMMIT_POLICY.md", + "ownership": "managed", + "installedAssetSha256": "a64067fb772fef987847bf3a1b0f31677522cd1d252031678bd2e95ac49504cf", + "installedTargetSha256": "a64067fb772fef987847bf3a1b0f31677522cd1d252031678bd2e95ac49504cf" + }, + { + "asset": "skills/repository-engineering-bootstrap/assets/profiles/python/DEPENDENCY_POLICY.md", + "target": "DEPENDENCY_POLICY.md", + "ownership": "managed", + "installedAssetSha256": "53b308d2dfddab7a5d4a77fc1e37b98119f39b3f289e9d09bad813238505bb22", + "installedTargetSha256": "53b308d2dfddab7a5d4a77fc1e37b98119f39b3f289e9d09bad813238505bb22" + }, + { + "asset": "skills/repository-engineering-bootstrap/assets/profiles/github/GITHUB_WORKFLOW.md", + "target": "GITHUB_WORKFLOW.md", + "ownership": "managed", + "installedAssetSha256": "3c021edd802e124c01670d01aa74bdc036ee7b5233b98fd9afefe7170cb22560", + "installedTargetSha256": "3c021edd802e124c01670d01aa74bdc036ee7b5233b98fd9afefe7170cb22560" + } + ] +} diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..31defe3 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,177 @@ +name: CI + +on: + pull_request: + push: + +permissions: + contents: read + +jobs: + verify: + name: Python ${{ matrix.python-version }} + runs-on: ubuntu-latest + timeout-minutes: 20 + strategy: + fail-fast: false + matrix: + python-version: + - "3.11" + - "3.14" + env: + MATRIX_PYTHON: ${{ matrix.python-version }} + + steps: + - name: Check out repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + with: + persist-credentials: false + + - name: Set up Python + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 + with: + python-version: ${{ matrix.python-version }} + + - name: Set up uv + uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d + with: + version: "0.12.5" + + - name: Configure external uv environment + run: | + set -euo pipefail + printf 'UV_PROJECT_ENVIRONMENT=%s\n' \ + "$RUNNER_TEMP/generic-rag-venv-$MATRIX_PYTHON" \ + >> "$GITHUB_ENV" + + - name: Verify locked environment + run: | + set -euo pipefail + uv lock --check + uv sync --locked --all-groups --python "$MATRIX_PYTHON" + + - name: Run unit and boundary tests + run: | + set -euo pipefail + uv run --frozen python -m unittest discover \ + -s tests \ + -p 'test_*.py' \ + -v + + - name: Run lint and format checks + if: matrix.python-version == '3.11' + run: | + set -euo pipefail + uv run --frozen ruff check src tests + uv run --frozen ruff format --check src tests + + - name: Run strict type checks + if: matrix.python-version == '3.11' + run: | + set -euo pipefail + uv run --frozen mypy --strict src tests + + - name: Compile source and tests + if: matrix.python-version == '3.11' + env: + PYTHONPYCACHEPREFIX: ${{ runner.temp }}/generic-rag-pycache + run: | + set -euo pipefail + uv run --frozen python -m compileall -q src tests + + - name: Build and verify clean distributions + if: matrix.python-version == '3.11' + run: | + set -euo pipefail + + rag_dist_dir="$RUNNER_TEMP/generic-rag-dist" + rag_extract_dir="$RUNNER_TEMP/generic-rag-sdist" + rag_rebuilt_dir="$RUNNER_TEMP/generic-rag-rebuilt" + rag_install_env="$RUNNER_TEMP/generic-rag-clean-venv" + rag_probe_dir="$RUNNER_TEMP/generic-rag-probe" + + mkdir -p \ + "$rag_dist_dir" \ + "$rag_extract_dir" \ + "$rag_rebuilt_dir" \ + "$rag_probe_dir" + + uv run --frozen python -m build \ + --no-isolation \ + --outdir "$rag_dist_dir" + uv run --frozen python tests/support/verify_artifacts.py \ + "$rag_dist_dir" + + rag_sdist_path="$( + find "$rag_dist_dir" \ + -maxdepth 1 \ + -type f \ + -name '*.tar.gz' \ + -print \ + -quit + )" + test -n "$rag_sdist_path" + uv run --frozen python -m tarfile \ + -e "$rag_sdist_path" "$rag_extract_dir" + + rag_source_root="$( + find "$rag_extract_dir" \ + -mindepth 1 \ + -maxdepth 1 \ + -type d \ + -print \ + -quit + )" + test -n "$rag_source_root" + uv run --frozen python -m build \ + --wheel \ + --no-isolation \ + --outdir "$rag_rebuilt_dir" \ + "$rag_source_root" + + rag_rebuilt_wheel="$( + find "$rag_rebuilt_dir" \ + -maxdepth 1 \ + -type f \ + -name '*.whl' \ + -print \ + -quit + )" + test -n "$rag_rebuilt_wheel" + + python -m venv "$rag_install_env" + rag_clean_python="$rag_install_env/bin/python" + "$rag_clean_python" -m pip install \ + --no-index \ + --no-deps \ + "$rag_rebuilt_wheel" + "$rag_clean_python" -m pip check + + ( + cd "$rag_probe_dir" + "$rag_clean_python" -I -B - <<'PY' + from importlib import metadata, resources + + assert metadata.version("generic-rag") == "0.1.0" + assert not metadata.requires("generic-rag") + assert resources.files("generic_rag").joinpath("py.typed").is_file() + PY + ) + + for rag_module in \ + generic_rag \ + generic_rag.errors \ + generic_rag.contracts \ + generic_rag.ports \ + generic_rag.projection \ + generic_rag.projection_integrity \ + generic_rag.retrieval + do + ( + cd "$rag_probe_dir" + "$rag_clean_python" -I -B \ + "$GITHUB_WORKSPACE/tests/support/clean_import_probe.py" \ + "$rag_module" \ + --forbid-path "$GITHUB_WORKSPACE" + ) + done diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000..6debfb3 --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,183 @@ + +## Repository Engineering Workflow + +### Working Method + +For every change: + +1. Inspect the repository and relevant existing behavior. +2. Clarify requirements, scope, constraints, acceptance criteria, and the + GitHub-planning and modularity gates. +3. Pass every required gate before a writer starts. +4. Make the smallest complete change that satisfies the requirement. +5. Run proportionate checks and review the final diff. + +For explanation, review, diagnosis, or planning, remain read-only unless the +request also authorizes implementation. For requests to build, change, or fix, +make the requested in-scope changes and verify them. Preserve unrelated +changes. Avoid speculative abstractions, unnecessary dependencies, and changes +to behavior outside the requested scope. Ask when a missing requirement would +materially change the result. Do not expand scope or make a breaking +compatibility decision without approval. + +### GitHub Planning Gate + +When the `github` profile is selected, record +`github_planning_gate = not_required | pending | passed | blocked` before a +task branch, worktree, write-capable agent, or tracked-file edit, together with +the classification reason. Read-only repository and GitHub inspection, +requirements, diagnosis, test planning, and modularity analysis may continue +while the gate is pending. + +Treat work as large when it has Size `L` or `XL`; two or more independently +complete implementation slices or parent integration criteria; at least three +production modules or two responsibility domains; modularity or migration +coordination; dependency sequencing; multiple owners, worktrees, or delivery +phases; or a parent integration branch. Companion tests and documentation do +not count toward these thresholds. Uncertainty defaults to large unless a +maintainer records a smaller classification. + +Large work must satisfy the selected GitHub workflow's planning requirements +before any writer starts. Verify the root Issue, all known descendants, native +relationships, canonical Project items, required metadata, approved base, and +first executable leaf. If a missing gate or new scope appears after writing +starts, stop further writes, preserve and report the existing diff, and +initialize that scope before continuing. + +### Development Agents + +Use project-scoped agents when their expertise improves the result: + +- `requirements_analyst`: inspect the repository, clarify requirements, and + define acceptance criteria before substantial or ambiguous work; classify + the modularity gate. +- `regression_diagnostician`: reproduce failures, trace regressions, and + determine whether their cause crosses a package or module boundary. +- `modularity_maintainer`: analyze, implement, or review cohesive module + boundaries under the applicable language policy and keep project module + indices synchronized. +- `runtime_implementer`: implement approved production-code, configuration, + and integration changes within accepted boundaries. +- `test_engineer`: design, add, update, and run focused verification. +- `documentation_maintainer`: update README files, policies, guides, + references, and examples after behavior and boundaries stabilize. + +These are repository-development agents. Product or domain agents, when +present, remain separate unless the task explicitly puts them in scope. + +### Modularity Gate + +Set `modularity_gate = required` when work adds, removes, renames, moves, +splits, or merges an importable package or module; redistributes responsibility, +state, resources, or lifecycle ownership; changes dependency direction or +introduces or resolves a cycle, replaces private cross-boundary access, or +changes a package export or public import path; introduces or materially +changes a shared abstraction, adapter, facade, or protocol; extracts code for +multiple consumers; or requires a structural compatibility or migration +decision. A regression caused by architectural coupling or a boundary-policy +violation also requires the gate. + +Use `modularity_gate = not_required` for localized work inside an established +module with unchanged imports, exports, ownership, dependencies, and public +contracts; tests-only or docs-only work; formatting, comments, or typing +cleanup; ordinary data or configuration edits; and dependency metadata alone. + +When required, call `modularity_maintainer` with an explicit mode: + +- `analysis`: read-only boundary design before writing. +- `implementation`: exclusive ownership of an accepted architecture-dominant + structural change or an index-only reconciliation. +- `review`: read-only comparison of an implemented diff with the accepted + brief, language policies, and module indices. + +If a writer discovers an unapproved gate trigger, stop that writer and run +modularity analysis before continuing. Never let an implementation agent make +an incidental architecture decision silently. + +Before modularity work, select one policy per affected implementation language. +Use a root `_MODULARITY_POLICY.md` override when present; otherwise +use `.codex/agents/modularity_maintainer/_MODULARITY_POLICY.md`. The +first match wins and policies are never merged. Each project module index is +the project-owned root `_MODULE_INDEX.md`. + +### Orchestration Sequence + +1. The main agent classifies the request and records preliminary gate + decisions. +2. For substantial or ambiguous work, run `requirements_analyst` read-only. + For a reported or observed failure, run `regression_diagnostician` + read-only. Independent investigations may run in parallel. Resolve an + uncertain modularity gate before writing. +3. When modularity is required, run `modularity_maintainer` in `analysis` mode, + pass it relevant requirements and diagnosis findings, and have it select and + verify the affected root `_MODULE_INDEX.md` files. Resolve user + decisions and accept its brief, including the index delta, before writing. +4. After requirements and any boundary brief are accepted, pass the GitHub + planning gate when selected and required. No writer starts while it is + `pending` or `blocked`. +5. Keep exactly one production writer active. Use `modularity_maintainer` for + architecture-dominant structural work and `runtime_implementer` for + behavior-dominant work constrained by the brief. +6. If both production specialists are necessary, assign explicit, + nonoverlapping files or symbols and run them sequentially. Prefer one owner + when separate passes would create an invalid intermediate state. +7. `modularity_maintainer` owns module-index accuracy. When it owns structural + implementation, update affected indices after the implementation is stable. + When `runtime_implementer` owns production code, run a later sequential + index-only implementation pass against the completed production diff. +8. When `runtime_implementer` implements a boundary brief, review the completed + production diff and reconciled indices with `modularity_maintainer` before + test or documentation edits. Return production deviations to the production + owner and index drift to `modularity_maintainer`, then repeat until resolved + or explicitly excepted. A separate self-review is unnecessary when + `modularity_maintainer` performed the implementation; its report and the + main agent's review provide the gate. +9. After production, APIs, and boundaries stabilize, `test_engineer` may plan + read-only and then owns test edits sequentially; + `documentation_maintainer` follows. +10. If tests or documentation change indexed verification or documentation + references or reveal drift, run a final sequential index-only + `modularity_maintainer` implementation pass followed by read-only review. +11. The main agent integrates results, checks index consistency, reviews the + complete diff, runs final checks, and reports to the user. + +The main agent is the coordination hub. Reuse specialist threads when +practical, pass distilled decisions rather than raw logs, and do not let peer +specialists delegate to, coordinate with, or silently redistribute ownership +among one another. + +### Handoff and Result Contract + +Every specialist handoff must include: + +- `mode = analysis | implementation | review`; +- objective and relevant evidence; +- `github_planning_gate`, its rationale, and verified Issue, Project, and + Iteration evidence when applicable; +- approved scope and explicit out-of-scope work; +- acceptance criteria and compatibility constraints; +- modularity-gate decision and accepted brief when applicable; +- applicable module-index paths, status, and accepted delta; +- read-only targets or exclusively owned files and symbols; +- upstream findings and accepted decisions; +- required checks and expected output; and +- stop conditions for missing or conflicting policy, breaking API choice, + scope deviation, unexpected structural work, or overlapping ownership. + +Every specialist result must report findings, files inspected or changed, +module indices consulted or updated, implementation/index consistency, public +API and compatibility impact, checks and results, blockers, and the recommended +next owner or action. Pass distilled decisions and evidence between agents +rather than raw logs. + +A missing language policy blocks modularity work. The modularity maintainer +reports the exact language and expected packaged-policy path; the main agent +asks the user whether to create it and then resumes the same specialist thread. +If creation is declined, record that decision and that no language policy +governs that part of the work before proceeding under general engineering +judgment. A missing module index requires no separate user permission and does +not block read-only analysis: derive the complete inventory, record the root +index path and contents in the brief, and later create the project-owned root +index in an exclusive implementation pass. Structural work is incomplete +while an affected index is missing or stale. + diff --git a/AGENT_TASK_RUNBOOK.md b/AGENT_TASK_RUNBOOK.md new file mode 100644 index 0000000..4b6b00b --- /dev/null +++ b/AGENT_TASK_RUNBOOK.md @@ -0,0 +1,405 @@ +# Agent Task Runbook + +Use this runbook for GitHub-tracked implementation. The rules in +[GITHUB_WORKFLOW.md](GITHUB_WORKFLOW.md) and +[COMMIT_POLICY.md](COMMIT_POLICY.md) remain authoritative. An explicit +implementation request bundles only the initial-planning mutations defined by +the workflow policy and creation of its native Development-linked task branch. +Obtain explicit authorization for every other GitHub mutation, push, pull +request, merge, Issue closure, staging, commit, or branch deletion. + +## Initialize GitHub Planning + +Before task branches, worktrees, writers, or tracked-file edits, read +`README.md` and the policies, classify the planning gate, and search all Issue +states for an existing matching scope. Preflight repository access, `gh` auth +with `project` scope, the canonical repository and Project declared in the +consumer repository instructions, Project fields, milestones, Iterations, +dependencies, and current worktrees. When an installation lock exists, use its +selected profiles as authoritative. Compare repository instructions only when +they independently declare a profile selection; this generic profile never +declares consumer profiles or GitHub identifiers. + +For large work, draft the complete root and all known descendants, including +nested parents and leaves, after read-only requirements and any modularity +analysis. Create the root first, then each descendant; add every URL to Project +identified by the canonical Project number declared in repository instructions +and establish native relationships. Execute and verify one mutation at a time; +stop on the first failure. This abbreviated flow uses stable outputs: + +```bash +set -euo pipefail + +: "${REPOSITORY_ENGINEERING_REPOSITORY:?Set canonical owner/repository from repository instructions}" +: "${REPOSITORY_ENGINEERING_PROJECT_OWNER:?Set from repository instructions}" +: "${REPOSITORY_ENGINEERING_PROJECT_NUMBER:?Set from repository instructions}" +repo_slug="$REPOSITORY_ENGINEERING_REPOSITORY" +project_owner_slug="$REPOSITORY_ENGINEERING_PROJECT_OWNER" +project_number_value="$REPOSITORY_ENGINEERING_PROJECT_NUMBER" + +test "$(gh repo view "$repo_slug" --json nameWithOwner --jq .nameWithOwner)" = "$repo_slug" + +parent_url="$(gh issue create --repo "$repo_slug" --title "" \ + --body-file "" --label "" --label "area:")" +parent_item_id="$(gh project item-add "$project_number_value" \ + --owner "$project_owner_slug" --url "$parent_url" --format json --jq .id)" + +child_url="$(gh issue create --repo "$repo_slug" --title "" \ + --body-file "" --label "" --label "area:" \ + --milestone "")" +child_item_id="$(gh project item-add "$project_number_value" \ + --owner "$project_owner_slug" --url "$child_url" --format json --jq .id)" +child_number="${child_url##*/}" +child_id="$(gh api --method GET "repos/$repo_slug/issues/$child_number" --jq .id)" +parent_number="${parent_url##*/}" +gh api --method POST "repos/$repo_slug/issues/$parent_number/sub_issues" \ + -F sub_issue_id="$child_id" +``` + +Set the three task-specific environment variables from the exact values in the +consumer repository instructions before execution, then preflight all three +resolved identifiers. Never infer a repository from the checkout remote or a +Project solely from its number. + +Use native `blocked by` relationships for dependencies. Resolve Project, item, +field, option, and Iteration IDs at runtime. Set one logical Iteration on the +root and every descendant: + +```bash +gh project item-edit --id --project-id \ + --field-id --iteration-id +``` + +Set all other required metadata from the policy. If no unused logical +Iteration exists for a new root, stop for explicit Project-schema +authorization. Set Priority through the organization Issue field, not the +Project field with the same display name. + +Re-read Issue bodies, native hierarchy and dependencies, Project membership, +fields, milestone, and the root-wide Iteration. Record stable URLs and IDs. +Set `github_planning_gate = passed` only when the graph is complete and one +unblocked leaf is `Ready`; otherwise set `blocked`, preserve successful +mutations, and repair only verified missing state. A standalone task uses the +same procedure without a parent. A later child must pass this check before its +scope is written. + +## Branch and Worktree Model + +```text + +└── issue-100-parent parent worktree; PR to default branch (merge commit) + ├── issue-101-first-task child worktree; PR to parent (squash merge) + └── issue-102-next-task child worktree; PR to parent (squash merge) +``` + +- Record the parent branch and exact starting commit in the parent Issue. + Normally create it from the current `$repo_remote/$default_branch`, after + resolving both the unique canonical-repository remote and `default_branch`; + never assume that `origin` or a same-named local branch is the intended base. +- Name parent and child branches `issue--`. +- Put task worktrees in a sibling root such as + `../-worktrees/issue--/`. Keep the primary + worktree on the resolved default branch and preserve its unrelated changes. +- Assign one integration owner to the parent worktree and one owner to each + child worktree. Worktrees may coexist, but only one write-capable owner is + active at a time. A dependent child waits until its prerequisite is + integrated. +- Do not commit feature work directly to the parent branch. It accepts child + squash merges and synchronization merges from the resolved default branch. + Handle an integration fix through a child Issue, branch, and worktree. +- Do not rebase or force-update a published parent or child branch. After a + fresh fetch and approval, merge `$repo_remote/$default_branch` into the + parent and the parent into active children; resolve a child's conflicts in + its own worktree. + +Every child PR targets the parent branch and uses **Squash and merge**. Child +commits still follow the commit policy, while the resulting squash commit must +be one coherent change that identifies the child Issue and PR. The final +parent PR targets the resolved default branch and uses a merge commit, +preserving one squash commit per child PR. + +For a standalone leaf Issue, create its branch in a sibling worktree from the +approved default-branch ref and target its PR directly to the resolved default +branch; omit the parent steps. + +## Create the Worktrees + +First fetch and inspect without changing branches: + +```bash +set -euo pipefail + +: "${REPOSITORY_ENGINEERING_REPOSITORY:?Set canonical owner/repository from repository instructions}" +repo_slug="$REPOSITORY_ENGINEERING_REPOSITORY" +canonical_repo_slug="$(gh repo view "$repo_slug" --json nameWithOwner --jq .nameWithOwner)" +normalize_repo_slug() { + printf '%s' "$1" | tr '[:upper:]' '[:lower:]' +} +test "$(normalize_repo_slug "$canonical_repo_slug")" = \ + "$(normalize_repo_slug "$repo_slug")" +repo_slug="$canonical_repo_slug" + +repo_remote="" +repo_remote_matches=0 +while IFS= read -r remote_name; do + remote_url="$(git remote get-url "$remote_name")" + remote_slug="$( + gh repo view "$remote_url" --json nameWithOwner --jq .nameWithOwner \ + 2>/dev/null || true + )" + if test -n "$remote_slug" && \ + test "$(normalize_repo_slug "$remote_slug")" = \ + "$(normalize_repo_slug "$repo_slug")"; then + repo_remote="$remote_name" + repo_remote_matches=$((repo_remote_matches + 1)) + fi +done < <(git remote) +test "$repo_remote_matches" -eq 1 +test -n "$repo_remote" + +git fetch "$repo_remote" +git worktree list +git status --short +default_branch="$(gh repo view "$repo_slug" \ + --json defaultBranchRef --jq .defaultBranchRef.name)" +test -n "$default_branch" +git show-ref --verify "refs/remotes/$repo_remote/$default_branch" +git show-ref --verify "refs/heads/$default_branch" +git log --left-right --count \ + "$repo_remote/$default_branch...$default_branch" +``` + +Fail if no local remote or more than one local remote resolves to the canonical +repository identity. Do not select a remote by conventional name or silently +fall back to `origin`. If the local default branch is ahead, behind, or +diverged, identify and record the approved base commit before continuing. +After the gate passes, create the remote task branch through the Issue's native +Development relationship. Resolve the consumer-owned repository slug, Issue, +exact approved base object, and branch name at runtime: + +```bash +issue_number= +branch=issue-- +base_oid= +repo_id="$(gh repo view "$repo_slug" --json id --jq .id)" +issue_id="$(gh issue view "$issue_number" --repo "$repo_slug" --json id --jq .id)" + +gh api graphql \ + -f query='mutation($repo:ID!,$issue:ID!,$oid:GitObjectID!,$name:String!){ + createLinkedBranch(input:{repositoryId:$repo,issueId:$issue,oid:$oid,name:$name}){ + linkedBranch{ref{name target{oid}}} + } + }' \ + -F repo="$repo_id" -F issue="$issue_id" -F oid="$base_oid" -f name="$branch" +``` + +Stop if the branch already exists but is not linked; do not silently recreate, +rename, or replace it. Use a linked PR to repair legacy traceability when that +action is separately authorized. The Development operation creates the remote +branch. Fetch it, then adapt the tracking-worktree template: + +```bash +branch_head_ref="refs/heads/$branch" +branch_remote_ref="refs/remotes/$repo_remote/$branch" +branch_fetch_refspec="$branch_head_ref:$branch_remote_ref" +git check-ref-format "$branch_head_ref" +git check-ref-format "$branch_remote_ref" +if ! git config --get-all "remote.$repo_remote.fetch" | \ + grep -Fqx -- "$branch_fetch_refspec"; then + git config --add "remote.$repo_remote.fetch" "$branch_fetch_refspec" +fi +git fetch "$repo_remote" "$branch_fetch_refspec" +git show-ref --verify "$branch_remote_ref" + +git worktree add --track -b "$branch" \ + ../-worktrees/issue--/issue-- \ + "$branch_remote_ref" +``` + +The narrow branch-specific fetch mapping makes the explicit remote-tracking +ref recognizable to `--track` even when the remote's existing fetch mappings +exclude the task branch. Do not replace or broaden other remote mappings. + +For a parent, use its parent-worktree path; for a child, use its child-worktree +path and the current clean parent commit as `base_oid`. If the linked branch +already exists locally, omit `--track -b` and add a worktree for that branch. +A branch can be checked out in only one worktree. + +Verify the Issue's Development link before writing: + +```bash +repo_owner="${repo_slug%%/*}" +repo_name="${repo_slug#*/}" + +linked_branch_pages="$(gh api graphql --paginate --slurp \ + -f query='query($owner:String!,$repo:String!,$number:Int!,$endCursor:String){ + repository(owner:$owner,name:$repo){ + nameWithOwner + issue(number:$number){ + number + repository{nameWithOwner} + linkedBranches(first:100,after:$endCursor){ + nodes{ref{name}} + pageInfo{hasNextPage endCursor} + } + } + } + }' \ + -F owner="$repo_owner" -F repo="$repo_name" \ + -F number="$issue_number")" + +test "$(printf '%s\n' "$linked_branch_pages" | jq -r \ + --arg repo "$repo_slug" --argjson issue "$issue_number" ' + length > 0 and all(.[]; + ((.data.repository.nameWithOwner | ascii_downcase) == + ($repo | ascii_downcase)) and + .data.repository.issue.number == $issue and + ((.data.repository.issue.repository.nameWithOwner | ascii_downcase) == + ($repo | ascii_downcase))) + ')" = true + +test "$(printf '%s\n' "$linked_branch_pages" | jq -r \ + --arg branch "$branch" ' + [.[].data.repository.issue.linkedBranches.nodes[].ref.name] | + index($branch) != null + ')" = true +``` + +The paginated query and both assertions must succeed before any tracked-file +write. A printed response, a bounded first page, or a similarly named unlinked +branch is not verification. + +After a PR is linked, use the ProjectV2 item node ID and Project node ID +resolved during planning. Query that exact item, prove that it belongs to the +declared Project and canonical repository Issue, paginate all linked pull +requests, and verify the expected canonical-repository PR. Do not select a +Project item from a bounded item list or by Issue number alone: + +```bash +: "${REPOSITORY_ENGINEERING_PROJECT_OWNER:?Set from repository instructions}" +: "${REPOSITORY_ENGINEERING_PROJECT_NUMBER:?Set from repository instructions}" +project_owner_slug="$REPOSITORY_ENGINEERING_PROJECT_OWNER" +project_number_value="$REPOSITORY_ENGINEERING_PROJECT_NUMBER" +project_id= +project_item_id= +pr_number= + +linked_pr_pages="$(gh api graphql --paginate --slurp \ + -f query='query($item:ID!,$endCursor:String){ + node(id:$item){ + __typename + ... on ProjectV2Item{ + id + project{ + id + number + owner{ + __typename + ... on Organization{login} + ... on User{login} + } + } + content{ + __typename + ... on Issue{number repository{nameWithOwner}} + } + fieldValueByName(name:"Linked pull requests"){ + __typename + ... on ProjectV2ItemFieldPullRequestValue{ + pullRequests(first:100,after:$endCursor){ + nodes{number url repository{nameWithOwner}} + pageInfo{hasNextPage endCursor} + } + } + } + } + } + }' \ + -F item="$project_item_id")" + +test "$(printf '%s\n' "$linked_pr_pages" | jq -r \ + --arg item "$project_item_id" \ + --arg project "$project_id" \ + --arg owner "$project_owner_slug" \ + --argjson project_number "$project_number_value" \ + --arg repo "$repo_slug" \ + --argjson issue "$issue_number" ' + length > 0 and all(.[]; + .data.node.__typename == "ProjectV2Item" and + .data.node.id == $item and + .data.node.project.id == $project and + .data.node.project.number == $project_number and + ((.data.node.project.owner.login | ascii_downcase) == + ($owner | ascii_downcase)) and + .data.node.content.__typename == "Issue" and + .data.node.content.number == $issue and + ((.data.node.content.repository.nameWithOwner | ascii_downcase) == + ($repo | ascii_downcase)) and + .data.node.fieldValueByName.__typename == + "ProjectV2ItemFieldPullRequestValue") + ')" = true + +test "$(printf '%s\n' "$linked_pr_pages" | jq -r \ + --arg repo "$repo_slug" --argjson pr "$pr_number" ' + [.[].data.node.fieldValueByName.pullRequests.nodes[] | + select(.number == $pr and + ((.repository.nameWithOwner | ascii_downcase) == + ($repo | ascii_downcase)))] | length > 0 + ')" = true +``` + +## Task Lifecycle + +1. **Pass the planning gate.** Complete or verify the initialization above. + Record the gate state, Issue graph, Project metadata, logical Iteration, + dependencies, approved base, and authorization. Stop before writing unless + the gate is `passed` or `not_required`. +2. **Prepare the parent.** Create or reuse its Development-linked branch and + worktree at the recorded base commit, verify the link, assign the + integration owner, and move the parent to the correct Project state. After + the first child lands, open the parent PR to the resolved default branch as + a draft with `Refs #` when authorized. +3. **Start one child.** Confirm that the child is `Ready`, assigned the root's + Iteration, and unblocked. Create its Development-linked branch and worktree + from the current parent commit, verify the link, assign its owner, and move + it to `In progress`. +4. **Implement and verify.** Change only the child scope. Follow the commit + policy, update tests and documentation, run relevant checks, and review the + final diff and worktree status. +5. **Review the child.** Merge the latest parent branch into the child without + rebasing, rerun affected checks, and open a draft PR whose base is the + parent branch and whose head is the child branch. Use `Refs #`. Mark + it ready and move the child to `In review` only when all evidence is present + and the PR appears in Development and Project `Linked pull requests`. +6. **Integrate the child.** Reconfirm the PR base and head, then use **Squash + and merge** when authorized. The child must contain the current parent tip; + otherwise merge the parent into it, rerun affected checks, and repeat + review. Record the child head SHA and resulting squash SHA, then fetch and + fast-forward the clean parent worktree from + `$repo_remote/` with `--ff-only`; stop on divergence. Verify + the squash diff and run focused integration checks there. On success, close + the child Issue manually and + verify `Done`. On failure, block sibling merges and keep the child open in + `In progress`; require an authorized revert PR or follow-up child fix instead + of editing the parent directly. If it was already closed, reopen it to + `Backlog` and replan it as required by the policy. +7. **Complete the parent.** After all required children are `Done`, merge the + exact approved `$repo_remote/$default_branch` tip into the parent after a + fresh fetch, resolve conflicts there, and run the complete parent + acceptance checks. Mark the parent PR ready, replace `Refs #` with + `Closes #`, move it to `In review`, and merge it to the resolved + default branch with a merge commit when authorization includes the resulting + parent closure. Record and verify the final merge SHA, Issue closure, + Project `Done`, and milestone state. +8. **Clean up and hand off.** Remove a worktree only after its status is clean + and its integration is verified. Never use forced worktree removal. Branch + deletion requires separate authorization; because squash merging does not + mark a child branch as Git-merged, verify PR and commit recoverability before + deleting it. Retain the parent worktree until the parent reaches the + resolved default branch. + +The handoff must include the planning-gate state and evidence, Issue and PR +links, base/head branches and SHAs, resulting squash SHAs, worktree paths, +Development and `Linked pull requests` evidence, Project metadata and +Iteration, completed and remaining criteria, checks, blockers, cleanup, GitHub +mutations, and the next owner or action. diff --git a/COMMIT_POLICY.md b/COMMIT_POLICY.md new file mode 100644 index 0000000..169094b --- /dev/null +++ b/COMMIT_POLICY.md @@ -0,0 +1,70 @@ +# Commit Policy + +## Principle + +A commit is a unit of a complete function or logic. + +Each commit should represent one coherent, working change that can be +understood, reviewed, and integrated independently. A commit should make +clear what was changed and why it belongs together. + +## Rules + +- Keep each commit focused on one feature, fix, refactor, or documentation + change. +- A commit must be complete and functional. Do not commit half-implemented + logic, broken intermediate states, or changes that depend on a later commit. +- Keep unrelated formatting, cleanup, and generated-file changes separate. +- Include related tests and documentation in the same commit when they are + part of the change. +- Prefer small commits, but do not split one logical change into artificial + pieces. +- Run the relevant checks before committing and review the diff for unintended + changes. + +## Commit Messages + +Use a concise imperative subject that describes the completed change. Start +with a verb and make the subject specific enough to be understood without +reading the entire diff. + +Examples: + +```text +Add request schema validation +Fix duplicate task assignments +Document session state transitions +``` + +Add a body when context is needed. Explain why the change was made and note +important implementation, compatibility, or migration details. Do not use the +body to combine unrelated changes. + +## History Hygiene + +Avoid vague messages such as `WIP`, `misc`, or `fix things`. Keep incomplete +work local until it forms a complete unit, or squash it before sharing the +branch. When correcting the immediately preceding commit, amend it when +appropriate; otherwise create a new, complete commit that clearly describes +the correction. + +### Amend Rules + +A commit may be amended only when all of these conditions apply: + +- It is the immediately preceding commit (`HEAD`). +- It has not been pushed or shared with other contributors. +- The amendment belongs to the same logical function or change. +- It does not add unrelated cleanup or new scope. +- Relevant checks are rerun and the final diff is reviewed. + +Create a new commit instead when any of these conditions apply: + +- The target commit is not `HEAD`. +- The commit has already been pushed or shared. +- The correction represents a separate logical change. +- Amending would obscure review history or remove useful context. +- The change requires coordination with other contributors. + +Amending a shared commit requires explicit agreement from all affected +contributors and confirmation that the branch can be safely force-updated. diff --git a/DEPENDENCY_POLICY.md b/DEPENDENCY_POLICY.md new file mode 100644 index 0000000..9552061 --- /dev/null +++ b/DEPENDENCY_POLICY.md @@ -0,0 +1,20 @@ +# Python Dependency Policy + +- Use `pyproject.toml` as the single source of truth for project metadata and + dependencies. +- Separate runtime, development, and test dependencies. +- Use one package manager consistently and commit its lock file. +- Install dependencies in a virtual environment; never rely on global + packages. +- Declare every directly imported package explicitly. +- Add dependencies only when the standard library or an existing dependency + cannot reasonably provide the required behavior. +- Review a package's maintenance, license, security history, size, and Python + compatibility before adding it. +- Keep `pyproject.toml` and the lock file synchronized. Do not edit the lock + file manually. +- Run tests, linting, type checks, and dependency audits after dependency + changes. +- Remove unused dependencies and keep dependency changes in focused commits. +- CI and deployment must install from the lock file and fail on dependency + metadata inconsistencies. diff --git a/GITHUB_WORKFLOW.md b/GITHUB_WORKFLOW.md new file mode 100644 index 0000000..91e0a5d --- /dev/null +++ b/GITHUB_WORKFLOW.md @@ -0,0 +1,209 @@ +# GitHub Workflow Policy + +This policy defines how contributors and coding agents use GitHub. It +supplements [AGENTS.md](AGENTS.md) and +[COMMIT_POLICY.md](COMMIT_POLICY.md). Agents execute it with +[AGENT_TASK_RUNBOOK.md](AGENT_TASK_RUNBOOK.md). + +## Authority and Authorization + +Code, tests, and detailed docs define behavior; an Issue defines scope and +completion; Project and organization Issue fields hold planning metadata; and +a PR holds the diff, review, and evidence. Record maintainer decisions for +conflicts. Planning data must not silently redefine an Issue. + +Relevant read-only inspection is allowed. An explicit implementation request +also authorizes only the initial-planning mutations below: create or reuse +in-scope Issues and a milestone, set native relationships, add items to the +canonical Project, set their planning metadata, and create the in-scope task +branch through the Issue's Development section after the gate passes. +Project-schema changes, unrelated Issue edits, staging, commits, other pushes, +PR actions, review, closure, merge, deletion, and cleanup still require +explicit authorization. + +## Initial Planning Gate + +Before task branches, worktrees, writers, or tracked edits, record +`github_planning_gate = not_required | pending | passed | blocked`. Only +read-only analysis may continue while it is pending. + +Read the canonical repository slug, Project owner, Project number, and any +independent profile declaration from the consumer repository instructions. +This reusable profile supplies none of those consumer facts and they must not +be inferred from its examples or installation source. When an installation +lock exists, its selected profiles are authoritative; compare repository +instructions only when they independently declare a profile selection. + +Work is large when any applies: estimated Size `L`/`XL`; two or more +independently complete implementation slices or parent integration criteria; +at least three production modules or two responsibility domains; modularity or +migration coordination; dependency sequencing; multiple owners, worktrees, or +delivery phases; or a parent integration branch. Companion tests/docs do not +count; uncertainty is large. + +Large work needs one root and all known descendants, including nested parents +and leaves, before implementation. Use native hierarchy and dependencies. Add +every item to the canonical Project declared in the repository instructions, +then set labels and organization Issue Priority plus Project Status and +Iteration on all items, and milestone and Project Size on each leaf. Verify +bodies, relationships, fields, approved base, and the first executable leaf +with a fresh read. If repository instructions do not declare the canonical +repository slug, Project owner, and Project number, the gate is blocked. + +On partial failure, preserve IDs, repair only missing state, and set `blocked`; +never create duplicates or implementation edits. Gate later scope too. +`not_required` is limited to read-only work or an approved mechanical +exception. A nontrivial standalone implementation passes with one Issue and +Project item; it does not waive planning metadata. + +## Issues + +Nontrivial feature, bug, dependency, packaging, security, architecture, +workflow, or public-documentation work requires an Issue. A maintainer may +approve an exception for a mechanical correction. + +Use parents for initiatives and leaves for independently complete functions. A +root defines its objective, non-goals, integration criteria, child inventory +and order, risks, evidence, and approved base. A leaf defines its root/parent, +Project, objective, context, in/out scope, testable completion, verification, +docs, components, dependencies, constraints, and risks. + +Use native parent/sub-issue relationships for hierarchy and `blocked by` for +dependencies. Do not start with an unfinished prerequisite or expand scope +without recorded approval. + +A parent closes only after all required children and its integration criteria +are complete. A child pull request must never close its parent. + +## Planning Metadata + +### Milestone + +A milestone groups a release or deliverable. Every leaf has one active +milestone before `Ready`; a parent links it but normally remains outside it. +Name the outcome and record owner, exit criteria, exclusions, and any due date. +Close it after its required leaves complete and record deferrals. + +### Iteration + +Iteration is a logical initiative group, not a schedule. Use sequential, +never-reused `Iteration NN`. GitHub's required internal dates/duration never +control assignment, readiness, priority, or rollover. + +Every Project item has one Iteration. The topmost root and all descendants +share it; standalone Issues are roots; linked PRs and new children inherit it. +`wave:NN` orders work inside it. Milestone remains the deliverable grouping. + +Status never changes Iteration; there is no calendar rollover. Replanning moves +the whole root tree. `Active work` contains `Ready`, `In progress`, and +`In review`; `Missing iteration` must stay empty. + +### Labels and Fields + +Use labels only for durable classification: + +- One work type: `bug`, `enhancement`, `documentation`, or `chore` +- One or more components: `area:` +- Delivery sequence, when needed: `wave:NN` +- Active blockage: `blocked` + +Do not create milestone, iteration, status, or priority labels. Use `wave:NN`, +not `priority:NN`. Use the organization Issue `Priority` field: `Urgent`, +`High`, `Medium`, or `Low`, defaulting to `Medium`. Use Project Size for +capacity and Estimate only with a defined unit. + +## Project Status + +| Status | Rule | +| --- | --- | +| `Backlog` | New, incomplete, reprioritized, or pre-start blocked work. | +| `Ready` | Accepted and unblocked, with required planning metadata assigned. | +| `In progress` | Authorized work started; assignee and a Development-linked branch or PR exist. Draft PRs remain here. | +| `In review` | A non-draft PR appears in Development and Project `Linked pull requests`, with verification evidence. | +| `Done` | Completion gates passed, approved work merged or exception completed, and Issue closed. | + +Requested implementation changes return an item to `In progress`. + +Blocked is an overlay, not a status. Add the native dependency, apply +`blocked`, and record the cause, unblock condition, owner, and next action. +Pre-start blocked work stays `Backlog`; interrupted active work stays +`In progress` unless reprioritized. Blocked work never becomes `Done`. + +## Branches, Commits, and Pull Requests + +Assign one accountable owner per branch and worktree, plus one integration +owner for a parent branch. Only one write-capable owner is active at a time. +Inspect each worktree first and preserve unrelated changes. + +Use `issue--` for nontrivial work. A standalone branch +identifies its leaf Issue. For a parent initiative, its branch is the +integration base for child Issue branches. Child PRs target the parent branch +and use **Squash and merge**; the final parent PR targets the repository's +resolved default branch and uses a merge commit. Resolve that branch from +GitHub before creating worktrees or pull requests; never assume its name. +Resolve exactly one local Git remote whose normalized GitHub repository +identity matches the canonical repository slug from consumer instructions. +Fail when no remote or multiple remotes match, and use the resolved remote for +every fetch, remote-tracking ref, worktree base, and divergence check; never +silently default to `origin`. +Follow [AGENT_TASK_RUNBOOK.md](AGENT_TASK_RUNBOOK.md) for the branch and +worktree procedure. + +Every Issue with implementation activity must have native GitHub Development +traceability. Create its task branch from the Issue's Development section, or +through the equivalent `createLinkedBranch` API, before tracked edits. Link a +parent integration branch to the parent Issue and each child branch to its +child Issue. A matching branch name, URL, comment, or `Refs #N` text alone is +not a native Development link. Before writing, query all pages of the +canonical Issue's linked branches and fail unless the exact expected task +branch is present; printing a response or checking only a bounded first page is +not verification. + +Once a PR exists, verify that it appears in the Issue's Development section +and the canonical Project's built-in `Linked pull requests` field. If either +link is missing, repair the authorized traceability before moving the item to +`In review`. Query the exact ProjectV2 item node ID resolved during planning, +verify that its content is the canonical repository's expected Issue, and +paginate the item's linked pull requests; a first-page or issue-number-only +Project lookup is not proof. Keep at least one linked branch or PR throughout +`In progress`; after branch cleanup, the merged PR is the durable link for +`Done`. + +Follow [COMMIT_POLICY.md](COMMIT_POLICY.md). An Issue may span multiple commits, +but each commit must remain coherent, functional, tested, and reviewable. + +Open a draft pull request while work or evidence is incomplete. Before marking +it ready: + +- Complete scope and acceptance criteria. +- Run and report relevant checks. +- Update required documentation. +- Review the final diff and intended base. + +Use `Refs #N` for partial work and for child PRs targeting a parent branch. +Close a child manually after its squash merge and integration checks. Use +`Closes #N` only in a PR to the resolved default branch when that merge +completes the named Issue; never target a parent for one child's work. Merge +requires explicit authorization. + +Issue closure is the completion trigger; automation may then set `Done`. Do not +use Project `Done` to close unfinished work. Verify Issue, PR, and Project state +after automation. + +## Completion and Exceptions + +An Issue is complete only when its scope and acceptance criteria are satisfied, +required checks pass, documentation is current, the diff is reviewed, required +review is complete, approved work is merged or an approved no-PR exception is +complete, and the Issue is closed with the Project item at `Done`. + +Close duplicate, invalid, superseded, cancelled, or rejected work as +`Not planned`. Explain the reason and link a replacement when applicable. When +reopening an Issue, clear `Done`, return it to `Backlog`, and replan it +explicitly. + +## Agent Handoff + +Follow the runbook handoff contract. Report the planning-gate state and +evidence, Issue and Project mutations, Development and linked-PR evidence, and +preserved unrelated worktree changes. diff --git a/PYTHON_MODULE_INDEX.md b/PYTHON_MODULE_INDEX.md new file mode 100644 index 0000000..d2d0c18 --- /dev/null +++ b/PYTHON_MODULE_INDEX.md @@ -0,0 +1,179 @@ +# Python Module Index + +## Inventory + +- Declared source root: `src` +- Packaging source of truth: `pyproject.toml` +- Importable production units: 7 +- Indexed production units: 7 +- Source/index parity: 7/7 +- Package data: `src/generic_rag/py.typed` +- Locked verification owner: `.github/workflows/ci.yml` (supporting workflow, + not an importable unit) + +## `generic_rag` + +- Source: `src/generic_rag/__init__.py` +- Responsibility: establish the side-effect-free package import namespace. +- Supported public imports: `import generic_rag`. +- Re-exports: none; `__all__ = ()` is intentional until a meaningful high-level + facade is implemented. +- Direct internal dependencies: none. +- Owned state or external resources: none. +- Material side effects: none. +- Verification: `tests/test_package_boundaries.py`, + `tests/support/clean_import_probe.py`, `tests/support/verify_artifacts.py`, and + the locked CI build, clean-install, import, compile, and artifact checks. +- Documentation: `README.md` and `docs/api.md`. +- Package data: declares and ships `py.typed`. + +## `generic_rag.errors` + +- Source: `src/generic_rag/errors.py` +- Responsibility: define the typed public failure categories exposed by generic + RAG contracts and workflows. +- Supported public imports: `GenericRagError`, `ContractValidationError`, + `CollaborationError`, and `StateCompatibilityError` from + `generic_rag.errors`. +- Re-exports: exactly the names in the module's `__all__`; none from the package + root. +- Direct internal dependencies: none. +- Owned state or external resources: none. +- Material side effects: none. +- Verification: `tests/test_errors.py`, `tests/test_package_boundaries.py`, + `tests/test_projection.py`, `tests/support/clean_import_probe.py`, and the + locked CI import and boundary checks. +- Documentation: `docs/api.md` and `docs/projection.md`. + +## `generic_rag.contracts` + +- Source: `src/generic_rag/contracts.py` +- Responsibility: define immutable validated values shared by generic RAG + workflows and provider ports. +- Supported public imports: `DocumentKey`, `DocumentIdentity`, `Document`, + `FragmentIdentity`, `Fragment`, `EmbeddingIdentity`, `EmbeddingVector`, + `VectorRecord`, `ProjectionIdentity`, `ProjectionCheckpoint`, + `ProjectionOutcome`, `ProjectionReceipt`, `ChunkingPolicy`, + `ProjectionLimits`, `ProjectionRequest`, `ProjectionManifestEntry`, + `ProjectionManifest`, `ProjectionStateAvailability`, + `ProjectionStateSnapshot`, `ProjectionStateStatus`, `ProjectionResult`, + `RetrievalLimits`, `RetrievalQuery`, `RetrievalOutcome`, `RetrievalHit`, and + `RetrievalResult` from `generic_rag.contracts`. +- Re-exports: exactly the names in the module's `__all__`; none from the package + root. +- Direct internal dependencies: `generic_rag.errors`. +- Owned state or external resources: none; instances own only immutable caller + values. +- Material side effects: none. +- Verification: `tests/test_contract_values.py`, + `tests/test_projection_contracts.py`, `tests/test_projection.py`, + `tests/test_retrieval_contracts.py`, `tests/test_package_boundaries.py`, + `tests/support/clean_import_probe.py`, `tests/support/verify_artifacts.py`, and + the locked CI import, boundary, and artifact checks. +- Documentation: `docs/api.md`, `docs/projection.md`, `docs/retrieval.md`, and + `docs/security-and-privacy.md`. + +## `generic_rag.ports` + +- Source: `src/generic_rag/ports.py` +- Responsibility: define synchronous injected collaborator interfaces and + explicit caller-owned borrowing semantics. +- Supported public imports: `Borrowed`, `Embedder`, `VectorIndexWriter`, + `VectorIndexResetter`, `VectorIndexReader`, and `LexicalRetriever` from + `generic_rag.ports`. +- Re-exports: exactly the names in the module's `__all__`; none from the package + root. +- Direct internal dependencies: `generic_rag.contracts`. +- Owned state or external resources: `Borrowed` retains a reference but never + owns, acquires, releases, closes, or shuts down the resource. +- Material side effects: none. +- Verification: `tests/test_ports.py`, `tests/test_package_boundaries.py`, + `tests/test_projection.py`, `tests/support/clean_import_probe.py`, and the + locked CI import and boundary checks. +- Documentation: `docs/api.md`, `docs/lifecycle.md`, `docs/projection.md`, + `docs/retrieval.md`, and `docs/security-and-privacy.md`. + +## `generic_rag.projection` + +- Source: `src/generic_rag/projection.py` +- Responsibility: deterministically plan and synchronously execute bounded, + revision-aware document projection against caller-supplied state. +- Supported public imports: `ProjectionFailureStage`, `ProjectionStateError`, + `ProjectionOperationError`, `project_documents`, and `rebuild_projection` + from `generic_rag.projection`. +- Re-exports: exactly the names in the module's `__all__`; none from the package + root. +- Direct internal dependencies: `generic_rag.contracts`, `generic_rag.errors`, + `generic_rag.ports`, and `generic_rag.projection_integrity`. +- Integrity ownership: delegates deterministic source-digest, fragment-ID, and + checkpoint-token derivation and checkpoint validation to + `generic_rag.projection_integrity`. +- Owned state or external resources: none; planning state is immutable and + local to each call, while every embedder, writer, and resetter remains + caller-owned through `Borrowed`. +- Material side effects: none at import time. At explicit workflow call time it + may invoke the borrowed embedder and vector writer, and full rebuild may + invoke the borrowed corpus resetter; it performs no persistence, network, + retry, acquisition, release, or lifecycle action itself. +- Verification: `tests/test_projection.py`, `tests/test_package_boundaries.py`, + `tests/support/clean_import_probe.py`, `tests/support/verify_artifacts.py`, and + the locked CI test, lint, type, build, clean-install, import, and artifact + checks. +- Documentation: `docs/projection.md`, `docs/api.md`, `docs/lifecycle.md`, and + `docs/security-and-privacy.md`. + +## `generic_rag.projection_integrity` + +- Source: `src/generic_rag/projection_integrity.py` +- Responsibility: own deterministic v1 derivation and validation algorithms + for projection integrity values. +- Supported public imports: `derive_source_digest`, `derive_fragment_id`, + `derive_projection_checkpoint_token`, and + `has_valid_projection_checkpoint` from `generic_rag.projection_integrity`. +- Re-exports: exactly the names in the module's `__all__`; none from the package + root. +- Direct internal dependencies: `generic_rag.contracts` and + `generic_rag.errors`. +- Owned state or external resources: none; each call derives or checks an + immutable value from exact caller-supplied contracts. +- Material side effects: none at import or call time. +- Verification: `tests/test_projection_integrity.py`, + `tests/test_projection.py`, `tests/test_retrieval.py`, + `tests/test_package_boundaries.py`, `tests/support/clean_import_probe.py`, + `tests/support/verify_artifacts.py`, and the locked CI test, lint, strict + type, compile, build, clean-install, import, and artifact checks. +- Documentation: `docs/api.md`, `docs/projection.md`, `docs/retrieval.md`, and + `docs/security-and-privacy.md`. + +## `generic_rag.retrieval` + +- Source: `src/generic_rag/retrieval.py` +- Responsibility: deterministically compose bounded semantic and hybrid + retrieval results from published caller-owned projection state. +- Supported public imports: `retrieve_semantic` and `retrieve_hybrid` from + `generic_rag.retrieval`. +- Re-exports: exactly the names in the module's `__all__`; none from the package + root. +- Direct internal dependencies: `generic_rag.contracts`, `generic_rag.errors`, + `generic_rag.ports`, and `generic_rag.projection_integrity`. +- Owned state or external resources: none; ranking state is local to each call, + while every embedder, vector reader, and lexical retriever remains + caller-owned through `Borrowed`. +- Material side effects: none at import time. At explicit workflow call time it + may read one borrowed embedder identity, embed one query, search one borrowed + vector reader, and for hybrid retrieval search one borrowed lexical + retriever; it performs no persistence, network, retry, acquisition, release, + logging, authorization, citation, or lifecycle action itself. +- Retrieval semantics: validates and deduplicates exact fragment identities, + filters candidates against the full current published document revision, + and preserves semantic provider order or fuses original semantic and lexical + ranks deterministically without comparing raw scores. Returned fragments + remain non-authoritative; this module does not authorize a query or source, + validate authoritative source text, or create citations. +- Verification: `tests/test_retrieval.py`, + `tests/test_retrieval_contracts.py`, `tests/test_ports.py`, + `tests/test_package_boundaries.py`, `tests/support/clean_import_probe.py`, + `tests/support/verify_artifacts.py`, and the locked CI test, lint, strict + type, compile, build, clean-install, import, and artifact checks. +- Documentation: `docs/retrieval.md`, `docs/api.md`, `docs/lifecycle.md`, + `docs/security-and-privacy.md`, and `README.md`. diff --git a/README.md b/README.md index b2c9de7..ec47813 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,198 @@ # generic-rag -Provider-neutral, dependency-free RAG projection and retrieval runtime + +`generic-rag` is a provider-neutral, runtime-dependency-free foundation for +retrieval-augmented generation (RAG). Version 0.1.0 requires Python 3.11 or +later and provides immutable contracts, typed error categories, synchronous +collaborator protocols, deterministic bounded projection orchestration, +semantic retrieval, rank-based hybrid retrieval, and explicit caller-owned +borrowing. + +The package has no built-in adapter, provider, factory, persistence, network +client, configuration system, authentication, authorization, citation +mechanism, or CLI. + +## Install from a checkout + +The project is not documented as a published package yet. From a repository +checkout, install it with: + +```console +python -m pip install . +``` + +The installed package has no runtime dependencies. Build and development tools +are separate locked dependency groups. + +## Project approved documents + +The host application remains responsible for authorization and policy checks. +After approving source content, construct a complete bounded target, wrap +application-owned adapters in `Borrowed`, and call a projection workflow: + +```python +from generic_rag.contracts import ( + ChunkingPolicy, + Document, + DocumentIdentity, + DocumentKey, + EmbeddingIdentity, + EmbeddingVector, + ProjectionIdentity, + ProjectionLimits, + ProjectionRequest, + ProjectionStateAvailability, + ProjectionStateSnapshot, +) +from generic_rag.ports import Borrowed +from generic_rag.projection import rebuild_projection + +class ExampleEmbedder: + identity = EmbeddingIdentity("example-model", 2) + + def embed(self, texts, /): + return tuple(EmbeddingVector((float(len(text)), 0.0)) for text in texts) + + +class ExampleWriter: + def replace_document(self, document, records, /): + # Replace the complete projection for this stable document key. + return None + + def delete_document(self, document, /): + return None + + +class ExampleResetter: + def reset_corpus(self, corpus_id, /): + # Remove every projected document for this corpus. + return None + + +request = ProjectionRequest( + "corpus-a", + ProjectionIdentity("schema-v1", ExampleEmbedder.identity), + ChunkingPolicy(max_fragment_codepoints=800, overlap_codepoints=80), + ProjectionLimits( + max_documents=100, + max_document_codepoints=100_000, + max_embedding_batch_size=32, + ), + ( + Document( + DocumentIdentity( + DocumentKey("corpus-a", "document-1"), + "revision-3", + ), + "Approved source text", + (("classification", "internal"),), + ), + ), +) + +# Bootstrap and recovery are explicit and destructive: reset, then replace. +result = rebuild_projection( + request, + ProjectionStateSnapshot(ProjectionStateAvailability.MISSING, None), + Borrowed(ExampleEmbedder()), + Borrowed(ExampleWriter()), + Borrowed(ExampleResetter()), +) + +# Persist result.manifest in application-owned state only after success. +``` + +For normal updates, load that manifest into a present +`ProjectionStateSnapshot` and call `project_documents`; it changes only added, +updated, removed, or rechunked documents. Use `rebuild_projection` only when an +explicit corpus-wide reset is intended. See the [projection guide](docs/projection.md) +for the complete lifecycle, state matrix, adapter obligations, and failure +behavior. + +Load the corresponding published manifest before retrieval, reauthorize each +query and source in the host, and call `retrieve_semantic` or +`retrieve_hybrid` with borrowed provider implementations. Returned fragment +text is non-authoritative: resolve each identity against the still-authorized +source revision and create host-owned citations before showing results to a +user or injecting bounded context into an agent. See the [retrieval +guide](docs/retrieval.md) for an executable independent-consumer example, +deterministic fusion behavior, outcome handling, and the complete host flow. + +Public values must be imported from their owning modules: + +- `generic_rag.contracts` +- `generic_rag.errors` +- `generic_rag.ports` +- `generic_rag.projection` +- `generic_rag.projection_integrity` +- `generic_rag.retrieval` + +The package root intentionally has no re-exports: `generic_rag.__all__ == ()`. +See the [API reference](docs/api.md) for every supported name and invariant. + +## Application RAG flow + +The package itself has no concept of a user or agent. A consuming application +decides which sources a user may approve, which queries may be submitted, which +provider implementations receive data, and whether retrieved fragments are +shown to a user or supplied to a downstream tool or agent. + +- Projection accepts caller-approved documents and explicitly injected + collaborators. It derives deterministic fragments under a bounded chunking + policy, embeds ordered fragment text, replaces or deletes complete document + projections, and returns a manifest and truthful receipt for caller-owned + persistence. +- Retrieval accepts the matching caller-loaded published state and injected + semantic or lexical providers. It validates and filters current-revision + candidates, deduplicates exact identities, and returns bounded score-free + hits. Hybrid retrieval fuses provider ranks deterministically; it does not + compare raw provider scores. +- The host reauthorizes every query, resolves each returned fragment identity + against authoritative source data, verifies the exact source slice, and + creates citations. It may then show cited results to a user or inject bounded + cited context into an agent; the agent must retain those citations. + +The package does not decide provider selection, authentication, authorization, +prompt or tool policy, retry, display, logging, or resource lifecycle. See the +[retrieval guide](docs/retrieval.md), [resource lifecycle](docs/lifecycle.md), +and [security and privacy](docs/security-and-privacy.md). + +## Compatibility + +Version 0.1.0 is pre-1.0. Consumers should pin a reviewed version and should not +assume compatibility across minor releases. For this release, direct imports +from the documented owning modules are the supported public paths; root-level +imports are not. + +The distribution includes `py.typed`. The wheel contains exactly the seven +importable modules `generic_rag`, `generic_rag.errors`, +`generic_rag.contracts`, `generic_rag.ports`, `generic_rag.projection`, +`generic_rag.projection_integrity`, and `generic_rag.retrieval`, plus the typing +marker. + +## Development verification + +The project commits a universal lock generated with uv 0.12.5. Reproduce the +locked environment and primary checks with: + +```console +uv lock --check +uv sync --locked --all-groups +uv run --frozen python -m unittest discover -s tests -p 'test_*.py' -v +uv run --frozen ruff check src tests +uv run --frozen ruff format --check src tests +uv run --frozen mypy --strict src tests +uv run --frozen python -m compileall -q src tests +``` + +Build both distributions into a temporary output directory and inspect them: + +```console +rag_dist_dir="$(mktemp -d)" +uv run --frozen python -m build --no-isolation --outdir "$rag_dist_dir" +uv run --frozen python tests/support/verify_artifacts.py "$rag_dist_dir" +``` + +CI is configured to run the tests on Python 3.11 and 3.14. On Python 3.11 it +also runs lint, format, strict type, compilation, artifact, source-rebuild, +clean-install, and isolated-import checks. The CI matrix is the authoritative +cross-version result. diff --git a/docs/api.md b/docs/api.md new file mode 100644 index 0000000..47d3c7c --- /dev/null +++ b/docs/api.md @@ -0,0 +1,407 @@ +# Public API + +Version 0.1.0 exposes immutable values, typed error categories, synchronous +collaborator protocols, deterministic bounded document projection, and +score-free semantic and hybrid retrieval. See the [projection guide](projection.md), +[retrieval guide](retrieval.md), [resource lifecycle](lifecycle.md), and +[security and privacy boundary](security-and-privacy.md) for the surrounding +usage contract. + +## Import boundary + +The supported package-root operation is `import generic_rag`. The root has an +intentionally empty export list, `generic_rag.__all__ == ()`, and re-exports no +public symbols. Import names from their owning modules instead. + +`generic_rag.errors` exports exactly: + +- `GenericRagError` +- `ContractValidationError` +- `CollaborationError` +- `StateCompatibilityError` + +`generic_rag.contracts` exports exactly: + +- `DocumentKey` +- `DocumentIdentity` +- `Document` +- `FragmentIdentity` +- `Fragment` +- `EmbeddingIdentity` +- `EmbeddingVector` +- `VectorRecord` +- `ProjectionIdentity` +- `ProjectionCheckpoint` +- `ProjectionOutcome` +- `ProjectionReceipt` +- `ChunkingPolicy` +- `ProjectionLimits` +- `ProjectionRequest` +- `ProjectionManifestEntry` +- `ProjectionManifest` +- `ProjectionStateAvailability` +- `ProjectionStateSnapshot` +- `ProjectionStateStatus` +- `ProjectionResult` +- `RetrievalLimits` +- `RetrievalQuery` +- `RetrievalOutcome` +- `RetrievalHit` +- `RetrievalResult` + +`generic_rag.ports` exports exactly: + +- `Borrowed` +- `Embedder` +- `VectorIndexWriter` +- `VectorIndexResetter` +- `VectorIndexReader` +- `LexicalRetriever` + +`generic_rag.projection` exports exactly: + +- `ProjectionFailureStage` +- `ProjectionStateError` +- `ProjectionOperationError` +- `project_documents` +- `rebuild_projection` + +`generic_rag.projection_integrity` exports exactly: + +- `derive_source_digest` +- `derive_fragment_id` +- `derive_projection_checkpoint_token` +- `has_valid_projection_checkpoint` + +`generic_rag.retrieval` exports exactly: + +- `retrieve_semantic` +- `retrieve_hybrid` + +The package does not support importing any of these names from the package +root. + +## Shared value rules + +Public contract dataclasses are frozen, slotted, hashable value objects. They +require exact concrete field types, including exact nested contract classes; +subclasses do not satisfy these runtime validators. In particular: + +- strings must be exact `str` values; +- discrete integers must be exact `int` values, so `bool`, floats, and integer + subclasses are rejected; +- tuple fields must be exact tuples, not lists, iterators, or tuple subclasses; + and +- field invariant violations detected during construction raise + `ContractValidationError`. + +Opaque identity strings must be nonempty and not whitespace-only. They are +otherwise stored exactly as supplied: the package does not trim, normalize, +canonicalize, parse, or resolve them. + +Attributes use `tuple[tuple[str, str], ...]`. Every pair must be an exact +two-element tuple containing exact strings. Empty keys and values are allowed, +and pair order and duplicates are preserved. + +## Errors + +The base error relationships used by projection are: + +```text +GenericRagError +├── ContractValidationError +├── CollaborationError +│ └── ProjectionOperationError +└── StateCompatibilityError + └── ProjectionStateError +``` + +- `ContractValidationError` reports a violated public value invariant or an + invalid top-level workflow input. +- `ProjectionStateError(status: ProjectionStateStatus)` reports state that an + incremental operation cannot use. Its exact `status` field gives the reason; + its message contains no state identifier. +- `ProjectionOperationError(stage, affected_document, receipt)` translates an + ordinary collaborator failure or invalid collaborator return. Its message is + content-free. `affected_document` is a stable key for document-specific + failures and otherwise `None`; `receipt` is truthful `FAILED` or `PARTIAL` + progress, or `None` when there were zero document attempts. + +The closed string enum `ProjectionFailureStage` has exact values: + +| Member | String value | +| --- | --- | +| `EMBEDDER_IDENTITY` | `"embedder_identity"` | +| `EMBEDDING` | `"embedding"` | +| `REPLACEMENT` | `"replacement"` | +| `DELETION` | `"deletion"` | +| `RESET` | `"reset"` | + +An ordinary collaborator exception is chained as the operation error's cause. +An invalid identity, malformed vector result, or non-`None` command result has +no internal cause. `KeyboardInterrupt` and `SystemExit` pass through unchanged. + +Retrieval defines no workflow-specific public exception. Invalid top-level +workflow inputs, including a query that exceeds `RetrievalLimits`, raise +`ContractValidationError` before collaborator effects. After a collaborator +boundary is entered, ordinary `Exception` failures and malformed collaborator +returns contribute a content-free failure state. A failed branch contributes no +fragment or exception text; a `PARTIAL` result can contain independently +validated hits from another branch. No exception cause crosses the result +boundary. `BaseException` subclasses pass through unchanged. + +## Documents and fragments + +| Type | Fields | Construction rules | +| --- | --- | --- | +| `DocumentKey` | `corpus_id: str`, `document_id: str` | Both values are nonblank opaque identities. | +| `DocumentIdentity` | `key: DocumentKey`, `revision_id: str` | `key` is the exact class; the revision is a nonblank opaque identity. | +| `Document` | `identity: DocumentIdentity`, `text: str`, `attributes: tuple[tuple[str, str], ...] = ()` | `text` is stored exactly and may be empty; attributes follow the shared tuple rules. | +| `FragmentIdentity` | `document: DocumentIdentity`, `fragment_id: str`, `start: int`, `end: int` | The ID is nonblank; offsets are exact nonnegative integers satisfying `start < end`. | +| `Fragment` | `identity: FragmentIdentity`, `text: str`, `attributes: tuple[tuple[str, str], ...] = ()` | `text` is nonempty and `len(text) == end - start`; attributes follow the shared tuple rules. | + +A fragment range is half-open, `[start, end)`, in Python Unicode code points. +It is not measured in bytes or user-perceived grapheme clusters. For example, +`"😀"` has one code point while `"e\u0301"` has two. + +The contract checks that fragment text length equals the range width. A +standalone fragment cannot prove that its text equals the indicated source +slice; projection establishes that correspondence for fragments it derives. + +## Embeddings and vector records + +| Type | Fields | Construction rules | +| --- | --- | --- | +| `EmbeddingIdentity` | `model_id: str`, `dimensions: int` | The model ID is nonblank and opaque; dimensions is a positive exact integer. | +| `EmbeddingVector` | `values: tuple[float, ...]` | The tuple is exact and nonempty. Every coordinate is an exact `int` or `float`, excluding `bool`, and must convert to a finite float without overflow. Accepted coordinates are stored canonically as floats. | +| `VectorRecord` | `fragment: Fragment`, `embedding: EmbeddingVector` | Both fields require their exact contract classes. | + +A standalone `EmbeddingVector` does not carry an `EmbeddingIdentity` and does +not itself compare length with declared dimensions. Projection validates the +embedder identity and every returned vector's exact dimensionality and +canonical finite-float representation. + +## Projection request and manifest values + +| Type | Fields | Construction rules | +| --- | --- | --- | +| `ProjectionIdentity` | `schema_id: str`, `embedding: EmbeddingIdentity` | The schema ID is nonblank and opaque; embedding requires its exact class. | +| `ChunkingPolicy` | `max_fragment_codepoints: int`, `overlap_codepoints: int` | Maximum is positive; overlap is nonnegative and smaller than maximum. | +| `ProjectionLimits` | `max_documents: int`, `max_document_codepoints: int`, `max_embedding_batch_size: int` | All three values are positive exact integers. | +| `ProjectionRequest` | `corpus_id: str`, `projection: ProjectionIdentity`, `chunking: ChunkingPolicy`, `limits: ProjectionLimits`, `documents: tuple[Document, ...]` | Documents must match the corpus, have unique stable keys, and remain within the count and per-document text caps. They are canonicalized by opaque `document_id`. | +| `ProjectionManifestEntry` | `document: DocumentIdentity`, `source_digest: str`, `fragment_count: int` | Digest must be lowercase `sha256:<64hex>` and fragment count is nonnegative. | +| `ProjectionCheckpoint` | `corpus_id: str`, `projection: ProjectionIdentity`, `token: str` | Corpus and token are nonblank; projection requires its exact class. | +| `ProjectionManifest` | `corpus_id: str`, `projection: ProjectionIdentity`, `chunking: ChunkingPolicy`, `entries: tuple[ProjectionManifestEntry, ...]`, `checkpoint: ProjectionCheckpoint` | Entries match the corpus, have unique stable keys, and are canonicalized by `document_id`; checkpoint corpus and projection match the manifest. | + +The package produces manifests; the caller owns their persistence. Source +digests, fragment IDs, and checkpoint tokens are deterministic under explicit +v1 domains described in the [projection guide](projection.md#deterministic-projection-values). + +The public integrity functions are synchronous and positional-only: + +```python +def derive_source_digest(document: Document, /) -> str: ... + +def derive_fragment_id( + document: DocumentIdentity, + start: int, + end: int, + /, +) -> str: ... + +def derive_projection_checkpoint_token( + corpus_id: str, + projection: ProjectionIdentity, + chunking: ChunkingPolicy, + entries: tuple[ProjectionManifestEntry, ...], + /, +) -> str: ... + +def has_valid_projection_checkpoint( + manifest: ProjectionManifest, + /, +) -> bool: ... +``` + +They reproduce the same v1 integrity values used by projection and retrieval. +They validate exact public contract shapes and canonical manifest-entry order; +invalid inputs raise `ContractValidationError`. Checkpoint validation returns +whether the supplied token equals the derived token. It does not inspect a +provider index or establish authorization. + +## Projection state and results + +`ProjectionStateAvailability` is a closed string enum: + +| Member | String value | Manifest rule | +| --- | --- | --- | +| `MISSING` | `"missing"` | Must be `None` | +| `PRESENT` | `"present"` | Must be an exact `ProjectionManifest` | +| `CORRUPT` | `"corrupt"` | Must be `None` | + +`ProjectionStateSnapshot(availability, manifest)` stores that caller-supplied +state. Projection evaluates it to a closed `ProjectionStateStatus`: + +| Member | String value | Meaning | +| --- | --- | --- | +| `MISSING` | `"missing"` | No state is available. | +| `CURRENT` | `"current"` | Valid state exactly matches the complete target. | +| `STALE` | `"stale"` | Valid compatible state requires mutations. | +| `CORRUPT` | `"corrupt"` | State is declared corrupt or fails integrity/consistency checks. | +| `SCHEMA_MISMATCH` | `"schema_mismatch"` | The schema ID differs. | +| `EMBEDDING_MISMATCH` | `"embedding_mismatch"` | The embedding identity differs. | + +`ProjectionResult(status_before, receipt, manifest)` represents only complete +success. Its receipt and manifest have the same corpus, projection, and +checkpoint. An `UNCHANGED` result requires `CURRENT` state and zero attempted +and completed documents. + +`ProjectionOutcome` is a closed string enum with these exact member values: + +| Member | String value | +| --- | --- | +| `COMPLETED` | `"completed"` | +| `UNCHANGED` | `"unchanged"` | +| `PARTIAL` | `"partial"` | +| `FAILED` | `"failed"` | + +`ProjectionReceipt` has fields `corpus_id`, `projection`, `outcome`, +`attempted_documents`, `completed_documents`, and `checkpoint`. It permits only +these truthful combinations: + +| Outcome | Counts | Checkpoint | +| --- | --- | --- | +| `COMPLETED` | `completed_documents == attempted_documents`, including zero attempts | Required | +| `UNCHANGED` | `completed_documents == attempted_documents`, including zero attempts | Required | +| `PARTIAL` | `0 < completed_documents < attempted_documents` | Forbidden | +| `FAILED` | `attempted_documents > 0` and `completed_documents == 0` | Forbidden | + +Any supplied checkpoint must have exactly the receipt's `corpus_id` and +`projection`. A successful workflow returns `ProjectionResult`; a failed +workflow exposes a failed or partial receipt only through +`ProjectionOperationError`. + +## Projection workflows + +Both public functions are synchronous and all parameters are positional-only: + +```python +def project_documents( + request: ProjectionRequest, + state: ProjectionStateSnapshot, + embedder: Borrowed[Embedder], + writer: Borrowed[VectorIndexWriter], + /, +) -> ProjectionResult: ... + +def rebuild_projection( + request: ProjectionRequest, + state: ProjectionStateSnapshot, + embedder: Borrowed[Embedder], + writer: Borrowed[VectorIndexWriter], + resetter: Borrowed[VectorIndexResetter], + /, +) -> ProjectionResult: ... +``` + +`project_documents` returns without collaborator effects when state is +`CURRENT`, applies only the canonical delta when state is `STALE`, and raises +`ProjectionStateError` before effects for every other status. + +`rebuild_projection` accepts every state status and always calls the resetter. +For a nonempty target it verifies embedder identity before reset, then replaces +every target document. For an empty target it resets without accessing the +embedder or writer. It is intentionally destructive and supplies no rollback +or retry. See the [projection guide](projection.md) for the full state and +failure matrices. + +## Retrieval values + +| Type | Fields | Construction rules | +| --- | --- | --- | +| `RetrievalLimits` | `max_query_codepoints: int` | The query-text cap is a positive exact integer. | +| `RetrievalQuery` | `corpus_id: str`, `text: str`, `hit_limit: int`, `candidate_limit: int` | Corpus and text are nonblank exact strings; limits are positive exact integers and `hit_limit <= candidate_limit`. Values are preserved exactly. | +| `RetrievalHit` | `fragment: Fragment`, `rank: int` | Fragment requires its exact class and rank is a positive exact integer. There is no score field. | +| `RetrievalResult` | `query: RetrievalQuery`, `outcome: RetrievalOutcome`, `hits: tuple[RetrievalHit, ...]`, `truncated: bool` | Nested values, the hit tuple, and the boolean require exact types. Hit count cannot exceed `query.hit_limit`. | + +`RetrievalOutcome` is a closed string enum with exact values `"complete"`, +`"partial"`, `"unavailable"`, `"stale"`, and `"failed"`. Within every result, +ranks are contiguous from one, fragment identities are unique, and every +fragment belongs to the query corpus. `PARTIAL` requires at least one hit; +`UNAVAILABLE`, `STALE`, and `FAILED` require no hits and `truncated=False`. + +The workflows set `truncated=True` exactly when validated, unique, +current-revision candidates exceed `query.hit_limit`. Hits and reader ports are +score-free; raw provider scores are neither represented nor promised +comparable. When constructing a `RetrievalResult` directly, callers remain +responsible for supplying a truthful `truncated` value because the value +contract cannot reconstruct discarded candidates. + +## Collaborator ports + +The protocols are synchronous, injected, structurally typed, and decorated +with `runtime_checkable`. Runtime protocol checks establish structural +presence, not the behavioral obligations below. Version 0.1.0 provides no +adapter, factory, or provider discovery. + +| Port | Exact public operation | Semantic obligation | +| --- | --- | --- | +| `Embedder` | `identity: EmbeddingIdentity` | Identify the exact model used for produced vectors. | +| `Embedder` | `embed(texts: tuple[str, ...], /) -> tuple[EmbeddingVector, ...]` | Return one same-order vector per input text, each with `identity.dimensions` coordinates; empty input returns an empty tuple. | +| `VectorIndexWriter` | `replace_document(document: DocumentIdentity, records: tuple[VectorRecord, ...], /) -> None` | Replace all derived vectors for the document's stable key. Every record carries the supplied full identity; an empty record tuple is valid. | +| `VectorIndexWriter` | `delete_document(document: DocumentKey, /) -> None` | Delete every derived revision for the stable document key. | +| `VectorIndexResetter` | `reset_corpus(corpus_id: str, /) -> None` | Remove the complete derived vector projection for the corpus. | +| `VectorIndexReader` | `search(query: RetrievalQuery, embedding: EmbeddingVector, /) -> tuple[Fragment, ...]` | Return fragments from the requested corpus, in provider rank order, with at most `query.candidate_limit` entries. | +| `LexicalRetriever` | `search(query: RetrievalQuery, /) -> tuple[Fragment, ...]` | Return fragments from the requested corpus, in provider rank order, with at most `query.candidate_limit` entries. | + +Projection enforces the embedder result rules and requires each writer or +resetter command to return exactly `None`. It cannot enforce external storage, +atomicity, authorization, concurrency, or lifecycle behavior. + +Retrieval checks its embedder identity and output, provider tuple types and +candidate bounds, fragment integrity, corpus and published revisions, and +cross-provider identity consistency. The caller still owns provider selection, +authorization, persistence, concurrency, retries, and authoritative source +validation. + +`Borrowed[T]` is the companion ownership marker, not a provider port. Its exact +behavior is documented in [resource lifecycle](lifecycle.md). + +## Retrieval workflows + +Both public functions are synchronous and all parameters are positional-only: + +```python +def retrieve_semantic( + query: RetrievalQuery, + state: ProjectionStateSnapshot, + limits: RetrievalLimits, + embedder: Borrowed[Embedder], + vector_reader: Borrowed[VectorIndexReader], + /, +) -> RetrievalResult: ... + +def retrieve_hybrid( + query: RetrievalQuery, + state: ProjectionStateSnapshot, + limits: RetrievalLimits, + embedder: Borrowed[Embedder], + vector_reader: Borrowed[VectorIndexReader], + lexical_retriever: Borrowed[LexicalRetriever], + /, +) -> RetrievalResult: ... +``` + +Semantic retrieval embeds the query once, validates at most +`candidate_limit` vector candidates, preserves provider order through +current-revision filtering, and returns at most `hit_limit` hits. Hybrid +retrieval also obtains at most `candidate_limit` lexical candidates, preserves +each provider's original ranks, and fuses exact identities using deterministic +reciprocal rank fusion with offset 60. It uses provider ranks rather than raw +scores and applies opaque identity ordering to ties. + +The workflows return `complete`, `partial`, `unavailable`, `stale`, or `failed` +according to published-state and collaborator results. They do not authorize, +cite, persist, log, retry, or manage collaborator resources. See the +[retrieval guide](retrieval.md) for candidate validation, exact outcome +handling, deterministic fusion, and the required user and agent host flow. diff --git a/docs/lifecycle.md b/docs/lifecycle.md new file mode 100644 index 0000000..167d06a --- /dev/null +++ b/docs/lifecycle.md @@ -0,0 +1,155 @@ +# Resource Lifecycle + +Version 0.1.0 uses an explicit caller-owned lifecycle. The package defines +collaborator protocols and `Borrowed[T]`; it does not acquire, configure, +discover, persist, synchronize, or release provider resources. Projection calls +and retrieval calls use borrowed collaborators only during an explicit +workflow invocation. See the [API reference](api.md) for exact port signatures. + +## Ownership rule + +The caller or provider integration owns every lifecycle decision: + +1. Select, acquire, and configure the embedder, vector index, lexical + retriever, projection-state store, credentials, network clients, and any + required synchronization. +2. Authorize the complete source set and construct a bounded + `ProjectionRequest`. +3. Wrap the application-owned collaborators in `Borrowed` and call + `project_documents` for a compatible incremental update, or deliberately + call `rebuild_projection` for bootstrap or destructive recovery. +4. On complete success, publish the returned manifest through caller-owned + persistence and synchronization. +5. Release, close, or shut down the real resources according to their provider + contracts. + +`generic-rag` performs none of the acquisition, manifest persistence, +publication, synchronization, or release steps implicitly. It provides no +transaction across collaborators and the manifest store. + +For retrieval, the caller also loads the manifest paired with the index, +reauthorizes the requester and source set for each query, applies retry and +timeout policy around package calls, resolves returned identities against +authoritative source revisions, creates citations, and decides whether cited +results may be shown to a user or supplied to an agent. + +## `Borrowed[T]` + +`Borrowed` is a frozen, slotted context manager that retains a reference. It +has exactly these context semantics: + +- `__enter__` returns the exact wrapped resource. +- `__exit__` returns `False`, so it never suppresses an exception. +- It never calls the resource's `__enter__`, `__exit__`, `close`, `shutdown`, + reset, acquisition, release, or synchronization operations. +- It does not translate a resource failure into `CollaborationError`. +- It provides no locking, thread safety, asynchronous context management, or + provider health checks. + +These guarantees still apply if the wrapped provider defines its own context +manager or lifecycle methods. Projection translates ordinary method failures +into `ProjectionOperationError`; retrieval represents an ordinary method +failure as a content-free failed branch and may return independent validated +hits from another branch. Those workflow behaviors do not change `Borrowed` +semantics. + +## Successful borrowed scope + +The resource that leaves the scope is still the application-owned object: + +```python +from generic_rag.ports import Borrowed + +provider = object() # Acquired and owned by the application. + +with Borrowed(provider) as active_provider: + assert active_provider is provider + +assert active_provider is provider # Borrowed did not close or replace it. +``` + +Application code decides when and how to release the real provider afterward. + +## Exceptional borrowed scope + +An exception leaves the borrowed scope unchanged and unsuppressed: + +```python +from generic_rag.ports import Borrowed + +provider = object() +failure = RuntimeError("provider failed") + +try: + with Borrowed(provider) as active_provider: + assert active_provider is provider + raise failure +except RuntimeError as caught: + assert caught is failure +``` + +If a caller needs cleanup after success or failure, it must arrange that +cleanup around the borrowed scope according to the provider's contract. + +## Projection call scope + +`project_documents` and `rebuild_projection` receive exact `Borrowed` wrappers. +They enter only those no-op wrappers; they never enter, close, or shut down the +underlying adapter objects. + +The caller must keep each adapter alive for the complete synchronous call. +Providers that require sessions, transactions, locks, or thread affinity must +be prepared before wrapping and must remain valid until the call returns or +raises. + +An incremental call may perform multiple ordered document mutations. A rebuild +performs a corpus reset before its document replacements. The package does not +roll back or retry earlier effects if a later operation fails. The +`ProjectionOperationError.receipt` reports completed document operations but +never supplies a checkpoint for partial work. Keep partially updated state out +of service and recover under application-owned coordination. + +The caller should persist `ProjectionResult.manifest` only after complete +success. The package does not retain it, and a vector index without the matching +published manifest cannot be used safely by the next incremental operation. +See the [projection guide](projection.md) for the state matrix and destructive +rebuild ordering. + +## Retrieval call scope + +`retrieve_semantic` and `retrieve_hybrid` receive exact `Borrowed` wrappers and +a caller-loaded `ProjectionStateSnapshot`. As with projection, they enter only +the no-op wrappers and never enter, close, or shut down the underlying provider +objects. Keep every provider alive for the complete synchronous call. + +Semantic retrieval reads the embedder identity, embeds one query, and then +searches the vector reader. Hybrid retrieval completes that semantic branch +and then calls the lexical retriever; an ordinary semantic failure does not +prevent the lexical call. The package does not retry. Ordinary provider +exceptions and malformed returns produce a failed branch: the overall result +is `failed` without hits or `partial` when the other branch supplies validated +hits. An identity or revision mismatch similarly produces `stale` without hits +or `partial` alongside another branch's hits. `BaseException` subclasses +propagate through the no-op borrowed scope. + +The host owns the complete operating lifecycle around those calls: + +- persist and publish each manifest with the provider index it describes; +- prevent projection, retrieval, rebuild, disable, and purge operations from + observing incompatible manifest/index combinations; +- select and acquire providers, credentials, and network resources and apply + provider-specific retry, timeout, and fallback policy; +- reauthorize each query and authoritative source, then revalidate returned + fragment identities and create citations; +- interpret `complete`, `partial`, `unavailable`, `stale`, and `failed` under + application policy and record any required audit events; and +- coordinate in-flight calls before disabling, rebuilding, purging, or shutting + down caller-owned state and resources. + +The package provides no daemon, background worker, registry, persistence, +dynamic loading, network discovery, lock, transaction, purge command, or +shutdown hook. See the [retrieval guide](retrieval.md) for the exact query flow +and outcome meanings. + +Review the [security and privacy boundary](security-and-privacy.md) before +passing content to any adapter implementation. diff --git a/docs/projection.md b/docs/projection.md new file mode 100644 index 0000000..933721b --- /dev/null +++ b/docs/projection.md @@ -0,0 +1,268 @@ +# Projection + +Version 0.1.0 can turn a complete, caller-approved document set into a +deterministic vector projection. It supplies orchestration, contracts, and +failure reporting; the caller supplies and owns the embedder, vector index, +projection-state persistence, authorization policy, and synchronization. + +Projection establishes the published manifest and derived vector index that +retrieval validates. The host must publish and load those as one matching +logical state, then reauthorize queries and authoritative sources. See the +[retrieval guide](retrieval.md) for semantic and hybrid use; neither workflow +implements authentication, authorization, citations, or user/agent policy. + +## Required adapters and state + +Implement application-specific objects that structurally satisfy these public +ports: + +- `Embedder` exposes an `EmbeddingIdentity` and returns one same-order vector + of exactly that dimensionality for every input text. +- `VectorIndexWriter` replaces the complete record set for one stable document + key or deletes every revision of that key. Both commands must return `None`. +- `VectorIndexResetter` removes every projected document for one corpus and + returns `None`. It is required only by `rebuild_projection`. + +The package does not provide adapters or a manifest store. Persist the +successful `ProjectionResult.manifest` in application-owned state, then load it +as a `ProjectionStateSnapshot` for the next operation. The stored manifest and +the vector index are one logical projection: publish them under application- +controlled synchronization so another operation cannot observe an unintended +combination. + +All workflow parameters are positional-only. Each collaborator must be wrapped +in the exact `Borrowed` class; borrowing never acquires, closes, resets, or +otherwise owns the wrapped resource. + +## Bootstrap, then update incrementally + +The first projection has no valid present manifest, so bootstrap it through the +explicitly destructive rebuild path: + +```python +from generic_rag.contracts import ( + ChunkingPolicy, + Document, + DocumentIdentity, + DocumentKey, + EmbeddingIdentity, + EmbeddingVector, + ProjectionIdentity, + ProjectionLimits, + ProjectionRequest, + ProjectionStateAvailability, + ProjectionStateSnapshot, +) +from generic_rag.ports import Borrowed +from generic_rag.projection import project_documents, rebuild_projection + + +class ApplicationEmbedder: + identity = EmbeddingIdentity("embedding-model-v1", 2) + + def embed(self, texts, /): + return tuple(EmbeddingVector((float(len(text)), 0.0)) for text in texts) + + +class ApplicationVectorIndex: + def replace_document(self, document, records, /): + # Replace all derived records for document.key in one adapter operation. + return None + + def delete_document(self, document, /): + # Delete every projected revision of this stable document key. + return None + + def reset_corpus(self, corpus_id, /): + # Delete the complete vector projection for this corpus. + return None + + +def target(revision_id, text): + return ProjectionRequest( + "approved-corpus", + ProjectionIdentity("schema-v1", ApplicationEmbedder.identity), + ChunkingPolicy(800, 80), + ProjectionLimits(100, 100_000, 32), + ( + Document( + DocumentIdentity( + DocumentKey("approved-corpus", "document-1"), + revision_id, + ), + text, + (("source", "application-authorized"),), + ), + ), + ) + + +embedder = ApplicationEmbedder() +index = ApplicationVectorIndex() +initial_request = target("revision-1", "First approved source text") + +initial_result = rebuild_projection( + initial_request, + ProjectionStateSnapshot(ProjectionStateAvailability.MISSING, None), + Borrowed(embedder), + Borrowed(index), + Borrowed(index), +) +# Persist initial_result.manifest only after the call succeeds. + +updated_request = target("revision-2", "Updated approved source text") +updated_result = project_documents( + updated_request, + ProjectionStateSnapshot( + ProjectionStateAvailability.PRESENT, + initial_result.manifest, + ), + Borrowed(embedder), + Borrowed(index), +) +# Atomically publish updated_result.manifest as the new application-owned state. +``` + +The example uses one object for the writer and resetter ports, but separate +objects are equally valid. Production adapters must implement the stated +complete replacement, deletion, reset, persistence, and synchronization +semantics; the example deliberately omits storage. + +## Incremental lifecycle + +`project_documents(request, state, embedder, writer)` applies a complete target +to compatible state: + +1. It validates the exact request, snapshot, and `Borrowed` wrapper types and + prepares the complete target before calling a collaborator. +2. It evaluates the supplied manifest against the target. `current` returns an + `UNCHANGED` result with zero attempts and no collaborator calls. Only + `stale` proceeds; every other status raises `ProjectionStateError` before + effects. +3. It sorts mutations by opaque `document_id`. Added documents, new revisions, + and documents affected by a chunking change are replaced; absent target + documents are deleted; unchanged entries are skipped. A source change under + an unchanged full document identity is corrupt state, not an incremental + replacement. +4. If any replacement is needed, it validates the embedder identity once, + embeds nonempty ordered batches no larger than + `max_embedding_batch_size`, and calls the writer once per document. A + delete-only update does not access the embedder. +5. After every mutation succeeds, it returns `COMPLETED` with the complete + target manifest and checkpoint. The caller may then publish that manifest. + +An empty document produces no fragments and is still replaced with an explicit +empty record tuple. An empty target incrementally deletes every document in the +previous valid manifest. + +## Destructive rebuild lifecycle + +`rebuild_projection(request, state, embedder, writer, resetter)` is the explicit +bootstrap and recovery operation. It accepts every state status, including +`current`, but always resets the requested corpus and recreates the complete +target: + +1. It validates inputs and computes `status_before` without effects. +2. For a nonempty target, it validates the embedder identity once *before* the + reset. A mismatch or invalid identity therefore leaves reset and writer + untouched. +3. It calls `reset_corpus`, then embeds and replaces every target document in + canonical `document_id` order. An empty target calls only the resetter. +4. Complete success returns a `COMPLETED` result, even when the target has zero + documents. + +Reset is intentionally destructive. The package provides no transaction, +rollback, retry, or two-phase publication across the resetter, writer, and +caller-owned manifest store. If reset succeeds and a later replacement fails, +the vector index can contain an incomplete rebuild and no successful +checkpoint is issued. Keep the prior manifest out of service and run an +application-controlled recovery, normally another full rebuild. + +## State matrix + +`ProjectionStateAvailability` describes what the caller could load. +`ProjectionStateStatus` is the workflow's evaluation of that snapshot against +the requested complete target. + +| Supplied state | Evaluated status | `project_documents` | `rebuild_projection` | +| --- | --- | --- | --- | +| `MISSING` with no manifest | `MISSING` | Raises before effects | Resets and builds target | +| `CORRUPT` with no manifest | `CORRUPT` | Raises before effects | Resets and builds target | +| Valid present manifest exactly matching target | `CURRENT` | Returns `UNCHANGED`; no effects | Resets and rebuilds target | +| Valid present manifest with compatible target differences | `STALE` | Applies incremental mutations | Resets and rebuilds target | +| Present manifest with invalid checkpoint, wrong corpus, or same-revision source/count inconsistency | `CORRUPT` | Raises before effects | Resets and rebuilds target | +| Present manifest with another schema ID | `SCHEMA_MISMATCH` | Raises before effects | Resets and rebuilds target | +| Present manifest with another embedding identity | `EMBEDDING_MISMATCH` | Raises before effects | Resets and rebuilds target | + +A revision change is a normal stale update. For the same full document +identity, changing the source digest—or the fragment count under unchanged +chunking—is treated as corrupt state rather than an unannounced rewrite. + +## Deterministic projection values + +Documents and manifest entries are canonicalized by opaque `document_id`. +Fragments use half-open Python Unicode code-point ranges and copy the source's +ordered attributes. Each fragment contains at most +`max_fragment_codepoints`; consecutive fragments overlap by +`overlap_codepoints`. Attribute order and duplicates remain significant. + +Source digests, fragment IDs, and checkpoint tokens use lowercase +`sha256:<64hex>` values. The current algorithms serialize tagged fields as +UTF-8 with `surrogatepass`, prefix each encoded field with its unsigned +eight-byte big-endian length, and hash them under these versioned domains: + +| Value | v1 domain | Bound inputs | +| --- | --- | --- | +| Source digest | `generic-rag:projection-source:v1` | Exact document text and ordered attributes | +| Fragment ID | `generic-rag:fragment-id:v1` | Corpus, document, revision, and fragment range | +| Checkpoint token | `generic-rag:projection-checkpoint:v1` | Corpus, projection and embedding identities, chunking policy, and ordered manifest entries | + +These values are reproducible for the same inputs and current v1 algorithm, +including across clean processes. The v1 domain names do not promise that a +future package version will retain the same algorithm or accept an old +manifest. Consumers that persist projection state should pin and review the +package version and use explicit rebuild for an incompatible upgrade. + +The positional-only public helpers `derive_source_digest`, +`derive_fragment_id`, `derive_projection_checkpoint_token`, and +`has_valid_projection_checkpoint` live in +`generic_rag.projection_integrity`. Projection and retrieval call the same +helpers; hosts may use them to construct or validate published state. They +validate contract structure but do not inspect provider storage. See the [API +reference](api.md) for their exact signatures. + +Hashes are deterministic comparison and identity values, not encryption, +authorization, or a proof of source ownership. See [security and +privacy](security-and-privacy.md). + +## Failures and receipts + +Invalid public values raise `ContractValidationError`. Incremental state that +is not `current` or `stale` raises `ProjectionStateError`, whose `status` gives +the evaluated reason. Both cases are detected before collaborator effects. + +An ordinary collaborator exception, or an invalid collaborator return, raises +`ProjectionOperationError` with: + +- `stage`: `EMBEDDER_IDENTITY`, `EMBEDDING`, `REPLACEMENT`, `DELETION`, or + `RESET`; +- `affected_document`: the stable key for a document-specific failure, else + `None`; and +- `receipt`: `FAILED` when no planned document completed, `PARTIAL` after one + or more but not all planned documents completed, or `None` when there were + zero document attempts. + +Failure receipts never contain a checkpoint. `attempted_documents` is the +total mutation or rebuild-document count; `completed_documents` counts only +fully completed document operations. The original ordinary exception is +chained as the cause. A structurally invalid identity, vector result, or +non-`None` writer/resetter return has no internal cause. `KeyboardInterrupt` +and `SystemExit` are neither translated nor retried. + +The public error messages do not include document or vector content. Adapter +exception messages remain reachable through exception chaining, so adapters +and application logging must avoid disclosing sensitive values. + +See the [API reference](api.md) for exact signatures and value invariants, +[resource lifecycle](lifecycle.md) for ownership details, and the [retrieval +guide](retrieval.md) for consuming a successfully published projection. diff --git a/docs/retrieval.md b/docs/retrieval.md new file mode 100644 index 0000000..3e25f21 --- /dev/null +++ b/docs/retrieval.md @@ -0,0 +1,324 @@ +# Retrieval + +`generic-rag` provides synchronous semantic and hybrid retrieval over a +host-published projection. The package composes caller-supplied collaborators; +it does not select, load, configure, retry, or close a provider. + +The package root intentionally exports nothing. Import retrieval contracts from +`generic_rag.contracts`, borrowed collaborator protocols from +`generic_rag.ports`, integrity helpers from +`generic_rag.projection_integrity`, and workflows from +`generic_rag.retrieval`. + +## Independent consumer example + +This complete example constructs a valid published state, injects local fake +providers, invokes both workflows, and resolves returned fragment identities +against a still-authorized authoritative source before creating citations. + +```python +from generic_rag.contracts import ( + ChunkingPolicy, + Document, + DocumentIdentity, + DocumentKey, + EmbeddingIdentity, + EmbeddingVector, + Fragment, + FragmentIdentity, + ProjectionCheckpoint, + ProjectionIdentity, + ProjectionManifest, + ProjectionManifestEntry, + ProjectionStateAvailability, + ProjectionStateSnapshot, + RetrievalLimits, + RetrievalOutcome, + RetrievalQuery, +) +from generic_rag.ports import Borrowed +from generic_rag.projection_integrity import ( + derive_fragment_id, + derive_projection_checkpoint_token, + derive_source_digest, +) +from generic_rag.retrieval import retrieve_hybrid, retrieve_semantic + +corpus_id = "corpus-a" +source_document = Document( + identity=DocumentIdentity( + key=DocumentKey(corpus_id=corpus_id, document_id="document-a"), + revision_id="revision-1", + ), + text="Approved source text.", + attributes=(("classification", "public"),), +) +chunking = ChunkingPolicy(max_fragment_codepoints=128, overlap_codepoints=0) +embedding_identity = EmbeddingIdentity(model_id="example-embedding", dimensions=2) +projection_identity = ProjectionIdentity( + schema_id="schema-v1", + embedding=embedding_identity, +) +entries = ( + ProjectionManifestEntry( + document=source_document.identity, + source_digest=derive_source_digest(source_document), + fragment_count=1, + ), +) +checkpoint_token = derive_projection_checkpoint_token( + corpus_id, + projection_identity, + chunking, + entries, +) +published_manifest = ProjectionManifest( + corpus_id=corpus_id, + projection=projection_identity, + chunking=chunking, + entries=entries, + checkpoint=ProjectionCheckpoint( + corpus_id=corpus_id, + projection=projection_identity, + token=checkpoint_token, + ), +) +published_state = ProjectionStateSnapshot( + availability=ProjectionStateAvailability.PRESENT, + manifest=published_manifest, +) + +start, end = 0, len(source_document.text) +fragment = Fragment( + identity=FragmentIdentity( + document=source_document.identity, + fragment_id=derive_fragment_id(source_document.identity, start, end), + start=start, + end=end, + ), + text=source_document.text[start:end], + attributes=source_document.attributes, +) + + +class ExampleEmbedder: + @property + def identity(self): + return embedding_identity + + def embed(self, texts, /): + assert texts == ("Where is the approved source?",) + return (EmbeddingVector(values=(1.0, 0.0)),) + + +class ExampleVectorReader: + def search(self, query, embedding, /): + assert embedding == EmbeddingVector(values=(1.0, 0.0)) + return (fragment,) + + +class ExampleLexicalRetriever: + def search(self, query, /): + return (fragment,) + + +query = RetrievalQuery( + corpus_id=corpus_id, + text="Where is the approved source?", + hit_limit=2, + candidate_limit=4, +) +limits = RetrievalLimits(max_query_codepoints=200) + +# The host reauthorizes the corpus and documents for this query before calling +# retrieval. The published state must describe the projection being searched. +authorized_revisions = {source_document.identity: source_document} + +semantic_result = retrieve_semantic( + query, + published_state, + limits, + Borrowed(ExampleEmbedder()), + Borrowed(ExampleVectorReader()), +) +hybrid_result = retrieve_hybrid( + query, + published_state, + limits, + Borrowed(ExampleEmbedder()), + Borrowed(ExampleVectorReader()), + Borrowed(ExampleLexicalRetriever()), +) + + +def host_citations(result): + if result.outcome not in ( + RetrievalOutcome.COMPLETE, + RetrievalOutcome.PARTIAL, + ): + return () + + citations = [] + for hit in result.hits: + identity = hit.fragment.identity + authoritative = authorized_revisions.get(identity.document) + if authoritative is None: + continue + authoritative_text = authoritative.text[identity.start : identity.end] + if authoritative_text != hit.fragment.text: + continue + citations.append( + ( + identity.document.key.document_id, + identity.document.revision_id, + identity.start, + identity.end, + ) + ) + return tuple(citations) + + +assert semantic_result.outcome is RetrievalOutcome.COMPLETE +assert hybrid_result.outcome is RetrievalOutcome.COMPLETE +assert host_citations(semantic_result) == (("document-a", "revision-1", 0, 21),) +assert host_citations(hybrid_result) == (("document-a", "revision-1", 0, 21),) +``` + +`authorized_revisions` represents a host authorization decision made for this +query; it is not package state. A real host also verifies that the requesting +user or agent may use the corpus before invoking retrieval. + +## Inputs and limits + +Both workflows require: + +- a `RetrievalQuery` with a nonblank corpus and query, a positive `hit_limit`, + and a positive `candidate_limit` where `hit_limit <= candidate_limit`; +- the matching `ProjectionStateSnapshot` loaded by the host; +- a `RetrievalLimits` whose positive `max_query_codepoints` bounds the query + before any collaborator call; and +- caller-owned collaborators wrapped in `Borrowed`. + +The limits are caller-selected work budgets, not tenant quotas, rate limits, +authorization rules, or guarantees about a provider's resource use. Each +provider may return at most `candidate_limit` candidates. The package validates +that bound but does not configure the provider or restrict work hidden behind +its interface. + +A missing state produces `unavailable`. A corrupt state or invalid checkpoint +produces `failed`. A state for another corpus produces `unavailable`. A valid +published state with no entries produces an empty `complete` result without +calling a provider. + +Every provider return must be an exact tuple with no more than +`candidate_limit` entries. Each entry must be an exact `Fragment` for the query +corpus, stay within the manifest's fragment-width bound, and carry the +deterministic fragment ID for its document identity and range. Retrieval looks +up each stable `DocumentKey` in the manifest and keeps it only when the full +published `DocumentIdentity`, including revision, matches. Missing and old +revisions are filtered as stale. These checks still cannot prove that candidate +text equals its claimed authoritative source slice. + +## Semantic retrieval + +`retrieve_semantic` performs these steps in order: + +1. Validate the top-level query, state, limits, and borrowed handles without + collaborator effects. +2. Check that the borrowed embedder identity exactly matches the manifest's + embedding identity. A mismatch produces `stale` without embedding or vector + search. +3. Embed the exact one-element tuple `(query.text,)` and require exactly one + finite vector with the published dimensions. +4. Ask the vector reader for up to `candidate_limit` fragments. +5. Validate, deduplicate, and filter the candidates against current manifest + revisions, then return at most `hit_limit` hits in provider order. + +Provider ranks used during validation and fusion are the candidates' original +one-based positions in the provider tuple. Filtering or exact-identity +deduplication does not close gaps in those source ranks. Semantic retrieval +preserves the remaining provider order, while public `RetrievalHit.rank` values +are final contiguous ranks from one. The first identical occurrence wins. +Conflicting payloads for one exact fragment identity fail that provider branch. + +## Hybrid retrieval + +`retrieve_hybrid` executes the semantic branch and then the lexical branch. +Semantic failure or staleness does not prevent the lexical call. Each provider +returns at most `candidate_limit` candidates, and its original ranks are +preserved. + +Validated current-revision candidates are fused by reciprocal rank fusion: + +```text +fused score = sum(1 / (60 + provider rank)) +``` + +Raw provider scores are neither accepted nor returned. Exact fragment +identities are deduplicated across providers. Identical payloads contribute +both ranks; conflicting payloads produce a content-free `failed` result. Hits +sort by descending fused score, then by this opaque identity tuple: + +```text +(corpus_id, document_id, revision_id, fragment_id, start, end) +``` + +That final comparison is deterministic raw string/integer ordering, not text +normalization or semantic relevance. `truncated` is true only when the number +of validated, unique, current-revision fragments exceeds `hit_limit`. + +## Outcomes + +| Outcome | Meaning and host response | +| --- | --- | +| `complete` | All required branches completed. Hits may be empty. Resolve every returned identity against current authorized source data before use. | +| `partial` | At least one validated hit is available, but a branch was stale or failed. Revalidate and cite usable hits; apply host policy before showing or injecting them. | +| `unavailable` | State is missing or the loaded manifest belongs to another corpus. Do not treat this as an empty authoritative answer. | +| `stale` | No hits are usable and a provider identity or candidate revision is stale. Reconcile or rebuild the projection. | +| `failed` | No hits are usable because state, collaborator output, or an ordinary collaborator call failed. Apply host retry, fallback, and audit policy. | + +If stale and failed conditions coexist without hits, `failed` takes precedence. +Ordinary `Exception` failures at collaborator boundaries contribute only a +content-free failure state: the failed branch contributes no fragment or +exception text. A terminal `failed` result is empty; a `partial` result contains +only independently validated hits from another branch. `BaseException` +subclasses such as cancellation signals propagate and must be handled by the +host runtime. + +## User and agent utilization + +The required host flow is: + +1. Authorize documents before projection and publish the resulting manifest + and provider index under host-controlled persistence and concurrency. +2. Reauthorize the user or agent, corpus, and documents for every query. +3. Load the published manifest that matches the provider index being searched. +4. Invoke semantic or hybrid retrieval with caller-owned collaborators. +5. Resolve every returned `FragmentIdentity` against the still-authorized, + authoritative source revision. Verify the exact source slice still equals + the returned text, then create a host citation. +6. Show cited results to the authorized user, or inject only bounded cited + context into an agent. Agents must retain the host citations in any derived + answer or artifact. + +Fragment text is derived, non-authoritative data. A collaborator candidate can +prove neither that its text equals the authoritative source slice nor that the +requester remains authorized. Matching deterministic identities and the +published revision is necessary but insufficient; host revalidation is +mandatory. + +The package never authenticates or authorizes users, creates citations, logs +queries or results, loads providers, opens network connections, owns credentials +or provider resources, or decides prompt, tool, display, retry, or fallback +policy. See [Lifecycle and ownership](lifecycle.md) and +[Security and privacy](security-and-privacy.md) for host responsibilities. + +## Operational ownership + +The host owns provider selection and acquisition, network and credential +handling, retry and timeout policy, manifest and index persistence, concurrency +control, audit, disable/rebuild/purge procedures, and orderly shutdown. +`Borrowed` is only a no-op ownership marker: retrieval neither enters nor exits +the underlying collaborator. See [Projection](projection.md) for how to create +the published state and [API reference](api.md) for exact imports and +signatures. diff --git a/docs/security-and-privacy.md b/docs/security-and-privacy.md new file mode 100644 index 0000000..179e188 --- /dev/null +++ b/docs/security-and-privacy.md @@ -0,0 +1,160 @@ +# Security and Privacy + +Version 0.1.0 defines in-process values, collaborator boundaries, and explicit +projection and retrieval workflows. The package itself performs no +persistence, network setup, provider discovery, credential loading, telemetry, +or background work. Projection passes derived fragment text and metadata to +caller-supplied providers. Retrieval passes query data to caller-supplied +providers and accepts fragment candidates from them. Those collaborator +effects are outside the package. + +## Caller responsibility + +Before constructing a `ProjectionRequest`, the caller must authorize every +source, revision, attribute, and intended destination. The package does not +authenticate an authoritative source or decide whether a user, tool, or agent +may project it. + +Before every retrieval, the caller must reauthorize the requester, corpus, and +source revisions, then load the published manifest paired with the index being +searched. After retrieval, it must resolve each fragment identity against +still-authorized authoritative data, verify the exact source slice, and create +a citation before display or agent use. An agent must retain those host +citations in derived output. + +The caller also controls: + +- corpus and tenant isolation; +- which adapter implementations receive document text, fragment text, + embeddings, attributes, identifiers, or queries; +- provider account, region, transport, and credential configuration; +- vector-index and manifest-store access control, retention, replacement, + deletion, backup, and recovery; +- synchronization between vector mutations and manifest publication; +- logging, tracing, metrics, exception rendering, redaction, and incident + response; and +- whether retrieved fragments are displayed, persisted, or supplied to another + tool or agent, and the prompt/tool policy applied to them. + +Do not place credentials or other secrets in attributes, opaque identifiers, +or checkpoint tokens. These fields preserve caller input and do not apply +redaction, escaping, authorization, or tenant isolation. + +## Adapter effects + +Calling `project_documents` can invoke the supplied embedder and writer. +Calling `rebuild_projection` can additionally reset all projected data for the +requested corpus. Those adapters may persist or transmit data according to +their implementations. Review their transport, storage, subprocess, network, +credential, and deletion behavior before use. + +Calling `retrieve_semantic` passes the exact query text to the embedder, then +passes the query and derived query embedding to the vector reader. Calling +`retrieve_hybrid` also passes the query to the lexical retriever. A provider may +transmit, retain, correlate, or log those values according to its +implementation. The package has no hidden network or runtime dependency, but +injected collaborators can have both. + +`Borrowed` only marks resources as caller-owned. It does not acquire, close, +authenticate, synchronize, sandbox, or reduce the privileges of an adapter. +The package supplies no transaction or rollback across the vector index and +caller-owned manifest store. See [resource lifecycle](lifecycle.md). + +## Sensitive source and derived data + +Treat all of the following as potentially sensitive: + +- document and query text; +- ordered attributes and opaque identities; +- fragments and their source ranges; +- embeddings and vector records; +- source digests, fragment IDs, manifests, and checkpoint tokens; +- retrieval ranks and outcomes; and +- adapter exceptions and host logs. + +Fragments and embeddings may reveal source information. Deterministic IDs and +digests may allow equality correlation or guessing attacks against predictable +content. Their `sha256:` representation provides neither encryption nor access +control and should not be used as proof of source ownership. + +Retrieval results expose no raw provider score, but their ranks, identities, +fragment text, truncation flag, and outcome can still reveal content, +correlation, or availability information. Do not treat the score-free boundary +as anonymization. + +Deleting an authoritative source does not automatically delete copies, +backups, logs, embeddings, or derived records held by an application or +provider. Incremental deletion and corpus reset cover only the behavior promised +by the supplied vector-index adapter. + +## Limits and resource policy + +`ProjectionLimits` rejects a request that exceeds its configured document count +or per-document code-point cap and bounds each embedding batch. +`ChunkingPolicy` bounds fragment size and overlap. These checks prevent one +accepted request from exceeding caller-selected values; they are not global +quotas, rate limits, memory isolation, provider billing controls, timeouts, or +admission control. + +`RetrievalLimits.max_query_codepoints` bounds query text before collaborator +effects. `candidate_limit` bounds the tuple accepted from each provider, and +`hit_limit` bounds returned hits. These are likewise caller-selected work +budgets, not quotas, authorization, cost controls, or proof that a provider did +only bounded internal work. + +Choose limits from trusted application policy rather than untrusted request +parameters. Account for the fact that a small fragment size and large permitted +document set can still produce many fragments and provider operations. Supply +external cancellation, concurrency, cost, and capacity controls where needed. + +## Errors and logging + +`ProjectionStateError` and the direct message of +`ProjectionOperationError` are content-free. Ordinary adapter exceptions are +preserved as chained causes, so rendering the full exception chain may expose +an adapter's message or fields. Adapter implementations must avoid placing +document text, fragments, embeddings, credentials, or sensitive identifiers in +exceptions and logs. Applications should apply redaction before exporting +traces or error reports. + +The package does not retry collaborator operations. A failure can leave earlier +document mutations in place, and a rebuild failure can occur after the corpus +was reset. Do not publish a failed or partial receipt as a completed checkpoint; +isolate the affected projection and recover under caller-owned policy. + +Retrieval maps ordinary collaborator exceptions and malformed collaborator +returns to a failed provider branch, producing `failed` when no validated hit +remains or `partial` when another branch supplies one. A `failed` result is +content-free, and a `partial` result contains only independently validated +hits; provider exception text and rejected candidates do not cross the result +boundary. Stale provider identity or revision data is reported separately as +`stale`, or as `partial` when another branch supplies a validated hit. The host +decides whether to retry, fall back, audit, or suppress a result. +`BaseException` subclasses propagate unchanged, so the host runtime must handle +its own cancellation and shutdown signals. The package does not log queries, +fragments, provider failures, or outcomes. + +## Trust boundary + +A fragment range is a half-open Python code-point range. Standalone contracts +check its width but do not prove that text came from the indicated source. +Projection derives its own fragment text from the supplied document, but the +package still cannot prove that the supplied document or revision was +authoritative or authorized. + +Retrieval candidates are collaborator-supplied derived data. Deterministic +fragment identity and a current manifest revision do not prove that candidate +text equals the authoritative source slice or that the requester remains +authorized. Host reauthorization, source resolution, exact-slice validation, +and citation creation are mandatory before use. + +Version 0.1.0 provides no built-in encryption, authentication, authorization, +ACL, content filter, persistence security, network security, citation +creation or validation, secret management, vendor guarantee, logging, or +user/agent policy. A consuming application must select and assess those +controls for its environment. + +The complete public value and collaborator boundaries are listed in the [API +reference](api.md), and deterministic projection behavior is documented in the +[projection guide](projection.md). Retrieval behavior and the required host +flow are documented in the [retrieval guide](retrieval.md). diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..09a30bb --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,53 @@ +[build-system] +requires = ["setuptools>=83,<85"] +build-backend = "setuptools.build_meta" + +[project] +name = "generic-rag" +version = "0.1.0" +description = "Provider-neutral, dependency-free RAG projection and retrieval runtime" +readme = "README.md" +requires-python = ">=3.11" +dependencies = [] + +[dependency-groups] +build = [ + "build>=1.5,<2", + "setuptools>=83,<85", +] +lint = [ + "ruff>=0.16,<0.17", +] +type = [ + "mypy>=2.3,<3", +] +dev = [ + { include-group = "build" }, + { include-group = "lint" }, + { include-group = "type" }, +] +test = [] + +[tool.setuptools] +package-dir = { "" = "src" } + +[tool.setuptools.packages.find] +where = ["src"] +include = ["generic_rag", "generic_rag.*"] + +[tool.setuptools.package-data] +generic_rag = ["py.typed"] + +[tool.ruff] +target-version = "py311" +line-length = 88 +src = ["src"] + +[tool.ruff.lint] +select = ["E", "F", "I", "UP"] + +[tool.mypy] +python_version = "3.11" +strict = true +mypy_path = "src" +packages = ["generic_rag"] diff --git a/src/generic_rag/__init__.py b/src/generic_rag/__init__.py new file mode 100644 index 0000000..46c466f --- /dev/null +++ b/src/generic_rag/__init__.py @@ -0,0 +1,7 @@ +"""Provider-neutral RAG contracts. + +Issue #2 intentionally exposes no root-level facade. Consumers import public +contracts, errors, and ports from their owning modules. +""" + +__all__: tuple[str, ...] = () diff --git a/src/generic_rag/contracts.py b/src/generic_rag/contracts.py new file mode 100644 index 0000000..077349f --- /dev/null +++ b/src/generic_rag/contracts.py @@ -0,0 +1,662 @@ +"""Immutable, validated values shared by generic RAG workflows and ports.""" + +from __future__ import annotations + +import re +from dataclasses import dataclass +from enum import StrEnum +from math import isfinite + +from .errors import ContractValidationError + +__all__ = ( + "DocumentKey", + "DocumentIdentity", + "Document", + "FragmentIdentity", + "Fragment", + "EmbeddingIdentity", + "EmbeddingVector", + "VectorRecord", + "ProjectionIdentity", + "ProjectionCheckpoint", + "ProjectionOutcome", + "ProjectionReceipt", + "ChunkingPolicy", + "ProjectionLimits", + "ProjectionRequest", + "ProjectionManifestEntry", + "ProjectionManifest", + "ProjectionStateAvailability", + "ProjectionStateSnapshot", + "ProjectionStateStatus", + "ProjectionResult", + "RetrievalLimits", + "RetrievalQuery", + "RetrievalOutcome", + "RetrievalHit", + "RetrievalResult", +) + +_SOURCE_DIGEST = re.compile(r"sha256:[0-9a-f]{64}") + + +def _require_exact_type(name: str, value: object, expected: type[object]) -> None: + if type(value) is not expected: + raise ContractValidationError( + f"{name} must be exactly {expected.__name__}, not {type(value).__name__}" + ) + + +def _require_nonblank_string(name: str, value: object) -> str: + _require_exact_type(name, value, str) + assert isinstance(value, str) + if not value or value.isspace(): + raise ContractValidationError(f"{name} must not be empty or whitespace-only") + return value + + +def _require_nonnegative_integer(name: str, value: object) -> int: + _require_exact_type(name, value, int) + assert isinstance(value, int) + if value < 0: + raise ContractValidationError(f"{name} must be nonnegative") + return value + + +def _require_positive_integer(name: str, value: object) -> int: + result = _require_nonnegative_integer(name, value) + if result == 0: + raise ContractValidationError(f"{name} must be positive") + return result + + +def _validate_attributes(name: str, attributes: object) -> tuple[tuple[str, str], ...]: + _require_exact_type(name, attributes, tuple) + assert isinstance(attributes, tuple) + for index, pair in enumerate(attributes): + if type(pair) is not tuple or len(pair) != 2: + raise ContractValidationError( + f"{name}[{index}] must be exactly a two-element tuple" + ) + key, value = pair + _require_exact_type(f"{name}[{index}][0]", key, str) + _require_exact_type(f"{name}[{index}][1]", value, str) + return attributes + + +@dataclass(frozen=True, slots=True) +class DocumentKey: + """Stable opaque corpus/document identity independent of revision.""" + + corpus_id: str + document_id: str + + def __post_init__(self) -> None: + _require_nonblank_string("corpus_id", self.corpus_id) + _require_nonblank_string("document_id", self.document_id) + + +@dataclass(frozen=True, slots=True) +class DocumentIdentity: + """A stable document key at one opaque authoritative revision.""" + + key: DocumentKey + revision_id: str + + def __post_init__(self) -> None: + _require_exact_type("key", self.key, DocumentKey) + _require_nonblank_string("revision_id", self.revision_id) + + +@dataclass(frozen=True, slots=True) +class Document: + """Exact caller-supplied document text and ordered opaque attributes.""" + + identity: DocumentIdentity + text: str + attributes: tuple[tuple[str, str], ...] = () + + def __post_init__(self) -> None: + _require_exact_type("identity", self.identity, DocumentIdentity) + _require_exact_type("text", self.text, str) + _validate_attributes("attributes", self.attributes) + + +@dataclass(frozen=True, slots=True) +class FragmentIdentity: + """Opaque fragment identity and half-open code-point source range.""" + + document: DocumentIdentity + fragment_id: str + start: int + end: int + + def __post_init__(self) -> None: + _require_exact_type("document", self.document, DocumentIdentity) + _require_nonblank_string("fragment_id", self.fragment_id) + start = _require_nonnegative_integer("start", self.start) + end = _require_nonnegative_integer("end", self.end) + if start >= end: + raise ContractValidationError("fragment range must satisfy start < end") + + +@dataclass(frozen=True, slots=True) +class Fragment: + """An exact nonempty document slice carried as non-authoritative derived data.""" + + identity: FragmentIdentity + text: str + attributes: tuple[tuple[str, str], ...] = () + + def __post_init__(self) -> None: + _require_exact_type("identity", self.identity, FragmentIdentity) + _require_exact_type("text", self.text, str) + if not self.text: + raise ContractValidationError("fragment text must not be empty") + if len(self.text) != self.identity.end - self.identity.start: + raise ContractValidationError( + "fragment text length must equal its half-open code-point range" + ) + _validate_attributes("attributes", self.attributes) + + +@dataclass(frozen=True, slots=True) +class EmbeddingIdentity: + """Opaque embedding-model identity and its required vector dimension.""" + + model_id: str + dimensions: int + + def __post_init__(self) -> None: + _require_nonblank_string("model_id", self.model_id) + _require_positive_integer("dimensions", self.dimensions) + + +@dataclass(frozen=True, slots=True) +class EmbeddingVector: + """A finite, nonempty embedding vector stored canonically as floats.""" + + values: tuple[float, ...] + + def __post_init__(self) -> None: + _require_exact_type("values", self.values, tuple) + if not self.values: + raise ContractValidationError("embedding vector must not be empty") + + canonical: list[float] = [] + for index, coordinate in enumerate(self.values): + if type(coordinate) not in (int, float): + raise ContractValidationError( + f"values[{index}] must be int or float, excluding bool" + ) + try: + numeric_coordinate = float(coordinate) + except OverflowError: + raise ContractValidationError( + f"values[{index}] must be finite" + ) from None + if not isfinite(numeric_coordinate): + raise ContractValidationError(f"values[{index}] must be finite") + canonical.append(numeric_coordinate) + object.__setattr__(self, "values", tuple(canonical)) + + +@dataclass(frozen=True, slots=True) +class VectorRecord: + """A fragment paired with the vector written for it.""" + + fragment: Fragment + embedding: EmbeddingVector + + def __post_init__(self) -> None: + _require_exact_type("fragment", self.fragment, Fragment) + _require_exact_type("embedding", self.embedding, EmbeddingVector) + + +@dataclass(frozen=True, slots=True) +class ProjectionIdentity: + """Schema and embedding identity that determine projection compatibility.""" + + schema_id: str + embedding: EmbeddingIdentity + + def __post_init__(self) -> None: + _require_nonblank_string("schema_id", self.schema_id) + _require_exact_type("embedding", self.embedding, EmbeddingIdentity) + + +@dataclass(frozen=True, slots=True) +class ProjectionCheckpoint: + """Opaque completed checkpoint bound to one corpus and projection identity.""" + + corpus_id: str + projection: ProjectionIdentity + token: str + + def __post_init__(self) -> None: + _require_nonblank_string("corpus_id", self.corpus_id) + _require_exact_type("projection", self.projection, ProjectionIdentity) + _require_nonblank_string("token", self.token) + + +class ProjectionOutcome(StrEnum): + """Truthful completion state for a projection attempt.""" + + COMPLETED = "completed" + UNCHANGED = "unchanged" + PARTIAL = "partial" + FAILED = "failed" + + @classmethod + def _missing_(cls, value: object) -> None: + raise ContractValidationError(f"{cls.__name__} value is not a defined member") + + +@dataclass(frozen=True, slots=True) +class ProjectionReceipt: + """Projection counts and checkpoint subject to completion invariants.""" + + corpus_id: str + projection: ProjectionIdentity + outcome: ProjectionOutcome + attempted_documents: int + completed_documents: int + checkpoint: ProjectionCheckpoint | None + + def __post_init__(self) -> None: + _require_nonblank_string("corpus_id", self.corpus_id) + _require_exact_type("projection", self.projection, ProjectionIdentity) + _require_exact_type("outcome", self.outcome, ProjectionOutcome) + attempted = _require_nonnegative_integer( + "attempted_documents", self.attempted_documents + ) + completed = _require_nonnegative_integer( + "completed_documents", self.completed_documents + ) + if completed > attempted: + raise ContractValidationError( + "completed_documents must not exceed attempted_documents" + ) + if self.checkpoint is not None: + _require_exact_type("checkpoint", self.checkpoint, ProjectionCheckpoint) + if self.checkpoint.corpus_id != self.corpus_id: + raise ContractValidationError( + "checkpoint corpus_id must match the receipt corpus_id" + ) + if self.checkpoint.projection != self.projection: + raise ContractValidationError( + "checkpoint projection must match the receipt projection" + ) + + successful = self.outcome in ( + ProjectionOutcome.COMPLETED, + ProjectionOutcome.UNCHANGED, + ) + if successful: + if completed != attempted: + raise ContractValidationError( + "completed and unchanged receipts must complete every attempt" + ) + if self.checkpoint is None: + raise ContractValidationError( + "completed and unchanged receipts require a checkpoint" + ) + elif self.outcome is ProjectionOutcome.PARTIAL: + if not 0 < completed < attempted: + raise ContractValidationError( + "partial receipts require 0 < completed_documents " + "< attempted_documents" + ) + if self.checkpoint is not None: + raise ContractValidationError( + "partial receipts must not claim a checkpoint" + ) + else: + if attempted == 0 or completed != 0: + raise ContractValidationError( + "failed receipts require attempts and zero completed documents" + ) + if self.checkpoint is not None: + raise ContractValidationError( + "failed receipts must not claim a checkpoint" + ) + + +@dataclass(frozen=True, slots=True) +class ChunkingPolicy: + """Code-point chunk size and overlap used by one projection.""" + + max_fragment_codepoints: int + overlap_codepoints: int + + def __post_init__(self) -> None: + maximum = _require_positive_integer( + "max_fragment_codepoints", + self.max_fragment_codepoints, + ) + overlap = _require_nonnegative_integer( + "overlap_codepoints", + self.overlap_codepoints, + ) + if overlap >= maximum: + raise ContractValidationError( + "overlap_codepoints must be smaller than max_fragment_codepoints" + ) + + +@dataclass(frozen=True, slots=True) +class ProjectionLimits: + """Independent document, text, and embedding-batch projection bounds.""" + + max_documents: int + max_document_codepoints: int + max_embedding_batch_size: int + + def __post_init__(self) -> None: + _require_positive_integer("max_documents", self.max_documents) + _require_positive_integer( + "max_document_codepoints", + self.max_document_codepoints, + ) + _require_positive_integer( + "max_embedding_batch_size", + self.max_embedding_batch_size, + ) + + +@dataclass(frozen=True, slots=True) +class ProjectionRequest: + """A complete bounded canonical target projection for one corpus.""" + + corpus_id: str + projection: ProjectionIdentity + chunking: ChunkingPolicy + limits: ProjectionLimits + documents: tuple[Document, ...] + + def __post_init__(self) -> None: + _require_nonblank_string("corpus_id", self.corpus_id) + _require_exact_type("projection", self.projection, ProjectionIdentity) + _require_exact_type("chunking", self.chunking, ChunkingPolicy) + _require_exact_type("limits", self.limits, ProjectionLimits) + _require_exact_type("documents", self.documents, tuple) + if len(self.documents) > self.limits.max_documents: + raise ContractValidationError("documents must not exceed max_documents") + + keys: set[DocumentKey] = set() + canonical: list[Document] = [] + for index, document in enumerate(self.documents): + _require_exact_type(f"documents[{index}]", document, Document) + if document.identity.key.corpus_id != self.corpus_id: + raise ContractValidationError( + "every document corpus_id must match the request corpus_id" + ) + if len(document.text) > self.limits.max_document_codepoints: + raise ContractValidationError( + "document text must not exceed max_document_codepoints" + ) + if document.identity.key in keys: + raise ContractValidationError( + "documents must have unique stable document keys" + ) + keys.add(document.identity.key) + canonical.append(document) + canonical.sort(key=lambda document: document.identity.key.document_id) + object.__setattr__(self, "documents", tuple(canonical)) + + +@dataclass(frozen=True, slots=True) +class ProjectionManifestEntry: + """One projected document revision, source digest, and fragment count.""" + + document: DocumentIdentity + source_digest: str + fragment_count: int + + def __post_init__(self) -> None: + _require_exact_type("document", self.document, DocumentIdentity) + _require_exact_type("source_digest", self.source_digest, str) + if _SOURCE_DIGEST.fullmatch(self.source_digest) is None: + raise ContractValidationError( + "source_digest must be lowercase sha256:<64hex>" + ) + _require_nonnegative_integer("fragment_count", self.fragment_count) + + +@dataclass(frozen=True, slots=True) +class ProjectionManifest: + """A complete canonical successful projection checkpoint manifest.""" + + corpus_id: str + projection: ProjectionIdentity + chunking: ChunkingPolicy + entries: tuple[ProjectionManifestEntry, ...] + checkpoint: ProjectionCheckpoint + + def __post_init__(self) -> None: + _require_nonblank_string("corpus_id", self.corpus_id) + _require_exact_type("projection", self.projection, ProjectionIdentity) + _require_exact_type("chunking", self.chunking, ChunkingPolicy) + _require_exact_type("entries", self.entries, tuple) + _require_exact_type("checkpoint", self.checkpoint, ProjectionCheckpoint) + if self.checkpoint.corpus_id != self.corpus_id: + raise ContractValidationError( + "checkpoint corpus_id must match the manifest corpus_id" + ) + if self.checkpoint.projection != self.projection: + raise ContractValidationError( + "checkpoint projection must match the manifest projection" + ) + + keys: set[DocumentKey] = set() + canonical: list[ProjectionManifestEntry] = [] + for index, entry in enumerate(self.entries): + _require_exact_type(f"entries[{index}]", entry, ProjectionManifestEntry) + if entry.document.key.corpus_id != self.corpus_id: + raise ContractValidationError( + "every manifest entry corpus_id must match the manifest corpus_id" + ) + if entry.document.key in keys: + raise ContractValidationError( + "manifest entries must have unique stable document keys" + ) + keys.add(entry.document.key) + canonical.append(entry) + canonical.sort(key=lambda entry: entry.document.key.document_id) + object.__setattr__(self, "entries", tuple(canonical)) + + +class ProjectionStateAvailability(StrEnum): + """Structural availability of a caller-supplied projection snapshot.""" + + MISSING = "missing" + PRESENT = "present" + CORRUPT = "corrupt" + + @classmethod + def _missing_(cls, value: object) -> None: + raise ContractValidationError(f"{cls.__name__} value is not a defined member") + + +@dataclass(frozen=True, slots=True) +class ProjectionStateSnapshot: + """Caller-supplied projection state without package-owned persistence.""" + + availability: ProjectionStateAvailability + manifest: ProjectionManifest | None + + def __post_init__(self) -> None: + _require_exact_type( + "availability", + self.availability, + ProjectionStateAvailability, + ) + if self.availability is ProjectionStateAvailability.PRESENT: + _require_exact_type("manifest", self.manifest, ProjectionManifest) + elif self.manifest is not None: + raise ContractValidationError( + "missing and corrupt state snapshots must not contain a manifest" + ) + + +class ProjectionStateStatus(StrEnum): + """Compatibility of supplied state with one complete projection target.""" + + MISSING = "missing" + CURRENT = "current" + STALE = "stale" + CORRUPT = "corrupt" + SCHEMA_MISMATCH = "schema_mismatch" + EMBEDDING_MISMATCH = "embedding_mismatch" + + @classmethod + def _missing_(cls, value: object) -> None: + raise ContractValidationError(f"{cls.__name__} value is not a defined member") + + +@dataclass(frozen=True, slots=True) +class ProjectionResult: + """One completely successful projection result and authoritative manifest.""" + + status_before: ProjectionStateStatus + receipt: ProjectionReceipt + manifest: ProjectionManifest + + def __post_init__(self) -> None: + _require_exact_type("status_before", self.status_before, ProjectionStateStatus) + _require_exact_type("receipt", self.receipt, ProjectionReceipt) + _require_exact_type("manifest", self.manifest, ProjectionManifest) + if self.receipt.outcome not in ( + ProjectionOutcome.COMPLETED, + ProjectionOutcome.UNCHANGED, + ): + raise ContractValidationError( + "projection results require a completed or unchanged receipt" + ) + if self.receipt.corpus_id != self.manifest.corpus_id: + raise ContractValidationError( + "receipt corpus_id must match the result manifest" + ) + if self.receipt.projection != self.manifest.projection: + raise ContractValidationError( + "receipt projection must match the result manifest" + ) + if self.receipt.checkpoint != self.manifest.checkpoint: + raise ContractValidationError( + "receipt checkpoint must equal the result manifest checkpoint" + ) + if self.receipt.outcome is ProjectionOutcome.UNCHANGED and ( + self.status_before is not ProjectionStateStatus.CURRENT + or self.receipt.attempted_documents != 0 + or self.receipt.completed_documents != 0 + ): + raise ContractValidationError( + "unchanged results require current state and zero document attempts" + ) + + +@dataclass(frozen=True, slots=True) +class RetrievalLimits: + """Independent bound on caller-supplied retrieval query text.""" + + max_query_codepoints: int + + def __post_init__(self) -> None: + _require_positive_integer("max_query_codepoints", self.max_query_codepoints) + + +@dataclass(frozen=True, slots=True) +class RetrievalQuery: + """A bounded retrieval request for one opaque corpus.""" + + corpus_id: str + text: str + hit_limit: int + candidate_limit: int + + def __post_init__(self) -> None: + _require_nonblank_string("corpus_id", self.corpus_id) + _require_nonblank_string("text", self.text) + hit_limit = _require_positive_integer("hit_limit", self.hit_limit) + candidate_limit = _require_positive_integer( + "candidate_limit", self.candidate_limit + ) + if hit_limit > candidate_limit: + raise ContractValidationError("hit_limit must not exceed candidate_limit") + + +class RetrievalOutcome(StrEnum): + """Explicit availability and completion state of retrieval.""" + + COMPLETE = "complete" + PARTIAL = "partial" + UNAVAILABLE = "unavailable" + STALE = "stale" + FAILED = "failed" + + @classmethod + def _missing_(cls, value: object) -> None: + raise ContractValidationError(f"{cls.__name__} value is not a defined member") + + +@dataclass(frozen=True, slots=True) +class RetrievalHit: + """A final score-free ranked fragment.""" + + fragment: Fragment + rank: int + + def __post_init__(self) -> None: + _require_exact_type("fragment", self.fragment, Fragment) + _require_positive_integer("rank", self.rank) + + +@dataclass(frozen=True, slots=True) +class RetrievalResult: + """A bounded, deduplicated, score-free retrieval result.""" + + query: RetrievalQuery + outcome: RetrievalOutcome + hits: tuple[RetrievalHit, ...] + truncated: bool + + def __post_init__(self) -> None: + _require_exact_type("query", self.query, RetrievalQuery) + _require_exact_type("outcome", self.outcome, RetrievalOutcome) + _require_exact_type("hits", self.hits, tuple) + _require_exact_type("truncated", self.truncated, bool) + if len(self.hits) > self.query.hit_limit: + raise ContractValidationError("hits must not exceed query.hit_limit") + + seen: set[FragmentIdentity] = set() + for expected_rank, hit in enumerate(self.hits, start=1): + _require_exact_type(f"hits[{expected_rank - 1}]", hit, RetrievalHit) + if hit.rank != expected_rank: + raise ContractValidationError("hit ranks must be contiguous from one") + if hit.fragment.identity.document.key.corpus_id != self.query.corpus_id: + raise ContractValidationError( + "every hit corpus_id must match the query corpus_id" + ) + if hit.fragment.identity in seen: + raise ContractValidationError( + "retrieval hits must have unique fragment identities" + ) + seen.add(hit.fragment.identity) + + empty_outcomes = ( + RetrievalOutcome.UNAVAILABLE, + RetrievalOutcome.STALE, + RetrievalOutcome.FAILED, + ) + if self.outcome in empty_outcomes: + if self.hits: + raise ContractValidationError( + "unavailable, stale, and failed results must not contain hits" + ) + if self.truncated: + raise ContractValidationError( + "unavailable, stale, and failed results must not be truncated" + ) + elif self.outcome is RetrievalOutcome.PARTIAL and not self.hits: + raise ContractValidationError("partial results require at least one hit") diff --git a/src/generic_rag/errors.py b/src/generic_rag/errors.py new file mode 100644 index 0000000..f3f7c0d --- /dev/null +++ b/src/generic_rag/errors.py @@ -0,0 +1,24 @@ +"""Public exception types for generic RAG contracts and collaboration.""" + +__all__ = ( + "GenericRagError", + "ContractValidationError", + "CollaborationError", + "StateCompatibilityError", +) + + +class GenericRagError(Exception): + """Base class for errors intentionally exposed by generic-rag.""" + + +class ContractValidationError(GenericRagError): + """Raised when a public immutable value violates its contract.""" + + +class CollaborationError(GenericRagError): + """Raised when orchestration translates a collaborator operation failure.""" + + +class StateCompatibilityError(GenericRagError): + """Raised when derived state has an incompatible projection identity.""" diff --git a/src/generic_rag/ports.py b/src/generic_rag/ports.py new file mode 100644 index 0000000..0eb7105 --- /dev/null +++ b/src/generic_rag/ports.py @@ -0,0 +1,114 @@ +"""Synchronous injected collaborator contracts and borrowing semantics.""" + +from __future__ import annotations + +from dataclasses import dataclass +from types import TracebackType +from typing import Generic, Literal, Protocol, TypeVar, runtime_checkable + +from .contracts import ( + DocumentIdentity, + DocumentKey, + EmbeddingIdentity, + EmbeddingVector, + Fragment, + RetrievalQuery, + VectorRecord, +) + +__all__ = ( + "Borrowed", + "Embedder", + "VectorIndexWriter", + "VectorIndexResetter", + "VectorIndexReader", + "LexicalRetriever", +) + +_T_co = TypeVar("_T_co", covariant=True) + + +@dataclass(frozen=True, slots=True) +class Borrowed(Generic[_T_co]): + """A no-op scope that marks a resource as caller-owned. + + Entry returns the exact wrapped resource. Exit never calls lifecycle methods + on it and never suppresses an exception. + """ + + resource: _T_co + + def __enter__(self) -> _T_co: + return self.resource + + def __exit__( + self, + exc_type: type[BaseException] | None, + exc_value: BaseException | None, + traceback: TracebackType | None, + ) -> Literal[False]: + return False + + +@runtime_checkable +class Embedder(Protocol): + """Synchronously embeds an ordered tuple without owning provider lifecycle.""" + + @property + def identity(self) -> EmbeddingIdentity: + """Return the exact model identity used for produced vectors.""" + ... + + def embed(self, texts: tuple[str, ...], /) -> tuple[EmbeddingVector, ...]: + """Return one same-order vector per input text.""" + ... + + +@runtime_checkable +class VectorIndexWriter(Protocol): + """Synchronously replaces and deletes complete document projections.""" + + def replace_document( + self, + document: DocumentIdentity, + records: tuple[VectorRecord, ...], + /, + ) -> None: + """Replace all derived vectors for the document's stable key.""" + ... + + def delete_document(self, document: DocumentKey, /) -> None: + """Delete every derived revision for a stable document key.""" + ... + + +@runtime_checkable +class VectorIndexResetter(Protocol): + """Synchronously removes every projected document for one corpus.""" + + def reset_corpus(self, corpus_id: str, /) -> None: + """Remove the complete derived vector projection for the corpus.""" + ... + + +@runtime_checkable +class VectorIndexReader(Protocol): + """Returns best-first vector candidates without exposing raw scores.""" + + def search( + self, + query: RetrievalQuery, + embedding: EmbeddingVector, + /, + ) -> tuple[Fragment, ...]: + """Return at most query.candidate_limit fragments in provider rank order.""" + ... + + +@runtime_checkable +class LexicalRetriever(Protocol): + """Returns best-first lexical candidates without exposing raw scores.""" + + def search(self, query: RetrievalQuery, /) -> tuple[Fragment, ...]: + """Return at most query.candidate_limit fragments in provider rank order.""" + ... diff --git a/src/generic_rag/projection.py b/src/generic_rag/projection.py new file mode 100644 index 0000000..019d79c --- /dev/null +++ b/src/generic_rag/projection.py @@ -0,0 +1,726 @@ +"""Deterministic bounded document projection orchestration.""" + +from __future__ import annotations + +from collections.abc import Callable +from dataclasses import dataclass +from enum import StrEnum +from math import isfinite +from typing import NoReturn, cast + +from .contracts import ( + ChunkingPolicy, + Document, + DocumentIdentity, + DocumentKey, + EmbeddingIdentity, + EmbeddingVector, + Fragment, + FragmentIdentity, + ProjectionCheckpoint, + ProjectionIdentity, + ProjectionLimits, + ProjectionManifest, + ProjectionManifestEntry, + ProjectionOutcome, + ProjectionReceipt, + ProjectionRequest, + ProjectionResult, + ProjectionStateAvailability, + ProjectionStateSnapshot, + ProjectionStateStatus, + VectorRecord, +) +from .errors import ( + CollaborationError, + ContractValidationError, + StateCompatibilityError, +) +from .ports import Borrowed, Embedder, VectorIndexResetter, VectorIndexWriter +from .projection_integrity import ( + derive_fragment_id, + derive_projection_checkpoint_token, + derive_source_digest, + has_valid_projection_checkpoint, +) + +__all__ = ( + "ProjectionFailureStage", + "ProjectionStateError", + "ProjectionOperationError", + "project_documents", + "rebuild_projection", +) + + +class ProjectionFailureStage(StrEnum): + """The collaborator stage at which a projection operation failed.""" + + EMBEDDER_IDENTITY = "embedder_identity" + EMBEDDING = "embedding" + REPLACEMENT = "replacement" + DELETION = "deletion" + RESET = "reset" + + @classmethod + def _missing_(cls, value: object) -> None: + raise ContractValidationError(f"{cls.__name__} value is not a defined member") + + +class ProjectionStateError(StateCompatibilityError): + """Raised before effects when incremental projection cannot use state.""" + + status: ProjectionStateStatus + + def __init__(self, status: ProjectionStateStatus) -> None: + _require_exact_type("status", status, ProjectionStateStatus) + self.status = status + super().__init__("projection state is not compatible with incremental update") + + +class ProjectionOperationError(CollaborationError): + """A content-free translation of one collaborator operation failure.""" + + stage: ProjectionFailureStage + affected_document: DocumentKey | None + receipt: ProjectionReceipt | None + + def __init__( + self, + stage: ProjectionFailureStage, + affected_document: DocumentKey | None, + receipt: ProjectionReceipt | None, + ) -> None: + _require_exact_type("stage", stage, ProjectionFailureStage) + if affected_document is not None: + _validate_document_key(affected_document) + if receipt is not None: + _require_exact_type("receipt", receipt, ProjectionReceipt) + if receipt.outcome not in ( + ProjectionOutcome.FAILED, + ProjectionOutcome.PARTIAL, + ): + raise ContractValidationError( + "operation error receipt must be failed or partial" + ) + self.stage = stage + self.affected_document = affected_document + self.receipt = receipt + super().__init__(f"projection collaborator failed during {stage.value}") + + +@dataclass(frozen=True, slots=True) +class _PreparedDocument: + source: Document + fragments: tuple[Fragment, ...] + entry: ProjectionManifestEntry + + +@dataclass(frozen=True, slots=True) +class _PreparedTarget: + request: ProjectionRequest + documents: tuple[_PreparedDocument, ...] + manifest: ProjectionManifest + + +@dataclass(frozen=True, slots=True) +class _Mutation: + key: DocumentKey + replacement: _PreparedDocument | None + + +def _require_exact_type(name: str, value: object, expected: type[object]) -> None: + if type(value) is not expected: + raise ContractValidationError( + f"{name} must be exactly {expected.__name__}, not {type(value).__name__}" + ) + + +def _validate_document_key(value: object) -> DocumentKey: + _require_exact_type("document key", value, DocumentKey) + assert isinstance(value, DocumentKey) + return DocumentKey(value.corpus_id, value.document_id) + + +def _validate_document_identity(value: object) -> DocumentIdentity: + _require_exact_type("document identity", value, DocumentIdentity) + assert isinstance(value, DocumentIdentity) + return DocumentIdentity( + _validate_document_key(value.key), + value.revision_id, + ) + + +def _validate_document(value: object) -> Document: + _require_exact_type("document", value, Document) + assert isinstance(value, Document) + return Document( + _validate_document_identity(value.identity), + value.text, + value.attributes, + ) + + +def _validate_embedding_identity(value: object) -> EmbeddingIdentity: + _require_exact_type("embedding identity", value, EmbeddingIdentity) + assert isinstance(value, EmbeddingIdentity) + return EmbeddingIdentity(value.model_id, value.dimensions) + + +def _validate_projection_identity(value: object) -> ProjectionIdentity: + _require_exact_type("projection identity", value, ProjectionIdentity) + assert isinstance(value, ProjectionIdentity) + return ProjectionIdentity( + value.schema_id, + _validate_embedding_identity(value.embedding), + ) + + +def _validate_chunking(value: object) -> ChunkingPolicy: + _require_exact_type("chunking", value, ChunkingPolicy) + assert isinstance(value, ChunkingPolicy) + return ChunkingPolicy( + value.max_fragment_codepoints, + value.overlap_codepoints, + ) + + +def _validate_limits(value: object) -> ProjectionLimits: + _require_exact_type("limits", value, ProjectionLimits) + assert isinstance(value, ProjectionLimits) + return ProjectionLimits( + value.max_documents, + value.max_document_codepoints, + value.max_embedding_batch_size, + ) + + +def _validate_request(value: object) -> ProjectionRequest: + _require_exact_type("request", value, ProjectionRequest) + assert isinstance(value, ProjectionRequest) + _require_exact_type("request documents", value.documents, tuple) + documents = tuple(_validate_document(document) for document in value.documents) + canonical = ProjectionRequest( + value.corpus_id, + _validate_projection_identity(value.projection), + _validate_chunking(value.chunking), + _validate_limits(value.limits), + documents, + ) + if documents != canonical.documents: + raise ContractValidationError("request documents must be in canonical order") + return canonical + + +def _validate_manifest_entry(value: object) -> ProjectionManifestEntry: + _require_exact_type("manifest entry", value, ProjectionManifestEntry) + assert isinstance(value, ProjectionManifestEntry) + return ProjectionManifestEntry( + _validate_document_identity(value.document), + value.source_digest, + value.fragment_count, + ) + + +def _validate_checkpoint(value: object) -> ProjectionCheckpoint: + _require_exact_type("checkpoint", value, ProjectionCheckpoint) + assert isinstance(value, ProjectionCheckpoint) + return ProjectionCheckpoint( + value.corpus_id, + _validate_projection_identity(value.projection), + value.token, + ) + + +def _validate_manifest(value: object) -> ProjectionManifest: + _require_exact_type("manifest", value, ProjectionManifest) + assert isinstance(value, ProjectionManifest) + _require_exact_type("manifest entries", value.entries, tuple) + entries = tuple(_validate_manifest_entry(entry) for entry in value.entries) + canonical = ProjectionManifest( + value.corpus_id, + _validate_projection_identity(value.projection), + _validate_chunking(value.chunking), + entries, + _validate_checkpoint(value.checkpoint), + ) + if entries != canonical.entries: + raise ContractValidationError("manifest entries must be in canonical order") + return canonical + + +def _validate_state(value: object) -> ProjectionStateSnapshot: + _require_exact_type("state", value, ProjectionStateSnapshot) + assert isinstance(value, ProjectionStateSnapshot) + _require_exact_type( + "state availability", + value.availability, + ProjectionStateAvailability, + ) + manifest = None if value.manifest is None else _validate_manifest(value.manifest) + return ProjectionStateSnapshot(value.availability, manifest) + + +def _validate_borrowed(name: str, value: object) -> None: + _require_exact_type(name, value, Borrowed) + + +def _fragments(document: Document, chunking: ChunkingPolicy) -> tuple[Fragment, ...]: + fragments: list[Fragment] = [] + start = 0 + text_length = len(document.text) + while start < text_length: + end = min(start + chunking.max_fragment_codepoints, text_length) + identity = FragmentIdentity( + document.identity, + derive_fragment_id(document.identity, start, end), + start, + end, + ) + fragments.append( + Fragment( + identity, + document.text[start:end], + document.attributes, + ) + ) + if end == text_length: + break + start = end - chunking.overlap_codepoints + return tuple(fragments) + + +def _prepare_target(request: ProjectionRequest) -> _PreparedTarget: + prepared: list[_PreparedDocument] = [] + for document in request.documents: + fragments = _fragments(document, request.chunking) + entry = ProjectionManifestEntry( + document.identity, + derive_source_digest(document), + len(fragments), + ) + prepared.append(_PreparedDocument(document, fragments, entry)) + entries = tuple(item.entry for item in prepared) + checkpoint = ProjectionCheckpoint( + request.corpus_id, + request.projection, + derive_projection_checkpoint_token( + request.corpus_id, + request.projection, + request.chunking, + entries, + ), + ) + manifest = ProjectionManifest( + request.corpus_id, + request.projection, + request.chunking, + entries, + checkpoint, + ) + return _PreparedTarget(request, tuple(prepared), manifest) + + +def _state_status( + state: ProjectionStateSnapshot, + target: _PreparedTarget, +) -> ProjectionStateStatus: + if state.availability is ProjectionStateAvailability.MISSING: + return ProjectionStateStatus.MISSING + if state.availability is ProjectionStateAvailability.CORRUPT: + return ProjectionStateStatus.CORRUPT + + manifest = state.manifest + assert manifest is not None + if not has_valid_projection_checkpoint(manifest): + return ProjectionStateStatus.CORRUPT + if manifest.corpus_id != target.request.corpus_id: + return ProjectionStateStatus.CORRUPT + if manifest.projection.schema_id != target.request.projection.schema_id: + return ProjectionStateStatus.SCHEMA_MISMATCH + if manifest.projection.embedding != target.request.projection.embedding: + return ProjectionStateStatus.EMBEDDING_MISMATCH + + target_by_key = {item.entry.document.key: item.entry for item in target.documents} + for previous in manifest.entries: + current = target_by_key.get(previous.document.key) + if current is None or previous.document != current.document: + continue + if previous.source_digest != current.source_digest: + return ProjectionStateStatus.CORRUPT + if ( + manifest.chunking == target.request.chunking + and previous.fragment_count != current.fragment_count + ): + return ProjectionStateStatus.CORRUPT + + if ( + manifest.chunking == target.request.chunking + and manifest.entries == target.manifest.entries + ): + return ProjectionStateStatus.CURRENT + return ProjectionStateStatus.STALE + + +def _incremental_plan( + manifest: ProjectionManifest, + target: _PreparedTarget, +) -> tuple[_Mutation, ...]: + previous_by_key = {entry.document.key: entry for entry in manifest.entries} + target_by_key = {item.entry.document.key: item for item in target.documents} + mutations: list[_Mutation] = [] + + for key, item in target_by_key.items(): + previous = previous_by_key.get(key) + if manifest.chunking != target.request.chunking or previous != item.entry: + mutations.append(_Mutation(key, item)) + for key in previous_by_key.keys() - target_by_key.keys(): + mutations.append(_Mutation(key, None)) + mutations.sort(key=lambda mutation: mutation.key.document_id) + return tuple(mutations) + + +def _rebuild_plan(target: _PreparedTarget) -> tuple[_Mutation, ...]: + return tuple(_Mutation(item.entry.document.key, item) for item in target.documents) + + +def _failure_receipt( + target: _PreparedTarget, + attempted: int, + completed: int, +) -> ProjectionReceipt | None: + if attempted == 0: + return None + outcome = ProjectionOutcome.FAILED if completed == 0 else ProjectionOutcome.PARTIAL + return ProjectionReceipt( + target.request.corpus_id, + target.request.projection, + outcome, + attempted, + completed, + None, + ) + + +def _raise_operation_error( + stage: ProjectionFailureStage, + affected_document: DocumentKey | None, + receipt: ProjectionReceipt | None, + cause: Exception | None = None, +) -> NoReturn: + error = ProjectionOperationError(stage, affected_document, receipt) + if cause is None: + raise error + raise error from cause + + +def _require_embedder_identity( + embedder: Embedder, + expected: EmbeddingIdentity, + receipt: ProjectionReceipt | None, +) -> None: + try: + identity = embedder.identity + except Exception as exc: + _raise_operation_error( + ProjectionFailureStage.EMBEDDER_IDENTITY, + None, + receipt, + exc, + ) + try: + validated = _validate_embedding_identity(identity) + except ContractValidationError: + _raise_operation_error( + ProjectionFailureStage.EMBEDDER_IDENTITY, + None, + receipt, + ) + if validated != expected: + _raise_operation_error( + ProjectionFailureStage.EMBEDDER_IDENTITY, + None, + receipt, + ) + + +def _validate_embeddings( + value: object, + expected_count: int, + expected_dimensions: int, +) -> tuple[EmbeddingVector, ...]: + if type(value) is not tuple or len(value) != expected_count: + raise ContractValidationError( + "embedder output must be an exact same-count tuple" + ) + assert isinstance(value, tuple) + for vector in value: + _require_exact_type("embedding", vector, EmbeddingVector) + assert isinstance(vector, EmbeddingVector) + if ( + type(vector.values) is not tuple + or len(vector.values) != expected_dimensions + ): + raise ContractValidationError( + "embedding vector must have the requested dimensions" + ) + if any( + type(coordinate) is not float or not isfinite(coordinate) + for coordinate in vector.values + ): + raise ContractValidationError( + "embedding vector coordinates must be canonical finite floats" + ) + return value + + +def _records_for_document( + embedder: Embedder, + item: _PreparedDocument, + batch_size: int, + target: _PreparedTarget, + attempted: int, + completed: int, +) -> tuple[VectorRecord, ...]: + records: list[VectorRecord] = [] + for start in range(0, len(item.fragments), batch_size): + fragments = item.fragments[start : start + batch_size] + texts = tuple(fragment.text for fragment in fragments) + try: + output = embedder.embed(texts) + except Exception as exc: + _raise_operation_error( + ProjectionFailureStage.EMBEDDING, + item.entry.document.key, + _failure_receipt(target, attempted, completed), + exc, + ) + try: + vectors = _validate_embeddings( + output, + len(fragments), + target.request.projection.embedding.dimensions, + ) + except ContractValidationError: + _raise_operation_error( + ProjectionFailureStage.EMBEDDING, + item.entry.document.key, + _failure_receipt(target, attempted, completed), + ) + records.extend( + VectorRecord(fragment, vector) + for fragment, vector in zip(fragments, vectors, strict=True) + ) + return tuple(records) + + +def _execute_prevalidated_mutations( + target: _PreparedTarget, + mutations: tuple[_Mutation, ...], + embedder: Embedder | None, + writer_scope: Borrowed[VectorIndexWriter], +) -> None: + if not mutations: + return + attempted = len(mutations) + completed = 0 + with writer_scope as writer: + for mutation in mutations: + if mutation.replacement is None: + try: + delete_document = cast( + Callable[[DocumentKey], object], + writer.delete_document, + ) + command_result = delete_document(mutation.key) + except Exception as exc: + _raise_operation_error( + ProjectionFailureStage.DELETION, + mutation.key, + _failure_receipt(target, attempted, completed), + exc, + ) + if command_result is not None: + _raise_operation_error( + ProjectionFailureStage.DELETION, + mutation.key, + _failure_receipt(target, attempted, completed), + ) + else: + assert embedder is not None + records = _records_for_document( + embedder, + mutation.replacement, + target.request.limits.max_embedding_batch_size, + target, + attempted, + completed, + ) + try: + replace_document = cast( + Callable[ + [DocumentIdentity, tuple[VectorRecord, ...]], + object, + ], + writer.replace_document, + ) + command_result = replace_document( + mutation.replacement.entry.document, + records, + ) + except Exception as exc: + _raise_operation_error( + ProjectionFailureStage.REPLACEMENT, + mutation.key, + _failure_receipt(target, attempted, completed), + exc, + ) + if command_result is not None: + _raise_operation_error( + ProjectionFailureStage.REPLACEMENT, + mutation.key, + _failure_receipt(target, attempted, completed), + ) + completed += 1 + + +def _execute_mutations( + target: _PreparedTarget, + mutations: tuple[_Mutation, ...], + embedder_scope: Borrowed[Embedder], + writer_scope: Borrowed[VectorIndexWriter], +) -> None: + needs_embedder = any(mutation.replacement is not None for mutation in mutations) + if not needs_embedder: + _execute_prevalidated_mutations(target, mutations, None, writer_scope) + return + + with embedder_scope as embedder: + _require_embedder_identity( + embedder, + target.request.projection.embedding, + _failure_receipt(target, len(mutations), 0), + ) + _execute_prevalidated_mutations(target, mutations, embedder, writer_scope) + + +def _reset_projection( + target: _PreparedTarget, + attempted: int, + resetter_scope: Borrowed[VectorIndexResetter], +) -> None: + try: + with resetter_scope as resetter: + reset_corpus = cast( + Callable[[str], object], + resetter.reset_corpus, + ) + command_result = reset_corpus(target.request.corpus_id) + except Exception as exc: + _raise_operation_error( + ProjectionFailureStage.RESET, + None, + _failure_receipt(target, attempted, 0), + exc, + ) + if command_result is not None: + _raise_operation_error( + ProjectionFailureStage.RESET, + None, + _failure_receipt(target, attempted, 0), + ) + + +def _successful_result( + target: _PreparedTarget, + status_before: ProjectionStateStatus, + outcome: ProjectionOutcome, + attempted: int, +) -> ProjectionResult: + receipt = ProjectionReceipt( + target.request.corpus_id, + target.request.projection, + outcome, + attempted, + attempted, + target.manifest.checkpoint, + ) + return ProjectionResult(status_before, receipt, target.manifest) + + +def project_documents( + request: ProjectionRequest, + state: ProjectionStateSnapshot, + embedder: Borrowed[Embedder], + writer: Borrowed[VectorIndexWriter], + /, +) -> ProjectionResult: + """Project a compatible present state to the complete target. + + Incompatible state raises ``ProjectionStateError`` before collaborator effects. + """ + + canonical_request = _validate_request(request) + canonical_state = _validate_state(state) + _validate_borrowed("embedder", embedder) + _validate_borrowed("writer", writer) + target = _prepare_target(canonical_request) + status = _state_status(canonical_state, target) + if status is ProjectionStateStatus.CURRENT: + return _successful_result(target, status, ProjectionOutcome.UNCHANGED, 0) + if status is not ProjectionStateStatus.STALE: + raise ProjectionStateError(status) + + manifest = canonical_state.manifest + assert manifest is not None + mutations = _incremental_plan(manifest, target) + _execute_mutations(target, mutations, embedder, writer) + return _successful_result( + target, + status, + ProjectionOutcome.COMPLETED, + len(mutations), + ) + + +def rebuild_projection( + request: ProjectionRequest, + state: ProjectionStateSnapshot, + embedder: Borrowed[Embedder], + writer: Borrowed[VectorIndexWriter], + resetter: Borrowed[VectorIndexResetter], + /, +) -> ProjectionResult: + """Reset one corpus and write its complete canonical target projection.""" + + canonical_request = _validate_request(request) + canonical_state = _validate_state(state) + _validate_borrowed("embedder", embedder) + _validate_borrowed("writer", writer) + _validate_borrowed("resetter", resetter) + target = _prepare_target(canonical_request) + status = _state_status(canonical_state, target) + mutations = _rebuild_plan(target) + + if mutations: + with embedder as embedder_resource: + _require_embedder_identity( + embedder_resource, + target.request.projection.embedding, + _failure_receipt(target, len(mutations), 0), + ) + _reset_projection(target, len(mutations), resetter) + _execute_prevalidated_mutations( + target, + mutations, + embedder_resource, + writer, + ) + else: + _reset_projection(target, 0, resetter) + return _successful_result( + target, + status, + ProjectionOutcome.COMPLETED, + len(mutations), + ) diff --git a/src/generic_rag/projection_integrity.py b/src/generic_rag/projection_integrity.py new file mode 100644 index 0000000..f03d30b --- /dev/null +++ b/src/generic_rag/projection_integrity.py @@ -0,0 +1,267 @@ +"""Deterministic integrity values shared by projection and retrieval.""" + +from __future__ import annotations + +import hashlib + +from .contracts import ( + ChunkingPolicy, + Document, + DocumentIdentity, + DocumentKey, + EmbeddingIdentity, + ProjectionCheckpoint, + ProjectionIdentity, + ProjectionManifest, + ProjectionManifestEntry, +) +from .errors import ContractValidationError + +__all__ = ( + "derive_source_digest", + "derive_fragment_id", + "derive_projection_checkpoint_token", + "has_valid_projection_checkpoint", +) + +_SOURCE_DIGEST_DOMAIN = "generic-rag:projection-source:v1" +_FRAGMENT_ID_DOMAIN = "generic-rag:fragment-id:v1" +_CHECKPOINT_DOMAIN = "generic-rag:projection-checkpoint:v1" + + +def _require_exact_type(name: str, value: object, expected: type[object]) -> None: + if type(value) is not expected: + raise ContractValidationError( + f"{name} must be exactly {expected.__name__}, not {type(value).__name__}" + ) + + +def _require_nonblank_string(name: str, value: object) -> str: + _require_exact_type(name, value, str) + assert isinstance(value, str) + if not value or value.isspace(): + raise ContractValidationError(f"{name} must not be empty or whitespace-only") + return value + + +def _require_nonnegative_integer(name: str, value: object) -> int: + _require_exact_type(name, value, int) + assert isinstance(value, int) + if value < 0: + raise ContractValidationError(f"{name} must be nonnegative") + return value + + +def _validate_document_key(value: object) -> DocumentKey: + _require_exact_type("document key", value, DocumentKey) + assert isinstance(value, DocumentKey) + return DocumentKey(value.corpus_id, value.document_id) + + +def _validate_document_identity(value: object) -> DocumentIdentity: + _require_exact_type("document identity", value, DocumentIdentity) + assert isinstance(value, DocumentIdentity) + return DocumentIdentity( + _validate_document_key(value.key), + value.revision_id, + ) + + +def _validate_document(value: object) -> Document: + _require_exact_type("document", value, Document) + assert isinstance(value, Document) + return Document( + _validate_document_identity(value.identity), + value.text, + value.attributes, + ) + + +def _validate_embedding_identity(value: object) -> EmbeddingIdentity: + _require_exact_type("embedding identity", value, EmbeddingIdentity) + assert isinstance(value, EmbeddingIdentity) + return EmbeddingIdentity(value.model_id, value.dimensions) + + +def _validate_projection_identity(value: object) -> ProjectionIdentity: + _require_exact_type("projection identity", value, ProjectionIdentity) + assert isinstance(value, ProjectionIdentity) + return ProjectionIdentity( + value.schema_id, + _validate_embedding_identity(value.embedding), + ) + + +def _validate_chunking(value: object) -> ChunkingPolicy: + _require_exact_type("chunking", value, ChunkingPolicy) + assert isinstance(value, ChunkingPolicy) + return ChunkingPolicy( + value.max_fragment_codepoints, + value.overlap_codepoints, + ) + + +def _validate_manifest_entry(value: object) -> ProjectionManifestEntry: + _require_exact_type("manifest entry", value, ProjectionManifestEntry) + assert isinstance(value, ProjectionManifestEntry) + return ProjectionManifestEntry( + _validate_document_identity(value.document), + value.source_digest, + value.fragment_count, + ) + + +def _validate_checkpoint(value: object) -> ProjectionCheckpoint: + _require_exact_type("checkpoint", value, ProjectionCheckpoint) + assert isinstance(value, ProjectionCheckpoint) + return ProjectionCheckpoint( + value.corpus_id, + _validate_projection_identity(value.projection), + value.token, + ) + + +def _validate_manifest(value: object) -> ProjectionManifest: + _require_exact_type("manifest", value, ProjectionManifest) + assert isinstance(value, ProjectionManifest) + _require_exact_type("manifest entries", value.entries, tuple) + entries = tuple(_validate_manifest_entry(entry) for entry in value.entries) + canonical = ProjectionManifest( + value.corpus_id, + _validate_projection_identity(value.projection), + _validate_chunking(value.chunking), + entries, + _validate_checkpoint(value.checkpoint), + ) + if entries != canonical.entries: + raise ContractValidationError("manifest entries must be in canonical order") + return canonical + + +def _sha256_fields(fields: tuple[str, ...]) -> str: + digest = hashlib.sha256() + for field in fields: + encoded = field.encode("utf-8", "surrogatepass") + digest.update(len(encoded).to_bytes(8, "big", signed=False)) + digest.update(encoded) + return f"sha256:{digest.hexdigest()}" + + +def derive_source_digest(document: Document, /) -> str: + """Derive the stable v1 digest of exact source text and attributes.""" + + canonical = _validate_document(document) + fields = [ + _SOURCE_DIGEST_DOMAIN, + "text", + canonical.text, + "attributes_count", + str(len(canonical.attributes)), + ] + for key, value in canonical.attributes: + fields.extend(("attribute_key", key, "attribute_value", value)) + return _sha256_fields(tuple(fields)) + + +def derive_fragment_id( + document: DocumentIdentity, + start: int, + end: int, + /, +) -> str: + """Derive the stable v1 identity for one valid half-open source range.""" + + canonical = _validate_document_identity(document) + canonical_start = _require_nonnegative_integer("start", start) + canonical_end = _require_nonnegative_integer("end", end) + if canonical_start >= canonical_end: + raise ContractValidationError("fragment range must satisfy start < end") + return _sha256_fields( + ( + _FRAGMENT_ID_DOMAIN, + "corpus_id", + canonical.key.corpus_id, + "document_id", + canonical.key.document_id, + "revision_id", + canonical.revision_id, + "start", + str(canonical_start), + "end", + str(canonical_end), + ) + ) + + +def derive_projection_checkpoint_token( + corpus_id: str, + projection: ProjectionIdentity, + chunking: ChunkingPolicy, + entries: tuple[ProjectionManifestEntry, ...], + /, +) -> str: + """Derive the stable v1 token for one canonical projection manifest.""" + + canonical_corpus_id = _require_nonblank_string("corpus_id", corpus_id) + canonical_projection = _validate_projection_identity(projection) + canonical_chunking = _validate_chunking(chunking) + _require_exact_type("entries", entries, tuple) + canonical_entries = tuple(_validate_manifest_entry(entry) for entry in entries) + ordering_witness = ProjectionManifest( + canonical_corpus_id, + canonical_projection, + canonical_chunking, + canonical_entries, + ProjectionCheckpoint( + canonical_corpus_id, + canonical_projection, + "integrity-validation", + ), + ) + if canonical_entries != ordering_witness.entries: + raise ContractValidationError("manifest entries must be in canonical order") + + fields = [ + _CHECKPOINT_DOMAIN, + "corpus_id", + canonical_corpus_id, + "schema_id", + canonical_projection.schema_id, + "embedding_model_id", + canonical_projection.embedding.model_id, + "embedding_dimensions", + str(canonical_projection.embedding.dimensions), + "max_fragment_codepoints", + str(canonical_chunking.max_fragment_codepoints), + "overlap_codepoints", + str(canonical_chunking.overlap_codepoints), + "entry_count", + str(len(canonical_entries)), + ] + for entry in canonical_entries: + fields.extend( + ( + "document_id", + entry.document.key.document_id, + "revision_id", + entry.document.revision_id, + "source_digest", + entry.source_digest, + "fragment_count", + str(entry.fragment_count), + ) + ) + return _sha256_fields(tuple(fields)) + + +def has_valid_projection_checkpoint(manifest: ProjectionManifest, /) -> bool: + """Return whether a structurally valid manifest has its exact v1 token.""" + + canonical = _validate_manifest(manifest) + expected = derive_projection_checkpoint_token( + canonical.corpus_id, + canonical.projection, + canonical.chunking, + canonical.entries, + ) + return canonical.checkpoint.token == expected diff --git a/src/generic_rag/py.typed b/src/generic_rag/py.typed new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/src/generic_rag/py.typed @@ -0,0 +1 @@ + diff --git a/src/generic_rag/retrieval.py b/src/generic_rag/retrieval.py new file mode 100644 index 0000000..6341631 --- /dev/null +++ b/src/generic_rag/retrieval.py @@ -0,0 +1,517 @@ +"""Bounded semantic retrieval and deterministic rank-based hybrid fusion.""" + +from __future__ import annotations + +from dataclasses import dataclass +from fractions import Fraction +from math import isfinite + +from .contracts import ( + ChunkingPolicy, + DocumentIdentity, + DocumentKey, + EmbeddingIdentity, + EmbeddingVector, + Fragment, + FragmentIdentity, + ProjectionCheckpoint, + ProjectionIdentity, + ProjectionManifest, + ProjectionManifestEntry, + ProjectionStateAvailability, + ProjectionStateSnapshot, + RetrievalHit, + RetrievalLimits, + RetrievalOutcome, + RetrievalQuery, + RetrievalResult, +) +from .errors import ContractValidationError +from .ports import Borrowed, Embedder, LexicalRetriever, VectorIndexReader +from .projection_integrity import ( + derive_fragment_id, + has_valid_projection_checkpoint, +) + +__all__ = ("retrieve_semantic", "retrieve_hybrid") + +_RRF_OFFSET = 60 + + +@dataclass(frozen=True, slots=True) +class _RankedFragment: + fragment: Fragment + source_rank: int + + +@dataclass(frozen=True, slots=True) +class _BranchResult: + all_candidates: tuple[_RankedFragment, ...] + current_candidates: tuple[_RankedFragment, ...] + stale: bool + failed: bool + + +def _require_exact_type(name: str, value: object, expected: type[object]) -> None: + if type(value) is not expected: + raise ContractValidationError( + f"{name} must be exactly {expected.__name__}, not {type(value).__name__}" + ) + + +def _validate_document_key(value: object) -> DocumentKey: + _require_exact_type("document key", value, DocumentKey) + assert isinstance(value, DocumentKey) + return DocumentKey(value.corpus_id, value.document_id) + + +def _validate_document_identity(value: object) -> DocumentIdentity: + _require_exact_type("document identity", value, DocumentIdentity) + assert isinstance(value, DocumentIdentity) + return DocumentIdentity( + _validate_document_key(value.key), + value.revision_id, + ) + + +def _validate_embedding_identity(value: object) -> EmbeddingIdentity: + _require_exact_type("embedding identity", value, EmbeddingIdentity) + assert isinstance(value, EmbeddingIdentity) + return EmbeddingIdentity(value.model_id, value.dimensions) + + +def _validate_projection_identity(value: object) -> ProjectionIdentity: + _require_exact_type("projection identity", value, ProjectionIdentity) + assert isinstance(value, ProjectionIdentity) + return ProjectionIdentity( + value.schema_id, + _validate_embedding_identity(value.embedding), + ) + + +def _validate_chunking(value: object) -> ChunkingPolicy: + _require_exact_type("chunking", value, ChunkingPolicy) + assert isinstance(value, ChunkingPolicy) + return ChunkingPolicy( + value.max_fragment_codepoints, + value.overlap_codepoints, + ) + + +def _validate_manifest_entry(value: object) -> ProjectionManifestEntry: + _require_exact_type("manifest entry", value, ProjectionManifestEntry) + assert isinstance(value, ProjectionManifestEntry) + return ProjectionManifestEntry( + _validate_document_identity(value.document), + value.source_digest, + value.fragment_count, + ) + + +def _validate_checkpoint(value: object) -> ProjectionCheckpoint: + _require_exact_type("checkpoint", value, ProjectionCheckpoint) + assert isinstance(value, ProjectionCheckpoint) + return ProjectionCheckpoint( + value.corpus_id, + _validate_projection_identity(value.projection), + value.token, + ) + + +def _validate_manifest(value: object) -> ProjectionManifest: + _require_exact_type("manifest", value, ProjectionManifest) + assert isinstance(value, ProjectionManifest) + _require_exact_type("manifest entries", value.entries, tuple) + entries = tuple(_validate_manifest_entry(entry) for entry in value.entries) + canonical = ProjectionManifest( + value.corpus_id, + _validate_projection_identity(value.projection), + _validate_chunking(value.chunking), + entries, + _validate_checkpoint(value.checkpoint), + ) + if entries != canonical.entries: + raise ContractValidationError("manifest entries must be in canonical order") + return canonical + + +def _validate_state(value: object) -> ProjectionStateSnapshot: + _require_exact_type("state", value, ProjectionStateSnapshot) + assert isinstance(value, ProjectionStateSnapshot) + _require_exact_type( + "state availability", + value.availability, + ProjectionStateAvailability, + ) + manifest = None if value.manifest is None else _validate_manifest(value.manifest) + return ProjectionStateSnapshot(value.availability, manifest) + + +def _validate_query(value: object) -> RetrievalQuery: + _require_exact_type("query", value, RetrievalQuery) + assert isinstance(value, RetrievalQuery) + return RetrievalQuery( + value.corpus_id, + value.text, + value.hit_limit, + value.candidate_limit, + ) + + +def _validate_limits(value: object) -> RetrievalLimits: + _require_exact_type("limits", value, RetrievalLimits) + assert isinstance(value, RetrievalLimits) + return RetrievalLimits(value.max_query_codepoints) + + +def _validate_borrowed(name: str, value: object) -> None: + _require_exact_type(name, value, Borrowed) + + +def _validate_embedding( + value: object, + expected_dimensions: int, +) -> EmbeddingVector: + _require_exact_type("embedding", value, EmbeddingVector) + assert isinstance(value, EmbeddingVector) + _require_exact_type("embedding values", value.values, tuple) + if len(value.values) != expected_dimensions: + raise ContractValidationError( + "embedding vector must have the requested dimensions" + ) + if any( + type(coordinate) is not float or not isfinite(coordinate) + for coordinate in value.values + ): + raise ContractValidationError( + "embedding vector coordinates must be canonical finite floats" + ) + return EmbeddingVector(value.values) + + +def _validate_fragment( + value: object, + query: RetrievalQuery, + chunking: ChunkingPolicy, +) -> Fragment: + _require_exact_type("candidate", value, Fragment) + assert isinstance(value, Fragment) + identity_value = value.identity + _require_exact_type("candidate identity", identity_value, FragmentIdentity) + assert isinstance(identity_value, FragmentIdentity) + identity = FragmentIdentity( + _validate_document_identity(identity_value.document), + identity_value.fragment_id, + identity_value.start, + identity_value.end, + ) + canonical = Fragment(identity, value.text, value.attributes) + if canonical.identity.document.key.corpus_id != query.corpus_id: + raise ContractValidationError("candidate corpus_id must match the query") + if len(canonical.text) > chunking.max_fragment_codepoints: + raise ContractValidationError( + "candidate width must not exceed the published chunk bound" + ) + if canonical.identity.fragment_id != derive_fragment_id( + canonical.identity.document, + canonical.identity.start, + canonical.identity.end, + ): + raise ContractValidationError( + "candidate fragment_id must match its deterministic identity" + ) + return canonical + + +def _validate_provider_candidates( + value: object, + query: RetrievalQuery, + manifest: ProjectionManifest, +) -> _BranchResult: + if type(value) is not tuple: + raise ContractValidationError("retriever output must be exactly tuple") + assert isinstance(value, tuple) + if len(value) > query.candidate_limit: + raise ContractValidationError( + "retriever output must not exceed query.candidate_limit" + ) + + manifest_by_key = {entry.document.key: entry for entry in manifest.entries} + first_by_identity: dict[FragmentIdentity, _RankedFragment] = {} + ordered: list[_RankedFragment] = [] + for source_rank, candidate in enumerate(value, start=1): + fragment = _validate_fragment(candidate, query, manifest.chunking) + previous = first_by_identity.get(fragment.identity) + if previous is not None: + if previous.fragment != fragment: + raise ContractValidationError( + "one provider returned conflicting candidate payloads" + ) + continue + ranked = _RankedFragment(fragment, source_rank) + first_by_identity[fragment.identity] = ranked + ordered.append(ranked) + + current: list[_RankedFragment] = [] + stale = False + for candidate in ordered: + published = manifest_by_key.get(candidate.fragment.identity.document.key) + if ( + published is None + or candidate.fragment.identity.document != published.document + ): + stale = True + continue + current.append(candidate) + return _BranchResult(tuple(ordered), tuple(current), stale, False) + + +def _failed_branch() -> _BranchResult: + return _BranchResult((), (), False, True) + + +def _stale_branch() -> _BranchResult: + return _BranchResult((), (), True, False) + + +def _semantic_branch( + query: RetrievalQuery, + manifest: ProjectionManifest, + embedder: Borrowed[Embedder], + vector_reader: Borrowed[VectorIndexReader], +) -> _BranchResult: + try: + with embedder as embedder_resource: + identity_value = embedder_resource.identity + identity = _validate_embedding_identity(identity_value) + if identity != manifest.projection.embedding: + return _stale_branch() + embedding_values = embedder_resource.embed((query.text,)) + if type(embedding_values) is not tuple or len(embedding_values) != 1: + raise ContractValidationError( + "embedder output must be an exact one-vector tuple" + ) + embedding = _validate_embedding( + embedding_values[0], + identity.dimensions, + ) + except Exception: + return _failed_branch() + + try: + with vector_reader as reader_resource: + candidates = reader_resource.search(query, embedding) + return _validate_provider_candidates(candidates, query, manifest) + except Exception: + return _failed_branch() + + +def _lexical_branch( + query: RetrievalQuery, + manifest: ProjectionManifest, + lexical_retriever: Borrowed[LexicalRetriever], +) -> _BranchResult: + try: + with lexical_retriever as lexical_resource: + candidates = lexical_resource.search(query) + return _validate_provider_candidates(candidates, query, manifest) + except Exception: + return _failed_branch() + + +def _terminal_result( + query: RetrievalQuery, + outcome: RetrievalOutcome, +) -> RetrievalResult: + return RetrievalResult(query, outcome, (), False) + + +def _state_manifest_or_result( + query: RetrievalQuery, + state: ProjectionStateSnapshot, +) -> ProjectionManifest | RetrievalResult: + if state.availability is ProjectionStateAvailability.MISSING: + return _terminal_result(query, RetrievalOutcome.UNAVAILABLE) + if state.availability is ProjectionStateAvailability.CORRUPT: + return _terminal_result(query, RetrievalOutcome.FAILED) + + manifest = state.manifest + assert manifest is not None + if manifest.corpus_id != query.corpus_id: + return _terminal_result(query, RetrievalOutcome.UNAVAILABLE) + if not has_valid_projection_checkpoint(manifest): + return _terminal_result(query, RetrievalOutcome.FAILED) + if not manifest.entries: + return _terminal_result(query, RetrievalOutcome.COMPLETE) + return manifest + + +def _outcome_for_hits( + has_hits: bool, + *, + stale: bool, + failed: bool, +) -> RetrievalOutcome: + if has_hits: + if stale or failed: + return RetrievalOutcome.PARTIAL + return RetrievalOutcome.COMPLETE + if failed: + return RetrievalOutcome.FAILED + if stale: + return RetrievalOutcome.STALE + return RetrievalOutcome.COMPLETE + + +def _ranked_result( + query: RetrievalQuery, + fragments: tuple[Fragment, ...], + *, + stale: bool, + failed: bool, +) -> RetrievalResult: + truncated = len(fragments) > query.hit_limit + selected = fragments[: query.hit_limit] + hits = tuple( + RetrievalHit(fragment, rank) for rank, fragment in enumerate(selected, start=1) + ) + outcome = _outcome_for_hits(bool(hits), stale=stale, failed=failed) + return RetrievalResult(query, outcome, hits, truncated) + + +def _identity_order(fragment: Fragment) -> tuple[str, str, str, str, int, int]: + identity = fragment.identity + return ( + identity.document.key.corpus_id, + identity.document.key.document_id, + identity.document.revision_id, + identity.fragment_id, + identity.start, + identity.end, + ) + + +def _has_cross_provider_conflict( + semantic: _BranchResult, + lexical: _BranchResult, +) -> bool: + semantic_by_identity = { + candidate.fragment.identity: candidate.fragment + for candidate in semantic.all_candidates + } + for candidate in lexical.all_candidates: + previous = semantic_by_identity.get(candidate.fragment.identity) + if previous is not None and previous != candidate.fragment: + return True + return False + + +def _fuse_candidates( + semantic: _BranchResult, + lexical: _BranchResult, +) -> tuple[Fragment, ...]: + fragments: dict[FragmentIdentity, Fragment] = {} + scores: dict[FragmentIdentity, Fraction] = {} + for branch in (semantic, lexical): + for candidate in branch.current_candidates: + identity = candidate.fragment.identity + fragments.setdefault(identity, candidate.fragment) + scores[identity] = scores.get(identity, Fraction()) + Fraction( + 1, + _RRF_OFFSET + candidate.source_rank, + ) + ranked_identities = sorted( + scores, + key=lambda identity: ( + -scores[identity], + _identity_order(fragments[identity]), + ), + ) + return tuple(fragments[identity] for identity in ranked_identities) + + +def retrieve_semantic( + query: RetrievalQuery, + state: ProjectionStateSnapshot, + limits: RetrievalLimits, + embedder: Borrowed[Embedder], + vector_reader: Borrowed[VectorIndexReader], + /, +) -> RetrievalResult: + """Retrieve bounded current fragments from one borrowed semantic index.""" + + canonical_query = _validate_query(query) + canonical_state = _validate_state(state) + canonical_limits = _validate_limits(limits) + _validate_borrowed("embedder", embedder) + _validate_borrowed("vector_reader", vector_reader) + if len(canonical_query.text) > canonical_limits.max_query_codepoints: + raise ContractValidationError( + "query text must not exceed limits.max_query_codepoints" + ) + + state_result = _state_manifest_or_result(canonical_query, canonical_state) + if type(state_result) is RetrievalResult: + return state_result + assert isinstance(state_result, ProjectionManifest) + branch = _semantic_branch( + canonical_query, + state_result, + embedder, + vector_reader, + ) + fragments = tuple(candidate.fragment for candidate in branch.current_candidates) + return _ranked_result( + canonical_query, + fragments, + stale=branch.stale, + failed=branch.failed, + ) + + +def retrieve_hybrid( + query: RetrievalQuery, + state: ProjectionStateSnapshot, + limits: RetrievalLimits, + embedder: Borrowed[Embedder], + vector_reader: Borrowed[VectorIndexReader], + lexical_retriever: Borrowed[LexicalRetriever], + /, +) -> RetrievalResult: + """Fuse bounded semantic and lexical ranks without comparing raw scores.""" + + canonical_query = _validate_query(query) + canonical_state = _validate_state(state) + canonical_limits = _validate_limits(limits) + _validate_borrowed("embedder", embedder) + _validate_borrowed("vector_reader", vector_reader) + _validate_borrowed("lexical_retriever", lexical_retriever) + if len(canonical_query.text) > canonical_limits.max_query_codepoints: + raise ContractValidationError( + "query text must not exceed limits.max_query_codepoints" + ) + + state_result = _state_manifest_or_result(canonical_query, canonical_state) + if type(state_result) is RetrievalResult: + return state_result + assert isinstance(state_result, ProjectionManifest) + semantic = _semantic_branch( + canonical_query, + state_result, + embedder, + vector_reader, + ) + lexical = _lexical_branch( + canonical_query, + state_result, + lexical_retriever, + ) + if _has_cross_provider_conflict(semantic, lexical): + return _terminal_result(canonical_query, RetrievalOutcome.FAILED) + fragments = _fuse_candidates(semantic, lexical) + return _ranked_result( + canonical_query, + fragments, + stale=semantic.stale or lexical.stale, + failed=semantic.failed or lexical.failed, + ) diff --git a/tests/support/clean_import_probe.py b/tests/support/clean_import_probe.py new file mode 100644 index 0000000..e61cdaa --- /dev/null +++ b/tests/support/clean_import_probe.py @@ -0,0 +1,240 @@ +"""Probe one installed generic_rag module in an isolated interpreter.""" + +from __future__ import annotations + +import argparse +import importlib +import multiprocessing +import os +import socket +import subprocess +import sys +import threading +from contextlib import ExitStack +from pathlib import Path +from types import FrameType, ModuleType +from typing import NoReturn, cast +from unittest.mock import patch + +_PACKAGE_ROOT = "generic_rag" +_FORBIDDEN_AUDIT_EVENTS: list[str] = [] +_AUDIT_HOOK_ACTIVE = False + + +def _called_from_package() -> bool: + frame: FrameType | None = sys._getframe(1) + while frame is not None: + module_name = frame.f_globals.get("__name__") + if isinstance(module_name, str) and ( + module_name == _PACKAGE_ROOT or module_name.startswith(f"{_PACKAGE_ROOT}.") + ): + return True + frame = frame.f_back + return False + + +def _is_loader_source_read(arguments: tuple[object, ...]) -> bool: + if not arguments: + return False + raw_path = arguments[0] + if not isinstance(raw_path, str | bytes | os.PathLike): + return False + path = Path(os.fsdecode(raw_path)) + if path.suffix not in {".py", ".pyc"}: + return False + + mode = arguments[1] if len(arguments) > 1 else None + if isinstance(mode, str) and any(marker in mode for marker in "wax+"): + return False + flags = arguments[2] if len(arguments) > 2 else None + write_flags = os.O_WRONLY | os.O_RDWR | os.O_CREAT | os.O_TRUNC | os.O_APPEND + if isinstance(flags, int) and flags & write_flags: + return False + + frame: FrameType | None = sys._getframe(1) + while frame is not None: + module_name = frame.f_globals.get("__name__") + if ( + module_name + in { + "_frozen_importlib_external", + "importlib._bootstrap_external", + } + and frame.f_code.co_name == "get_data" + ): + return True + frame = frame.f_back + return False + + +def _audit_hook(event: str, arguments: tuple[object, ...]) -> None: + global _AUDIT_HOOK_ACTIVE + + if _AUDIT_HOOK_ACTIVE: + return + _AUDIT_HOOK_ACTIVE = True + try: + if not _called_from_package(): + return + if event == "open" and _is_loader_source_read(arguments): + return + forbidden = ( + event == "open" + or event.startswith("socket.") + or event.startswith("subprocess.") + or event == "os.system" + or event.startswith("os.spawn") + ) + if forbidden: + _FORBIDDEN_AUDIT_EVENTS.append(event) + raise RuntimeError(f"forbidden import-time operation: {event}") + finally: + _AUDIT_HOOK_ACTIVE = False + + +def _forbidden_operation(*arguments: object, **keywords: object) -> NoReturn: + del arguments, keywords + raise RuntimeError("forbidden import-time process, network, or thread operation") + + +def _import_with_guards(module_name: str) -> ModuleType: + sys.addaudithook(_audit_hook) + with ExitStack() as stack: + stack.enter_context( + patch.object(threading.Thread, "start", _forbidden_operation) + ) + stack.enter_context( + patch.object(multiprocessing.Process, "start", _forbidden_operation) + ) + stack.enter_context(patch.object(socket, "socket", _forbidden_operation)) + stack.enter_context( + patch.object(socket, "create_connection", _forbidden_operation) + ) + stack.enter_context(patch.object(subprocess, "Popen", _forbidden_operation)) + stack.enter_context(patch.object(os, "system", _forbidden_operation)) + stack.enter_context(patch.object(os, "popen", _forbidden_operation)) + for name in ( + "spawnl", + "spawnle", + "spawnlp", + "spawnlpe", + "spawnv", + "spawnve", + "spawnvp", + "spawnvpe", + ): + if hasattr(os, name): + stack.enter_context(patch.object(os, name, _forbidden_operation)) + imported = importlib.import_module(module_name) + + if _FORBIDDEN_AUDIT_EVENTS: + joined = ", ".join(_FORBIDDEN_AUDIT_EVENTS) + raise AssertionError(f"package import attempted forbidden events: {joined}") + return imported + + +def _assert_no_external_imports(before: set[str], after: set[str]) -> None: + allowed_roots = set(sys.stdlib_module_names) + allowed_roots.update({"builtins", _PACKAGE_ROOT}) + unexpected = sorted( + module_name + for module_name in after - before + if module_name.partition(".")[0] not in allowed_roots + ) + if unexpected: + raise AssertionError( + "package import loaded non-stdlib modules: " + ", ".join(unexpected) + ) + + +def _assert_forbidden_paths_absent(paths: list[str]) -> None: + resolved_entries = [Path(entry).resolve() for entry in sys.path if entry] + for raw_path in paths: + forbidden = Path(raw_path).resolve() + for entry in resolved_entries: + try: + entry.relative_to(forbidden) + except ValueError: + continue + raise AssertionError( + f"forbidden checkout path leaked into sys.path: {entry}" + ) + + +def _bootstrap_source_root(raw_source_root: str | None) -> Path | None: + if raw_source_root is None: + return None + source_root = Path(raw_source_root).resolve(strict=True) + if not source_root.is_dir(): + raise AssertionError(f"source root is not a directory: {source_root}") + sys.path.insert(0, os.fspath(source_root)) + return source_root + + +def _assert_import_origin(imported: ModuleType, source_root: Path | None) -> None: + if source_root is None: + return + raw_origin = getattr(imported, "__file__", None) + if not isinstance(raw_origin, str): + raise AssertionError(f"source import has no file origin: {imported.__name__}") + origin = Path(raw_origin).resolve(strict=True) + expected_package = source_root / _PACKAGE_ROOT + try: + origin.relative_to(expected_package) + except ValueError: + raise AssertionError( + f"{imported.__name__} came from {origin}, not {expected_package}" + ) from None + + +def _parse_arguments() -> argparse.Namespace: + parser = argparse.ArgumentParser() + parser.add_argument("module") + parser.add_argument( + "--forbid-path", + action="append", + default=[], + help="Fail when this path or one of its children occurs on sys.path.", + ) + parser.add_argument( + "--source-root", + help="Explicit resolved src root for an isolated source-tree probe.", + ) + return parser.parse_args() + + +def main() -> int: + arguments = _parse_arguments() + module_name = cast(str, arguments.module) + forbidden_paths = cast(list[str], arguments.forbid_path) + raw_source_root = cast(str | None, arguments.source_root) + if module_name != _PACKAGE_ROOT and not module_name.startswith(f"{_PACKAGE_ROOT}."): + raise ValueError(f"probe is restricted to {_PACKAGE_ROOT} modules") + + _assert_forbidden_paths_absent(forbidden_paths) + source_root = _bootstrap_source_root(raw_source_root) + before = set(sys.modules) + imported = _import_with_guards(module_name) + after = set(sys.modules) + _assert_no_external_imports(before, after) + _assert_import_origin(imported, source_root) + + if imported.__name__ != module_name: + raise AssertionError(f"requested {module_name}, imported {imported.__name__}") + if module_name == _PACKAGE_ROOT: + if getattr(imported, "__all__", None) != (): + raise AssertionError("generic_rag root __all__ must be empty") + loaded_submodules = sorted( + name for name in after if name.startswith(f"{_PACKAGE_ROOT}.") + ) + if loaded_submodules: + raise AssertionError( + "root import loaded submodules: " + ", ".join(loaded_submodules) + ) + + print(f"clean import passed: {module_name}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/support/verify_artifacts.py b/tests/support/verify_artifacts.py new file mode 100644 index 0000000..667753d --- /dev/null +++ b/tests/support/verify_artifacts.py @@ -0,0 +1,243 @@ +"""Validate generic-rag wheel and source-distribution artifacts.""" + +from __future__ import annotations + +import argparse +import stat +import tarfile +import zipfile +from email import message_from_bytes +from email.message import Message +from pathlib import Path, PurePosixPath +from typing import cast + +_PACKAGE_FILES = { + "generic_rag/__init__.py", + "generic_rag/contracts.py", + "generic_rag/errors.py", + "generic_rag/ports.py", + "generic_rag/projection.py", + "generic_rag/projection_integrity.py", + "generic_rag/retrieval.py", +} +_PACKAGE_DATA = {"generic_rag/py.typed"} + + +class ArtifactVerificationError(RuntimeError): + pass + + +def _require(condition: bool, message: str) -> None: + if not condition: + raise ArtifactVerificationError(message) + + +def _safe_archive_name(name: str) -> PurePosixPath: + path = PurePosixPath(name) + _require(bool(path.parts), "archive contains an empty member name") + _require(not path.is_absolute(), f"archive member is absolute: {name}") + _require(".." not in path.parts, f"archive member traverses upward: {name}") + _require("\\" not in name, f"archive member uses a backslash: {name}") + return path + + +def _required_header(metadata: Message, name: str) -> str: + value = metadata.get(name) + _require(value is not None and bool(value.strip()), f"missing {name} metadata") + assert value is not None + return value + + +def _assert_metadata(metadata: Message) -> str: + _require(_required_header(metadata, "Name") == "generic-rag", "wrong Name") + version = _required_header(metadata, "Version") + _require( + _required_header(metadata, "Requires-Python") == ">=3.11", + "Requires-Python must be exactly >=3.11", + ) + requirements = metadata.get_all("Requires-Dist") + _require( + not requirements, + "runtime dependencies leaked into metadata: " + ", ".join(requirements or []), + ) + return version + + +def _assert_no_shipped_tests(names: set[str], artifact: str) -> None: + shipped = sorted( + name + for name in names + if any( + part.lower() in {"test", "tests"} or part.lower().startswith("test_") + for part in PurePosixPath(name).parts + ) + ) + _require( + not shipped, + f"{artifact} ships test files: {', '.join(shipped)}", + ) + + +def _verify_wheel(wheel: Path) -> str: + with zipfile.ZipFile(wheel) as archive: + entries = archive.infolist() + names = {entry.filename.rstrip("/") for entry in entries} + for entry in entries: + _safe_archive_name(entry.filename) + file_type = (entry.external_attr >> 16) & 0o170000 + _require( + file_type != stat.S_IFLNK, + f"wheel contains a symlink: {entry.filename}", + ) + + _assert_no_shipped_tests(names, "wheel") + _require( + _PACKAGE_FILES <= names, + "wheel is missing package modules: " + + ", ".join(sorted(_PACKAGE_FILES - names)), + ) + _require( + _PACKAGE_DATA <= names, + "wheel is missing py.typed", + ) + shipped_python = { + name + for name in names + if name.startswith("generic_rag/") and name.endswith(".py") + } + _require( + shipped_python == _PACKAGE_FILES, + "wheel has an unexpected production module set: " + + ", ".join(sorted(shipped_python)), + ) + _require( + not any("__pycache__" in PurePosixPath(name).parts for name in names), + "wheel contains bytecode caches", + ) + _require( + not any(name.endswith((".so", ".pyd", ".dylib", ".pyc")) for name in names), + "wheel is not a clean pure-Python artifact", + ) + + metadata_names = sorted( + name for name in names if name.endswith(".dist-info/METADATA") + ) + _require( + len(metadata_names) == 1, + "wheel must contain exactly one METADATA file", + ) + metadata = message_from_bytes(archive.read(metadata_names[0])) + version = _assert_metadata(metadata) + + allowed_roots = { + "generic_rag", + metadata_names[0].partition("/")[0], + } + unexpected_roots = sorted( + { + PurePosixPath(name).parts[0] + for name in names + if PurePosixPath(name).parts[0] not in allowed_roots + } + ) + _require( + not unexpected_roots, + "wheel contains unexpected roots: " + ", ".join(unexpected_roots), + ) + + expected_name = f"generic_rag-{version}-py3-none-any.whl" + _require( + wheel.name == expected_name, + f"wheel filename must be {expected_name}, got {wheel.name}", + ) + return version + + +def _verify_sdist(sdist: Path, expected_version: str) -> None: + expected_root = f"generic_rag-{expected_version}" + with tarfile.open(sdist, mode="r:gz") as archive: + members = archive.getmembers() + names = {member.name.rstrip("/") for member in members} + for member in members: + path = _safe_archive_name(member.name) + _require( + path.parts[0] == expected_root, + f"sdist has an unexpected root: {member.name}", + ) + _require( + not member.issym() and not member.islnk(), + f"sdist contains a link: {member.name}", + ) + _require( + member.isfile() or member.isdir(), + f"sdist contains a special file: {member.name}", + ) + + required = { + f"{expected_root}/README.md", + f"{expected_root}/pyproject.toml", + f"{expected_root}/PKG-INFO", + *{ + f"{expected_root}/src/{package_file}" + for package_file in _PACKAGE_FILES | _PACKAGE_DATA + }, + } + _require( + required <= names, + "sdist is missing required files: " + ", ".join(sorted(required - names)), + ) + + metadata_member = archive.getmember(f"{expected_root}/PKG-INFO") + metadata_file = archive.extractfile(metadata_member) + _require(metadata_file is not None, "sdist PKG-INFO is not readable") + assert metadata_file is not None + with metadata_file: + metadata = message_from_bytes(metadata_file.read()) + _require( + _assert_metadata(metadata) == expected_version, + "wheel and sdist versions differ", + ) + + expected_name = f"{expected_root}.tar.gz" + _require( + sdist.name == expected_name, + f"sdist filename must be {expected_name}, got {sdist.name}", + ) + + +def _parse_arguments() -> argparse.Namespace: + parser = argparse.ArgumentParser() + parser.add_argument("dist_directory", type=Path) + return parser.parse_args() + + +def main() -> int: + arguments = _parse_arguments() + dist_directory = cast(Path, arguments.dist_directory) + _require( + dist_directory.is_dir(), + f"distribution directory does not exist: {dist_directory}", + ) + wheel_files = sorted(dist_directory.glob("*.whl")) + sdist_files = sorted(dist_directory.glob("*.tar.gz")) + _require(len(wheel_files) == 1, "expected exactly one wheel") + _require(len(sdist_files) == 1, "expected exactly one sdist") + unexpected = sorted( + path.name + for path in dist_directory.iterdir() + if path.is_file() and path not in {*wheel_files, *sdist_files} + ) + _require( + not unexpected, + "distribution directory contains unexpected files: " + ", ".join(unexpected), + ) + + version = _verify_wheel(wheel_files[0]) + _verify_sdist(sdist_files[0], version) + print(f"verified wheel: {wheel_files[0]}") + print(f"verified sdist: {sdist_files[0]}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/test_contract_values.py b/tests/test_contract_values.py new file mode 100644 index 0000000..ea50fed --- /dev/null +++ b/tests/test_contract_values.py @@ -0,0 +1,366 @@ +"""Contract tests for immutable document, fragment, and embedding values.""" + +from __future__ import annotations + +import unittest +from dataclasses import FrozenInstanceError, fields +from typing import cast + +from generic_rag.contracts import ( + Document, + DocumentIdentity, + DocumentKey, + EmbeddingIdentity, + EmbeddingVector, + Fragment, + FragmentIdentity, + VectorRecord, +) +from generic_rag.errors import ContractValidationError + + +class _StringSubclass(str): + pass + + +class _IntegerSubclass(int): + pass + + +class _FloatSubclass(float): + pass + + +class _TupleSubclass(tuple[object, ...]): + pass + + +def _document_identity( + *, + corpus_id: str = "corpus", + document_id: str = "document", + revision_id: str = "revision", +) -> DocumentIdentity: + return DocumentIdentity( + key=DocumentKey(corpus_id=corpus_id, document_id=document_id), + revision_id=revision_id, + ) + + +def _fragment_identity( + *, + corpus_id: str = "corpus", + document_id: str = "document", + revision_id: str = "revision", + fragment_id: str = "fragment", + start: int = 0, + end: int = 2, +) -> FragmentIdentity: + return FragmentIdentity( + document=_document_identity( + corpus_id=corpus_id, + document_id=document_id, + revision_id=revision_id, + ), + fragment_id=fragment_id, + start=start, + end=end, + ) + + +class ContractValueTests(unittest.TestCase): + def assert_frozen_and_slotted( + self, + instance: object, + field_name: str, + ) -> None: + self.assertFalse(hasattr(instance, "__dict__")) + with self.assertRaises(FrozenInstanceError): + setattr(instance, field_name, object()) + + def test_exact_field_layouts(self) -> None: + self.assertEqual( + tuple(field.name for field in fields(DocumentKey)), + ("corpus_id", "document_id"), + ) + self.assertEqual( + tuple(field.name for field in fields(DocumentIdentity)), + ("key", "revision_id"), + ) + self.assertEqual( + tuple(field.name for field in fields(Document)), + ("identity", "text", "attributes"), + ) + self.assertEqual( + tuple(field.name for field in fields(FragmentIdentity)), + ("document", "fragment_id", "start", "end"), + ) + self.assertEqual( + tuple(field.name for field in fields(Fragment)), + ("identity", "text", "attributes"), + ) + self.assertEqual( + tuple(field.name for field in fields(EmbeddingIdentity)), + ("model_id", "dimensions"), + ) + self.assertEqual( + tuple(field.name for field in fields(EmbeddingVector)), + ("values",), + ) + self.assertEqual( + tuple(field.name for field in fields(VectorRecord)), + ("fragment", "embedding"), + ) + + def test_values_are_frozen_slotted_hashable_value_objects(self) -> None: + document_key = DocumentKey("corpus", "document") + document_identity = DocumentIdentity(document_key, "revision") + document = Document(document_identity, "") + fragment_identity = FragmentIdentity( + document_identity, + "fragment", + 0, + 1, + ) + fragment = Fragment(fragment_identity, "x") + embedding_identity = EmbeddingIdentity("model", 1) + embedding = EmbeddingVector((1,)) + record = VectorRecord(fragment, embedding) + cases = ( + (document_key, "corpus_id"), + (document_identity, "revision_id"), + (document, "text"), + (fragment_identity, "start"), + (fragment, "text"), + (embedding_identity, "dimensions"), + (embedding, "values"), + (record, "fragment"), + ) + + for instance, field_name in cases: + with self.subTest(instance=type(instance).__name__): + self.assert_frozen_and_slotted(instance, field_name) + self.assertIsInstance(hash(instance), int) + self.assertEqual(instance, instance) + + def test_opaque_identity_strings_are_preserved_exactly(self) -> None: + corpus_id = " Corpus/../e\u0301 " + document_id = " Document:ABC " + revision_id = " Rev/001 " + fragment_id = " Fragment/../001 " + key = DocumentKey(corpus_id, document_id) + document = DocumentIdentity(key, revision_id) + fragment = FragmentIdentity(document, fragment_id, 4, 5) + + self.assertEqual(key.corpus_id, corpus_id) + self.assertEqual(key.document_id, document_id) + self.assertEqual(document.revision_id, revision_id) + self.assertEqual(fragment.fragment_id, fragment_id) + + def test_identity_strings_reject_empty_whitespace_and_inexact_types(self) -> None: + invalid_values: tuple[object, ...] = ( + "", + " \t\n", + None, + b"value", + 1, + _StringSubclass("value"), + ) + + for invalid in invalid_values: + with self.subTest(field="corpus_id", value=invalid): + with self.assertRaises(ContractValidationError): + DocumentKey(cast(str, invalid), "document") + with self.subTest(field="document_id", value=invalid): + with self.assertRaises(ContractValidationError): + DocumentKey("corpus", cast(str, invalid)) + with self.subTest(field="revision_id", value=invalid): + with self.assertRaises(ContractValidationError): + DocumentIdentity( + DocumentKey("corpus", "document"), + cast(str, invalid), + ) + with self.subTest(field="fragment_id", value=invalid): + with self.assertRaises(ContractValidationError): + FragmentIdentity( + _document_identity(), + cast(str, invalid), + 0, + 1, + ) + with self.subTest(field="model_id", value=invalid): + with self.assertRaises(ContractValidationError): + EmbeddingIdentity(cast(str, invalid), 1) + + def test_nested_value_fields_require_the_exact_declared_classes(self) -> None: + document_identity = _document_identity() + fragment_identity = _fragment_identity() + fragment = Fragment(fragment_identity, "ab") + embedding = EmbeddingVector((1.0,)) + + with self.assertRaises(ContractValidationError): + DocumentIdentity(cast(DocumentKey, object()), "revision") + with self.assertRaises(ContractValidationError): + Document(cast(DocumentIdentity, object()), "text") + with self.assertRaises(ContractValidationError): + FragmentIdentity( + cast(DocumentIdentity, object()), + "fragment", + 0, + 1, + ) + with self.assertRaises(ContractValidationError): + Fragment(cast(FragmentIdentity, object()), "x") + with self.assertRaises(ContractValidationError): + VectorRecord(cast(Fragment, object()), embedding) + with self.assertRaises(ContractValidationError): + VectorRecord(fragment, cast(EmbeddingVector, object())) + + class DocumentKeySubclass(DocumentKey): + pass + + subclass_key = DocumentKeySubclass("corpus", "document") + with self.assertRaises(ContractValidationError): + DocumentIdentity(subclass_key, "revision") + self.assertEqual(document_identity.key.corpus_id, "corpus") + + def test_document_text_is_exact_and_may_be_empty(self) -> None: + identity = _document_identity() + + self.assertEqual(Document(identity, "").text, "") + text = " \u0000 e\u0301 \n" + self.assertEqual(Document(identity, text).text, text) + for invalid in (None, b"text", _StringSubclass("text")): + with self.subTest(value=invalid): + with self.assertRaises(ContractValidationError): + Document(identity, cast(str, invalid)) + + def test_attributes_preserve_empty_values_duplicates_and_order(self) -> None: + attributes = ( + ("", ""), + ("key", "first"), + ("key", "first"), + ("key", "second"), + ) + document = Document(_document_identity(), "text", attributes) + fragment = Fragment(_fragment_identity(), "ab", attributes) + + self.assertEqual(document.attributes, attributes) + self.assertEqual(fragment.attributes, attributes) + + def test_attributes_reject_inexact_or_malformed_containers(self) -> None: + invalid_attributes: tuple[object, ...] = ( + [("key", "value")], + {"key": "value"}, + iter((("key", "value"),)), + _TupleSubclass((("key", "value"),)), + (["key", "value"],), + (("key",),), + (("key", "value", "extra"),), + ((1, "value"),), + (("key", 1),), + ((_StringSubclass("key"), "value"),), + (("key", _StringSubclass("value")),), + ) + + for invalid in invalid_attributes: + with self.subTest(value=invalid): + with self.assertRaises(ContractValidationError): + Document( + _document_identity(), + "text", + cast(tuple[tuple[str, str], ...], invalid), + ) + with self.assertRaises(ContractValidationError): + Fragment( + _fragment_identity(), + "ab", + cast(tuple[tuple[str, str], ...], invalid), + ) + + def test_fragment_ranges_use_exact_half_open_code_point_offsets(self) -> None: + emoji_identity = _fragment_identity(start=10, end=11) + combining_identity = _fragment_identity(start=20, end=22) + + self.assertEqual(Fragment(emoji_identity, "\U0001f600").text, "\U0001f600") + combining = "e\u0301" + self.assertEqual(len(combining), 2) + self.assertEqual(Fragment(combining_identity, combining).text, combining) + + invalid_ranges = ( + (-1, 1), + (0, 0), + (2, 1), + ) + for start, end in invalid_ranges: + with self.subTest(start=start, end=end): + with self.assertRaises(ContractValidationError): + _fragment_identity(start=start, end=end) + + for invalid in (True, 1.0, _IntegerSubclass(1)): + with self.subTest(field="start", value=invalid): + with self.assertRaises(ContractValidationError): + _fragment_identity(start=cast(int, invalid), end=2) + with self.subTest(field="end", value=invalid): + with self.assertRaises(ContractValidationError): + _fragment_identity(start=0, end=cast(int, invalid)) + + def test_fragment_text_is_nonempty_and_matches_the_range_length(self) -> None: + identity = _fragment_identity(start=4, end=6) + + self.assertEqual(Fragment(identity, "xy").text, "xy") + for invalid in ("", "x", "xyz"): + with self.subTest(value=invalid): + with self.assertRaises(ContractValidationError): + Fragment(identity, invalid) + for invalid_text_type in (None, b"xy", _StringSubclass("xy")): + with self.subTest(value=invalid_text_type): + with self.assertRaises(ContractValidationError): + Fragment(identity, cast(str, invalid_text_type)) + + def test_embedding_dimensions_are_positive_exact_integers(self) -> None: + self.assertEqual(EmbeddingIdentity("model", 3).dimensions, 3) + + for invalid in (0, -1, True, 1.0, _IntegerSubclass(1)): + with self.subTest(value=invalid): + with self.assertRaises(ContractValidationError): + EmbeddingIdentity("model", cast(int, invalid)) + + def test_embedding_vectors_are_nonempty_exact_tuples_of_finite_numbers( + self, + ) -> None: + vector = EmbeddingVector((1, -0.0, 2.5)) + + self.assertEqual(vector.values, (1.0, -0.0, 2.5)) + self.assertTrue(all(type(value) is float for value in vector.values)) + + invalid_containers: tuple[object, ...] = ( + [], + iter((1.0,)), + _TupleSubclass((1.0,)), + (), + ) + for invalid in invalid_containers: + with self.subTest(value=invalid): + with self.assertRaises(ContractValidationError): + EmbeddingVector(cast(tuple[float, ...], invalid)) + + invalid_coordinates: tuple[object, ...] = ( + True, + None, + "1", + _IntegerSubclass(1), + _FloatSubclass(1.0), + float("nan"), + float("inf"), + float("-inf"), + 10**10000, + ) + for invalid in invalid_coordinates: + with self.subTest(value=type(invalid).__name__): + with self.assertRaises(ContractValidationError): + EmbeddingVector(cast(tuple[float, ...], (cast(float, invalid),))) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_errors.py b/tests/test_errors.py new file mode 100644 index 0000000..9281643 --- /dev/null +++ b/tests/test_errors.py @@ -0,0 +1,53 @@ +"""Contract tests for the public generic RAG exception hierarchy.""" + +from __future__ import annotations + +import unittest + +import generic_rag.errors as errors +from generic_rag.errors import ( + CollaborationError, + ContractValidationError, + GenericRagError, + StateCompatibilityError, +) + + +class ErrorContractTests(unittest.TestCase): + def test_exports_are_exact_and_owned_by_the_module(self) -> None: + expected = ( + "GenericRagError", + "ContractValidationError", + "CollaborationError", + "StateCompatibilityError", + ) + + self.assertEqual(errors.__all__, expected) + for name in expected: + exported = getattr(errors, name) + self.assertEqual(exported.__module__, "generic_rag.errors") + + def test_specialized_errors_share_the_public_base(self) -> None: + specialized = ( + ContractValidationError, + CollaborationError, + StateCompatibilityError, + ) + + for error_type in specialized: + with self.subTest(error_type=error_type.__name__): + self.assertTrue(issubclass(error_type, GenericRagError)) + self.assertIsNot(error_type, GenericRagError) + + def test_specialized_errors_remain_distinct_categories(self) -> None: + error_types = { + ContractValidationError, + CollaborationError, + StateCompatibilityError, + } + + self.assertEqual(len(error_types), 3) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_package_boundaries.py b/tests/test_package_boundaries.py new file mode 100644 index 0000000..5ac439d --- /dev/null +++ b/tests/test_package_boundaries.py @@ -0,0 +1,418 @@ +"""Architecture, export, source-index, and clean-import contract tests.""" + +from __future__ import annotations + +import ast +import importlib +import os +import re +import subprocess +import sys +import tempfile +import unittest +from pathlib import Path + +import generic_rag +import generic_rag.contracts as contracts +import generic_rag.errors as errors +import generic_rag.ports as ports +import generic_rag.projection as projection +import generic_rag.projection_integrity as projection_integrity +import generic_rag.retrieval as retrieval + +_PROJECT_ROOT = Path(__file__).resolve().parents[1] +_SOURCE_ROOT = _PROJECT_ROOT / "src" +_PACKAGE_ROOT = _SOURCE_ROOT / "generic_rag" +_CLEAN_IMPORT_PROBE = _PROJECT_ROOT / "tests" / "support" / "clean_import_probe.py" +_EXPECTED_SOURCES = { + "generic_rag": "src/generic_rag/__init__.py", + "generic_rag.contracts": "src/generic_rag/contracts.py", + "generic_rag.errors": "src/generic_rag/errors.py", + "generic_rag.ports": "src/generic_rag/ports.py", + "generic_rag.projection": "src/generic_rag/projection.py", + "generic_rag.projection_integrity": ("src/generic_rag/projection_integrity.py"), + "generic_rag.retrieval": "src/generic_rag/retrieval.py", +} +_EXPECTED_DEPENDENCIES = { + "generic_rag": set(), + "generic_rag.contracts": {"generic_rag.errors"}, + "generic_rag.errors": set(), + "generic_rag.ports": {"generic_rag.contracts"}, + "generic_rag.projection": { + "generic_rag.contracts", + "generic_rag.errors", + "generic_rag.ports", + "generic_rag.projection_integrity", + }, + "generic_rag.projection_integrity": { + "generic_rag.contracts", + "generic_rag.errors", + }, + "generic_rag.retrieval": { + "generic_rag.contracts", + "generic_rag.errors", + "generic_rag.ports", + "generic_rag.projection_integrity", + }, +} +_EXPECTED_EXPORTS = { + "generic_rag": (), + "generic_rag.errors": ( + "GenericRagError", + "ContractValidationError", + "CollaborationError", + "StateCompatibilityError", + ), + "generic_rag.contracts": ( + "DocumentKey", + "DocumentIdentity", + "Document", + "FragmentIdentity", + "Fragment", + "EmbeddingIdentity", + "EmbeddingVector", + "VectorRecord", + "ProjectionIdentity", + "ProjectionCheckpoint", + "ProjectionOutcome", + "ProjectionReceipt", + "ChunkingPolicy", + "ProjectionLimits", + "ProjectionRequest", + "ProjectionManifestEntry", + "ProjectionManifest", + "ProjectionStateAvailability", + "ProjectionStateSnapshot", + "ProjectionStateStatus", + "ProjectionResult", + "RetrievalLimits", + "RetrievalQuery", + "RetrievalOutcome", + "RetrievalHit", + "RetrievalResult", + ), + "generic_rag.ports": ( + "Borrowed", + "Embedder", + "VectorIndexWriter", + "VectorIndexResetter", + "VectorIndexReader", + "LexicalRetriever", + ), + "generic_rag.projection": ( + "ProjectionFailureStage", + "ProjectionStateError", + "ProjectionOperationError", + "project_documents", + "rebuild_projection", + ), + "generic_rag.projection_integrity": ( + "derive_source_digest", + "derive_fragment_id", + "derive_projection_checkpoint_token", + "has_valid_projection_checkpoint", + ), + "generic_rag.retrieval": ( + "retrieve_semantic", + "retrieve_hybrid", + ), +} + + +def _production_sources() -> tuple[Path, ...]: + return tuple(sorted(_PACKAGE_ROOT.rglob("*.py"))) + + +def _module_name(path: Path) -> str: + relative = path.relative_to(_SOURCE_ROOT) + parts = list(relative.parts) + if parts[-1] == "__init__.py": + parts.pop() + else: + parts[-1] = path.stem + return ".".join(parts) + + +def _syntax_tree(path: Path) -> ast.Module: + return ast.parse(path.read_text(encoding="utf-8"), filename=str(path)) + + +def _resolve_import_from(module_name: str, node: ast.ImportFrom) -> str: + if node.level == 0: + return node.module or "" + package_parts = module_name.split(".")[:-1] + upward_steps = node.level - 1 + if upward_steps > len(package_parts): + return "" + prefix = package_parts[: len(package_parts) - upward_steps] + if node.module: + prefix.extend(node.module.split(".")) + return ".".join(prefix) + + +def _internal_dependencies(module_name: str, tree: ast.Module) -> set[str]: + dependencies: set[str] = set() + for node in ast.walk(tree): + if isinstance(node, ast.Import): + for alias in node.names: + if alias.name == "generic_rag" or alias.name.startswith("generic_rag."): + dependencies.add(alias.name) + elif isinstance(node, ast.ImportFrom): + imported_module = _resolve_import_from(module_name, node) + if imported_module == "generic_rag" or imported_module.startswith( + "generic_rag." + ): + dependencies.add(imported_module) + return dependencies + + +def _probe_environment() -> dict[str, str]: + credential_markers = ( + "AUTH", + "CREDENTIAL", + "KEY", + "PASSWORD", + "SECRET", + "TOKEN", + ) + environment = { + name: value + for name, value in os.environ.items() + if not name.startswith("PYTHON") + and not any(marker in name.upper() for marker in credential_markers) + } + environment["PYTHONDONTWRITEBYTECODE"] = "1" + return environment + + +class PackageBoundaryTests(unittest.TestCase): + def test_production_module_inventory_is_exact(self) -> None: + actual = { + _module_name(path): path.relative_to(_PROJECT_ROOT).as_posix() + for path in _production_sources() + } + + self.assertEqual(len(actual), 7) + self.assertEqual(actual, _EXPECTED_SOURCES) + + def test_supported_exports_are_exact_and_owned(self) -> None: + modules = { + "generic_rag": generic_rag, + "generic_rag.errors": errors, + "generic_rag.contracts": contracts, + "generic_rag.ports": ports, + "generic_rag.projection": projection, + "generic_rag.projection_integrity": projection_integrity, + "generic_rag.retrieval": retrieval, + } + + for module_name, expected_exports in _EXPECTED_EXPORTS.items(): + module = modules[module_name] + with self.subTest(module=module_name): + self.assertEqual(module.__all__, expected_exports) + for name in expected_exports: + exported = getattr(module, name) + self.assertEqual(exported.__module__, module_name) + + for name in ( + *_EXPECTED_EXPORTS["generic_rag.errors"], + *_EXPECTED_EXPORTS["generic_rag.contracts"], + *_EXPECTED_EXPORTS["generic_rag.ports"], + *_EXPECTED_EXPORTS["generic_rag.projection"], + *_EXPECTED_EXPORTS["generic_rag.projection_integrity"], + *_EXPECTED_EXPORTS["generic_rag.retrieval"], + ): + with self.subTest(root_reexport=name): + self.assertFalse(hasattr(generic_rag, name)) + + def test_each_supported_module_imports_from_its_owning_path(self) -> None: + for module_name in _EXPECTED_EXPORTS: + with self.subTest(module=module_name): + module = importlib.import_module(module_name) + self.assertEqual(module.__name__, module_name) + self.assertEqual(module.__all__, _EXPECTED_EXPORTS[module_name]) + + def test_internal_dependency_graph_is_exact_and_acyclic(self) -> None: + actual = { + module_name: _internal_dependencies( + module_name, + _syntax_tree(_PROJECT_ROOT / relative_path), + ) + for module_name, relative_path in _EXPECTED_SOURCES.items() + } + + self.assertEqual(sum(map(len, actual.values())), 12) + self.assertEqual(actual, _EXPECTED_DEPENDENCIES) + root_tree = _syntax_tree(_PROJECT_ROOT / _EXPECTED_SOURCES["generic_rag"]) + root_imports = [ + node + for node in ast.walk(root_tree) + if isinstance(node, ast.Import | ast.ImportFrom) + ] + self.assertEqual(root_imports, []) + + def test_production_imports_are_stdlib_or_declared_internal_dependencies( + self, + ) -> None: + allowed_roots = set(sys.stdlib_module_names) + allowed_roots.update({"__future__", "generic_rag"}) + forbidden_roots = { + "MySQLdb", + "mysql", + "pymysql", + "sqlalchemy", + "sqlite3", + "story_writing_agents", + "tests", + } + + for path in _production_sources(): + module_name = _module_name(path) + tree = _syntax_tree(path) + for node in ast.walk(tree): + imported_modules: tuple[str, ...] = () + if isinstance(node, ast.Import): + imported_modules = tuple(alias.name for alias in node.names) + elif isinstance(node, ast.ImportFrom): + imported_modules = (_resolve_import_from(module_name, node),) + for alias in node.names: + self.assertNotEqual( + alias.name, + "*", + f"{module_name} uses a wildcard import", + ) + if imported_modules[0].startswith("generic_rag"): + self.assertFalse( + alias.name.startswith("_"), + f"{module_name} imports private name {alias.name}", + ) + + for imported_module in imported_modules: + if not imported_module: + continue + root = imported_module.partition(".")[0] + with self.subTest(module=module_name, imported=imported_module): + self.assertNotIn(root, forbidden_roots) + self.assertIn( + root, + allowed_roots, + f"{module_name} imports third-party module " + f"{imported_module}", + ) + + def test_production_has_no_any_dynamic_import_or_sys_path_access(self) -> None: + dynamic_functions = { + "__import__", + "import_module", + "module_from_spec", + "spec_from_file_location", + } + + for path in _production_sources(): + module_name = _module_name(path) + tree = _syntax_tree(path) + importlib_aliases = {"importlib"} + sys_aliases = {"sys"} + direct_dynamic_aliases = set(dynamic_functions) + for node in ast.walk(tree): + if isinstance(node, ast.Import): + for alias in node.names: + if alias.name == "importlib": + importlib_aliases.add(alias.asname or alias.name) + if alias.name == "sys": + sys_aliases.add(alias.asname or alias.name) + elif isinstance(node, ast.ImportFrom): + if node.module == "importlib": + for alias in node.names: + if alias.name in dynamic_functions: + direct_dynamic_aliases.add(alias.asname or alias.name) + + violations: list[str] = [] + for node in ast.walk(tree): + if isinstance(node, ast.Name) and node.id == "Any": + violations.append("typing.Any") + elif isinstance(node, ast.Attribute) and node.attr == "Any": + violations.append("typing.Any") + elif ( + isinstance(node, ast.Attribute) + and node.attr == "path" + and isinstance(node.value, ast.Name) + and node.value.id in sys_aliases + ): + violations.append("sys.path") + elif isinstance(node, ast.Call): + function = node.func + if ( + isinstance(function, ast.Name) + and function.id in direct_dynamic_aliases + ): + violations.append(f"dynamic import {function.id}") + elif ( + isinstance(function, ast.Attribute) + and isinstance(function.value, ast.Name) + and function.value.id in importlib_aliases + and function.attr in dynamic_functions + ): + violations.append(f"dynamic import {function.attr}") + + self.assertEqual( + violations, + [], + f"{module_name} has forbidden boundary operations", + ) + + def test_module_index_has_exact_source_path_parity(self) -> None: + index_path = _PROJECT_ROOT / "PYTHON_MODULE_INDEX.md" + entries: dict[str, str] = {} + current_module: str | None = None + heading_pattern = re.compile(r"^## \x60(generic_rag(?:\.[a-z_]+)?)\x60$") + source_pattern = re.compile(r"^- Source: \x60([^ \x60]+)\x60$") + + for line in index_path.read_text(encoding="utf-8").splitlines(): + heading_match = heading_pattern.fullmatch(line) + if heading_match: + current_module = heading_match.group(1) + self.assertNotIn(current_module, entries) + continue + source_match = source_pattern.fullmatch(line) + if source_match and current_module is not None: + entries[current_module] = source_match.group(1) + current_module = None + + self.assertEqual(entries, _EXPECTED_SOURCES) + + def test_each_module_imports_in_an_isolated_side_effect_guarded_process( + self, + ) -> None: + environment = _probe_environment() + for module_name in _EXPECTED_EXPORTS: + with self.subTest(module=module_name): + with tempfile.TemporaryDirectory( + prefix="generic-rag-import-" + ) as working_directory: + completed = subprocess.run( + ( + sys.executable, + "-I", + "-B", + str(_CLEAN_IMPORT_PROBE), + module_name, + "--source-root", + str(_SOURCE_ROOT), + ), + cwd=working_directory, + env=environment, + check=False, + capture_output=True, + text=True, + timeout=10, + ) + self.assertEqual( + completed.returncode, + 0, + f"stdout:\n{completed.stdout}\nstderr:\n{completed.stderr}", + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_ports.py b/tests/test_ports.py new file mode 100644 index 0000000..4033810 --- /dev/null +++ b/tests/test_ports.py @@ -0,0 +1,434 @@ +"""Contract tests and deterministic witnesses for injected collaborator ports.""" + +from __future__ import annotations + +import inspect +import unittest +from collections.abc import Callable +from dataclasses import FrozenInstanceError, fields +from types import TracebackType +from typing import cast, get_type_hints + +import generic_rag.ports as ports +from generic_rag.contracts import ( + DocumentIdentity, + DocumentKey, + EmbeddingIdentity, + EmbeddingVector, + Fragment, + FragmentIdentity, + RetrievalQuery, + VectorRecord, +) +from generic_rag.ports import ( + Borrowed, + Embedder, + LexicalRetriever, + VectorIndexReader, + VectorIndexResetter, + VectorIndexWriter, +) + + +def _document_identity( + *, + corpus_id: str = "corpus", + document_id: str = "document", + revision_id: str = "revision", +) -> DocumentIdentity: + return DocumentIdentity( + DocumentKey(corpus_id, document_id), + revision_id, + ) + + +def _fragment( + fragment_id: str, + *, + corpus_id: str = "corpus", + document_id: str = "document", +) -> Fragment: + document = _document_identity( + corpus_id=corpus_id, + document_id=document_id, + ) + identity = FragmentIdentity(document, fragment_id, 0, 1) + return Fragment(identity, "x") + + +def _query( + *, + corpus_id: str = "corpus", + candidate_limit: int = 3, +) -> RetrievalQuery: + return RetrievalQuery(corpus_id, "query", 2, candidate_limit) + + +class _FakeEmbedder: + def __init__(self) -> None: + self._identity = EmbeddingIdentity("fake-model", 2) + self.calls: list[tuple[str, ...]] = [] + + @property + def identity(self) -> EmbeddingIdentity: + return self._identity + + def embed(self, texts: tuple[str, ...], /) -> tuple[EmbeddingVector, ...]: + self.calls.append(texts) + return tuple( + EmbeddingVector((index, len(text))) for index, text in enumerate(texts) + ) + + +class _ShapeOnlyInvalidEmbedder: + @property + def identity(self) -> EmbeddingIdentity: + return EmbeddingIdentity("shape-only", 2) + + def embed(self, texts: tuple[str, ...], /) -> tuple[EmbeddingVector, ...]: + del texts + return (EmbeddingVector((1.0,)),) + + +class _FakeVectorWriter: + def __init__(self) -> None: + self.replacements: list[tuple[DocumentIdentity, tuple[VectorRecord, ...]]] = [] + self.deletions: list[DocumentKey] = [] + + def replace_document( + self, + document: DocumentIdentity, + records: tuple[VectorRecord, ...], + /, + ) -> None: + if any(record.fragment.identity.document != document for record in records): + raise AssertionError("records must belong to the supplied document") + self.replacements.append((document, records)) + + def delete_document(self, document: DocumentKey, /) -> None: + self.deletions.append(document) + + +class _FakeVectorResetter: + def __init__(self) -> None: + self.corpora: list[str] = [] + + def reset_corpus(self, corpus_id: str, /) -> None: + self.corpora.append(corpus_id) + + +class _FakeVectorReader: + def __init__(self, ranked: tuple[Fragment, ...]) -> None: + self.ranked = ranked + self.calls: list[tuple[RetrievalQuery, EmbeddingVector]] = [] + + def search( + self, + query: RetrievalQuery, + embedding: EmbeddingVector, + /, + ) -> tuple[Fragment, ...]: + self.calls.append((query, embedding)) + matching = tuple( + fragment + for fragment in self.ranked + if fragment.identity.document.key.corpus_id == query.corpus_id + ) + return matching[: query.candidate_limit] + + +class _FakeLexicalRetriever: + def __init__(self, ranked: tuple[Fragment, ...]) -> None: + self.ranked = ranked + self.calls: list[RetrievalQuery] = [] + + def search(self, query: RetrievalQuery, /) -> tuple[Fragment, ...]: + self.calls.append(query) + matching = tuple( + fragment + for fragment in self.ranked + if fragment.identity.document.key.corpus_id == query.corpus_id + ) + return matching[: query.candidate_limit] + + +class _RaisingLexicalRetriever: + def __init__(self, failure: RuntimeError) -> None: + self.failure = failure + + def search(self, query: RetrievalQuery, /) -> tuple[Fragment, ...]: + del query + raise self.failure + + +class _LifecycleSentinel: + def __init__(self) -> None: + self.enter_calls = 0 + self.exit_calls = 0 + self.close_calls = 0 + self.shutdown_calls = 0 + + def __enter__(self) -> _LifecycleSentinel: + self.enter_calls += 1 + return self + + def __exit__( + self, + exc_type: type[BaseException] | None, + exc_value: BaseException | None, + traceback: TracebackType | None, + ) -> bool: + del exc_type, exc_value, traceback + self.exit_calls += 1 + return True + + def close(self) -> None: + self.close_calls += 1 + + def shutdown(self) -> None: + self.shutdown_calls += 1 + + +class _MissingMethods: + pass + + +class PortContractTests(unittest.TestCase): + def test_exports_are_exact_and_owned_by_the_module(self) -> None: + expected = ( + "Borrowed", + "Embedder", + "VectorIndexWriter", + "VectorIndexResetter", + "VectorIndexReader", + "LexicalRetriever", + ) + + self.assertEqual(ports.__all__, expected) + for name in expected: + exported = getattr(ports, name) + self.assertEqual(exported.__module__, "generic_rag.ports") + + def test_protocols_are_runtime_checkable_structural_shapes(self) -> None: + self.assertIsInstance(_FakeEmbedder(), Embedder) + self.assertIsInstance(_FakeVectorWriter(), VectorIndexWriter) + self.assertIsInstance(_FakeVectorResetter(), VectorIndexResetter) + self.assertIsInstance(_FakeVectorReader(()), VectorIndexReader) + self.assertIsInstance(_FakeLexicalRetriever(()), LexicalRetriever) + + missing = _MissingMethods() + self.assertNotIsInstance(missing, Embedder) + self.assertNotIsInstance(missing, VectorIndexWriter) + self.assertNotIsInstance(missing, VectorIndexResetter) + self.assertNotIsInstance(missing, VectorIndexReader) + self.assertNotIsInstance(missing, LexicalRetriever) + + def test_runtime_protocol_check_does_not_claim_semantic_enforcement( + self, + ) -> None: + shape_only = _ShapeOnlyInvalidEmbedder() + + self.assertIsInstance(shape_only, Embedder) + self.assertNotEqual( + len(shape_only.embed(())), + 0, + "runtime_checkable verifies names, not the empty-input obligation", + ) + self.assertNotEqual( + len(shape_only.embed(("text",))[0].values), + shape_only.identity.dimensions, + "runtime_checkable does not verify embedding dimensions", + ) + + def test_protocol_methods_are_synchronous_and_positional_only(self) -> None: + methods = ( + (Embedder.embed, ("self", "texts")), + ( + VectorIndexWriter.replace_document, + ("self", "document", "records"), + ), + (VectorIndexWriter.delete_document, ("self", "document")), + (VectorIndexResetter.reset_corpus, ("self", "corpus_id")), + ( + VectorIndexReader.search, + ("self", "query", "embedding"), + ), + (LexicalRetriever.search, ("self", "query")), + ) + for method, names in methods: + with self.subTest(method=method.__qualname__): + self.assertFalse(inspect.iscoroutinefunction(method)) + parameters = tuple(inspect.signature(method).parameters.values()) + self.assertEqual( + tuple(parameter.name for parameter in parameters), + names, + ) + self.assertTrue( + all( + parameter.kind is inspect.Parameter.POSITIONAL_ONLY + for parameter in parameters + ) + ) + + def test_protocol_annotations_keep_scores_and_lifecycle_out(self) -> None: + self.assertEqual( + get_type_hints(Embedder.embed), + { + "texts": tuple[str, ...], + "return": tuple[EmbeddingVector, ...], + }, + ) + self.assertEqual( + get_type_hints(VectorIndexWriter.replace_document), + { + "document": DocumentIdentity, + "records": tuple[VectorRecord, ...], + "return": type(None), + }, + ) + self.assertEqual( + get_type_hints(VectorIndexWriter.delete_document), + { + "document": DocumentKey, + "return": type(None), + }, + ) + self.assertEqual( + get_type_hints(VectorIndexResetter.reset_corpus), + { + "corpus_id": str, + "return": type(None), + }, + ) + self.assertEqual( + get_type_hints(VectorIndexReader.search), + { + "query": RetrievalQuery, + "embedding": EmbeddingVector, + "return": tuple[Fragment, ...], + }, + ) + self.assertEqual( + get_type_hints(LexicalRetriever.search), + { + "query": RetrievalQuery, + "return": tuple[Fragment, ...], + }, + ) + lifecycle_names = {"close", "shutdown", "__enter__", "__exit__"} + for protocol in ( + Embedder, + VectorIndexWriter, + VectorIndexResetter, + VectorIndexReader, + LexicalRetriever, + ): + with self.subTest(protocol=protocol.__name__): + self.assertTrue(lifecycle_names.isdisjoint(protocol.__dict__)) + + def test_embedder_witness_preserves_order_count_dimensions_and_empty(self) -> None: + embedder = _FakeEmbedder() + + self.assertEqual(embedder.embed(()), ()) + vectors = embedder.embed(("a", "longer")) + + self.assertEqual(embedder.calls, [(), ("a", "longer")]) + self.assertEqual(len(vectors), 2) + self.assertEqual(vectors[0].values, (0.0, 1.0)) + self.assertEqual(vectors[1].values, (1.0, 6.0)) + self.assertTrue( + all( + len(vector.values) == embedder.identity.dimensions for vector in vectors + ) + ) + + def test_writer_witness_replaces_complete_sets_and_accepts_empty(self) -> None: + writer = _FakeVectorWriter() + document = _document_identity() + fragment = _fragment("fragment") + records = (VectorRecord(fragment, EmbeddingVector((1.0, 2.0))),) + + writer.replace_document(document, records) + writer.replace_document(document, ()) + writer.delete_document(document.key) + + self.assertEqual( + writer.replacements, + [(document, records), (document, ())], + ) + self.assertEqual(writer.deletions, [document.key]) + + wrong_document = _document_identity(document_id="other") + with self.assertRaises(AssertionError): + writer.replace_document(wrong_document, records) + + def test_resetter_witness_removes_one_exact_corpus_and_returns_none(self) -> None: + resetter = _FakeVectorResetter() + + result = cast(Callable[[str], object], resetter.reset_corpus)(" Corpus/../A ") + + self.assertIsNone(result) + self.assertEqual(resetter.corpora, [" Corpus/../A "]) + + def test_reader_witnesses_preserve_rank_and_enforce_candidate_bound(self) -> None: + ranked = ( + _fragment("first"), + _fragment("wrong-corpus", corpus_id="other"), + _fragment("second", document_id="second-document"), + _fragment("third", document_id="third-document"), + ) + query = _query(candidate_limit=2) + embedding = EmbeddingVector((1.0, 2.0)) + vector_reader = _FakeVectorReader(ranked) + lexical_reader = _FakeLexicalRetriever(ranked) + + self.assertEqual( + vector_reader.search(query, embedding), + (ranked[0], ranked[2]), + ) + self.assertEqual( + lexical_reader.search(query), + (ranked[0], ranked[2]), + ) + self.assertEqual(vector_reader.calls, [(query, embedding)]) + self.assertEqual(lexical_reader.calls, [query]) + + def test_borrowed_never_invokes_or_owns_resource_lifecycle(self) -> None: + resource = _LifecycleSentinel() + borrowed = Borrowed(resource) + + self.assertEqual( + tuple(field.name for field in fields(Borrowed)), + ("resource",), + ) + self.assertFalse(hasattr(borrowed, "__dict__")) + self.assertIs(borrowed.resource, resource) + with self.assertRaises(FrozenInstanceError): + setattr(borrowed, "resource", object()) + + with borrowed as entered: + self.assertIs(entered, resource) + + self.assertEqual( + ( + resource.enter_calls, + resource.exit_calls, + resource.close_calls, + resource.shutdown_calls, + ), + (0, 0, 0, 0), + ) + + def test_borrowed_never_suppresses_collaborator_failure(self) -> None: + failure = RuntimeError("sentinel collaborator failure") + retriever = _RaisingLexicalRetriever(failure) + + with self.assertRaises(RuntimeError) as raised: + with Borrowed(retriever) as borrowed: + borrowed.search(_query()) + + self.assertIs(raised.exception, failure) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_projection.py b/tests/test_projection.py new file mode 100644 index 0000000..700f2df --- /dev/null +++ b/tests/test_projection.py @@ -0,0 +1,1661 @@ +"""Deterministic projection orchestration, failure, and lifecycle tests.""" + +from __future__ import annotations + +import hashlib +import inspect +import json +import os +import subprocess +import sys +import textwrap +import unittest +from collections.abc import Callable +from pathlib import Path +from typing import cast + +import generic_rag.projection as projection_module +from generic_rag.contracts import ( + ChunkingPolicy, + Document, + DocumentIdentity, + DocumentKey, + EmbeddingIdentity, + EmbeddingVector, + ProjectionCheckpoint, + ProjectionIdentity, + ProjectionLimits, + ProjectionManifest, + ProjectionManifestEntry, + ProjectionOutcome, + ProjectionReceipt, + ProjectionRequest, + ProjectionStateAvailability, + ProjectionStateSnapshot, + ProjectionStateStatus, + VectorRecord, +) +from generic_rag.errors import ( + CollaborationError, + ContractValidationError, + StateCompatibilityError, +) +from generic_rag.ports import ( + Borrowed, + Embedder, + VectorIndexResetter, + VectorIndexWriter, +) +from generic_rag.projection import ( + ProjectionFailureStage, + ProjectionOperationError, + ProjectionStateError, + project_documents, + rebuild_projection, +) + +_SOURCE_ROOT = Path(__file__).resolve().parents[1] / "src" + + +def _document( + document_id: str, + *, + revision_id: str = "revision-1", + text: str = "abcdefgh", + corpus_id: str = "corpus", + attributes: tuple[tuple[str, str], ...] = (), +) -> Document: + return Document( + DocumentIdentity(DocumentKey(corpus_id, document_id), revision_id), + text, + attributes, + ) + + +def _identity( + *, + schema_id: str = "schema-v1", + model_id: str = "model-v1", + dimensions: int = 2, +) -> ProjectionIdentity: + return ProjectionIdentity( + schema_id, + EmbeddingIdentity(model_id, dimensions), + ) + + +def _request( + documents: tuple[Document, ...], + *, + projection: ProjectionIdentity | None = None, + chunking: ChunkingPolicy | None = None, + batch_size: int = 2, + corpus_id: str = "corpus", +) -> ProjectionRequest: + return ProjectionRequest( + corpus_id, + projection or _identity(), + chunking or ChunkingPolicy(4, 1), + ProjectionLimits(max(1, len(documents)), 100, batch_size), + documents, + ) + + +def _hash_fields(fields_to_hash: tuple[str, ...]) -> str: + """Independent length-delimited SHA-256 oracle from the public contract.""" + + digest = hashlib.sha256() + for value in fields_to_hash: + encoded = value.encode("utf-8", "surrogatepass") + digest.update(len(encoded).to_bytes(8, "big", signed=False)) + digest.update(encoded) + return f"sha256:{digest.hexdigest()}" + + +def _source_digest(document: Document) -> str: + source_fields = [ + "generic-rag:projection-source:v1", + "text", + document.text, + "attributes_count", + str(len(document.attributes)), + ] + for key, value in document.attributes: + source_fields.extend(("attribute_key", key, "attribute_value", value)) + return _hash_fields(tuple(source_fields)) + + +def _fragment_id(document: DocumentIdentity, start: int, end: int) -> str: + return _hash_fields( + ( + "generic-rag:fragment-id:v1", + "corpus_id", + document.key.corpus_id, + "document_id", + document.key.document_id, + "revision_id", + document.revision_id, + "start", + str(start), + "end", + str(end), + ) + ) + + +def _expected_ranges( + text: str, chunking: ChunkingPolicy +) -> tuple[tuple[int, int], ...]: + ranges: list[tuple[int, int]] = [] + start = 0 + while start < len(text): + end = min(start + chunking.max_fragment_codepoints, len(text)) + ranges.append((start, end)) + if end == len(text): + break + start = end - chunking.overlap_codepoints + return tuple(ranges) + + +def _checkpoint_token( + request: ProjectionRequest, + entries: tuple[ProjectionManifestEntry, ...], +) -> str: + token_fields = [ + "generic-rag:projection-checkpoint:v1", + "corpus_id", + request.corpus_id, + "schema_id", + request.projection.schema_id, + "embedding_model_id", + request.projection.embedding.model_id, + "embedding_dimensions", + str(request.projection.embedding.dimensions), + "max_fragment_codepoints", + str(request.chunking.max_fragment_codepoints), + "overlap_codepoints", + str(request.chunking.overlap_codepoints), + "entry_count", + str(len(entries)), + ] + for entry in entries: + token_fields.extend( + ( + "document_id", + entry.document.key.document_id, + "revision_id", + entry.document.revision_id, + "source_digest", + entry.source_digest, + "fragment_count", + str(entry.fragment_count), + ) + ) + return _hash_fields(tuple(token_fields)) + + +def _manifest(request: ProjectionRequest) -> ProjectionManifest: + entries = tuple( + ProjectionManifestEntry( + document.identity, + _source_digest(document), + len(_expected_ranges(document.text, request.chunking)), + ) + for document in request.documents + ) + checkpoint = ProjectionCheckpoint( + request.corpus_id, + request.projection, + _checkpoint_token(request, entries), + ) + return ProjectionManifest( + request.corpus_id, + request.projection, + request.chunking, + entries, + checkpoint, + ) + + +def _present(request: ProjectionRequest) -> ProjectionStateSnapshot: + return ProjectionStateSnapshot( + ProjectionStateAvailability.PRESENT, + _manifest(request), + ) + + +class _FakeEmbedder: + def __init__( + self, + expected_identity: EmbeddingIdentity, + events: list[str], + *, + identity_value: object | None = None, + identity_failure: BaseException | None = None, + output: Callable[[tuple[str, ...]], object] | None = None, + embed_failure_at: int | None = None, + embed_failure: BaseException | None = None, + ) -> None: + self.expected_identity = expected_identity + self.events = events + self.identity_value = ( + expected_identity if identity_value is None else identity_value + ) + self.identity_failure = identity_failure + self.output = output + self.embed_failure_at = embed_failure_at + self.embed_failure = embed_failure or RuntimeError("embedding failed") + self.identity_calls = 0 + self.embed_calls: list[tuple[str, ...]] = [] + self.lifecycle_calls: list[str] = [] + + @property + def identity(self) -> EmbeddingIdentity: + self.identity_calls += 1 + self.events.append("identity") + if self.identity_failure is not None: + raise self.identity_failure + return cast(EmbeddingIdentity, self.identity_value) + + def embed(self, texts: tuple[str, ...], /) -> tuple[EmbeddingVector, ...]: + call_index = len(self.embed_calls) + self.embed_calls.append(texts) + self.events.append("embed:" + "|".join(texts)) + if self.embed_failure_at == call_index: + raise self.embed_failure + if self.output is not None: + return cast(tuple[EmbeddingVector, ...], self.output(texts)) + return tuple( + EmbeddingVector((float(len(text)), float(index))) + for index, text in enumerate(texts) + ) + + def __enter__(self) -> _FakeEmbedder: + self.lifecycle_calls.append("enter") + return self + + def __exit__(self, *arguments: object) -> bool: + del arguments + self.lifecycle_calls.append("exit") + return True + + def close(self) -> None: + self.lifecycle_calls.append("close") + + def shutdown(self) -> None: + self.lifecycle_calls.append("shutdown") + + +class _FakeWriter: + def __init__( + self, + events: list[str], + *, + replace_failure_at: int | None = None, + replace_failure: BaseException | None = None, + replace_result: object = None, + delete_failure_at: int | None = None, + delete_failure: BaseException | None = None, + delete_result: object = None, + ) -> None: + self.events = events + self.replace_failure_at = replace_failure_at + self.replace_failure = replace_failure or RuntimeError("replacement failed") + self.replace_result = replace_result + self.delete_failure_at = delete_failure_at + self.delete_failure = delete_failure or RuntimeError("deletion failed") + self.delete_result = delete_result + self.replacements: list[tuple[DocumentIdentity, tuple[VectorRecord, ...]]] = [] + self.deletions: list[DocumentKey] = [] + self.lifecycle_calls: list[str] = [] + + def replace_document( + self, + document: DocumentIdentity, + records: tuple[VectorRecord, ...], + /, + ) -> None: + call_index = len(self.replacements) + self.replacements.append((document, records)) + self.events.append(f"replace:{document.key.document_id}") + if self.replace_failure_at == call_index: + raise self.replace_failure + return cast(None, self.replace_result) + + def delete_document(self, document: DocumentKey, /) -> None: + call_index = len(self.deletions) + self.deletions.append(document) + self.events.append(f"delete:{document.document_id}") + if self.delete_failure_at == call_index: + raise self.delete_failure + return cast(None, self.delete_result) + + def __enter__(self) -> _FakeWriter: + self.lifecycle_calls.append("enter") + return self + + def __exit__(self, *arguments: object) -> bool: + del arguments + self.lifecycle_calls.append("exit") + return True + + def close(self) -> None: + self.lifecycle_calls.append("close") + + def shutdown(self) -> None: + self.lifecycle_calls.append("shutdown") + + +class _FakeResetter: + def __init__( + self, + events: list[str], + *, + failure: BaseException | None = None, + result: object = None, + ) -> None: + self.events = events + self.failure = failure + self.result = result + self.corpora: list[str] = [] + self.lifecycle_calls: list[str] = [] + + def reset_corpus(self, corpus_id: str, /) -> None: + self.corpora.append(corpus_id) + self.events.append(f"reset:{corpus_id}") + if self.failure is not None: + raise self.failure + return cast(None, self.result) + + def __enter__(self) -> _FakeResetter: + self.lifecycle_calls.append("enter") + return self + + def __exit__(self, *arguments: object) -> bool: + del arguments + self.lifecycle_calls.append("exit") + return True + + def close(self) -> None: + self.lifecycle_calls.append("close") + + def shutdown(self) -> None: + self.lifecycle_calls.append("shutdown") + + +def _borrow_embedder(embedder: _FakeEmbedder) -> Borrowed[Embedder]: + return Borrowed(cast(Embedder, embedder)) + + +def _borrow_writer(writer: _FakeWriter) -> Borrowed[VectorIndexWriter]: + return Borrowed(cast(VectorIndexWriter, writer)) + + +def _borrow_resetter(resetter: _FakeResetter) -> Borrowed[VectorIndexResetter]: + return Borrowed(cast(VectorIndexResetter, resetter)) + + +def _constant_output(value: object) -> Callable[[tuple[str, ...]], object]: + def output(texts: tuple[str, ...]) -> object: + del texts + return value + + return output + + +def _collaborators( + request: ProjectionRequest, +) -> tuple[_FakeEmbedder, _FakeWriter, _FakeResetter, list[str]]: + events: list[str] = [] + return ( + _FakeEmbedder(request.projection.embedding, events), + _FakeWriter(events), + _FakeResetter(events), + events, + ) + + +class ProjectionPublicApiTests(unittest.TestCase): + def test_exports_are_exact_and_owned_by_projection_module(self) -> None: + expected = ( + "ProjectionFailureStage", + "ProjectionStateError", + "ProjectionOperationError", + "project_documents", + "rebuild_projection", + ) + + self.assertEqual(projection_module.__all__, expected) + for name in expected: + self.assertEqual( + getattr(projection_module, name).__module__, + "generic_rag.projection", + ) + + def test_public_workflows_are_synchronous_and_positional_only(self) -> None: + expected_parameters = { + project_documents: ("request", "state", "embedder", "writer"), + rebuild_projection: ( + "request", + "state", + "embedder", + "writer", + "resetter", + ), + } + for function, expected in expected_parameters.items(): + with self.subTest(function=function.__name__): + self.assertFalse(inspect.iscoroutinefunction(function)) + parameters = tuple( + inspect.signature( + cast(Callable[..., object], function) + ).parameters.values() + ) + self.assertEqual(tuple(item.name for item in parameters), expected) + self.assertTrue( + all( + item.kind is inspect.Parameter.POSITIONAL_ONLY + for item in parameters + ) + ) + + def test_failure_stage_is_an_exact_closed_string_enum(self) -> None: + self.assertEqual( + tuple((member.name, member.value) for member in ProjectionFailureStage), + ( + ("EMBEDDER_IDENTITY", "embedder_identity"), + ("EMBEDDING", "embedding"), + ("REPLACEMENT", "replacement"), + ("DELETION", "deletion"), + ("RESET", "reset"), + ), + ) + for value in ("EMBEDDING", "unknown", "", None, 1, object()): + with self.subTest(value=value): + with self.assertRaises(ContractValidationError): + ProjectionFailureStage(cast(str, value)) + + def test_state_error_is_typed_content_free_and_requires_exact_status(self) -> None: + error = ProjectionStateError(ProjectionStateStatus.SCHEMA_MISMATCH) + + self.assertIsInstance(error, StateCompatibilityError) + self.assertIs(error.status, ProjectionStateStatus.SCHEMA_MISMATCH) + self.assertNotIn("schema", str(error).lower()) + with self.assertRaises(ContractValidationError): + ProjectionStateError(cast(ProjectionStateStatus, "schema_mismatch")) + + def test_operation_error_fields_and_receipt_are_exact_and_truthful(self) -> None: + request = _request((_document("alpha"),)) + receipt = ProjectionReceipt( + request.corpus_id, + request.projection, + ProjectionOutcome.FAILED, + 1, + 0, + None, + ) + key = request.documents[0].identity.key + error = ProjectionOperationError( + ProjectionFailureStage.REPLACEMENT, + key, + receipt, + ) + + self.assertIsInstance(error, CollaborationError) + self.assertEqual( + ProjectionOperationError.__annotations__, + { + "stage": "ProjectionFailureStage", + "affected_document": "DocumentKey | None", + "receipt": "ProjectionReceipt | None", + }, + ) + self.assertIs(error.stage, ProjectionFailureStage.REPLACEMENT) + self.assertEqual(error.affected_document, key) + self.assertIs(error.receipt, receipt) + for stage, affected, value_receipt in ( + ("replacement", key, receipt), + (ProjectionFailureStage.REPLACEMENT, object(), receipt), + ( + ProjectionFailureStage.REPLACEMENT, + key, + ProjectionReceipt( + request.corpus_id, + request.projection, + ProjectionOutcome.COMPLETED, + 1, + 1, + _manifest(request).checkpoint, + ), + ), + (ProjectionFailureStage.REPLACEMENT, key, object()), + ): + with self.subTest(stage=stage, affected=affected): + with self.assertRaises(ContractValidationError): + ProjectionOperationError( + cast(ProjectionFailureStage, stage), + cast(DocumentKey, affected), + cast(ProjectionReceipt, value_receipt), + ) + + def test_invalid_top_level_inputs_are_rejected_before_any_effect(self) -> None: + request = _request((_document("alpha"),)) + state = _present(_request((), projection=request.projection)) + embedder, writer, resetter, events = _collaborators(request) + invalid_calls: tuple[Callable[[], object], ...] = ( + lambda: project_documents( + cast(ProjectionRequest, object()), + state, + _borrow_embedder(embedder), + _borrow_writer(writer), + ), + lambda: project_documents( + request, + cast(ProjectionStateSnapshot, object()), + _borrow_embedder(embedder), + _borrow_writer(writer), + ), + lambda: project_documents( + request, + state, + cast(Borrowed[Embedder], object()), + _borrow_writer(writer), + ), + lambda: project_documents( + request, + state, + _borrow_embedder(embedder), + cast(Borrowed[VectorIndexWriter], object()), + ), + lambda: rebuild_projection( + request, + state, + _borrow_embedder(embedder), + _borrow_writer(writer), + cast(Borrowed[VectorIndexResetter], object()), + ), + ) + + for call in invalid_calls: + with self.subTest(call=call): + with self.assertRaises(ContractValidationError): + call() + self.assertEqual(events, []) + + +class ProjectionDeterminismTests(unittest.TestCase): + def test_unicode_codepoint_chunks_overlap_and_preserve_source_attributes( + self, + ) -> None: + document = _document( + "unicode", + text="A😀e\u0301한Z", + attributes=(("", ""), ("tag", "one"), ("tag", "one")), + ) + request = _request( + (document,), + chunking=ChunkingPolicy(3, 1), + batch_size=8, + ) + previous = _request( + (), projection=request.projection, chunking=request.chunking + ) + embedder, writer, _, _ = _collaborators(request) + + result = project_documents( + request, + _present(previous), + _borrow_embedder(embedder), + _borrow_writer(writer), + ) + + records = writer.replacements[0][1] + self.assertEqual( + tuple( + (record.fragment.identity.start, record.fragment.identity.end) + for record in records + ), + ((0, 3), (2, 5), (4, 6)), + ) + self.assertEqual( + tuple(record.fragment.text for record in records), + ("A😀e", "e\u0301한", "한Z"), + ) + self.assertTrue( + all(record.fragment.attributes == document.attributes for record in records) + ) + self.assertTrue( + all( + record.fragment.identity.document == document.identity + for record in records + ) + ) + self.assertEqual(result.manifest.entries[0].fragment_count, 3) + + def test_fragment_ids_source_digest_and_checkpoint_match_independent_oracle( + self, + ) -> None: + document = _document( + "doc/../A", + revision_id=" rev e\u0301 ", + text="abcdef", + attributes=(("k", "v"), ("k", "v"), ("", "")), + ) + request = _request((document,), chunking=ChunkingPolicy(4, 1)) + embedder, writer, resetter, _ = _collaborators(request) + + result = rebuild_projection( + request, + ProjectionStateSnapshot(ProjectionStateAvailability.MISSING, None), + _borrow_embedder(embedder), + _borrow_writer(writer), + _borrow_resetter(resetter), + ) + + expected_manifest = _manifest(request) + self.assertEqual(result.manifest, expected_manifest) + self.assertEqual( + tuple( + record.fragment.identity.fragment_id + for record in writer.replacements[0][1] + ), + tuple( + _fragment_id(document.identity, start, end) + for start, end in _expected_ranges(document.text, request.chunking) + ), + ) + self.assertEqual( + result.manifest.entries[0].source_digest, + _source_digest(document), + ) + self.assertEqual( + result.manifest.checkpoint.token, + _checkpoint_token(request, expected_manifest.entries), + ) + + def test_source_digest_distinguishes_attribute_order_duplicates_and_text( + self, + ) -> None: + base = _document("doc", text="same", attributes=(("a", "1"), ("b", "2"))) + variants = ( + _document("doc", text="same", attributes=(("b", "2"), ("a", "1"))), + _document("doc", text="same", attributes=(("a", "1"), ("a", "1"))), + _document("doc", text="same!", attributes=(("a", "1"), ("b", "2"))), + ) + + self.assertEqual(len({_source_digest(base), *map(_source_digest, variants)}), 4) + + def test_embedding_batches_are_bounded_ordered_and_never_empty(self) -> None: + document = _document("doc", text="abcdefghijklmn") + request = _request( + (document,), + chunking=ChunkingPolicy(3, 0), + batch_size=2, + ) + previous = _request( + (), projection=request.projection, chunking=request.chunking + ) + embedder, writer, _, _ = _collaborators(request) + + project_documents( + request, + _present(previous), + _borrow_embedder(embedder), + _borrow_writer(writer), + ) + + self.assertEqual( + embedder.embed_calls, + [("abc", "def"), ("ghi", "jkl"), ("mn",)], + ) + self.assertTrue(all(batch for batch in embedder.embed_calls)) + self.assertEqual( + tuple(record.fragment.text for record in writer.replacements[0][1]), + ("abc", "def", "ghi", "jkl", "mn"), + ) + + def test_empty_document_is_replaced_by_an_explicit_empty_record_tuple(self) -> None: + document = _document("doc", revision_id="revision-2", text="") + request = _request((document,)) + previous = _request( + (_document("doc", revision_id="revision-1", text="old"),), + projection=request.projection, + chunking=request.chunking, + ) + embedder, writer, _, _ = _collaborators(request) + + result = project_documents( + request, + _present(previous), + _borrow_embedder(embedder), + _borrow_writer(writer), + ) + + self.assertEqual(embedder.embed_calls, []) + self.assertEqual(writer.replacements, [(document.identity, ())]) + self.assertEqual(result.manifest.entries[0].fragment_count, 0) + + def test_document_and_mutation_order_is_canonical_by_opaque_document_id( + self, + ) -> None: + request = _request( + ( + _document("c", revision_id="new"), + _document("a", revision_id="new"), + ) + ) + previous = _request( + ( + _document("d", revision_id="old"), + _document("b", revision_id="old"), + ), + projection=request.projection, + chunking=request.chunking, + ) + embedder, writer, _, events = _collaborators(request) + + project_documents( + request, + _present(previous), + _borrow_embedder(embedder), + _borrow_writer(writer), + ) + + mutation_events = [ + event for event in events if event.startswith(("replace:", "delete:")) + ] + self.assertEqual( + mutation_events, + ["replace:a", "delete:b", "replace:c", "delete:d"], + ) + + def test_fixed_projection_is_identical_in_clean_processes(self) -> None: + script = textwrap.dedent( + f""" + import json + import sys + sys.path.insert(0, {os.fspath(_SOURCE_ROOT)!r}) + from generic_rag.contracts import * + from generic_rag.ports import Borrowed + from generic_rag.projection import rebuild_projection + + class Embedder: + identity = EmbeddingIdentity('model-v1', 2) + def embed(self, texts, /): + return tuple( + EmbeddingVector((len(text), index)) + for index, text in enumerate(texts) + ) + class Writer: + def __init__(self): self.records = () + def replace_document(self, document, records, /): self.records = records + def delete_document(self, document, /): return None + class Resetter: + def reset_corpus(self, corpus_id, /): return None + + document = Document( + DocumentIdentity(DocumentKey('corpus', 'doc/../A'), ' rev e\\u0301 '), + 'abcdef', + (('k', 'v'), ('k', 'v'), ('', '')), + ) + request = ProjectionRequest( + 'corpus', + ProjectionIdentity('schema-v1', EmbeddingIdentity('model-v1', 2)), + ChunkingPolicy(4, 1), + ProjectionLimits(1, 100, 2), + (document,), + ) + writer = Writer() + result = rebuild_projection( + request, + ProjectionStateSnapshot(ProjectionStateAvailability.MISSING, None), + Borrowed(Embedder()), + Borrowed(writer), + Borrowed(Resetter()), + ) + print(json.dumps({{ + 'source': result.manifest.entries[0].source_digest, + 'checkpoint': result.manifest.checkpoint.token, + 'fragments': [ + record.fragment.identity.fragment_id + for record in writer.records + ], + }}, sort_keys=True)) + """ + ) + outputs: list[dict[str, object]] = [] + for _ in range(2): + completed = subprocess.run( + (sys.executable, "-I", "-B", "-c", script), + check=False, + capture_output=True, + text=True, + timeout=10, + ) + self.assertEqual(completed.returncode, 0, completed.stderr) + outputs.append(cast(dict[str, object], json.loads(completed.stdout))) + + request = _request( + ( + _document( + "doc/../A", + revision_id=" rev e\u0301 ", + text="abcdef", + attributes=(("k", "v"), ("k", "v"), ("", "")), + ), + ), + chunking=ChunkingPolicy(4, 1), + ) + expected = _manifest(request) + self.assertEqual(outputs[0], outputs[1]) + self.assertEqual(outputs[0]["source"], expected.entries[0].source_digest) + self.assertEqual(outputs[0]["checkpoint"], expected.checkpoint.token) + self.assertEqual( + outputs[0]["fragments"], + [ + _fragment_id(request.documents[0].identity, start, end) + for start, end in _expected_ranges( + request.documents[0].text, + request.chunking, + ) + ], + ) + + +class IncrementalProjectionTests(unittest.TestCase): + def test_current_state_is_unchanged_without_touching_collaborators(self) -> None: + request = _request((_document("alpha"), _document("beta"))) + embedder, writer, _, events = _collaborators(request) + embedder.identity_failure = AssertionError("identity must not be read") + writer.replace_failure_at = 0 + + result = project_documents( + request, + _present(request), + _borrow_embedder(embedder), + _borrow_writer(writer), + ) + + self.assertIs(result.status_before, ProjectionStateStatus.CURRENT) + self.assertIs(result.receipt.outcome, ProjectionOutcome.UNCHANGED) + self.assertEqual(result.receipt.attempted_documents, 0) + self.assertEqual(result.receipt.completed_documents, 0) + self.assertEqual(result.manifest, _manifest(request)) + self.assertEqual(events, []) + + def test_incompatible_state_statuses_fail_before_collaborator_effects(self) -> None: + request = _request((_document("alpha"),)) + schema_request = _request( + request.documents, + projection=_identity(schema_id="other-schema"), + ) + embedding_request = _request( + request.documents, + projection=_identity(model_id="other-model"), + ) + corpus_request = _request( + (_document("alpha", corpus_id="other"),), + projection=request.projection, + corpus_id="other", + ) + cases = ( + ( + ProjectionStateSnapshot(ProjectionStateAvailability.MISSING, None), + ProjectionStateStatus.MISSING, + ), + ( + ProjectionStateSnapshot(ProjectionStateAvailability.CORRUPT, None), + ProjectionStateStatus.CORRUPT, + ), + (_present(schema_request), ProjectionStateStatus.SCHEMA_MISMATCH), + (_present(embedding_request), ProjectionStateStatus.EMBEDDING_MISMATCH), + (_present(corpus_request), ProjectionStateStatus.CORRUPT), + ) + + for state, expected_status in cases: + embedder, writer, _, events = _collaborators(request) + with self.subTest(status=expected_status): + with self.assertRaises(ProjectionStateError) as raised: + project_documents( + request, + state, + _borrow_embedder(embedder), + _borrow_writer(writer), + ) + self.assertIs(raised.exception.status, expected_status) + self.assertEqual(events, []) + + def test_invalid_checkpoint_is_corrupt_before_any_effect(self) -> None: + request = _request((_document("alpha"),)) + valid = _manifest(request) + corrupt = ProjectionManifest( + valid.corpus_id, + valid.projection, + valid.chunking, + valid.entries, + ProjectionCheckpoint(valid.corpus_id, valid.projection, "wrong-token"), + ) + embedder, writer, _, events = _collaborators(request) + + with self.assertRaises(ProjectionStateError) as raised: + project_documents( + request, + ProjectionStateSnapshot( + ProjectionStateAvailability.PRESENT, + corrupt, + ), + _borrow_embedder(embedder), + _borrow_writer(writer), + ) + + self.assertIs(raised.exception.status, ProjectionStateStatus.CORRUPT) + self.assertEqual(events, []) + + def test_same_revision_digest_or_fragment_count_drift_is_corrupt(self) -> None: + request = _request((_document("alpha"),)) + valid = _manifest(request) + original = valid.entries[0] + changed_entries = ( + ( + ProjectionManifestEntry( + original.document, + "sha256:" + "f" * 64, + original.fragment_count, + ), + ), + ( + ProjectionManifestEntry( + original.document, + original.source_digest, + original.fragment_count + 1, + ), + ), + ) + + for entries in changed_entries: + previous_request = _request( + request.documents, + projection=request.projection, + chunking=request.chunking, + ) + manifest = ProjectionManifest( + request.corpus_id, + request.projection, + request.chunking, + entries, + ProjectionCheckpoint( + request.corpus_id, + request.projection, + _checkpoint_token(previous_request, entries), + ), + ) + embedder, writer, _, events = _collaborators(request) + with self.subTest(entries=entries): + with self.assertRaises(ProjectionStateError) as raised: + project_documents( + request, + ProjectionStateSnapshot( + ProjectionStateAvailability.PRESENT, + manifest, + ), + _borrow_embedder(embedder), + _borrow_writer(writer), + ) + self.assertIs( + raised.exception.status, + ProjectionStateStatus.CORRUPT, + ) + self.assertEqual(events, []) + + def test_revision_and_chunking_changes_are_stale_and_replaced(self) -> None: + target = _request( + (_document("alpha", revision_id="revision-2", text="new text"),), + chunking=ChunkingPolicy(4, 1), + ) + previous_requests = ( + _request( + (_document("alpha", revision_id="revision-1", text="new text"),), + projection=target.projection, + chunking=target.chunking, + ), + _request( + target.documents, + projection=target.projection, + chunking=ChunkingPolicy(5, 0), + ), + ) + + for previous in previous_requests: + embedder, writer, _, _ = _collaborators(target) + with self.subTest(previous=previous): + result = project_documents( + target, + _present(previous), + _borrow_embedder(embedder), + _borrow_writer(writer), + ) + self.assertIs(result.status_before, ProjectionStateStatus.STALE) + self.assertEqual( + [identity for identity, _ in writer.replacements], + [target.documents[0].identity], + ) + + def test_same_revision_source_change_is_corrupt_not_stale(self) -> None: + target = _request( + (_document("alpha", revision_id="revision-2", text="new text"),) + ) + previous = _request( + (_document("alpha", revision_id="revision-2", text="old text"),), + projection=target.projection, + chunking=target.chunking, + ) + embedder, writer, _, events = _collaborators(target) + + with self.assertRaises(ProjectionStateError) as raised: + project_documents( + target, + _present(previous), + _borrow_embedder(embedder), + _borrow_writer(writer), + ) + + self.assertIs(raised.exception.status, ProjectionStateStatus.CORRUPT) + self.assertEqual(events, []) + + def test_delete_only_plan_does_not_access_embedder(self) -> None: + target = _request(()) + previous = _request( + (_document("zeta"), _document("alpha")), + projection=target.projection, + chunking=target.chunking, + ) + embedder, writer, _, events = _collaborators(target) + embedder.identity_failure = AssertionError("delete-only must not embed") + + result = project_documents( + target, + _present(previous), + _borrow_embedder(embedder), + _borrow_writer(writer), + ) + + self.assertEqual( + writer.deletions, + [DocumentKey("corpus", "alpha"), DocumentKey("corpus", "zeta")], + ) + self.assertEqual(embedder.identity_calls, 0) + self.assertFalse(any(event.startswith("embed:") for event in events)) + self.assertEqual(result.receipt.attempted_documents, 2) + self.assertEqual(result.receipt.completed_documents, 2) + + def test_success_receipt_counts_only_changed_and_removed_documents(self) -> None: + unchanged = _document("same", revision_id="r1", text="same") + target = _request( + ( + unchanged, + _document("added", revision_id="r1", text="added"), + _document("changed", revision_id="r2", text="new"), + ) + ) + previous = _request( + ( + unchanged, + _document("changed", revision_id="r1", text="old"), + _document("removed", revision_id="r1", text="gone"), + ), + projection=target.projection, + chunking=target.chunking, + ) + embedder, writer, _, _ = _collaborators(target) + + result = project_documents( + target, + _present(previous), + _borrow_embedder(embedder), + _borrow_writer(writer), + ) + + self.assertIs(result.receipt.outcome, ProjectionOutcome.COMPLETED) + self.assertEqual(result.receipt.attempted_documents, 3) + self.assertEqual(result.receipt.completed_documents, 3) + self.assertEqual(result.receipt.checkpoint, result.manifest.checkpoint) + self.assertNotIn(unchanged.identity, [item[0] for item in writer.replacements]) + + +class RebuildProjectionTests(unittest.TestCase): + def test_nonempty_rebuild_validates_identity_once_before_reset_and_reuses_it( + self, + ) -> None: + request = _request((_document("alpha"), _document("beta"))) + embedder, writer, resetter, events = _collaborators(request) + + result = rebuild_projection( + request, + ProjectionStateSnapshot(ProjectionStateAvailability.MISSING, None), + _borrow_embedder(embedder), + _borrow_writer(writer), + _borrow_resetter(resetter), + ) + + self.assertEqual(embedder.identity_calls, 1) + self.assertEqual(resetter.corpora, ["corpus"]) + self.assertEqual( + [identity.key.document_id for identity, _ in writer.replacements], + ["alpha", "beta"], + ) + self.assertEqual(events[0:2], ["identity", "reset:corpus"]) + self.assertEqual(sum(event == "identity" for event in events), 1) + self.assertIs(result.status_before, ProjectionStateStatus.MISSING) + self.assertEqual(result.receipt.attempted_documents, 2) + + def test_wrong_embedder_identity_fails_before_reset_without_a_cause(self) -> None: + request = _request((_document("alpha"),)) + events: list[str] = [] + embedder = _FakeEmbedder( + request.projection.embedding, + events, + identity_value=EmbeddingIdentity("other-model", 2), + ) + writer = _FakeWriter(events) + resetter = _FakeResetter(events) + + with self.assertRaises(ProjectionOperationError) as raised: + rebuild_projection( + request, + ProjectionStateSnapshot(ProjectionStateAvailability.MISSING, None), + _borrow_embedder(embedder), + _borrow_writer(writer), + _borrow_resetter(resetter), + ) + + self.assertIs( + raised.exception.stage, + ProjectionFailureStage.EMBEDDER_IDENTITY, + ) + self.assertIsNone(raised.exception.affected_document) + self.assertIsNone(raised.exception.__cause__) + self.assertIsNotNone(raised.exception.receipt) + assert raised.exception.receipt is not None + self.assertIs(raised.exception.receipt.outcome, ProjectionOutcome.FAILED) + self.assertEqual(resetter.corpora, []) + self.assertEqual(writer.replacements, []) + + def test_identity_exception_is_preserved_as_cause_before_reset(self) -> None: + request = _request((_document("alpha"),)) + failure = RuntimeError("identity provider failed") + events: list[str] = [] + embedder = _FakeEmbedder( + request.projection.embedding, + events, + identity_failure=failure, + ) + writer = _FakeWriter(events) + resetter = _FakeResetter(events) + + with self.assertRaises(ProjectionOperationError) as raised: + rebuild_projection( + request, + ProjectionStateSnapshot(ProjectionStateAvailability.MISSING, None), + _borrow_embedder(embedder), + _borrow_writer(writer), + _borrow_resetter(resetter), + ) + + self.assertIs(raised.exception.__cause__, failure) + self.assertEqual(resetter.corpora, []) + + def test_empty_rebuild_resets_only_without_embedder_or_writer_access(self) -> None: + request = _request(()) + embedder, writer, resetter, events = _collaborators(request) + embedder.identity_failure = AssertionError("empty rebuild must not embed") + writer.replace_failure_at = 0 + + result = rebuild_projection( + request, + ProjectionStateSnapshot(ProjectionStateAvailability.CORRUPT, None), + _borrow_embedder(embedder), + _borrow_writer(writer), + _borrow_resetter(resetter), + ) + + self.assertEqual(events, ["reset:corpus"]) + self.assertEqual(embedder.identity_calls, 0) + self.assertEqual(writer.replacements, []) + self.assertIs(result.status_before, ProjectionStateStatus.CORRUPT) + self.assertEqual(result.receipt.attempted_documents, 0) + self.assertEqual(result.receipt.completed_documents, 0) + self.assertIs(result.receipt.outcome, ProjectionOutcome.COMPLETED) + + def test_rebuild_accepts_every_state_status_and_reports_it_truthfully(self) -> None: + request = _request((_document("alpha"),)) + other_schema = _request( + request.documents, + projection=_identity(schema_id="other"), + ) + other_embedding = _request( + request.documents, + projection=_identity(model_id="other"), + ) + stale = _request( + (_document("alpha", revision_id="old"),), + projection=request.projection, + ) + states = ( + ( + ProjectionStateSnapshot(ProjectionStateAvailability.MISSING, None), + ProjectionStateStatus.MISSING, + ), + ( + ProjectionStateSnapshot(ProjectionStateAvailability.CORRUPT, None), + ProjectionStateStatus.CORRUPT, + ), + (_present(request), ProjectionStateStatus.CURRENT), + (_present(stale), ProjectionStateStatus.STALE), + (_present(other_schema), ProjectionStateStatus.SCHEMA_MISMATCH), + (_present(other_embedding), ProjectionStateStatus.EMBEDDING_MISMATCH), + ) + + for state, status in states: + embedder, writer, resetter, _ = _collaborators(request) + with self.subTest(status=status): + result = rebuild_projection( + request, + state, + _borrow_embedder(embedder), + _borrow_writer(writer), + _borrow_resetter(resetter), + ) + self.assertIs(result.status_before, status) + self.assertEqual(resetter.corpora, ["corpus"]) + self.assertIs(result.receipt.outcome, ProjectionOutcome.COMPLETED) + + def test_reset_exception_has_failed_receipt_and_preserves_cause(self) -> None: + request = _request((_document("alpha"), _document("beta"))) + failure = RuntimeError("reset failed") + events: list[str] = [] + embedder = _FakeEmbedder(request.projection.embedding, events) + writer = _FakeWriter(events) + resetter = _FakeResetter(events, failure=failure) + + with self.assertRaises(ProjectionOperationError) as raised: + rebuild_projection( + request, + ProjectionStateSnapshot(ProjectionStateAvailability.MISSING, None), + _borrow_embedder(embedder), + _borrow_writer(writer), + _borrow_resetter(resetter), + ) + + error = raised.exception + self.assertIs(error.stage, ProjectionFailureStage.RESET) + self.assertIsNone(error.affected_document) + self.assertIs(error.__cause__, failure) + self.assertIsNotNone(error.receipt) + assert error.receipt is not None + self.assertIs(error.receipt.outcome, ProjectionOutcome.FAILED) + self.assertEqual(error.receipt.attempted_documents, 2) + self.assertEqual(error.receipt.completed_documents, 0) + self.assertIsNone(error.receipt.checkpoint) + self.assertEqual(writer.replacements, []) + + def test_non_none_reset_result_is_failure_without_cause(self) -> None: + for documents, expected_receipt in ( + ((_document("alpha"),), True), + ((), False), + ): + request = _request(documents) + events: list[str] = [] + embedder = _FakeEmbedder(request.projection.embedding, events) + writer = _FakeWriter(events) + resetter = _FakeResetter(events, result=False) + with self.subTest(documents=documents): + with self.assertRaises(ProjectionOperationError) as raised: + rebuild_projection( + request, + ProjectionStateSnapshot( + ProjectionStateAvailability.MISSING, + None, + ), + _borrow_embedder(embedder), + _borrow_writer(writer), + _borrow_resetter(resetter), + ) + self.assertIs(raised.exception.stage, ProjectionFailureStage.RESET) + self.assertIsNone(raised.exception.__cause__) + self.assertEqual( + raised.exception.receipt is not None, + expected_receipt, + ) + + +class ProjectionFailureTests(unittest.TestCase): + def _incremental_target( + self, + documents: tuple[Document, ...], + *, + batch_size: int = 2, + ) -> tuple[ProjectionRequest, ProjectionStateSnapshot]: + target = _request(documents, batch_size=batch_size) + previous = _request( + (), + projection=target.projection, + chunking=target.chunking, + batch_size=batch_size, + ) + return target, _present(previous) + + def test_embedding_exception_is_called_once_and_preserved_as_cause(self) -> None: + request, state = self._incremental_target((_document("alpha", text="x"),)) + failure = RuntimeError("provider failed") + events: list[str] = [] + embedder = _FakeEmbedder( + request.projection.embedding, + events, + embed_failure_at=0, + embed_failure=failure, + ) + writer = _FakeWriter(events) + + with self.assertRaises(ProjectionOperationError) as raised: + project_documents( + request, + state, + _borrow_embedder(embedder), + _borrow_writer(writer), + ) + + error = raised.exception + self.assertIs(error.stage, ProjectionFailureStage.EMBEDDING) + self.assertEqual(error.affected_document, DocumentKey("corpus", "alpha")) + self.assertIs(error.__cause__, failure) + self.assertEqual(embedder.embed_calls, [("x",)]) + self.assertEqual(writer.replacements, []) + self.assertIsNotNone(error.receipt) + assert error.receipt is not None + self.assertIs(error.receipt.outcome, ProjectionOutcome.FAILED) + self.assertEqual(error.receipt.completed_documents, 0) + self.assertIsNone(error.receipt.checkpoint) + + def test_malformed_embedding_outputs_fail_without_internal_cause(self) -> None: + request, state = self._incremental_target((_document("alpha", text="x"),)) + nonfinite = EmbeddingVector((1.0, 2.0)) + object.__setattr__(nonfinite, "values", (float("nan"), 2.0)) + noncanonical = EmbeddingVector((1.0, 2.0)) + object.__setattr__(noncanonical, "values", (1, 2.0)) + outputs: tuple[object, ...] = ( + [EmbeddingVector((1.0, 2.0))], + (), + (object(),), + (EmbeddingVector((1.0,)),), + (EmbeddingVector((1.0, 2.0, 3.0)),), + (nonfinite,), + (noncanonical,), + ) + + for output in outputs: + events: list[str] = [] + embedder = _FakeEmbedder( + request.projection.embedding, + events, + output=_constant_output(output), + ) + writer = _FakeWriter(events) + with self.subTest(output=output): + with self.assertRaises(ProjectionOperationError) as raised: + project_documents( + request, + state, + _borrow_embedder(embedder), + _borrow_writer(writer), + ) + self.assertIs( + raised.exception.stage, + ProjectionFailureStage.EMBEDDING, + ) + self.assertIsNone(raised.exception.__cause__) + self.assertEqual(len(embedder.embed_calls), 1) + self.assertEqual(writer.replacements, []) + + def test_second_document_embedding_failure_reports_partial_completion(self) -> None: + request, state = self._incremental_target( + (_document("alpha"), _document("beta")), + batch_size=2, + ) + events: list[str] = [] + failure = RuntimeError("second document failed") + embedder = _FakeEmbedder( + request.projection.embedding, + events, + embed_failure_at=2, + embed_failure=failure, + ) + writer = _FakeWriter(events) + + with self.assertRaises(ProjectionOperationError) as raised: + project_documents( + request, + state, + _borrow_embedder(embedder), + _borrow_writer(writer), + ) + + error = raised.exception + self.assertEqual(error.affected_document, DocumentKey("corpus", "beta")) + self.assertIsNotNone(error.receipt) + assert error.receipt is not None + self.assertIs(error.receipt.outcome, ProjectionOutcome.PARTIAL) + self.assertEqual(error.receipt.attempted_documents, 2) + self.assertEqual(error.receipt.completed_documents, 1) + self.assertEqual( + [identity.key.document_id for identity, _ in writer.replacements], + ["alpha"], + ) + self.assertIs(error.__cause__, failure) + + def test_replacement_exception_has_matching_failed_receipt_and_cause(self) -> None: + request, state = self._incremental_target((_document("alpha", text="x"),)) + events: list[str] = [] + failure = RuntimeError("writer failed") + embedder = _FakeEmbedder(request.projection.embedding, events) + writer = _FakeWriter( + events, + replace_failure_at=0, + replace_failure=failure, + ) + + with self.assertRaises(ProjectionOperationError) as raised: + project_documents( + request, + state, + _borrow_embedder(embedder), + _borrow_writer(writer), + ) + + error = raised.exception + self.assertIs(error.stage, ProjectionFailureStage.REPLACEMENT) + self.assertEqual(error.affected_document, DocumentKey("corpus", "alpha")) + self.assertIs(error.__cause__, failure) + self.assertIsNotNone(error.receipt) + assert error.receipt is not None + self.assertIs(error.receipt.outcome, ProjectionOutcome.FAILED) + self.assertEqual(len(writer.replacements), 1) + + def test_non_none_replacement_result_is_failure_without_cause(self) -> None: + request, state = self._incremental_target((_document("alpha", text="x"),)) + events: list[str] = [] + embedder = _FakeEmbedder(request.projection.embedding, events) + writer = _FakeWriter(events, replace_result=False) + + with self.assertRaises(ProjectionOperationError) as raised: + project_documents( + request, + state, + _borrow_embedder(embedder), + _borrow_writer(writer), + ) + + self.assertIs(raised.exception.stage, ProjectionFailureStage.REPLACEMENT) + self.assertIsNone(raised.exception.__cause__) + self.assertIsNotNone(raised.exception.receipt) + assert raised.exception.receipt is not None + self.assertEqual(raised.exception.receipt.completed_documents, 0) + + def test_deletion_exception_after_replacement_reports_partial_receipt(self) -> None: + target = _request((_document("alpha", text="x"),)) + previous = _request( + (_document("beta", text="x"),), + projection=target.projection, + chunking=target.chunking, + ) + events: list[str] = [] + failure = RuntimeError("delete failed") + embedder = _FakeEmbedder(target.projection.embedding, events) + writer = _FakeWriter( + events, + delete_failure_at=0, + delete_failure=failure, + ) + + with self.assertRaises(ProjectionOperationError) as raised: + project_documents( + target, + _present(previous), + _borrow_embedder(embedder), + _borrow_writer(writer), + ) + + error = raised.exception + self.assertIs(error.stage, ProjectionFailureStage.DELETION) + self.assertEqual(error.affected_document, DocumentKey("corpus", "beta")) + self.assertIs(error.__cause__, failure) + self.assertIsNotNone(error.receipt) + assert error.receipt is not None + self.assertIs(error.receipt.outcome, ProjectionOutcome.PARTIAL) + self.assertEqual(error.receipt.attempted_documents, 2) + self.assertEqual(error.receipt.completed_documents, 1) + self.assertIsNone(error.receipt.checkpoint) + self.assertEqual( + [event for event in events if event.startswith(("replace:", "delete:"))], + ["replace:alpha", "delete:beta"], + ) + + def test_non_none_deletion_result_is_failed_without_cause(self) -> None: + target = _request(()) + previous = _request( + (_document("alpha", text="x"),), + projection=target.projection, + chunking=target.chunking, + ) + events: list[str] = [] + embedder = _FakeEmbedder(target.projection.embedding, events) + writer = _FakeWriter(events, delete_result=0) + + with self.assertRaises(ProjectionOperationError) as raised: + project_documents( + target, + _present(previous), + _borrow_embedder(embedder), + _borrow_writer(writer), + ) + + self.assertIs(raised.exception.stage, ProjectionFailureStage.DELETION) + self.assertIsNone(raised.exception.__cause__) + self.assertIsNotNone(raised.exception.receipt) + assert raised.exception.receipt is not None + self.assertIs(raised.exception.receipt.outcome, ProjectionOutcome.FAILED) + + def test_rebuild_replacement_failure_never_claims_a_checkpoint(self) -> None: + request = _request((_document("alpha"), _document("beta"))) + events: list[str] = [] + embedder = _FakeEmbedder(request.projection.embedding, events) + writer = _FakeWriter(events, replace_failure_at=1) + resetter = _FakeResetter(events) + + with self.assertRaises(ProjectionOperationError) as raised: + rebuild_projection( + request, + ProjectionStateSnapshot(ProjectionStateAvailability.CORRUPT, None), + _borrow_embedder(embedder), + _borrow_writer(writer), + _borrow_resetter(resetter), + ) + + error = raised.exception + self.assertIs(error.stage, ProjectionFailureStage.REPLACEMENT) + self.assertEqual(error.affected_document, DocumentKey("corpus", "beta")) + self.assertIsNotNone(error.receipt) + assert error.receipt is not None + self.assertIs(error.receipt.outcome, ProjectionOutcome.PARTIAL) + self.assertEqual(error.receipt.completed_documents, 1) + self.assertIsNone(error.receipt.checkpoint) + self.assertEqual(resetter.corpora, ["corpus"]) + + def test_control_flow_exceptions_are_never_translated_or_retried(self) -> None: + for failure in (KeyboardInterrupt("interrupt"), SystemExit("exit")): + request, state = self._incremental_target((_document("alpha", text="x"),)) + events: list[str] = [] + embedder = _FakeEmbedder( + request.projection.embedding, + events, + embed_failure_at=0, + embed_failure=failure, + ) + writer = _FakeWriter(events) + with self.subTest(failure=type(failure).__name__): + with self.assertRaises(type(failure)) as raised: + project_documents( + request, + state, + _borrow_embedder(embedder), + _borrow_writer(writer), + ) + self.assertIs(raised.exception, failure) + self.assertEqual(len(embedder.embed_calls), 1) + self.assertEqual(writer.replacements, []) + + def test_reset_control_flow_exception_is_not_translated_or_retried(self) -> None: + request = _request((_document("alpha"),)) + failure = KeyboardInterrupt("reset interrupted") + events: list[str] = [] + embedder = _FakeEmbedder(request.projection.embedding, events) + writer = _FakeWriter(events) + resetter = _FakeResetter(events, failure=failure) + + with self.assertRaises(KeyboardInterrupt) as raised: + rebuild_projection( + request, + ProjectionStateSnapshot(ProjectionStateAvailability.MISSING, None), + _borrow_embedder(embedder), + _borrow_writer(writer), + _borrow_resetter(resetter), + ) + + self.assertIs(raised.exception, failure) + self.assertEqual(resetter.corpora, ["corpus"]) + self.assertEqual(writer.replacements, []) + + def test_workflows_never_enter_close_shutdown_or_suppress_resources(self) -> None: + request, state = self._incremental_target((_document("alpha", text="x"),)) + embedder, writer, resetter, _ = _collaborators(request) + + project_documents( + request, + state, + _borrow_embedder(embedder), + _borrow_writer(writer), + ) + rebuild_projection( + request, + ProjectionStateSnapshot(ProjectionStateAvailability.MISSING, None), + _borrow_embedder(embedder), + _borrow_writer(writer), + _borrow_resetter(resetter), + ) + + self.assertEqual(embedder.lifecycle_calls, []) + self.assertEqual(writer.lifecycle_calls, []) + self.assertEqual(resetter.lifecycle_calls, []) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_projection_contracts.py b/tests/test_projection_contracts.py new file mode 100644 index 0000000..e4c2cb1 --- /dev/null +++ b/tests/test_projection_contracts.py @@ -0,0 +1,919 @@ +"""Contract tests for immutable projection values and truthful state.""" + +from __future__ import annotations + +import unittest +from dataclasses import FrozenInstanceError, fields +from typing import cast + +from generic_rag.contracts import ( + ChunkingPolicy, + Document, + DocumentIdentity, + DocumentKey, + EmbeddingIdentity, + ProjectionCheckpoint, + ProjectionIdentity, + ProjectionLimits, + ProjectionManifest, + ProjectionManifestEntry, + ProjectionOutcome, + ProjectionReceipt, + ProjectionRequest, + ProjectionResult, + ProjectionStateAvailability, + ProjectionStateSnapshot, + ProjectionStateStatus, +) +from generic_rag.errors import ContractValidationError + + +class _StringSubclass(str): + pass + + +class _IntegerSubclass(int): + pass + + +def _projection( + *, + schema_id: str = "schema", + model_id: str = "model", + dimensions: int = 2, +) -> ProjectionIdentity: + return ProjectionIdentity( + schema_id=schema_id, + embedding=EmbeddingIdentity(model_id, dimensions), + ) + + +def _checkpoint( + *, + corpus_id: str = "corpus", + projection: ProjectionIdentity | None = None, + token: str = "checkpoint", +) -> ProjectionCheckpoint: + return ProjectionCheckpoint( + corpus_id=corpus_id, + projection=projection or _projection(), + token=token, + ) + + +def _document( + document_id: str = "document", + *, + corpus_id: str = "corpus", + revision_id: str = "revision", + text: str = "text", +) -> Document: + return Document( + DocumentIdentity(DocumentKey(corpus_id, document_id), revision_id), + text, + ) + + +def _entry( + document_id: str = "document", + *, + corpus_id: str = "corpus", + revision_id: str = "revision", + digest_digit: str = "0", + fragment_count: int = 1, +) -> ProjectionManifestEntry: + return ProjectionManifestEntry( + _document( + document_id, + corpus_id=corpus_id, + revision_id=revision_id, + ).identity, + f"sha256:{digest_digit * 64}", + fragment_count, + ) + + +def _manifest( + *, + corpus_id: str = "corpus", + projection: ProjectionIdentity | None = None, + chunking: ChunkingPolicy | None = None, + entries: tuple[ProjectionManifestEntry, ...] | None = None, + token: str = "checkpoint", +) -> ProjectionManifest: + resolved_projection = projection or _projection() + return ProjectionManifest( + corpus_id, + resolved_projection, + chunking or ChunkingPolicy(4, 1), + (_entry(corpus_id=corpus_id),) if entries is None else entries, + _checkpoint( + corpus_id=corpus_id, + projection=resolved_projection, + token=token, + ), + ) + + +class ProjectionContractTests(unittest.TestCase): + def test_projection_fields_are_exact_frozen_and_slotted(self) -> None: + self.assertEqual( + tuple(field.name for field in fields(ProjectionIdentity)), + ("schema_id", "embedding"), + ) + self.assertEqual( + tuple(field.name for field in fields(ProjectionCheckpoint)), + ("corpus_id", "projection", "token"), + ) + self.assertEqual( + tuple(field.name for field in fields(ProjectionReceipt)), + ( + "corpus_id", + "projection", + "outcome", + "attempted_documents", + "completed_documents", + "checkpoint", + ), + ) + + projection = _projection() + checkpoint = _checkpoint(projection=projection) + receipt = ProjectionReceipt( + "corpus", + projection, + ProjectionOutcome.COMPLETED, + 0, + 0, + checkpoint, + ) + for instance, field_name in ( + (projection, "schema_id"), + (checkpoint, "token"), + (receipt, "completed_documents"), + ): + with self.subTest(instance=type(instance).__name__): + self.assertFalse(hasattr(instance, "__dict__")) + with self.assertRaises(FrozenInstanceError): + setattr(instance, field_name, object()) + self.assertIsInstance(hash(instance), int) + + def test_projection_outcomes_are_exact_closed_string_enums(self) -> None: + self.assertEqual( + tuple((member.name, member.value) for member in ProjectionOutcome), + ( + ("COMPLETED", "completed"), + ("UNCHANGED", "unchanged"), + ("PARTIAL", "partial"), + ("FAILED", "failed"), + ), + ) + self.assertEqual(ProjectionOutcome("completed"), ProjectionOutcome.COMPLETED) + self.assertIsInstance(ProjectionOutcome.COMPLETED, str) + + for invalid in ("COMPLETED", "", "unknown", None, 1, object()): + with self.subTest(value=invalid): + with self.assertRaises(ContractValidationError): + ProjectionOutcome(cast(str, invalid)) + + def test_projection_identity_preserves_opaque_strings(self) -> None: + schema_id = " Schema/../e\u0301 " + model_id = " Model:ABC " + projection = _projection(schema_id=schema_id, model_id=model_id) + + self.assertEqual(projection.schema_id, schema_id) + self.assertEqual(projection.embedding.model_id, model_id) + + def test_projection_identity_rejects_invalid_fields(self) -> None: + invalid_strings: tuple[object, ...] = ( + "", + " \t", + None, + b"schema", + _StringSubclass("schema"), + ) + for invalid in invalid_strings: + with self.subTest(value=invalid): + with self.assertRaises(ContractValidationError): + ProjectionIdentity( + cast(str, invalid), + EmbeddingIdentity("model", 2), + ) + + with self.assertRaises(ContractValidationError): + ProjectionIdentity("schema", cast(EmbeddingIdentity, object())) + + def test_checkpoint_requires_exact_opaque_identity_values(self) -> None: + corpus_id = " Corpus/../A " + token = " Token/../001 " + projection = _projection() + checkpoint = _checkpoint( + corpus_id=corpus_id, + projection=projection, + token=token, + ) + + self.assertEqual(checkpoint.corpus_id, corpus_id) + self.assertEqual(checkpoint.token, token) + self.assertIs(checkpoint.projection, projection) + + for invalid in ("", " \n", None, _StringSubclass("value")): + with self.subTest(field="corpus_id", value=invalid): + with self.assertRaises(ContractValidationError): + ProjectionCheckpoint( + cast(str, invalid), + projection, + "token", + ) + with self.subTest(field="token", value=invalid): + with self.assertRaises(ContractValidationError): + ProjectionCheckpoint( + "corpus", + projection, + cast(str, invalid), + ) + with self.assertRaises(ContractValidationError): + ProjectionCheckpoint( + "corpus", + cast(ProjectionIdentity, object()), + "token", + ) + + def test_completed_and_unchanged_receipts_require_truthful_checkpoints( + self, + ) -> None: + projection = _projection() + checkpoint = _checkpoint(projection=projection) + + for outcome in ( + ProjectionOutcome.COMPLETED, + ProjectionOutcome.UNCHANGED, + ): + with self.subTest(outcome=outcome): + empty = ProjectionReceipt( + "corpus", + projection, + outcome, + 0, + 0, + checkpoint, + ) + complete = ProjectionReceipt( + "corpus", + projection, + outcome, + 3, + 3, + checkpoint, + ) + self.assertEqual(empty.completed_documents, 0) + self.assertEqual(complete.completed_documents, 3) + self.assertIs(complete.checkpoint, checkpoint) + + with self.assertRaises(ContractValidationError): + ProjectionReceipt( + "corpus", + projection, + outcome, + 3, + 2, + checkpoint, + ) + with self.assertRaises(ContractValidationError): + ProjectionReceipt( + "corpus", + projection, + outcome, + 3, + 3, + None, + ) + + def test_partial_and_failed_receipts_cannot_claim_completed_work(self) -> None: + projection = _projection() + partial = ProjectionReceipt( + "corpus", + projection, + ProjectionOutcome.PARTIAL, + 3, + 1, + None, + ) + failed = ProjectionReceipt( + "corpus", + projection, + ProjectionOutcome.FAILED, + 2, + 0, + None, + ) + + self.assertEqual(partial.completed_documents, 1) + self.assertIsNone(partial.checkpoint) + self.assertEqual(failed.completed_documents, 0) + self.assertIsNone(failed.checkpoint) + + invalid_partial_counts = ((0, 0), (1, 0), (1, 1), (2, 2)) + for attempted, completed in invalid_partial_counts: + with self.subTest(attempted=attempted, completed=completed): + with self.assertRaises(ContractValidationError): + ProjectionReceipt( + "corpus", + projection, + ProjectionOutcome.PARTIAL, + attempted, + completed, + None, + ) + + invalid_failed_counts = ((0, 0), (1, 1), (2, 1)) + for attempted, completed in invalid_failed_counts: + with self.subTest(attempted=attempted, completed=completed): + with self.assertRaises(ContractValidationError): + ProjectionReceipt( + "corpus", + projection, + ProjectionOutcome.FAILED, + attempted, + completed, + None, + ) + + for outcome, attempted, completed in ( + (ProjectionOutcome.PARTIAL, 2, 1), + (ProjectionOutcome.FAILED, 2, 0), + ): + with self.subTest(outcome=outcome): + with self.assertRaises(ContractValidationError): + ProjectionReceipt( + "corpus", + projection, + outcome, + attempted, + completed, + _checkpoint(projection=projection), + ) + + def test_receipt_checkpoint_must_match_corpus_and_projection(self) -> None: + projection = _projection() + mismatched_projection = _projection(schema_id="other-schema") + + for checkpoint in ( + _checkpoint(corpus_id="other-corpus", projection=projection), + _checkpoint(corpus_id="corpus", projection=mismatched_projection), + ): + with self.subTest(checkpoint=checkpoint): + with self.assertRaises(ContractValidationError): + ProjectionReceipt( + "corpus", + projection, + ProjectionOutcome.COMPLETED, + 1, + 1, + checkpoint, + ) + + def test_receipt_rejects_inexact_nested_and_discrete_types(self) -> None: + projection = _projection() + checkpoint = _checkpoint(projection=projection) + + with self.assertRaises(ContractValidationError): + ProjectionReceipt( + "corpus", + cast(ProjectionIdentity, object()), + ProjectionOutcome.COMPLETED, + 1, + 1, + checkpoint, + ) + with self.assertRaises(ContractValidationError): + ProjectionReceipt( + "corpus", + projection, + cast(ProjectionOutcome, "completed"), + 1, + 1, + checkpoint, + ) + with self.assertRaises(ContractValidationError): + ProjectionReceipt( + "corpus", + projection, + ProjectionOutcome.COMPLETED, + 1, + 1, + cast(ProjectionCheckpoint, object()), + ) + + for invalid in (-1, True, 1.0, _IntegerSubclass(1)): + with self.subTest(field="attempted_documents", value=invalid): + with self.assertRaises(ContractValidationError): + ProjectionReceipt( + "corpus", + projection, + ProjectionOutcome.COMPLETED, + cast(int, invalid), + 1, + checkpoint, + ) + with self.subTest(field="completed_documents", value=invalid): + with self.assertRaises(ContractValidationError): + ProjectionReceipt( + "corpus", + projection, + ProjectionOutcome.COMPLETED, + 1, + cast(int, invalid), + checkpoint, + ) + + with self.assertRaises(ContractValidationError): + ProjectionReceipt( + "corpus", + projection, + ProjectionOutcome.COMPLETED, + 1, + 2, + checkpoint, + ) + + +class ProjectionWorkflowContractTests(unittest.TestCase): + def test_new_projection_values_have_exact_frozen_slotted_fields(self) -> None: + expected_fields = { + ChunkingPolicy: ("max_fragment_codepoints", "overlap_codepoints"), + ProjectionLimits: ( + "max_documents", + "max_document_codepoints", + "max_embedding_batch_size", + ), + ProjectionRequest: ( + "corpus_id", + "projection", + "chunking", + "limits", + "documents", + ), + ProjectionManifestEntry: ( + "document", + "source_digest", + "fragment_count", + ), + ProjectionManifest: ( + "corpus_id", + "projection", + "chunking", + "entries", + "checkpoint", + ), + ProjectionStateSnapshot: ("availability", "manifest"), + ProjectionResult: ("status_before", "receipt", "manifest"), + } + projection = _projection() + manifest = _manifest(projection=projection) + instances = ( + ChunkingPolicy(4, 1), + ProjectionLimits(2, 20, 2), + ProjectionRequest( + "corpus", + projection, + ChunkingPolicy(4, 1), + ProjectionLimits(2, 20, 2), + (_document(),), + ), + manifest.entries[0], + manifest, + ProjectionStateSnapshot(ProjectionStateAvailability.PRESENT, manifest), + ProjectionResult( + ProjectionStateStatus.STALE, + ProjectionReceipt( + "corpus", + projection, + ProjectionOutcome.COMPLETED, + 1, + 1, + manifest.checkpoint, + ), + manifest, + ), + ) + + for instance in instances: + with self.subTest(value=type(instance).__name__): + self.assertEqual( + tuple(field.name for field in fields(type(instance))), + expected_fields[type(instance)], + ) + self.assertFalse(hasattr(instance, "__dict__")) + with self.assertRaises(FrozenInstanceError): + setattr(instance, expected_fields[type(instance)][0], object()) + self.assertIsInstance(hash(instance), int) + + def test_chunking_policy_uses_positive_codepoint_size_and_bounded_overlap( + self, + ) -> None: + self.assertEqual(ChunkingPolicy(4, 0), ChunkingPolicy(4, 0)) + self.assertEqual(ChunkingPolicy(4, 3).overlap_codepoints, 3) + + for maximum, overlap in ( + (0, 0), + (-1, 0), + (4, -1), + (4, 4), + (4, 5), + (True, 0), + (4, True), + (4.0, 0), + (4, 1.0), + (_IntegerSubclass(4), 0), + ): + with self.subTest(maximum=maximum, overlap=overlap): + with self.assertRaises(ContractValidationError): + ChunkingPolicy(cast(int, maximum), cast(int, overlap)) + + def test_projection_limits_require_three_positive_exact_integers(self) -> None: + self.assertEqual(ProjectionLimits(1, 1, 1), ProjectionLimits(1, 1, 1)) + + for field_index in range(3): + for invalid in (0, -1, True, 1.0, _IntegerSubclass(1)): + values: list[object] = [2, 20, 3] + values[field_index] = invalid + with self.subTest(field=field_index, value=invalid): + with self.assertRaises(ContractValidationError): + ProjectionLimits( + cast(int, values[0]), + cast(int, values[1]), + cast(int, values[2]), + ) + + def test_request_preserves_opaque_values_and_canonicalizes_document_order( + self, + ) -> None: + documents = ( + _document("zeta", text=""), + _document("alpha", revision_id=" revision/../2 ", text="e\u0301"), + ) + request = ProjectionRequest( + "corpus", + _projection(), + ChunkingPolicy(2, 1), + ProjectionLimits(2, 2, 1), + documents, + ) + + self.assertEqual( + tuple(document.identity.key.document_id for document in request.documents), + ("alpha", "zeta"), + ) + self.assertEqual(request.documents[0].identity.revision_id, " revision/../2 ") + self.assertEqual(request.documents[0].text, "e\u0301") + self.assertEqual( + ProjectionRequest( + "corpus", + _projection(), + ChunkingPolicy(2, 1), + ProjectionLimits(2, 2, 1), + (), + ).documents, + (), + ) + + def test_request_rejects_inexact_fields_and_document_bound_violations( + self, + ) -> None: + projection = _projection() + chunking = ChunkingPolicy(4, 1) + limits = ProjectionLimits(1, 4, 2) + valid = _document(text="four") + invalid_requests = ( + ("", projection, chunking, limits, (valid,)), + (_StringSubclass("corpus"), projection, chunking, limits, (valid,)), + ("corpus", object(), chunking, limits, (valid,)), + ("corpus", projection, object(), limits, (valid,)), + ("corpus", projection, chunking, object(), (valid,)), + ("corpus", projection, chunking, limits, [valid]), + ("corpus", projection, chunking, limits, (object(),)), + ("corpus", projection, chunking, limits, (valid, _document("two"))), + ("corpus", projection, chunking, limits, (_document(text="12345"),)), + ( + "corpus", + projection, + chunking, + limits, + (_document(corpus_id="other"),), + ), + ( + "corpus", + projection, + chunking, + ProjectionLimits(2, 4, 2), + (_document(revision_id="r1"), _document(revision_id="r2")), + ), + ) + for values in invalid_requests: + with self.subTest(values=values): + with self.assertRaises(ContractValidationError): + ProjectionRequest( + values[0], + cast(ProjectionIdentity, values[1]), + cast(ChunkingPolicy, values[2]), + cast(ProjectionLimits, values[3]), + cast(tuple[Document, ...], values[4]), + ) + + def test_manifest_entry_requires_lowercase_sha256_and_nonnegative_count( + self, + ) -> None: + entry = _entry(fragment_count=0) + self.assertEqual(entry.source_digest, "sha256:" + "0" * 64) + self.assertEqual(entry.fragment_count, 0) + + for digest in ( + "", + "0" * 64, + "sha256:" + "A" * 64, + "sha256:" + "0" * 63, + "sha256:" + "0" * 65, + _StringSubclass("sha256:" + "0" * 64), + ): + with self.subTest(digest=digest): + with self.assertRaises(ContractValidationError): + ProjectionManifestEntry( + _document().identity, + digest, + 1, + ) + for count in (-1, True, 1.0, _IntegerSubclass(1)): + with self.subTest(count=count): + with self.assertRaises(ContractValidationError): + ProjectionManifestEntry( + _document().identity, + "sha256:" + "0" * 64, + cast(int, count), + ) + with self.assertRaises(ContractValidationError): + ProjectionManifestEntry( + cast(DocumentIdentity, object()), + "sha256:" + "0" * 64, + 1, + ) + + def test_manifest_canonicalizes_entries_and_preserves_checkpoint(self) -> None: + projection = _projection() + checkpoint = _checkpoint(projection=projection, token=" token/../x ") + zeta = _entry("zeta", digest_digit="1") + alpha = _entry("alpha", digest_digit="2") + manifest = ProjectionManifest( + "corpus", + projection, + ChunkingPolicy(4, 1), + (zeta, alpha), + checkpoint, + ) + + self.assertEqual(manifest.entries, (alpha, zeta)) + self.assertIs(manifest.checkpoint, checkpoint) + self.assertEqual(manifest.checkpoint.token, " token/../x ") + + def test_manifest_rejects_mismatched_inexact_or_duplicate_content(self) -> None: + projection = _projection() + chunking = ChunkingPolicy(4, 1) + checkpoint = _checkpoint(projection=projection) + entry = _entry() + invalid_values = ( + ("", projection, chunking, (entry,), checkpoint), + ("corpus", object(), chunking, (entry,), checkpoint), + ("corpus", projection, object(), (entry,), checkpoint), + ("corpus", projection, chunking, [entry], checkpoint), + ("corpus", projection, chunking, (object(),), checkpoint), + ("corpus", projection, chunking, (entry,), object()), + ( + "corpus", + projection, + chunking, + (_entry(corpus_id="other"),), + checkpoint, + ), + ("corpus", projection, chunking, (entry, entry), checkpoint), + ( + "corpus", + projection, + chunking, + (entry,), + _checkpoint(corpus_id="other", projection=projection), + ), + ( + "corpus", + projection, + chunking, + (entry,), + _checkpoint(projection=_projection(schema_id="other")), + ), + ) + for values in invalid_values: + with self.subTest(values=values): + with self.assertRaises(ContractValidationError): + ProjectionManifest( + values[0], + cast(ProjectionIdentity, values[1]), + cast(ChunkingPolicy, values[2]), + cast(tuple[ProjectionManifestEntry, ...], values[3]), + cast(ProjectionCheckpoint, values[4]), + ) + + def test_state_availability_is_an_exact_closed_string_enum(self) -> None: + self.assertEqual( + tuple( + (member.name, member.value) for member in ProjectionStateAvailability + ), + (("MISSING", "missing"), ("PRESENT", "present"), ("CORRUPT", "corrupt")), + ) + for invalid in ("MISSING", "", "unknown", None, 1, object()): + with self.subTest(value=invalid): + with self.assertRaises(ContractValidationError): + ProjectionStateAvailability(cast(str, invalid)) + + def test_state_snapshot_requires_manifest_only_when_present(self) -> None: + manifest = _manifest() + self.assertIs( + ProjectionStateSnapshot( + ProjectionStateAvailability.PRESENT, + manifest, + ).manifest, + manifest, + ) + for availability in ( + ProjectionStateAvailability.MISSING, + ProjectionStateAvailability.CORRUPT, + ): + self.assertIsNone(ProjectionStateSnapshot(availability, None).manifest) + with self.assertRaises(ContractValidationError): + ProjectionStateSnapshot(availability, manifest) + with self.assertRaises(ContractValidationError): + ProjectionStateSnapshot(ProjectionStateAvailability.PRESENT, None) + with self.assertRaises(ContractValidationError): + ProjectionStateSnapshot( + cast(ProjectionStateAvailability, "present"), + manifest, + ) + + def test_state_status_is_an_exact_closed_string_enum(self) -> None: + self.assertEqual( + tuple((member.name, member.value) for member in ProjectionStateStatus), + ( + ("MISSING", "missing"), + ("CURRENT", "current"), + ("STALE", "stale"), + ("CORRUPT", "corrupt"), + ("SCHEMA_MISMATCH", "schema_mismatch"), + ("EMBEDDING_MISMATCH", "embedding_mismatch"), + ), + ) + for invalid in ("CURRENT", "", "unknown", None, 1, object()): + with self.subTest(value=invalid): + with self.assertRaises(ContractValidationError): + ProjectionStateStatus(cast(str, invalid)) + + def test_result_accepts_matching_completed_and_truthful_unchanged_values( + self, + ) -> None: + manifest = _manifest() + completed = ProjectionResult( + ProjectionStateStatus.STALE, + ProjectionReceipt( + "corpus", + manifest.projection, + ProjectionOutcome.COMPLETED, + 2, + 2, + manifest.checkpoint, + ), + manifest, + ) + unchanged = ProjectionResult( + ProjectionStateStatus.CURRENT, + ProjectionReceipt( + "corpus", + manifest.projection, + ProjectionOutcome.UNCHANGED, + 0, + 0, + manifest.checkpoint, + ), + manifest, + ) + + self.assertEqual(completed.status_before, ProjectionStateStatus.STALE) + self.assertEqual(unchanged.receipt.attempted_documents, 0) + + def test_result_rejects_inexact_mismatched_or_untruthful_values(self) -> None: + manifest = _manifest() + completed = ProjectionReceipt( + "corpus", + manifest.projection, + ProjectionOutcome.COMPLETED, + 1, + 1, + manifest.checkpoint, + ) + invalid_values = ( + ("stale", completed, manifest), + (ProjectionStateStatus.STALE, object(), manifest), + (ProjectionStateStatus.STALE, completed, object()), + ( + ProjectionStateStatus.STALE, + ProjectionReceipt( + "other", + manifest.projection, + ProjectionOutcome.COMPLETED, + 1, + 1, + _checkpoint(corpus_id="other", projection=manifest.projection), + ), + manifest, + ), + ( + ProjectionStateStatus.STALE, + ProjectionReceipt( + "corpus", + manifest.projection, + ProjectionOutcome.PARTIAL, + 2, + 1, + None, + ), + manifest, + ), + ( + ProjectionStateStatus.STALE, + ProjectionReceipt( + "corpus", + manifest.projection, + ProjectionOutcome.FAILED, + 1, + 0, + None, + ), + manifest, + ), + ( + ProjectionStateStatus.STALE, + ProjectionReceipt( + "corpus", + _projection(schema_id="other"), + ProjectionOutcome.COMPLETED, + 1, + 1, + _checkpoint(projection=_projection(schema_id="other")), + ), + manifest, + ), + ( + ProjectionStateStatus.STALE, + ProjectionReceipt( + "corpus", + manifest.projection, + ProjectionOutcome.COMPLETED, + 1, + 1, + _checkpoint(projection=manifest.projection, token="other"), + ), + manifest, + ), + ( + ProjectionStateStatus.STALE, + ProjectionReceipt( + "corpus", + manifest.projection, + ProjectionOutcome.UNCHANGED, + 0, + 0, + manifest.checkpoint, + ), + manifest, + ), + ( + ProjectionStateStatus.CURRENT, + ProjectionReceipt( + "corpus", + manifest.projection, + ProjectionOutcome.UNCHANGED, + 1, + 1, + manifest.checkpoint, + ), + manifest, + ), + ) + for status, receipt, value_manifest in invalid_values: + with self.subTest(status=status, receipt=receipt): + with self.assertRaises(ContractValidationError): + ProjectionResult( + cast(ProjectionStateStatus, status), + cast(ProjectionReceipt, receipt), + cast(ProjectionManifest, value_manifest), + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_projection_integrity.py b/tests/test_projection_integrity.py new file mode 100644 index 0000000..5e3c67a --- /dev/null +++ b/tests/test_projection_integrity.py @@ -0,0 +1,464 @@ +"""Independent v1 goldens and validation for projection integrity values.""" + +from __future__ import annotations + +import hashlib +import inspect +import unittest +from collections.abc import Callable +from typing import cast + +import generic_rag.projection_integrity as integrity +from generic_rag.contracts import ( + ChunkingPolicy, + Document, + DocumentIdentity, + DocumentKey, + EmbeddingIdentity, + EmbeddingVector, + ProjectionCheckpoint, + ProjectionIdentity, + ProjectionLimits, + ProjectionManifest, + ProjectionManifestEntry, + ProjectionRequest, + ProjectionStateAvailability, + ProjectionStateSnapshot, + VectorRecord, +) +from generic_rag.errors import ContractValidationError +from generic_rag.ports import Borrowed, Embedder, VectorIndexResetter, VectorIndexWriter +from generic_rag.projection import rebuild_projection +from generic_rag.projection_integrity import ( + derive_fragment_id, + derive_projection_checkpoint_token, + derive_source_digest, + has_valid_projection_checkpoint, +) + +_CORPUS_ID = " Corpus/../e\u0301 " +_SOURCE_GOLDEN = ( + "sha256:19390f5d911704efab45a06821296d2309454cfb8bdca63799d1c451afa1a9ea" +) +_FRAGMENT_GOLDEN = ( + "sha256:49d28767699855dbf6778396488946303a62dc8394c46adcdd22bef653950293" +) +_CHECKPOINT_GOLDEN = ( + "sha256:f62fca95057be15ab4e4bab8eebde94b7f92eb87773ae2d82d849664c80ba59f" +) + + +class _TupleSubclass(tuple[object, ...]): + pass + + +def _golden_document() -> Document: + return Document( + DocumentIdentity( + DocumentKey(_CORPUS_ID, "doc/\0😀"), + "rev/한", + ), + "A\0😀e\u0301\n", + (("k", "v"), ("k", "v"), ("", "한")), + ) + + +def _projection() -> ProjectionIdentity: + return ProjectionIdentity( + "schema/😀", + EmbeddingIdentity("model/e\u0301", 3), + ) + + +def _checkpoint_entries() -> tuple[ProjectionManifestEntry, ...]: + return ( + ProjectionManifestEntry( + DocumentIdentity(DocumentKey(_CORPUS_ID, "alpha/e\u0301"), "r1😀"), + _SOURCE_GOLDEN, + 2, + ), + ProjectionManifestEntry( + DocumentIdentity(DocumentKey(_CORPUS_ID, "한"), "r2\0"), + "sha256:" + "b" * 64, + 0, + ), + ) + + +def _manifest(token: str = _CHECKPOINT_GOLDEN) -> ProjectionManifest: + projection = _projection() + return ProjectionManifest( + _CORPUS_ID, + projection, + ChunkingPolicy(4, 1), + _checkpoint_entries(), + ProjectionCheckpoint(_CORPUS_ID, projection, token), + ) + + +def _independent_hash(fields: tuple[str, ...]) -> str: + digest = hashlib.sha256() + for field in fields: + encoded = field.encode("utf-8", "surrogatepass") + digest.update(len(encoded).to_bytes(8, "big", signed=False)) + digest.update(encoded) + return f"sha256:{digest.hexdigest()}" + + +class _ProjectionEmbedder: + identity = _projection().embedding + + def embed(self, texts: tuple[str, ...], /) -> tuple[EmbeddingVector, ...]: + return tuple(EmbeddingVector((float(len(text)), 0.0, 1.0)) for text in texts) + + +class _ProjectionWriter: + def __init__(self) -> None: + self.records: tuple[VectorRecord, ...] = () + + def replace_document( + self, + document: DocumentIdentity, + records: tuple[VectorRecord, ...], + /, + ) -> None: + del document + self.records = records + + def delete_document(self, document: DocumentKey, /) -> None: + del document + + +class _ProjectionResetter: + def reset_corpus(self, corpus_id: str, /) -> None: + del corpus_id + + +class ProjectionIntegrityPublicTests(unittest.TestCase): + def test_exports_are_exact_owned_synchronous_and_positional_only(self) -> None: + expected = ( + "derive_source_digest", + "derive_fragment_id", + "derive_projection_checkpoint_token", + "has_valid_projection_checkpoint", + ) + expected_parameters = { + derive_source_digest: ("document",), + derive_fragment_id: ("document", "start", "end"), + derive_projection_checkpoint_token: ( + "corpus_id", + "projection", + "chunking", + "entries", + ), + has_valid_projection_checkpoint: ("manifest",), + } + + self.assertEqual(integrity.__all__, expected) + for function, parameter_names in expected_parameters.items(): + with self.subTest(function=function.__name__): + self.assertEqual( + function.__module__, "generic_rag.projection_integrity" + ) + self.assertFalse(inspect.iscoroutinefunction(function)) + parameters = tuple( + inspect.signature( + cast(Callable[..., object], function) + ).parameters.values() + ) + self.assertEqual( + tuple(parameter.name for parameter in parameters), + parameter_names, + ) + self.assertTrue( + all( + parameter.kind is inspect.Parameter.POSITIONAL_ONLY + for parameter in parameters + ) + ) + + def test_fixed_unicode_nul_and_duplicate_attribute_v1_goldens(self) -> None: + document = _golden_document() + entries = _checkpoint_entries() + + source_fields = [ + "generic-rag:projection-source:v1", + "text", + document.text, + "attributes_count", + str(len(document.attributes)), + ] + for key, value in document.attributes: + source_fields.extend(("attribute_key", key, "attribute_value", value)) + self.assertEqual(_independent_hash(tuple(source_fields)), _SOURCE_GOLDEN) + self.assertEqual(derive_source_digest(document), _SOURCE_GOLDEN) + + fragment_fields = ( + "generic-rag:fragment-id:v1", + "corpus_id", + document.identity.key.corpus_id, + "document_id", + document.identity.key.document_id, + "revision_id", + document.identity.revision_id, + "start", + "1", + "end", + "5", + ) + self.assertEqual(_independent_hash(fragment_fields), _FRAGMENT_GOLDEN) + self.assertEqual(derive_fragment_id(document.identity, 1, 5), _FRAGMENT_GOLDEN) + + checkpoint_fields = [ + "generic-rag:projection-checkpoint:v1", + "corpus_id", + _CORPUS_ID, + "schema_id", + _projection().schema_id, + "embedding_model_id", + _projection().embedding.model_id, + "embedding_dimensions", + "3", + "max_fragment_codepoints", + "4", + "overlap_codepoints", + "1", + "entry_count", + "2", + ] + for entry in entries: + checkpoint_fields.extend( + ( + "document_id", + entry.document.key.document_id, + "revision_id", + entry.document.revision_id, + "source_digest", + entry.source_digest, + "fragment_count", + str(entry.fragment_count), + ) + ) + self.assertEqual( + _independent_hash(tuple(checkpoint_fields)), _CHECKPOINT_GOLDEN + ) + self.assertEqual( + derive_projection_checkpoint_token( + _CORPUS_ID, + _projection(), + ChunkingPolicy(4, 1), + entries, + ), + _CHECKPOINT_GOLDEN, + ) + + def test_checkpoint_validation_is_exact_and_does_not_mutate_manifest(self) -> None: + manifest = _manifest() + before = repr(manifest) + + self.assertTrue(has_valid_projection_checkpoint(manifest)) + self.assertFalse(has_valid_projection_checkpoint(_manifest("wrong-token"))) + self.assertEqual(repr(manifest), before) + + def test_projection_workflow_preserves_the_extracted_v1_algorithms(self) -> None: + document = _golden_document() + projection = _projection() + chunking = ChunkingPolicy(4, 1) + request = ProjectionRequest( + _CORPUS_ID, + projection, + chunking, + ProjectionLimits(1, len(document.text), 4), + (document,), + ) + writer = _ProjectionWriter() + + result = rebuild_projection( + request, + ProjectionStateSnapshot(ProjectionStateAvailability.MISSING, None), + Borrowed(cast(Embedder, _ProjectionEmbedder())), + Borrowed(cast(VectorIndexWriter, writer)), + Borrowed(cast(VectorIndexResetter, _ProjectionResetter())), + ) + + entry = result.manifest.entries[0] + self.assertEqual(entry.source_digest, derive_source_digest(document)) + self.assertEqual( + tuple(record.fragment.identity.fragment_id for record in writer.records), + tuple( + derive_fragment_id(document.identity, start, end) + for start, end in ((0, 4), (3, 6)) + ), + ) + self.assertEqual( + result.manifest.checkpoint.token, + derive_projection_checkpoint_token( + request.corpus_id, + request.projection, + request.chunking, + result.manifest.entries, + ), + ) + + +class ProjectionIntegrityValidationTests(unittest.TestCase): + def assert_contract_failure(self, operation: Callable[[], object]) -> None: + with self.assertRaises(ContractValidationError) as raised: + operation() + self.assertIs(type(raised.exception), ContractValidationError) + + def test_top_level_values_require_exact_public_types(self) -> None: + operations: tuple[Callable[[], object], ...] = ( + lambda: derive_source_digest(cast(Document, object())), + lambda: derive_fragment_id( + cast(DocumentIdentity, object()), + 0, + 1, + ), + lambda: derive_projection_checkpoint_token( + cast(str, object()), + _projection(), + ChunkingPolicy(4, 1), + (), + ), + lambda: derive_projection_checkpoint_token( + _CORPUS_ID, + cast(ProjectionIdentity, object()), + ChunkingPolicy(4, 1), + (), + ), + lambda: derive_projection_checkpoint_token( + _CORPUS_ID, + _projection(), + cast(ChunkingPolicy, object()), + (), + ), + lambda: derive_projection_checkpoint_token( + _CORPUS_ID, + _projection(), + ChunkingPolicy(4, 1), + cast(tuple[ProjectionManifestEntry, ...], []), + ), + lambda: has_valid_projection_checkpoint(cast(ProjectionManifest, object())), + ) + + for operation in operations: + with self.subTest(operation=operation): + self.assert_contract_failure(operation) + + def test_corrupted_frozen_nested_values_fail_as_contract_errors(self) -> None: + malformed_documents = (_golden_document(), _golden_document()) + object.__setattr__(malformed_documents[0], "identity", object()) + object.__setattr__(malformed_documents[1], "attributes", [("k", "v")]) + + malformed_identity = _golden_document().identity + object.__setattr__(malformed_identity, "key", object()) + + malformed_projection = _projection() + object.__setattr__(malformed_projection.embedding, "dimensions", True) + + malformed_chunking = ChunkingPolicy(4, 1) + object.__setattr__(malformed_chunking, "overlap_codepoints", 4) + + malformed_entry = _checkpoint_entries()[0] + object.__setattr__(malformed_entry, "fragment_count", -1) + + operations: tuple[Callable[[], object], ...] = ( + *( + lambda value=value: derive_source_digest(value) + for value in malformed_documents + ), + lambda: derive_fragment_id(malformed_identity, 0, 1), + lambda: derive_projection_checkpoint_token( + _CORPUS_ID, + malformed_projection, + ChunkingPolicy(4, 1), + (), + ), + lambda: derive_projection_checkpoint_token( + _CORPUS_ID, + _projection(), + malformed_chunking, + (), + ), + lambda: derive_projection_checkpoint_token( + _CORPUS_ID, + _projection(), + ChunkingPolicy(4, 1), + (malformed_entry,), + ), + ) + + for operation in operations: + with self.subTest(operation=operation): + self.assert_contract_failure(operation) + + def test_fragment_ranges_are_nonnegative_ordered_exact_integers(self) -> None: + document = _golden_document().identity + for start, end in ( + (-1, 1), + (0, 0), + (2, 1), + (True, 1), + (0, False), + (0.0, 1), + (0, 1.0), + ): + with self.subTest(start=start, end=end): + self.assert_contract_failure( + lambda: derive_fragment_id( + document, + cast(int, start), + cast(int, end), + ) + ) + + def test_checkpoint_entries_must_be_an_exact_canonical_tuple(self) -> None: + entries = _checkpoint_entries() + wrong_corpus = ProjectionManifestEntry( + DocumentIdentity(DocumentKey("other", "document"), "revision"), + _SOURCE_GOLDEN, + 1, + ) + duplicate = ProjectionManifestEntry( + entries[0].document, + "sha256:" + "c" * 64, + 1, + ) + malformed_cases: tuple[object, ...] = ( + list(entries), + _TupleSubclass(entries), + tuple(reversed(entries)), + (entries[0], duplicate), + (wrong_corpus,), + (object(),), + ) + + for malformed in malformed_cases: + with self.subTest(container=type(malformed).__name__, value=malformed): + self.assert_contract_failure( + lambda: derive_projection_checkpoint_token( + _CORPUS_ID, + _projection(), + ChunkingPolicy(4, 1), + cast(tuple[ProjectionManifestEntry, ...], malformed), + ) + ) + + def test_checkpoint_validation_rejects_corrupted_manifest_shapes(self) -> None: + malformed_manifests = (_manifest(), _manifest(), _manifest()) + object.__setattr__( + malformed_manifests[0], "entries", list(_checkpoint_entries()) + ) + object.__setattr__(malformed_manifests[1], "checkpoint", object()) + object.__setattr__(malformed_manifests[2].projection, "schema_id", 1) + + for malformed in malformed_manifests: + with self.subTest(manifest=malformed): + self.assert_contract_failure( + lambda: has_valid_projection_checkpoint(malformed) + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_retrieval.py b/tests/test_retrieval.py new file mode 100644 index 0000000..5f66da1 --- /dev/null +++ b/tests/test_retrieval.py @@ -0,0 +1,1312 @@ +"""Behavioral tests for bounded semantic and deterministic hybrid retrieval.""" + +from __future__ import annotations + +import inspect +import itertools +import json +import os +import subprocess +import sys +import textwrap +import unittest +from collections.abc import Callable +from fractions import Fraction +from pathlib import Path +from types import TracebackType +from typing import cast + +import generic_rag.retrieval as retrieval_module +from generic_rag.contracts import ( + ChunkingPolicy, + DocumentIdentity, + DocumentKey, + EmbeddingIdentity, + EmbeddingVector, + Fragment, + FragmentIdentity, + ProjectionCheckpoint, + ProjectionIdentity, + ProjectionManifest, + ProjectionManifestEntry, + ProjectionStateAvailability, + ProjectionStateSnapshot, + RetrievalHit, + RetrievalLimits, + RetrievalOutcome, + RetrievalQuery, + RetrievalResult, +) +from generic_rag.errors import ContractValidationError +from generic_rag.ports import Borrowed, Embedder, LexicalRetriever, VectorIndexReader +from generic_rag.projection_integrity import ( + derive_fragment_id, + derive_projection_checkpoint_token, +) +from generic_rag.retrieval import retrieve_hybrid, retrieve_semantic + +_SOURCE_ROOT = Path(__file__).resolve().parents[1] / "src" +_DEFAULT = object() + + +class _TupleSubclass(tuple[object, ...]): + pass + + +class _ControlFlow(BaseException): + pass + + +def _projection( + *, + model_id: str = "model-v1", + dimensions: int = 2, +) -> ProjectionIdentity: + return ProjectionIdentity( + "schema-v1", + EmbeddingIdentity(model_id, dimensions), + ) + + +def _document( + document_id: str, + *, + corpus_id: str = "corpus", + revision_id: str = "revision-1", +) -> DocumentIdentity: + return DocumentIdentity(DocumentKey(corpus_id, document_id), revision_id) + + +def _manifest( + documents: tuple[DocumentIdentity, ...], + *, + corpus_id: str = "corpus", + projection: ProjectionIdentity | None = None, + chunking: ChunkingPolicy | None = None, + token: str | None = None, +) -> ProjectionManifest: + selected_projection = projection or _projection() + selected_chunking = chunking or ChunkingPolicy(4, 1) + entries = tuple( + sorted( + ( + ProjectionManifestEntry( + document, + "sha256:" + f"{index:064x}", + 1, + ) + for index, document in enumerate(documents, start=1) + ), + key=lambda entry: entry.document.key.document_id, + ) + ) + checkpoint_token = token or derive_projection_checkpoint_token( + corpus_id, + selected_projection, + selected_chunking, + entries, + ) + return ProjectionManifest( + corpus_id, + selected_projection, + selected_chunking, + entries, + ProjectionCheckpoint(corpus_id, selected_projection, checkpoint_token), + ) + + +def _present(manifest: ProjectionManifest) -> ProjectionStateSnapshot: + return ProjectionStateSnapshot(ProjectionStateAvailability.PRESENT, manifest) + + +def _fragment( + document: DocumentIdentity, + *, + text: str = "x", + start: int = 0, + fragment_id: str | None = None, + attributes: tuple[tuple[str, str], ...] = (), +) -> Fragment: + end = start + len(text) + identity = FragmentIdentity( + document, + fragment_id or derive_fragment_id(document, start, end), + start, + end, + ) + return Fragment(identity, text, attributes) + + +def _query( + *, + corpus_id: str = "corpus", + text: str = "query", + hit_limit: int = 4, + candidate_limit: int = 8, +) -> RetrievalQuery: + return RetrievalQuery(corpus_id, text, hit_limit, candidate_limit) + + +def _limits(max_query_codepoints: int = 100) -> RetrievalLimits: + return RetrievalLimits(max_query_codepoints) + + +class _LifecycleResource: + def __init__(self, events: list[str]) -> None: + self.events = events + self.lifecycle_calls: list[str] = [] + + def __enter__(self) -> _LifecycleResource: + self.lifecycle_calls.append("enter") + return self + + def __exit__( + self, + exc_type: type[BaseException] | None, + exc_value: BaseException | None, + traceback: TracebackType | None, + ) -> bool: + del exc_type, exc_value, traceback + self.lifecycle_calls.append("exit") + return True + + def close(self) -> None: + self.lifecycle_calls.append("close") + + def shutdown(self) -> None: + self.lifecycle_calls.append("shutdown") + + +class _FakeEmbedder(_LifecycleResource): + def __init__( + self, + expected_identity: EmbeddingIdentity, + events: list[str], + *, + identity_value: object = _DEFAULT, + output: object = _DEFAULT, + identity_failure: BaseException | None = None, + embed_failure: BaseException | None = None, + ) -> None: + super().__init__(events) + self.expected_identity = expected_identity + self.identity_value = identity_value + self.output = output + self.identity_failure = identity_failure + self.embed_failure = embed_failure + self.identity_calls = 0 + self.embed_calls: list[tuple[str, ...]] = [] + + @property + def identity(self) -> EmbeddingIdentity: + self.identity_calls += 1 + self.events.append("identity") + if self.identity_failure is not None: + raise self.identity_failure + if self.identity_value is _DEFAULT: + return self.expected_identity + return cast(EmbeddingIdentity, self.identity_value) + + def embed(self, texts: tuple[str, ...], /) -> tuple[EmbeddingVector, ...]: + self.embed_calls.append(texts) + self.events.append("embed:" + "|".join(texts)) + if self.embed_failure is not None: + raise self.embed_failure + if self.output is _DEFAULT: + return (EmbeddingVector((1.0, 2.0)),) + return cast(tuple[EmbeddingVector, ...], self.output) + + +class _FakeVectorReader(_LifecycleResource): + def __init__( + self, + events: list[str], + output: object = (), + *, + failure: BaseException | None = None, + ) -> None: + super().__init__(events) + self.output = output + self.failure = failure + self.calls: list[tuple[RetrievalQuery, EmbeddingVector]] = [] + + def search( + self, + query: RetrievalQuery, + embedding: EmbeddingVector, + /, + ) -> tuple[Fragment, ...]: + self.calls.append((query, embedding)) + self.events.append("vector") + if self.failure is not None: + raise self.failure + return cast(tuple[Fragment, ...], self.output) + + +class _FakeLexicalRetriever(_LifecycleResource): + def __init__( + self, + events: list[str], + output: object = (), + *, + failure: BaseException | None = None, + ) -> None: + super().__init__(events) + self.output = output + self.failure = failure + self.calls: list[RetrievalQuery] = [] + + def search(self, query: RetrievalQuery, /) -> tuple[Fragment, ...]: + self.calls.append(query) + self.events.append("lexical") + if self.failure is not None: + raise self.failure + return cast(tuple[Fragment, ...], self.output) + + +def _borrow_embedder(value: _FakeEmbedder) -> Borrowed[Embedder]: + return Borrowed(cast(Embedder, value)) + + +def _borrow_vector(value: _FakeVectorReader) -> Borrowed[VectorIndexReader]: + return Borrowed(cast(VectorIndexReader, value)) + + +def _borrow_lexical(value: _FakeLexicalRetriever) -> Borrowed[LexicalRetriever]: + return Borrowed(cast(LexicalRetriever, value)) + + +def _hit_fragments(result: RetrievalResult) -> tuple[Fragment, ...]: + return tuple(hit.fragment for hit in result.hits) + + +def _identity_order(fragment: Fragment) -> tuple[str, str, str, str, int, int]: + identity = fragment.identity + return ( + identity.document.key.corpus_id, + identity.document.key.document_id, + identity.document.revision_id, + identity.fragment_id, + identity.start, + identity.end, + ) + + +def _independent_fusion( + semantic: tuple[Fragment, ...], + lexical: tuple[Fragment, ...], +) -> tuple[Fragment, ...]: + fragments: dict[FragmentIdentity, Fragment] = {} + scores: dict[FragmentIdentity, Fraction] = {} + for candidates in (semantic, lexical): + seen: set[FragmentIdentity] = set() + for rank, fragment in enumerate(candidates, start=1): + if fragment.identity in seen: + continue + seen.add(fragment.identity) + fragments.setdefault(fragment.identity, fragment) + scores[fragment.identity] = scores.get( + fragment.identity, + Fraction(), + ) + Fraction(1, 60 + rank) + identities = sorted( + scores, + key=lambda identity: ( + -scores[identity], + _identity_order(fragments[identity]), + ), + ) + return tuple(fragments[identity] for identity in identities) + + +class RetrievalPublicAndInputTests(unittest.TestCase): + def test_exports_and_function_shapes_are_exact_and_owned(self) -> None: + expected = ("retrieve_semantic", "retrieve_hybrid") + signatures = { + retrieve_semantic: ( + "query", + "state", + "limits", + "embedder", + "vector_reader", + ), + retrieve_hybrid: ( + "query", + "state", + "limits", + "embedder", + "vector_reader", + "lexical_retriever", + ), + } + + self.assertEqual(retrieval_module.__all__, expected) + for function, names in signatures.items(): + with self.subTest(function=function.__name__): + self.assertEqual(function.__module__, "generic_rag.retrieval") + self.assertFalse(inspect.iscoroutinefunction(function)) + parameters = tuple( + inspect.signature( + cast(Callable[..., object], function) + ).parameters.values() + ) + self.assertEqual(tuple(item.name for item in parameters), names) + self.assertTrue( + all( + item.kind is inspect.Parameter.POSITIONAL_ONLY + for item in parameters + ) + ) + + def test_invalid_top_level_inputs_fail_before_any_collaborator_effect(self) -> None: + document = _document("document") + state = _present(_manifest((document,))) + query = _query(hit_limit=1, candidate_limit=1) + events: list[str] = [] + embedder = _FakeEmbedder(_projection().embedding, events) + vector = _FakeVectorReader(events, (_fragment(document),)) + operations: tuple[Callable[[], object], ...] = ( + lambda: retrieve_semantic( + cast(RetrievalQuery, object()), + state, + _limits(), + _borrow_embedder(embedder), + _borrow_vector(vector), + ), + lambda: retrieve_semantic( + query, + cast(ProjectionStateSnapshot, object()), + _limits(), + _borrow_embedder(embedder), + _borrow_vector(vector), + ), + lambda: retrieve_semantic( + query, + state, + cast(RetrievalLimits, object()), + _borrow_embedder(embedder), + _borrow_vector(vector), + ), + lambda: retrieve_semantic( + query, + state, + _limits(), + cast(Borrowed[Embedder], object()), + _borrow_vector(vector), + ), + lambda: retrieve_semantic( + query, + state, + _limits(), + _borrow_embedder(embedder), + cast(Borrowed[VectorIndexReader], object()), + ), + lambda: retrieve_hybrid( + query, + state, + _limits(), + _borrow_embedder(embedder), + _borrow_vector(vector), + cast(Borrowed[LexicalRetriever], object()), + ), + ) + + for operation in operations: + with self.subTest(operation=operation): + with self.assertRaises(ContractValidationError): + operation() + self.assertEqual(events, []) + + def test_corrupted_frozen_query_and_state_fail_before_effects(self) -> None: + document = _document("document") + query = _query(hit_limit=1, candidate_limit=1) + object.__setattr__(query, "candidate_limit", True) + state = _present(_manifest((document,))) + malformed_state = _present(_manifest((document,))) + assert malformed_state.manifest is not None + object.__setattr__(malformed_state.manifest, "entries", []) + + for malformed_query, malformed_snapshot in ( + (query, state), + (_query(hit_limit=1, candidate_limit=1), malformed_state), + ): + events: list[str] = [] + embedder = _FakeEmbedder(_projection().embedding, events) + vector = _FakeVectorReader(events) + with self.subTest(query=malformed_query, state=malformed_snapshot): + with self.assertRaises(ContractValidationError): + retrieve_semantic( + malformed_query, + malformed_snapshot, + _limits(), + _borrow_embedder(embedder), + _borrow_vector(vector), + ) + self.assertEqual(events, []) + + def test_query_codepoint_cap_is_independent_and_checked_before_effects( + self, + ) -> None: + document = _document("document") + fragment = _fragment(document) + state = _present(_manifest((document,))) + exact_text = "😀e\u0301\0" + self.assertEqual(len(exact_text), 4) + + events: list[str] = [] + embedder = _FakeEmbedder(_projection().embedding, events) + vector = _FakeVectorReader(events, (fragment,)) + result = retrieve_semantic( + _query(text=exact_text, hit_limit=1, candidate_limit=1), + state, + RetrievalLimits(4), + _borrow_embedder(embedder), + _borrow_vector(vector), + ) + self.assertIs(result.outcome, RetrievalOutcome.COMPLETE) + self.assertEqual(embedder.embed_calls, [(exact_text,)]) + + blocked_events: list[str] = [] + blocked_embedder = _FakeEmbedder(_projection().embedding, blocked_events) + blocked_vector = _FakeVectorReader(blocked_events, (fragment,)) + with self.assertRaises(ContractValidationError): + retrieve_semantic( + _query(text=exact_text + "Z", hit_limit=1, candidate_limit=1), + state, + RetrievalLimits(4), + _borrow_embedder(blocked_embedder), + _borrow_vector(blocked_vector), + ) + self.assertEqual(blocked_events, []) + + +class RetrievalStateAndEffectTests(unittest.TestCase): + def test_terminal_state_outcomes_never_touch_collaborators(self) -> None: + document = _document("document") + valid = _manifest((document,)) + invalid_checkpoint = ProjectionManifest( + valid.corpus_id, + valid.projection, + valid.chunking, + valid.entries, + ProjectionCheckpoint(valid.corpus_id, valid.projection, "wrong-token"), + ) + other_document = _document("document", corpus_id="other") + cases = ( + ( + ProjectionStateSnapshot(ProjectionStateAvailability.MISSING, None), + RetrievalOutcome.UNAVAILABLE, + ), + ( + ProjectionStateSnapshot(ProjectionStateAvailability.CORRUPT, None), + RetrievalOutcome.FAILED, + ), + ( + _present(_manifest((other_document,), corpus_id="other")), + RetrievalOutcome.UNAVAILABLE, + ), + (_present(invalid_checkpoint), RetrievalOutcome.FAILED), + (_present(_manifest(())), RetrievalOutcome.COMPLETE), + ) + + for state, expected in cases: + events: list[str] = [] + embedder = _FakeEmbedder( + _projection().embedding, + events, + identity_failure=AssertionError("must not run"), + ) + vector = _FakeVectorReader( + events, + failure=AssertionError("must not run"), + ) + lexical = _FakeLexicalRetriever( + events, + failure=AssertionError("must not run"), + ) + with self.subTest(outcome=expected): + result = retrieve_hybrid( + _query(), + state, + _limits(), + _borrow_embedder(embedder), + _borrow_vector(vector), + _borrow_lexical(lexical), + ) + self.assertIs(result.outcome, expected) + self.assertEqual(result.hits, ()) + self.assertFalse(result.truncated) + self.assertEqual(events, []) + + def test_semantic_and_hybrid_effect_order_and_call_bounds_are_exact(self) -> None: + semantic_document = _document("semantic") + lexical_document = _document("lexical") + state = _present(_manifest((semantic_document, lexical_document))) + query = _query(hit_limit=2, candidate_limit=2) + + semantic_events: list[str] = [] + semantic_embedder = _FakeEmbedder(_projection().embedding, semantic_events) + semantic_vector = _FakeVectorReader( + semantic_events, + (_fragment(semantic_document),), + ) + semantic_result = retrieve_semantic( + query, + state, + _limits(), + _borrow_embedder(semantic_embedder), + _borrow_vector(semantic_vector), + ) + self.assertEqual(semantic_events, ["identity", "embed:query", "vector"]) + self.assertEqual(semantic_embedder.identity_calls, 1) + self.assertEqual(semantic_embedder.embed_calls, [(query.text,)]) + self.assertEqual(len(semantic_vector.calls), 1) + self.assertEqual(semantic_vector.calls[0][0], query) + self.assertEqual(semantic_vector.calls[0][1], EmbeddingVector((1.0, 2.0))) + self.assertIs(semantic_result.outcome, RetrievalOutcome.COMPLETE) + + hybrid_events: list[str] = [] + hybrid_embedder = _FakeEmbedder(_projection().embedding, hybrid_events) + hybrid_vector = _FakeVectorReader( + hybrid_events, + (_fragment(semantic_document),), + ) + hybrid_lexical = _FakeLexicalRetriever( + hybrid_events, + (_fragment(lexical_document),), + ) + hybrid_result = retrieve_hybrid( + query, + state, + _limits(), + _borrow_embedder(hybrid_embedder), + _borrow_vector(hybrid_vector), + _borrow_lexical(hybrid_lexical), + ) + self.assertEqual( + hybrid_events, + ["identity", "embed:query", "vector", "lexical"], + ) + self.assertEqual(hybrid_embedder.identity_calls, 1) + self.assertEqual(hybrid_embedder.embed_calls, [(query.text,)]) + self.assertEqual(len(hybrid_vector.calls), 1) + self.assertEqual(hybrid_lexical.calls, [query]) + self.assertIs(hybrid_result.outcome, RetrievalOutcome.COMPLETE) + self.assertEqual(tuple(hit.rank for hit in hybrid_result.hits), (1, 2)) + + +class RetrievalProviderValidationTests(unittest.TestCase): + def test_embedder_identity_and_vector_outputs_are_strictly_validated(self) -> None: + document = _document("document") + state = _present(_manifest((document,))) + malformed_identity = EmbeddingIdentity("model-v1", 2) + object.__setattr__(malformed_identity, "dimensions", True) + wrong_dimensions = EmbeddingVector((1.0,)) + noncanonical_number = EmbeddingVector((1.0, 2.0)) + object.__setattr__(noncanonical_number, "values", (1, 2.0)) + nonfinite = EmbeddingVector((1.0, 2.0)) + object.__setattr__(nonfinite, "values", (float("nan"), 2.0)) + malformed_container = EmbeddingVector((1.0, 2.0)) + object.__setattr__(malformed_container, "values", [1.0, 2.0]) + cases = ( + ("identity type", object(), _DEFAULT), + ("identity shape", malformed_identity, _DEFAULT), + ("output list", _DEFAULT, [EmbeddingVector((1.0, 2.0))]), + ("output empty", _DEFAULT, ()), + ( + "output count", + _DEFAULT, + (EmbeddingVector((1.0, 2.0)), EmbeddingVector((3.0, 4.0))), + ), + ("vector type", _DEFAULT, (object(),)), + ("dimensions", _DEFAULT, (wrong_dimensions,)), + ("coordinate type", _DEFAULT, (noncanonical_number,)), + ("nonfinite", _DEFAULT, (nonfinite,)), + ("values container", _DEFAULT, (malformed_container,)), + ) + + for label, identity_value, output in cases: + events: list[str] = [] + embedder = _FakeEmbedder( + _projection().embedding, + events, + identity_value=identity_value, + output=output, + ) + vector = _FakeVectorReader(events, (_fragment(document),)) + with self.subTest(case=label): + result = retrieve_semantic( + _query(hit_limit=1, candidate_limit=1), + state, + _limits(), + _borrow_embedder(embedder), + _borrow_vector(vector), + ) + self.assertIs(result.outcome, RetrievalOutcome.FAILED) + self.assertEqual(result.hits, ()) + self.assertFalse(result.truncated) + self.assertLessEqual(embedder.identity_calls, 1) + self.assertLessEqual(len(embedder.embed_calls), 1) + self.assertEqual(vector.calls, []) + + def test_vector_and_lexical_candidate_outputs_share_strict_validation(self) -> None: + documents = tuple(_document(name) for name in ("a", "b", "c")) + state = _present(_manifest(documents)) + valid = tuple(_fragment(document) for document in documents) + wrong_corpus = _fragment(_document("wrong", corpus_id="other")) + too_wide = _fragment(documents[0], text="abcde") + wrong_id = _fragment(documents[0], fragment_id="wrong-fragment-id") + malformed_outputs: tuple[tuple[str, object], ...] = ( + ("list", list(valid[:1])), + ("tuple subclass", _TupleSubclass(valid[:1])), + ("candidate count", valid), + ("candidate type", (object(),)), + ("wrong corpus", (wrong_corpus,)), + ("fragment width", (too_wide,)), + ("fragment id", (wrong_id,)), + ) + query = _query(hit_limit=2, candidate_limit=2) + + for pathway in ("vector", "lexical"): + for label, malformed in malformed_outputs: + events: list[str] = [] + embedder = _FakeEmbedder(_projection().embedding, events) + vector = _FakeVectorReader( + events, + malformed if pathway == "vector" else (), + ) + lexical = _FakeLexicalRetriever( + events, + malformed if pathway == "lexical" else (), + ) + with self.subTest(pathway=pathway, case=label): + if pathway == "vector": + result = retrieve_semantic( + query, + state, + _limits(), + _borrow_embedder(embedder), + _borrow_vector(vector), + ) + else: + result = retrieve_hybrid( + query, + state, + _limits(), + _borrow_embedder(embedder), + _borrow_vector(vector), + _borrow_lexical(lexical), + ) + self.assertIs(result.outcome, RetrievalOutcome.FAILED) + self.assertEqual(result.hits, ()) + self.assertFalse(result.truncated) + if pathway == "vector": + self.assertEqual(len(vector.calls), 1) + self.assertEqual(lexical.calls, []) + else: + self.assertEqual(len(vector.calls), 1) + self.assertEqual(len(lexical.calls), 1) + + def test_revision_and_document_membership_staleness_preserve_current_hits( + self, + ) -> None: + published_a = _document("a", revision_id="current") + published_c = _document("c", revision_id="current") + stale_revision = _fragment(_document("a", revision_id="old")) + absent_document = _fragment(_document("b", revision_id="current")) + current = _fragment(published_c) + state = _present(_manifest((published_a, published_c))) + events: list[str] = [] + embedder = _FakeEmbedder(_projection().embedding, events) + vector = _FakeVectorReader( + events, + (stale_revision, absent_document, current), + ) + + result = retrieve_semantic( + _query(hit_limit=2, candidate_limit=3), + state, + _limits(), + _borrow_embedder(embedder), + _borrow_vector(vector), + ) + + self.assertIs(result.outcome, RetrievalOutcome.PARTIAL) + self.assertEqual(_hit_fragments(result), (current,)) + self.assertEqual(result.hits[0], RetrievalHit(current, 1)) + self.assertFalse(result.truncated) + + def test_embedder_identity_mismatch_is_stale_without_embedding_or_vector_read( + self, + ) -> None: + document = _document("document") + state = _present(_manifest((document,))) + events: list[str] = [] + embedder = _FakeEmbedder( + _projection().embedding, + events, + identity_value=EmbeddingIdentity("other-model", 2), + ) + vector = _FakeVectorReader(events, (_fragment(document),)) + + result = retrieve_semantic( + _query(hit_limit=1, candidate_limit=1), + state, + _limits(), + _borrow_embedder(embedder), + _borrow_vector(vector), + ) + + self.assertIs(result.outcome, RetrievalOutcome.STALE) + self.assertEqual(events, ["identity"]) + self.assertEqual(embedder.embed_calls, []) + self.assertEqual(vector.calls, []) + + +class RetrievalDeduplicationAndFusionTests(unittest.TestCase): + def test_identical_provider_duplicates_are_deduplicated_without_truncation( + self, + ) -> None: + documents = (_document("a"), _document("b")) + first, second = tuple(_fragment(document) for document in documents) + events: list[str] = [] + result = retrieve_semantic( + _query(hit_limit=2, candidate_limit=3), + _present(_manifest(documents)), + _limits(), + _borrow_embedder(_FakeEmbedder(_projection().embedding, events)), + _borrow_vector(_FakeVectorReader(events, (first, first, second))), + ) + + self.assertIs(result.outcome, RetrievalOutcome.COMPLETE) + self.assertEqual(_hit_fragments(result), (first, second)) + self.assertFalse(result.truncated) + + def test_within_and_cross_provider_payload_conflicts_fail_content_free( + self, + ) -> None: + document = _document("document") + original = _fragment(document, attributes=(("variant", "one"),)) + conflict = _fragment(document, attributes=(("sentinel-secret", "two"),)) + state = _present(_manifest((document,))) + + events: list[str] = [] + within = retrieve_semantic( + _query(hit_limit=2, candidate_limit=2), + state, + _limits(), + _borrow_embedder(_FakeEmbedder(_projection().embedding, events)), + _borrow_vector(_FakeVectorReader(events, (original, conflict))), + ) + self.assertIs(within.outcome, RetrievalOutcome.FAILED) + self.assertEqual(within.hits, ()) + self.assertNotIn("sentinel-secret", repr(within)) + + cross_events: list[str] = [] + cross = retrieve_hybrid( + _query(hit_limit=1, candidate_limit=1), + state, + _limits(), + _borrow_embedder(_FakeEmbedder(_projection().embedding, cross_events)), + _borrow_vector(_FakeVectorReader(cross_events, (original,))), + _borrow_lexical(_FakeLexicalRetriever(cross_events, (conflict,))), + ) + self.assertIs(cross.outcome, RetrievalOutcome.FAILED) + self.assertEqual(cross.hits, ()) + self.assertFalse(cross.truncated) + self.assertNotIn("sentinel-secret", repr(cross)) + + def test_exact_cross_provider_duplicate_is_one_fused_hit(self) -> None: + document = _document("document") + fragment = _fragment(document) + events: list[str] = [] + result = retrieve_hybrid( + _query(hit_limit=1, candidate_limit=1), + _present(_manifest((document,))), + _limits(), + _borrow_embedder(_FakeEmbedder(_projection().embedding, events)), + _borrow_vector(_FakeVectorReader(events, (fragment,))), + _borrow_lexical(_FakeLexicalRetriever(events, (fragment,))), + ) + + self.assertIs(result.outcome, RetrievalOutcome.COMPLETE) + self.assertEqual(_hit_fragments(result), (fragment,)) + self.assertFalse(result.truncated) + + def test_duplicate_gaps_retain_original_provider_ranks(self) -> None: + documents = tuple(_document(name) for name in ("a", "b", "c", "d")) + by_id = { + document.key.document_id: _fragment(document) for document in documents + } + semantic = (by_id["d"], by_id["d"], by_id["b"]) + lexical = (by_id["a"], by_id["c"]) + events: list[str] = [] + result = retrieve_hybrid( + _query(hit_limit=4, candidate_limit=4), + _present(_manifest(documents)), + _limits(), + _borrow_embedder(_FakeEmbedder(_projection().embedding, events)), + _borrow_vector(_FakeVectorReader(events, semantic)), + _borrow_lexical(_FakeLexicalRetriever(events, lexical)), + ) + + self.assertEqual(_hit_fragments(result), _independent_fusion(semantic, lexical)) + document_ids = tuple( + hit.fragment.identity.document.key.document_id for hit in result.hits + ) + self.assertLess(document_ids.index("c"), document_ids.index("b")) + + def test_rrf_uses_exact_fraction_offset_sixty_witness(self) -> None: + semantic_ids = ( + "A", + "s02", + "s03", + "s04", + "B", + "s06", + "s07", + "s08", + "s09", + "s10", + "s11", + "s12", + "s13", + ) + lexical_ids = ( + "l01", + "l02", + "l03", + "l04", + "l05", + "l06", + "l07", + "B", + "l09", + "l10", + "l11", + "l12", + "A", + ) + all_ids = tuple(sorted(set((*semantic_ids, *lexical_ids)))) + documents = tuple(_document(document_id) for document_id in all_ids) + fragments = { + document.key.document_id: _fragment(document) for document in documents + } + semantic = tuple(fragments[document_id] for document_id in semantic_ids) + lexical = tuple(fragments[document_id] for document_id in lexical_ids) + events: list[str] = [] + result = retrieve_hybrid( + _query(hit_limit=13, candidate_limit=13), + _present(_manifest(documents)), + _limits(), + _borrow_embedder(_FakeEmbedder(_projection().embedding, events)), + _borrow_vector(_FakeVectorReader(events, semantic)), + _borrow_lexical(_FakeLexicalRetriever(events, lexical)), + ) + + expected = _independent_fusion(semantic, lexical) + self.assertEqual(_hit_fragments(result), expected[:13]) + self.assertEqual( + tuple( + hit.fragment.identity.document.key.document_id + for hit in result.hits[:2] + ), + ("A", "B"), + ) + self.assertGreater( + Fraction(1, 60 + 1) + Fraction(1, 60 + 13), + Fraction(1, 60 + 5) + Fraction(1, 60 + 8), + ) + self.assertLess( + Fraction(1, 61 + 1) + Fraction(1, 61 + 13), + Fraction(1, 61 + 5) + Fraction(1, 61 + 8), + ) + self.assertTrue(result.truncated) + + def test_unicode_ties_and_branch_permutation_use_identity_order(self) -> None: + decomposed = _document("e\u0301") + composed = _document("é") + first = _fragment(decomposed) + second = _fragment(composed) + state = _present(_manifest((decomposed, composed))) + + outputs: list[tuple[Fragment, ...]] = [] + for semantic, lexical in (((first,), (second,)), ((second,), (first,))): + events: list[str] = [] + result = retrieve_hybrid( + _query(hit_limit=2, candidate_limit=2), + state, + _limits(), + _borrow_embedder(_FakeEmbedder(_projection().embedding, events)), + _borrow_vector(_FakeVectorReader(events, semantic)), + _borrow_lexical(_FakeLexicalRetriever(events, lexical)), + ) + outputs.append(_hit_fragments(result)) + + self.assertEqual(outputs, [(first, second), (first, second)]) + + def test_all_small_provider_permutations_match_fraction_oracle(self) -> None: + documents = tuple(_document(name) for name in ("a", "b", "c")) + fragments = tuple(_fragment(document) for document in documents) + state = _present(_manifest(documents)) + + for semantic in itertools.permutations(fragments): + for lexical in itertools.permutations(fragments): + events: list[str] = [] + with self.subTest( + semantic=tuple( + item.identity.document.key.document_id for item in semantic + ), + lexical=tuple( + item.identity.document.key.document_id for item in lexical + ), + ): + result = retrieve_hybrid( + _query(hit_limit=3, candidate_limit=3), + state, + _limits(), + _borrow_embedder( + _FakeEmbedder(_projection().embedding, events) + ), + _borrow_vector(_FakeVectorReader(events, semantic)), + _borrow_lexical(_FakeLexicalRetriever(events, lexical)), + ) + self.assertEqual( + _hit_fragments(result), + _independent_fusion(semantic, lexical), + ) + + def test_truncation_reflects_post_validation_unique_current_candidates( + self, + ) -> None: + documents = tuple(_document(name) for name in ("a", "b", "c")) + current = tuple(_fragment(document) for document in documents) + + events: list[str] = [] + truncated = retrieve_semantic( + _query(hit_limit=2, candidate_limit=3), + _present(_manifest(documents)), + _limits(), + _borrow_embedder(_FakeEmbedder(_projection().embedding, events)), + _borrow_vector(_FakeVectorReader(events, current)), + ) + self.assertEqual(len(truncated.hits), 2) + self.assertTrue(truncated.truncated) + + stale = _fragment(_document("b", revision_id="old")) + filtered_events: list[str] = [] + filtered = retrieve_semantic( + _query(hit_limit=1, candidate_limit=3), + _present(_manifest(documents[:2])), + _limits(), + _borrow_embedder(_FakeEmbedder(_projection().embedding, filtered_events)), + _borrow_vector( + _FakeVectorReader(filtered_events, (current[0], current[0], stale)) + ), + ) + self.assertIs(filtered.outcome, RetrievalOutcome.PARTIAL) + self.assertEqual(_hit_fragments(filtered), (current[0],)) + self.assertFalse(filtered.truncated) + + +class RetrievalFailureAndLifecycleTests(unittest.TestCase): + def assert_no_lifecycle(self, *resources: _LifecycleResource) -> None: + for resource in resources: + self.assertEqual(resource.lifecycle_calls, []) + + def test_failure_and_stale_precedence_is_truthful(self) -> None: + document = _document("document") + fragment = _fragment(document) + state = _present(_manifest((document,))) + wrong_identity = EmbeddingIdentity("other-model", 2) + cases = ( + (wrong_identity, None, (), RetrievalOutcome.STALE), + ( + wrong_identity, + RuntimeError("lexical sentinel"), + (), + RetrievalOutcome.FAILED, + ), + (wrong_identity, None, (fragment,), RetrievalOutcome.PARTIAL), + ( + RuntimeError("identity sentinel"), + None, + (fragment,), + RetrievalOutcome.PARTIAL, + ), + (_DEFAULT, None, (), RetrievalOutcome.COMPLETE), + ) + + for semantic_state, lexical_failure, lexical_output, expected in cases: + events: list[str] = [] + embedder = _FakeEmbedder( + _projection().embedding, + events, + identity_value=( + semantic_state + if isinstance(semantic_state, EmbeddingIdentity) + else _DEFAULT + ), + identity_failure=( + semantic_state + if isinstance(semantic_state, BaseException) + else None + ), + ) + vector = _FakeVectorReader(events) + lexical = _FakeLexicalRetriever( + events, + lexical_output, + failure=lexical_failure, + ) + with self.subTest(expected=expected, semantic=semantic_state): + result = retrieve_hybrid( + _query(hit_limit=1, candidate_limit=1), + state, + _limits(), + _borrow_embedder(embedder), + _borrow_vector(vector), + _borrow_lexical(lexical), + ) + self.assertIs(result.outcome, expected) + self.assertEqual( + bool(result.hits), expected is RetrievalOutcome.PARTIAL + ) + self.assertNotIn("sentinel", repr(result)) + + def test_ordinary_exceptions_are_content_free_and_never_retried(self) -> None: + document = _document("document") + state = _present(_manifest((document,))) + stages = ("identity", "embedding", "vector", "lexical") + expected_calls = { + "identity": (1, 0, 0, 0), + "embedding": (1, 1, 0, 0), + "vector": (1, 1, 1, 0), + "lexical": (1, 1, 1, 1), + } + + for stage in stages: + failure = RuntimeError(f"secret-{stage}-sentinel") + events: list[str] = [] + embedder = _FakeEmbedder( + _projection().embedding, + events, + identity_failure=failure if stage == "identity" else None, + embed_failure=failure if stage == "embedding" else None, + ) + vector = _FakeVectorReader( + events, + failure=failure if stage == "vector" else None, + ) + lexical = _FakeLexicalRetriever( + events, + failure=failure if stage == "lexical" else None, + ) + with self.subTest(stage=stage): + if stage == "lexical": + result = retrieve_hybrid( + _query(hit_limit=1, candidate_limit=1), + state, + _limits(), + _borrow_embedder(embedder), + _borrow_vector(vector), + _borrow_lexical(lexical), + ) + else: + result = retrieve_semantic( + _query(hit_limit=1, candidate_limit=1), + state, + _limits(), + _borrow_embedder(embedder), + _borrow_vector(vector), + ) + self.assertIs(result.outcome, RetrievalOutcome.FAILED) + self.assertEqual(result.hits, ()) + self.assertFalse(result.truncated) + self.assertNotIn(str(failure), repr(result)) + self.assertEqual( + ( + embedder.identity_calls, + len(embedder.embed_calls), + len(vector.calls), + len(lexical.calls), + ), + expected_calls[stage], + ) + self.assert_no_lifecycle(embedder, vector, lexical) + + def test_base_exceptions_propagate_exactly_once_without_lifecycle_actions( + self, + ) -> None: + document = _document("document") + state = _present(_manifest((document,))) + expected_calls = { + "identity": (1, 0, 0, 0), + "embedding": (1, 1, 0, 0), + "vector": (1, 1, 1, 0), + "lexical": (1, 1, 1, 1), + } + for stage in ("identity", "embedding", "vector", "lexical"): + failure = _ControlFlow(f"control-{stage}") + events: list[str] = [] + embedder = _FakeEmbedder( + _projection().embedding, + events, + identity_failure=failure if stage == "identity" else None, + embed_failure=failure if stage == "embedding" else None, + ) + vector = _FakeVectorReader( + events, + failure=failure if stage == "vector" else None, + ) + lexical = _FakeLexicalRetriever( + events, + failure=failure if stage == "lexical" else None, + ) + with self.subTest(stage=stage): + with self.assertRaises(_ControlFlow) as raised: + if stage == "lexical": + retrieve_hybrid( + _query(hit_limit=1, candidate_limit=1), + state, + _limits(), + _borrow_embedder(embedder), + _borrow_vector(vector), + _borrow_lexical(lexical), + ) + else: + retrieve_semantic( + _query(hit_limit=1, candidate_limit=1), + state, + _limits(), + _borrow_embedder(embedder), + _borrow_vector(vector), + ) + self.assertIs(raised.exception, failure) + self.assertEqual( + ( + embedder.identity_calls, + len(embedder.embed_calls), + len(vector.calls), + len(lexical.calls), + ), + expected_calls[stage], + ) + self.assert_no_lifecycle(embedder, vector, lexical) + + def test_success_never_enters_closes_or_shuts_down_borrowed_resources(self) -> None: + document = _document("document") + fragment = _fragment(document) + events: list[str] = [] + embedder = _FakeEmbedder(_projection().embedding, events) + vector = _FakeVectorReader(events, (fragment,)) + lexical = _FakeLexicalRetriever(events, (fragment,)) + + result = retrieve_hybrid( + _query(hit_limit=1, candidate_limit=1), + _present(_manifest((document,))), + _limits(), + _borrow_embedder(embedder), + _borrow_vector(vector), + _borrow_lexical(lexical), + ) + + self.assertIs(result.outcome, RetrievalOutcome.COMPLETE) + self.assert_no_lifecycle(embedder, vector, lexical) + + +class RetrievalProcessDeterminismTests(unittest.TestCase): + def test_hash_seed_does_not_change_hybrid_rank_order(self) -> None: + script = textwrap.dedent( + f""" + import json + import sys + sys.path.insert(0, {os.fspath(_SOURCE_ROOT)!r}) + from generic_rag.contracts import ( + ChunkingPolicy, DocumentIdentity, DocumentKey, + EmbeddingIdentity, EmbeddingVector, Fragment, FragmentIdentity, + ProjectionCheckpoint, ProjectionIdentity, ProjectionManifest, + ProjectionManifestEntry, ProjectionStateAvailability, + ProjectionStateSnapshot, RetrievalLimits, RetrievalQuery, + ) + from generic_rag.ports import Borrowed + from generic_rag.projection_integrity import ( + derive_fragment_id, derive_projection_checkpoint_token, + ) + from generic_rag.retrieval import retrieve_hybrid + + projection = ProjectionIdentity( + 'schema-v1', EmbeddingIdentity('model-v1', 2) + ) + chunking = ChunkingPolicy(4, 1) + documents = tuple( + DocumentIdentity(DocumentKey('corpus', name), 'revision-1') + for name in ('e\\u0301', 'é', '한') + ) + entries = tuple( + ProjectionManifestEntry( + document, 'sha256:' + format(index, '064x'), 1 + ) + for index, document in enumerate(documents, start=1) + ) + token = derive_projection_checkpoint_token( + 'corpus', projection, chunking, entries + ) + manifest = ProjectionManifest( + 'corpus', projection, chunking, entries, + ProjectionCheckpoint('corpus', projection, token), + ) + fragments = tuple( + Fragment( + FragmentIdentity( + document, derive_fragment_id(document, 0, 1), 0, 1 + ), + 'x', + ) + for document in documents + ) + + class E: + identity = projection.embedding + def embed(self, texts, /): + return (EmbeddingVector((1.0, 2.0)),) + class V: + def search(self, query, embedding, /): + return (fragments[2], fragments[0], fragments[1]) + class L: + def search(self, query, /): + return (fragments[1], fragments[0], fragments[2]) + + result = retrieve_hybrid( + RetrievalQuery('corpus', 'query', 3, 3), + ProjectionStateSnapshot(ProjectionStateAvailability.PRESENT, manifest), + RetrievalLimits(100), Borrowed(E()), Borrowed(V()), Borrowed(L()), + ) + print(json.dumps([ + hit.fragment.identity.document.key.document_id + for hit in result.hits + ], ensure_ascii=False)) + """ + ) + outputs: list[list[str]] = [] + for seed in ("1", "17", "987654"): + environment = dict(os.environ) + environment["PYTHONHASHSEED"] = seed + completed = subprocess.run( + (sys.executable, "-B", "-c", script), + env=environment, + check=False, + capture_output=True, + text=True, + timeout=10, + ) + self.assertEqual(completed.returncode, 0, completed.stderr) + outputs.append(cast(list[str], json.loads(completed.stdout))) + + self.assertEqual(outputs[0], outputs[1]) + self.assertEqual(outputs[1], outputs[2]) + self.assertEqual(outputs[0], ["é", "한", "e\u0301"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_retrieval_contracts.py b/tests/test_retrieval_contracts.py new file mode 100644 index 0000000..2fe397c --- /dev/null +++ b/tests/test_retrieval_contracts.py @@ -0,0 +1,356 @@ +"""Contract tests for bounded score-free retrieval values.""" + +from __future__ import annotations + +import inspect +import unittest +from dataclasses import FrozenInstanceError, fields +from typing import cast + +from generic_rag.contracts import ( + DocumentIdentity, + DocumentKey, + Fragment, + FragmentIdentity, + RetrievalHit, + RetrievalLimits, + RetrievalOutcome, + RetrievalQuery, + RetrievalResult, +) +from generic_rag.errors import ContractValidationError + + +class _StringSubclass(str): + pass + + +class _IntegerSubclass(int): + pass + + +class _TupleSubclass(tuple[object, ...]): + pass + + +def _fragment( + fragment_id: str, + *, + corpus_id: str = "corpus", + document_id: str = "document", + revision_id: str = "revision", + start: int = 0, + attributes: tuple[tuple[str, str], ...] = (), +) -> Fragment: + document = DocumentIdentity( + DocumentKey(corpus_id, document_id), + revision_id, + ) + identity = FragmentIdentity( + document=document, + fragment_id=fragment_id, + start=start, + end=start + 1, + ) + return Fragment(identity, "x", attributes) + + +def _query( + *, + corpus_id: str = "corpus", + text: str = "query", + hit_limit: int = 3, + candidate_limit: int = 5, +) -> RetrievalQuery: + return RetrievalQuery( + corpus_id=corpus_id, + text=text, + hit_limit=hit_limit, + candidate_limit=candidate_limit, + ) + + +class RetrievalContractTests(unittest.TestCase): + def test_retrieval_fields_are_exact_frozen_and_slotted(self) -> None: + self.assertEqual( + tuple(field.name for field in fields(RetrievalLimits)), + ("max_query_codepoints",), + ) + self.assertEqual( + tuple(field.name for field in fields(RetrievalQuery)), + ("corpus_id", "text", "hit_limit", "candidate_limit"), + ) + self.assertEqual( + tuple(field.name for field in fields(RetrievalHit)), + ("fragment", "rank"), + ) + self.assertEqual( + tuple(field.name for field in fields(RetrievalResult)), + ("query", "outcome", "hits", "truncated"), + ) + + limits = RetrievalLimits(100) + query = _query() + hit = RetrievalHit(_fragment("fragment"), 1) + result = RetrievalResult( + query, + RetrievalOutcome.COMPLETE, + (hit,), + False, + ) + for instance, field_name in ( + (limits, "max_query_codepoints"), + (query, "text"), + (hit, "rank"), + (result, "truncated"), + ): + with self.subTest(instance=type(instance).__name__): + self.assertFalse(hasattr(instance, "__dict__")) + with self.assertRaises(FrozenInstanceError): + setattr(instance, field_name, object()) + self.assertIsInstance(hash(instance), int) + + def test_retrieval_limits_are_independent_positive_exact_integers(self) -> None: + limits = RetrievalLimits(max_query_codepoints=7) + + self.assertEqual(limits.max_query_codepoints, 7) + self.assertNotIn("hit_limit", inspect.signature(RetrievalLimits).parameters) + self.assertNotIn( + "candidate_limit", + inspect.signature(RetrievalLimits).parameters, + ) + for invalid in (0, -1, True, 1.0, _IntegerSubclass(1)): + with self.subTest(value=invalid): + with self.assertRaises(ContractValidationError): + RetrievalLimits(cast(int, invalid)) + + def test_retrieval_outcomes_are_exact_closed_string_enums(self) -> None: + self.assertEqual( + tuple((member.name, member.value) for member in RetrievalOutcome), + ( + ("COMPLETE", "complete"), + ("PARTIAL", "partial"), + ("UNAVAILABLE", "unavailable"), + ("STALE", "stale"), + ("FAILED", "failed"), + ), + ) + self.assertEqual(RetrievalOutcome("complete"), RetrievalOutcome.COMPLETE) + self.assertIsInstance(RetrievalOutcome.COMPLETE, str) + + for invalid in ("COMPLETE", "", "unknown", None, 1, object()): + with self.subTest(value=invalid): + with self.assertRaises(ContractValidationError): + RetrievalOutcome(cast(str, invalid)) + + def test_query_preserves_opaque_corpus_and_nonblank_text(self) -> None: + corpus_id = " Corpus/../e\u0301 " + text = " Query/../A " + query = _query(corpus_id=corpus_id, text=text) + + self.assertEqual(query.corpus_id, corpus_id) + self.assertEqual(query.text, text) + + for invalid in ("", " \t\n", None, b"value", _StringSubclass("value")): + with self.subTest(field="corpus_id", value=invalid): + with self.assertRaises(ContractValidationError): + _query(corpus_id=cast(str, invalid)) + with self.subTest(field="text", value=invalid): + with self.assertRaises(ContractValidationError): + _query(text=cast(str, invalid)) + + def test_query_limits_are_positive_exact_integers_and_bounded(self) -> None: + query = _query(hit_limit=2, candidate_limit=2) + self.assertEqual((query.hit_limit, query.candidate_limit), (2, 2)) + + for invalid in (0, -1, True, 1.0, _IntegerSubclass(1)): + with self.subTest(field="hit_limit", value=invalid): + with self.assertRaises(ContractValidationError): + _query(hit_limit=cast(int, invalid)) + with self.subTest(field="candidate_limit", value=invalid): + with self.assertRaises(ContractValidationError): + _query(candidate_limit=cast(int, invalid)) + + with self.assertRaises(ContractValidationError): + _query(hit_limit=3, candidate_limit=2) + + def test_hit_is_positive_ranked_exact_fragment_without_score(self) -> None: + fragment = _fragment("fragment") + hit = RetrievalHit(fragment, 1) + + self.assertIs(hit.fragment, fragment) + self.assertEqual(hit.rank, 1) + self.assertNotIn("score", inspect.signature(RetrievalHit).parameters) + self.assertFalse(hasattr(hit, "score")) + + with self.assertRaises(ContractValidationError): + RetrievalHit(cast(Fragment, object()), 1) + for invalid in (0, -1, True, 1.0, _IntegerSubclass(1)): + with self.subTest(rank=invalid): + with self.assertRaises(ContractValidationError): + RetrievalHit(fragment, cast(int, invalid)) + + def test_result_accepts_contiguous_unique_bounded_matching_hits(self) -> None: + query = _query(hit_limit=3, candidate_limit=5) + hits = ( + RetrievalHit(_fragment("first"), 1), + RetrievalHit(_fragment("second", document_id="other"), 2), + ) + result = RetrievalResult( + query, + RetrievalOutcome.COMPLETE, + hits, + False, + ) + + self.assertEqual(result.hits, hits) + self.assertFalse(result.truncated) + + explicitly_truncated = RetrievalResult( + query, + RetrievalOutcome.COMPLETE, + hits, + True, + ) + self.assertTrue(explicitly_truncated.truncated) + + def test_result_rejects_noncontiguous_or_duplicate_hits(self) -> None: + query = _query() + first = _fragment("first") + second = _fragment("second") + + invalid_rank_sequences = ( + (RetrievalHit(first, 2),), + (RetrievalHit(first, 1), RetrievalHit(second, 3)), + (RetrievalHit(first, 1), RetrievalHit(second, 1)), + ) + for hits in invalid_rank_sequences: + with self.subTest(ranks=tuple(hit.rank for hit in hits)): + with self.assertRaises(ContractValidationError): + RetrievalResult( + query, + RetrievalOutcome.COMPLETE, + hits, + False, + ) + + duplicate_identity = _fragment( + "duplicate", + attributes=(("variant", "one"),), + ) + duplicate_identity_other_value = _fragment( + "duplicate", + attributes=(("variant", "two"),), + ) + with self.assertRaises(ContractValidationError): + RetrievalResult( + query, + RetrievalOutcome.COMPLETE, + ( + RetrievalHit(duplicate_identity, 1), + RetrievalHit(duplicate_identity_other_value, 2), + ), + False, + ) + + def test_result_rejects_wrong_corpus_and_excess_hits(self) -> None: + with self.assertRaises(ContractValidationError): + RetrievalResult( + _query(corpus_id="corpus"), + RetrievalOutcome.COMPLETE, + (RetrievalHit(_fragment("hit", corpus_id="other"), 1),), + False, + ) + + with self.assertRaises(ContractValidationError): + RetrievalResult( + _query(hit_limit=1), + RetrievalOutcome.COMPLETE, + ( + RetrievalHit(_fragment("first"), 1), + RetrievalHit(_fragment("second"), 2), + ), + False, + ) + + def test_terminal_outcomes_cannot_claim_hits_or_truncation(self) -> None: + query = _query() + hit = RetrievalHit(_fragment("fragment"), 1) + + for outcome in ( + RetrievalOutcome.UNAVAILABLE, + RetrievalOutcome.STALE, + RetrievalOutcome.FAILED, + ): + with self.subTest(outcome=outcome): + result = RetrievalResult(query, outcome, (), False) + self.assertEqual(result.hits, ()) + self.assertFalse(result.truncated) + + with self.assertRaises(ContractValidationError): + RetrievalResult(query, outcome, (hit,), False) + with self.assertRaises(ContractValidationError): + RetrievalResult(query, outcome, (), True) + + def test_partial_results_require_at_least_one_valid_hit(self) -> None: + query = _query() + + with self.assertRaises(ContractValidationError): + RetrievalResult(query, RetrievalOutcome.PARTIAL, (), False) + + hit = RetrievalHit(_fragment("fragment"), 1) + for truncated in (False, True): + with self.subTest(truncated=truncated): + result = RetrievalResult( + query, + RetrievalOutcome.PARTIAL, + (hit,), + truncated, + ) + self.assertEqual(result.hits, (hit,)) + + def test_result_requires_exact_nested_container_and_bool_types(self) -> None: + query = _query() + hit = RetrievalHit(_fragment("fragment"), 1) + + with self.assertRaises(ContractValidationError): + RetrievalResult( + cast(RetrievalQuery, object()), + RetrievalOutcome.COMPLETE, + (), + False, + ) + with self.assertRaises(ContractValidationError): + RetrievalResult( + query, + cast(RetrievalOutcome, "complete"), + (), + False, + ) + for invalid_hits in ( + [hit], + iter((hit,)), + _TupleSubclass((hit,)), + (object(),), + ): + with self.subTest(hits=type(invalid_hits).__name__): + with self.assertRaises(ContractValidationError): + RetrievalResult( + query, + RetrievalOutcome.COMPLETE, + cast(tuple[RetrievalHit, ...], invalid_hits), + False, + ) + for invalid_truncated in (0, 1, None, "false"): + with self.subTest(truncated=invalid_truncated): + with self.assertRaises(ContractValidationError): + RetrievalResult( + query, + RetrievalOutcome.COMPLETE, + (), + cast(bool, invalid_truncated), + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/uv.lock b/uv.lock new file mode 100644 index 0000000..e174e64 --- /dev/null +++ b/uv.lock @@ -0,0 +1,389 @@ +version = 1 +revision = 3 +requires-python = ">=3.11" +resolution-markers = [ + "python_full_version >= '3.15'", + "python_full_version < '3.15'", +] + +[[package]] +name = "ast-serialize" +version = "0.8.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/e1/a9/11851c3e02a3fea2ddc9932d1fdc7d2edaeecc0d2e11bc5f2a7fde2b0934/ast_serialize-0.8.0.tar.gz", hash = "sha256:6c37c43e4004dfb42d321ddedc569dc17ff4259296f3af577c9ea46a809bc010", size = 845638, upload-time = "2026-08-07T11:29:02.152Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/34/16/6e520b57cd8c75914b38c670ad4593d13c22911e4306cc7165dab8b0789b/ast_serialize-0.8.0-cp314-cp314-pyemscripten_2026_0_wasm32.whl", hash = "sha256:3d822605fa7bb326ef868d25fafced7fc660fa46d9b90c02ea86d5e2f5d325f7", size = 863924, upload-time = "2026-08-07T11:27:34.579Z" }, + { url = "https://files.pythonhosted.org/packages/03/e1/48802de9b22a2bcad42ec80601a17e3f69172fe4f590e6311bcc2b323aeb/ast_serialize-0.8.0-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:2efa40b068197d5efb62655b43baadb842ed71c4958cccd3e8b86a35726f0119", size = 1177662, upload-time = "2026-08-07T11:27:36.196Z" }, + { url = "https://files.pythonhosted.org/packages/38/d4/323438db76bded3a1f3523a3167b8325916b2ddceb2107a330c6ec9fcf4d/ast_serialize-0.8.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:db1b957291bca08c7e72f43a12357b2948e20775d970e3fc3dac0aa3160ab725", size = 1167072, upload-time = "2026-08-07T11:27:37.646Z" }, + { url = "https://files.pythonhosted.org/packages/77/82/53c5400b54144b56de8ed7f957fd1ccd97e42482009292ab46121d15f8dd/ast_serialize-0.8.0-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fdc0d5b18ff8fb364e87923e47c0a91d0d69dbcaeaa274591f7fd26892cc3a3a", size = 1225497, upload-time = "2026-08-07T11:27:39.225Z" }, + { url = "https://files.pythonhosted.org/packages/44/5f/36c07327a8b91303fbf1382c7c3e8a2902072dbe1b9546138a5288e75ff0/ast_serialize-0.8.0-cp314-cp314t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:9da7330f3e235bf7da89b8d39205c6350fc0c08a85379743f2df9fff87d6d980", size = 1227101, upload-time = "2026-08-07T11:27:40.799Z" }, + { url = "https://files.pythonhosted.org/packages/9d/48/5adf5c67addc7ddb328122208c6d375a84cf154984f412b4087330a157bd/ast_serialize-0.8.0-cp314-cp314t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f3186969ee66a9863b00acc6523ace44c56974eecb348a7ea4b228d9f0b80e19", size = 1424001, upload-time = "2026-08-07T11:27:42.708Z" }, + { url = "https://files.pythonhosted.org/packages/38/a1/70074dd3869d2b0e934f91891d8d6b734361cd3b80f85ca7ece2e668ecdd/ast_serialize-0.8.0-cp314-cp314t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:40a57b73731be45da4fa41430c4d5dc94a24b3a4faba7b9e069978c0402064ea", size = 1245545, upload-time = "2026-08-07T11:27:44.4Z" }, + { url = "https://files.pythonhosted.org/packages/e3/be/53b9c0a8a6399950c2e3546bdfab96d2b299d5b114b47eb94fd3c49c4054/ast_serialize-0.8.0-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5075b9da3ef807eda752502446dfecea3b381c4900b7e27a5d5f4f899eb39951", size = 1248961, upload-time = "2026-08-07T11:27:45.781Z" }, + { url = "https://files.pythonhosted.org/packages/eb/13/3651d3812548a2bda15e26e5dd51aadb48cf682d0865370255fcf0e367dd/ast_serialize-0.8.0-cp314-cp314t-manylinux_2_31_riscv64.whl", hash = "sha256:293cc1c5bfa741f8e3fbe8175b9c07beee487c9a6fdbb25a5acad9f1df2d30a9", size = 1243877, upload-time = "2026-08-07T11:27:47.325Z" }, + { url = "https://files.pythonhosted.org/packages/21/a0/521f0bf000f675e9312a4aae2c8ba7a992405d072a85c485e08fd59433b9/ast_serialize-0.8.0-cp314-cp314t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:e0910c3442a75216dde0f102d854ba2aaa71d2482e0ee213630b9bf29584fba3", size = 1293903, upload-time = "2026-08-07T11:27:49.264Z" }, + { url = "https://files.pythonhosted.org/packages/b1/7e/402fc902568aa2ee65865a3e151f000db0153da8ce6b1be4c9c349025f8d/ast_serialize-0.8.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:43dd6d596879bb1cb8a12cc9dae7bb10090a39a35883026c24f82488a195619a", size = 1401070, upload-time = "2026-08-07T11:27:50.947Z" }, + { url = "https://files.pythonhosted.org/packages/ff/7c/97d4b66c057f1706fc8be6dd532cc77c988794357c8f4ffdb6adabb39562/ast_serialize-0.8.0-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:8c9d537f59e936392cfd3597789d1390304dd659efc3c486ce7f40fb6b8a9f53", size = 1502602, upload-time = "2026-08-07T11:27:52.364Z" }, + { url = "https://files.pythonhosted.org/packages/89/6f/72cc3b71562001bba46e898ccfbf1844f7939b3e28912736206102f2e5a8/ast_serialize-0.8.0-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:f0190a33d7f97c65e9069f7a7f40499eea6b5cbe260c558378109caf20ce934b", size = 1495848, upload-time = "2026-08-07T11:27:53.803Z" }, + { url = "https://files.pythonhosted.org/packages/a0/53/d6f629d1e49308b2f363dae028baa213ec222c9106fa1f7f0d1f7b41499a/ast_serialize-0.8.0-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:77308ae6c5cf5264cc0f01a7c556ec77a9e68eb1f61b093534d698139fdc3b14", size = 1556556, upload-time = "2026-08-07T11:27:55.342Z" }, + { url = "https://files.pythonhosted.org/packages/ee/22/340f35dd8dfc6d412d53dc20699ca014b8d228db923e8ed4759c512b162c/ast_serialize-0.8.0-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:8d53a23f27e1ed3a36b2d26fd2a1a6228c8e85a1ed62ff7cdb44bd610769f20a", size = 1417822, upload-time = "2026-08-07T11:27:56.712Z" }, + { url = "https://files.pythonhosted.org/packages/11/29/6dde5c13fbebc051d3a6df4ec0a6fd1d5359333cc1193f7f609f3410b4d8/ast_serialize-0.8.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:ffa5e7cb08f96fed9121f77b224151e41caf88feab9d652bb46c78202b6fbeda", size = 1445153, upload-time = "2026-08-07T11:27:58.275Z" }, + { url = "https://files.pythonhosted.org/packages/62/c5/f473a8ed030f7a0ca24b9849cca184677a50c053867a7b808c2e1289bbd3/ast_serialize-0.8.0-cp314-cp314t-win32.whl", hash = "sha256:fa70ed4dea0bb18b30a1789c77baa701d0ef30c474f2ccabdea61e25623a8827", size = 1063711, upload-time = "2026-08-07T11:27:59.793Z" }, + { url = "https://files.pythonhosted.org/packages/23/63/39e171fcd38ca057c2e1979d5ee81ac7a3502784abe3d83df7454f7a0978/ast_serialize-0.8.0-cp314-cp314t-win_amd64.whl", hash = "sha256:d8b3c8eee4c1baef9d4e84d2a59a805501617127be42615cb48970b15b0892b6", size = 1103740, upload-time = "2026-08-07T11:28:01.405Z" }, + { url = "https://files.pythonhosted.org/packages/21/1c/d00762b399e7726d68d0a088cc946e3a4c60f1c6176f557608f672f627f3/ast_serialize-0.8.0-cp314-cp314t-win_arm64.whl", hash = "sha256:ac4f0a83c55a9b782f79ad55a5247b7db123c1db405959791c2ef886e9710c9f", size = 1076021, upload-time = "2026-08-07T11:28:02.947Z" }, + { url = "https://files.pythonhosted.org/packages/4c/11/911210c3c78923273a9211a2b6cfc4c8aa723b30dab3e1c8d19afb983b40/ast_serialize-0.8.0-cp315-abi3.abi3t-macosx_10_12_x86_64.whl", hash = "sha256:86b8a1e6d90467345356098b040150e82fbc26d24a7a202224b13dc1f6264ca0", size = 1177715, upload-time = "2026-08-07T11:28:04.654Z" }, + { url = "https://files.pythonhosted.org/packages/77/89/6282881c8587606638db153cbe21e1e0c4d1f3970dee1aa0610a1c62a026/ast_serialize-0.8.0-cp315-abi3.abi3t-macosx_11_0_arm64.whl", hash = "sha256:39e92ff8e8cb45947fe9007174b2950e1fb098e6abd00266a13cd3bcf6675068", size = 1169347, upload-time = "2026-08-07T11:28:06.1Z" }, + { url = "https://files.pythonhosted.org/packages/97/78/a9f846a03a340ff3728c915f23338ca742742f3292700559cdb3ad999b1e/ast_serialize-0.8.0-cp315-abi3.abi3t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c85d8d18db5b2dfcb3b7e38a4d600ca35504c0ed8a6f75cd1c811e4ffe248a15", size = 1225916, upload-time = "2026-08-07T11:28:07.654Z" }, + { url = "https://files.pythonhosted.org/packages/c0/15/aba6ef8a988a6eceb6f0359589aac509e29ae2dba67fd9bfd5af0c3f13e7/ast_serialize-0.8.0-cp315-abi3.abi3t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:9830ff7e764f74d9eefb01170c61a9f0fd2c027dac5fcb72e064decd57d56371", size = 1227135, upload-time = "2026-08-07T11:28:09.504Z" }, + { url = "https://files.pythonhosted.org/packages/94/29/3f63d696ea7c5b8abadcecc3505be51bd900daaccc522ed8322fa5b05a93/ast_serialize-0.8.0-cp315-abi3.abi3t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:6479d9722a4cd21b578f5478074c41e6169f04811996ec881655560f703a5bba", size = 1425040, upload-time = "2026-08-07T11:28:11.044Z" }, + { url = "https://files.pythonhosted.org/packages/e2/5d/0aac338604ff59df5774d4304307898982252f325ff7cafe31d52fedcb65/ast_serialize-0.8.0-cp315-abi3.abi3t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a63bed264e818cd83eec11feed0f50aa162542b91132ef58afebc857182763a5", size = 1246278, upload-time = "2026-08-07T11:28:12.519Z" }, + { url = "https://files.pythonhosted.org/packages/23/ca/9f1ef795bb724719532bd86dbec11e5b66857d3fbe9b6772baec0191a6ed/ast_serialize-0.8.0-cp315-abi3.abi3t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9d187197d234aa45d6cfa2b096be5f666e8cc2e7eb3722d0ab8926293cf5720c", size = 1250029, upload-time = "2026-08-07T11:28:13.896Z" }, + { url = "https://files.pythonhosted.org/packages/dc/25/5e061372d2ed953b9ba3b9c4f73de3b8e9234cda3f6c088db4686801d0e1/ast_serialize-0.8.0-cp315-abi3.abi3t-manylinux_2_31_riscv64.whl", hash = "sha256:2d39a56282cfcc0d8eeea37267c754be59c98d48505c23b1dae5c6011f3813dd", size = 1243575, upload-time = "2026-08-07T11:28:15.37Z" }, + { url = "https://files.pythonhosted.org/packages/a8/c1/ae7da218053120635a4ca802366c69f707203641af95372eeb83f70dfd52/ast_serialize-0.8.0-cp315-abi3.abi3t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:f7cc5f10386994c0f4844f1e6d6a97127e9b478660eb6dec2b257644f0acab64", size = 1294396, upload-time = "2026-08-07T11:28:16.813Z" }, + { url = "https://files.pythonhosted.org/packages/2e/89/271d1f49c5269fcddcc789ea3f25be401f6723fc1138aeda539f4d05516d/ast_serialize-0.8.0-cp315-abi3.abi3t-musllinux_1_2_aarch64.whl", hash = "sha256:6102f2f985c2e542be85cd857678ec9356fefa792b93cadfadd31139f5696f27", size = 1401987, upload-time = "2026-08-07T11:28:18.333Z" }, + { url = "https://files.pythonhosted.org/packages/55/be/4e7d77fcf571ac7cb5cf7115a20c36642bd7d29473b45dfaaefeb9618f90/ast_serialize-0.8.0-cp315-abi3.abi3t-musllinux_1_2_armv7l.whl", hash = "sha256:3a8660fe66667b76a6e9dccd1d33e66b229fde3b308db991c041609226c005b6", size = 1502904, upload-time = "2026-08-07T11:28:20.039Z" }, + { url = "https://files.pythonhosted.org/packages/8b/ae/ed1de2db7e019d4236fbc164ffa5ef9a6022a300a342bbf142d21b7c141e/ast_serialize-0.8.0-cp315-abi3.abi3t-musllinux_1_2_i686.whl", hash = "sha256:e7266307e5fba39836edb79def8608887af48820508bff3c5f2941e1e04d1534", size = 1496967, upload-time = "2026-08-07T11:28:21.734Z" }, + { url = "https://files.pythonhosted.org/packages/92/89/5fea507fae5c5f18b7dc7f95e5c00956574b8c717b8fd2049c504fab0b18/ast_serialize-0.8.0-cp315-abi3.abi3t-musllinux_1_2_ppc64le.whl", hash = "sha256:4ca7e6fd1ad845d1cc649dc2ecd499db2f8f46af5bf8da7b70dd858774cc038b", size = 1559041, upload-time = "2026-08-07T11:28:23.194Z" }, + { url = "https://files.pythonhosted.org/packages/42/71/478d69df21b64e064554a68134c94be304270316ca676a94e63c389a636a/ast_serialize-0.8.0-cp315-abi3.abi3t-musllinux_1_2_riscv64.whl", hash = "sha256:2880350b13d3eae69a0d70bc1fb6c9bfaca4dbd0e20ba8cd1aa483080b56ff06", size = 1417367, upload-time = "2026-08-07T11:28:24.601Z" }, + { url = "https://files.pythonhosted.org/packages/5e/2d/8962dc8d5b3a9dc27b36f9db199afa25264c741505469d9ec10ffbfd2ba7/ast_serialize-0.8.0-cp315-abi3.abi3t-musllinux_1_2_x86_64.whl", hash = "sha256:ab0f9a59f7d63d0d441b56b9a818b273705264352d5115cfee12e940e816d958", size = 1446178, upload-time = "2026-08-07T11:28:26.152Z" }, + { url = "https://files.pythonhosted.org/packages/4f/22/14d2ad4fd1d1bcd0dc687ca268e0630069f45162496260c0efb70ee0ea72/ast_serialize-0.8.0-cp315-abi3.abi3t-win32.whl", hash = "sha256:0485a25ef519c62e749ee3c1ad8070e591b380d67226349eb5a70b228dc1ac4a", size = 1063811, upload-time = "2026-08-07T11:28:27.864Z" }, + { url = "https://files.pythonhosted.org/packages/18/1d/84a327c0202a41aa5fdba3ade33904d6d8f3b9e6806fa83568d835395850/ast_serialize-0.8.0-cp315-abi3.abi3t-win_amd64.whl", hash = "sha256:bd84d60bca7079e741be4ac5dbe237751a59d7f6f9f0126b11880d63822cbe16", size = 1105518, upload-time = "2026-08-07T11:28:29.691Z" }, + { url = "https://files.pythonhosted.org/packages/8c/92/74556dec52fde85a2ad84ed159991b916241043788609c15d8b77e14570b/ast_serialize-0.8.0-cp315-abi3.abi3t-win_arm64.whl", hash = "sha256:057769b5921336eb2d9124f2a731b42ed05ffdac559b840dbdf6f3937cf153dc", size = 1076319, upload-time = "2026-08-07T11:28:31.282Z" }, + { url = "https://files.pythonhosted.org/packages/d1/5d/c650b1f2cc1e75193358da95a080261422e8cd10b66d7370b1688c9915c5/ast_serialize-0.8.0-cp315-cp315-pyemscripten_2026_5_wasm32.whl", hash = "sha256:a02cbed7d8bfdcdee88edaac12bd50d53d9953aaa2e1852ef078625be5f1c0b5", size = 852914, upload-time = "2026-08-07T11:28:32.929Z" }, + { url = "https://files.pythonhosted.org/packages/d9/e3/6142e920fec6ef7bccabd8c24ed8ed99f8bdc6cb8b065e1df7c6a3b2d667/ast_serialize-0.8.0-cp39-abi3-macosx_10_12_x86_64.whl", hash = "sha256:e1bd223df0f6c96b396975fa604cb33bce53d9b4a0185490be4c4a289f7c9c87", size = 1184007, upload-time = "2026-08-07T11:28:34.654Z" }, + { url = "https://files.pythonhosted.org/packages/a6/e9/6e8be8df02b35d85e2b8809f7f1cfa290bdf5882b55127a539d049482db0/ast_serialize-0.8.0-cp39-abi3-macosx_11_0_arm64.whl", hash = "sha256:ddd3b61f45c132da66c5476b281891e08c1fd87fbdabe8a6973e1622efc85f06", size = 1177588, upload-time = "2026-08-07T11:28:36.318Z" }, + { url = "https://files.pythonhosted.org/packages/8c/80/7e0fd2e2e2aba257820db4a8657c4c356844d36b914b20a4af294bcfb902/ast_serialize-0.8.0-cp39-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1f9caa63fad8241257ae401b5ff0a64026c6adb36b8e86cbe8782d9ea505daf6", size = 1234575, upload-time = "2026-08-07T11:28:37.772Z" }, + { url = "https://files.pythonhosted.org/packages/b0/6a/3bae0af06f9b1bae3001c44d64215f5b567877e7aae9ffd45db11c3a7647/ast_serialize-0.8.0-cp39-abi3-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:3926fa117b5e65019853a2969966d11c7175af377a3425991f3fe73784412405", size = 1236015, upload-time = "2026-08-07T11:28:39.14Z" }, + { url = "https://files.pythonhosted.org/packages/6f/c4/ce2d41a1bc22508e82618901f7e10f2a5e2f9556553fea90624daf9875e2/ast_serialize-0.8.0-cp39-abi3-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:485f1113af805e9e170b95ef993ca3fbd4f89c04bab25c58b4fc632d854801ab", size = 1432808, upload-time = "2026-08-07T11:28:40.664Z" }, + { url = "https://files.pythonhosted.org/packages/1a/90/f5058f209756dd70e958b7538aaa82d25d24944baf9ec8ae6f27b06fcacc/ast_serialize-0.8.0-cp39-abi3-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3ccebbed24f1281062d5852353c72c47502955926cfcb8345ffb3a44d87ff3d3", size = 1256251, upload-time = "2026-08-07T11:28:42.223Z" }, + { url = "https://files.pythonhosted.org/packages/bf/32/7f77ea87fa0836daab706ed5cb7f903bb25fa26a77439011aee626af11d8/ast_serialize-0.8.0-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:252f883290d1cdb728eb7fe1d9a7221b88af5a329aae0bc91ddee4dafb820331", size = 1258574, upload-time = "2026-08-07T11:28:43.751Z" }, + { url = "https://files.pythonhosted.org/packages/eb/5a/75b82ad2725b5e8e8c742732f9e76c6738a292d0709e1f60d10a973730b4/ast_serialize-0.8.0-cp39-abi3-manylinux_2_31_riscv64.whl", hash = "sha256:96abc072ad29db8d02194afd47d68987322622787daceae82398d7b69f3ba2e6", size = 1254075, upload-time = "2026-08-07T11:28:45.28Z" }, + { url = "https://files.pythonhosted.org/packages/4e/54/8c20ed4eea805516a3fd23dd4a721ce28c64f50f0e4b359969f60a8c97a6/ast_serialize-0.8.0-cp39-abi3-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:9118ad3e369727060b2696fc4078f250ecffca4248ba87f537f55cea9f9dce06", size = 1301018, upload-time = "2026-08-07T11:28:46.851Z" }, + { url = "https://files.pythonhosted.org/packages/cb/5b/9f14430f12fe830b656fb38f8e2e05ee13b02a88967660bef46af0ab22a8/ast_serialize-0.8.0-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:f359df4bd921918af8bebd142a376c77511d7151cc8ba852760b587b5a4a54f3", size = 1409951, upload-time = "2026-08-07T11:28:48.312Z" }, + { url = "https://files.pythonhosted.org/packages/2d/3d/084882eca93c842bd4262591a071ec7f825340644035e51501208cc5a8d4/ast_serialize-0.8.0-cp39-abi3-musllinux_1_2_armv7l.whl", hash = "sha256:e94f9121d13fa36cbf21314783c77d05ae3a0868decd18cf5233fdcc6de49ac8", size = 1509544, upload-time = "2026-08-07T11:28:49.847Z" }, + { url = "https://files.pythonhosted.org/packages/ce/73/ea84852096c2036c61cc0b2f97b90242207419f534dc671060ee1c8e05cb/ast_serialize-0.8.0-cp39-abi3-musllinux_1_2_i686.whl", hash = "sha256:54f95b486018d262bcb387a9afd96f0da74508b442762b80c769454a6fbb3ee3", size = 1505671, upload-time = "2026-08-07T11:28:51.239Z" }, + { url = "https://files.pythonhosted.org/packages/cb/88/287b9a5300c1f2f651d259f670931b63110adc265b7613c885b44c5bc53d/ast_serialize-0.8.0-cp39-abi3-musllinux_1_2_ppc64le.whl", hash = "sha256:4c38b915511e32bc718c49dbce98ff9af36bac0ad6a604f58000cd5e3aecdba7", size = 1563685, upload-time = "2026-08-07T11:28:53.112Z" }, + { url = "https://files.pythonhosted.org/packages/ee/f3/1bc3a79afcf0c2a8d2c37182d0d659d1545a9d7f7f6dc9cf3e63d6c17135/ast_serialize-0.8.0-cp39-abi3-musllinux_1_2_riscv64.whl", hash = "sha256:9a2ef9cf12f2de4f1028c42c1dd7d775255e0fb3e5bb48896c97e35ef52366fe", size = 1427977, upload-time = "2026-08-07T11:28:54.418Z" }, + { url = "https://files.pythonhosted.org/packages/5c/cd/440c798957e14e31776bfeb024d8fafe0bb1d5b89c51c2f067e69938f7b0/ast_serialize-0.8.0-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:6f18048fe9f6dd266bd577cdec48bdcecb74faaa01fe941324435483b013ed2a", size = 1454335, upload-time = "2026-08-07T11:28:55.968Z" }, + { url = "https://files.pythonhosted.org/packages/4f/4a/587eb36dcc240a54c8660f599464516b469ecad96f0dbdb6bccbedb50745/ast_serialize-0.8.0-cp39-abi3-win32.whl", hash = "sha256:31883542dd6c94d178f5db3d32fbd69c5eb88b3a7c018e7ac8cc0c45195ddbed", size = 1068858, upload-time = "2026-08-07T11:28:57.541Z" }, + { url = "https://files.pythonhosted.org/packages/5f/a4/3e887bbd92164e183cb6e412c6a3e9198ddd446d7fe405958293ef5ef49c/ast_serialize-0.8.0-cp39-abi3-win_amd64.whl", hash = "sha256:861794565b06337005c1447ef23103a3d5a627d08bdc827870d00d0b28ef5f51", size = 1111839, upload-time = "2026-08-07T11:28:59Z" }, + { url = "https://files.pythonhosted.org/packages/25/6c/b400476d3ceba681ab929787edc9554f6d88fcc69435eb681b00fc0457a5/ast_serialize-0.8.0-cp39-abi3-win_arm64.whl", hash = "sha256:b2a5978662fd4db463dfb4b974d2b10ac6430b98f5333aabc7051909df3561d0", size = 1083655, upload-time = "2026-08-07T11:29:00.349Z" }, +] + +[[package]] +name = "build" +version = "1.5.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama", marker = "os_name == 'nt'" }, + { name = "packaging" }, + { name = "pyproject-hooks" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/78/e0/df5e171f685f82f37b12e1f208064e24244911079d7b767447d1af7e0d70/build-1.5.0.tar.gz", hash = "sha256:302c22c3ba2a0fd5f3911918651341ebb3896176cbdec15bd421f80b1afc7647", size = 89796, upload-time = "2026-04-30T03:18:25.17Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0d/fe/6bea5c9162869c5beba5d9c8abbed835ec85bf1ec1fba05a3822325c45f3/build-1.5.0-py3-none-any.whl", hash = "sha256:13f3eecb844759ab66efec90ca17639bbf14dc06cb2fdf37a9010322d9c50a6f", size = 26018, upload-time = "2026-04-30T03:18:23.644Z" }, +] + +[[package]] +name = "colorama" +version = "0.4.6" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d8/53/6f443c9a4a8358a93a6792e2acffb9d9d5cb0a5cfd8802644b7b1c9a02e4/colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44", size = 27697, upload-time = "2022-10-25T02:36:22.414Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", size = 25335, upload-time = "2022-10-25T02:36:20.889Z" }, +] + +[[package]] +name = "generic-rag" +version = "0.1.0" +source = { editable = "." } + +[package.dev-dependencies] +build = [ + { name = "build" }, + { name = "setuptools" }, +] +dev = [ + { name = "build" }, + { name = "mypy" }, + { name = "ruff" }, + { name = "setuptools" }, +] +lint = [ + { name = "ruff" }, +] +type = [ + { name = "mypy" }, +] + +[package.metadata] + +[package.metadata.requires-dev] +build = [ + { name = "build", specifier = ">=1.5,<2" }, + { name = "setuptools", specifier = ">=83,<85" }, +] +dev = [ + { name = "build", specifier = ">=1.5,<2" }, + { name = "mypy", specifier = ">=2.3,<3" }, + { name = "ruff", specifier = ">=0.16,<0.17" }, + { name = "setuptools", specifier = ">=83,<85" }, +] +lint = [{ name = "ruff", specifier = ">=0.16,<0.17" }] +test = [] +type = [{ name = "mypy", specifier = ">=2.3,<3" }] + +[[package]] +name = "librt" +version = "0.15.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/36/9b/356320fbae2ac8467e21c5e73e1389c80468e4998c62cc7d3536cc51b614/librt-0.15.0.tar.gz", hash = "sha256:4e66cbe84437497d951b799d3e1551291b6fb3d643820a7014b3655d57a59162", size = 214338, upload-time = "2026-08-07T10:49:42.663Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/59/52/06790ced2ac7117f890c21bda43c39c958ec82aa665c0718e821d33ff939/librt-0.15.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:823b92cf3c18ecd08afc70c42473888b41b6e8ef5046f3b82c05c154a2fa3d22", size = 148039, upload-time = "2026-08-07T10:46:41.165Z" }, + { url = "https://files.pythonhosted.org/packages/e7/1d/8e150b7fc449a1f33c8a760965cc1f43b14fc1577d9d0b50ab2701420e74/librt-0.15.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:c70bc1b602cf59917e8f0c7a2cbc8bcc6fbc14d5486136b00707a79619121d63", size = 153067, upload-time = "2026-08-07T10:46:42.418Z" }, + { url = "https://files.pythonhosted.org/packages/51/87/a162bc5a66a35599dc619ecb215145f4de7d68e886b479b6d12593139f7c/librt-0.15.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:814ff83a25b5fce8b9c80c4dd803153fb5c5599fc74db9e022466938368957ef", size = 493087, upload-time = "2026-08-07T10:46:43.657Z" }, + { url = "https://files.pythonhosted.org/packages/e5/3a/aeea1fc620cf48060d3065b37614edbf97043c099d0f50782bc8ca61d897/librt-0.15.0-cp311-cp311-manylinux2014_i686.manylinux_2_17_i686.manylinux_2_28_i686.whl", hash = "sha256:57f5eeb6ad4c180de583b1038e61fe5fbd9796bb69a8a1c1a0c7ddbec4c8c60f", size = 485608, upload-time = "2026-08-07T10:46:45.038Z" }, + { url = "https://files.pythonhosted.org/packages/52/ff/fe571ad416f0856fd0d5578ffc2e6dc531891e586e36b647bcf50569cab8/librt-0.15.0-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:82909c8f7eb9952656b65d3147afde4cf8e6d5a991eebc86418b5e65843b0ab8", size = 498723, upload-time = "2026-08-07T10:46:46.35Z" }, + { url = "https://files.pythonhosted.org/packages/0f/e1/7a65eb5dedb1f00aebd948cdd8e17add48bf066cab3514e9daf84ab45a6c/librt-0.15.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f779070399f991400fc451719e0ea388eb7de313388bada2c127a35de05f798a", size = 516002, upload-time = "2026-08-07T10:46:47.599Z" }, + { url = "https://files.pythonhosted.org/packages/5f/45/59832b0ebfbd08c2742e6ece372ceb53f18bf1faef5d33c8daf3abebf749/librt-0.15.0-cp311-cp311-manylinux_2_34_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:bac89069bc496ebdf4f79ebb57bbd10d0b214c8454225deb672d91002bd17e18", size = 508607, upload-time = "2026-08-07T10:46:48.873Z" }, + { url = "https://files.pythonhosted.org/packages/ea/0d/37fa73f3b43ebd8259f91ae9102a15e5a54e65d581e48dea72df3e81d7a4/librt-0.15.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:e0d00c708fb2f5822b152429b1ac80a58dbbbc3f6c232c4d13a3f7fcf2ea5b4c", size = 530422, upload-time = "2026-08-07T10:46:50.45Z" }, + { url = "https://files.pythonhosted.org/packages/26/02/e046c6fe7a5881ac34623242192f484426ba8a75595fd18f22c53a3f530f/librt-0.15.0-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:6c6624fe268625869485553dd7cc1daf30d22558215bb2a4ff16f67a9801a31a", size = 534303, upload-time = "2026-08-07T10:46:51.693Z" }, + { url = "https://files.pythonhosted.org/packages/95/32/d5e6d861ab0366f3edf74f887ab0c9eb9f535aaf01d32b80b4f734daa179/librt-0.15.0-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:f56b397858a23dacf35ede366ed2212fdc03a6a57a1ad36468ad6e9dc5fac091", size = 536084, upload-time = "2026-08-07T10:46:52.951Z" }, + { url = "https://files.pythonhosted.org/packages/2a/de/d69d725513fe53fc90c6d7a1f86e4428939bad2fb905b17fe4c18d413dde/librt-0.15.0-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:4388184646efe2054911c5b00a1077d6d1ee86a95b7e8ba96dc7850a809f3f40", size = 514307, upload-time = "2026-08-07T10:46:54.194Z" }, + { url = "https://files.pythonhosted.org/packages/36/93/f8aded0d6682b4f25820fa86e0690f87f01df9fd7bd09ddb04d9167ad021/librt-0.15.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:97335f59082f9fe2ce6c2a9cc6433a0114bbb6cd4d5c09dd76c95c68b9f9a8b0", size = 557686, upload-time = "2026-08-07T10:46:55.443Z" }, + { url = "https://files.pythonhosted.org/packages/74/09/ffeb6bdeb6cd862b4272fddc8ad05f938dd25d020ed517e631813917d80a/librt-0.15.0-cp311-cp311-win32.whl", hash = "sha256:83380ffde38062a2e9bb55d83e74474f6614665528b98a6928720fc006dfffbb", size = 104917, upload-time = "2026-08-07T10:46:56.605Z" }, + { url = "https://files.pythonhosted.org/packages/96/28/7e2313a3ffbf0b4de7ba3da58a09e488507b4bd1ea2b5e69378354a23415/librt-0.15.0-cp311-cp311-win_amd64.whl", hash = "sha256:f75720477ee05d509a310e856cacc8d909adc182f7b91193c207bcc26d7ee6db", size = 125886, upload-time = "2026-08-07T10:46:57.729Z" }, + { url = "https://files.pythonhosted.org/packages/39/9e/04b8c3cde014ef255ee785730425268354543acc38902093a40afa0dc164/librt-0.15.0-cp311-cp311-win_arm64.whl", hash = "sha256:256237037a3ab001ae8d9803b2d43562a4c3aa38739843694349e4d5ebb0fd56", size = 111885, upload-time = "2026-08-07T10:46:58.787Z" }, + { url = "https://files.pythonhosted.org/packages/ba/39/99c25030e782bdfb7a21be8c05254806a2e4bbb05c8d50c2a2130acbfa05/librt-0.15.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:e87bc679f86a99aa3b26e3c78eeb821a247c9a28eae48eaafcc32c3bf4c3bb9e", size = 151021, upload-time = "2026-08-07T10:47:00.057Z" }, + { url = "https://files.pythonhosted.org/packages/14/43/f4b1bd1b2888798a1409808889a25ea1ba49eaabce7d681ed27734c2df9d/librt-0.15.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:71599e011ac880e8e45d46047d714871894c7d4ab6f25626f8d4f89da21f368d", size = 155267, upload-time = "2026-08-07T10:47:01.311Z" }, + { url = "https://files.pythonhosted.org/packages/0c/db/3ad9c965c72f1e1d6beeec44ec10a54e17be8ae042fbb4baade16cbadced/librt-0.15.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c802434092b769b1d613ed2e13fac15fbfce1934a74bd10283b03c0fae231cd1", size = 503136, upload-time = "2026-08-07T10:47:02.45Z" }, + { url = "https://files.pythonhosted.org/packages/4b/07/5888a6d76acd62ebce66c61b74d94e9370b9c32929f111e487bb6546f8ed/librt-0.15.0-cp312-cp312-manylinux2014_i686.manylinux_2_17_i686.manylinux_2_28_i686.whl", hash = "sha256:5500eeae393a184d14e1f35645962c27129d20c81afa4069e6ef826ebc2b3aaa", size = 496670, upload-time = "2026-08-07T10:47:03.675Z" }, + { url = "https://files.pythonhosted.org/packages/29/39/ab57cc2f5b276156da02bb7f5a8921bada1cb1993ffec99acf811c602c23/librt-0.15.0-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:6ecfc32dfb46fb7b565bcd6abf9412acf978775a998273d22888a6d7953730dd", size = 513688, upload-time = "2026-08-07T10:47:04.981Z" }, + { url = "https://files.pythonhosted.org/packages/a7/b9/bdbb0b648b5c2befb031f4c6f3b1dd857415e8fb492a25a3c764a6681e6c/librt-0.15.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:89cc46cfd15022e35084355478c9ac809d90b1152222706ac9a7655ec21df6fa", size = 531904, upload-time = "2026-08-07T10:47:06.211Z" }, + { url = "https://files.pythonhosted.org/packages/93/26/473c2e4b6c104e9e58e27ce95fc8005c8bd4fc36cae4f254371125a92db8/librt-0.15.0-cp312-cp312-manylinux_2_34_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:d5f51401d102c885b9ca509e62c79b1dbff286e1b9b047fde6f763780789356d", size = 524427, upload-time = "2026-08-07T10:47:07.592Z" }, + { url = "https://files.pythonhosted.org/packages/26/60/03b3abb82b41714671b907bf6989b228e31e6a8af52dec82b5b0728dc250/librt-0.15.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:cc30523e3f1a23fb7511cc659834a0d01a1042bb9de359bc1c131cc4ec6c9656", size = 543155, upload-time = "2026-08-07T10:47:08.866Z" }, + { url = "https://files.pythonhosted.org/packages/f2/0e/9bb1f0a4affbd0a1888f4f79dc03ed2a299d9a2c26c59ab2a97dcbf11903/librt-0.15.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:59fe030d8ae4a57e3fb7756bf35a858de74e04066fc8555c53d0af979132af81", size = 546890, upload-time = "2026-08-07T10:47:10.327Z" }, + { url = "https://files.pythonhosted.org/packages/dc/84/6937a280d461f7de6e031ffb02edc2b7c3c90d49d630565ce8ff27cbc5f2/librt-0.15.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:5a6526a2a956bbb1e4ae3568c82e650fc99119c66bb011ea60715744955a2b4d", size = 555163, upload-time = "2026-08-07T10:47:11.798Z" }, + { url = "https://files.pythonhosted.org/packages/bc/95/2a2853c1ee014bf102116e7f897a04beeaeb2461b45b79af98bdfb95f1ef/librt-0.15.0-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:85ea21ec6730194d67156b0e0b5430ccb1d61f8b8b907e39b37f9812b74a13f0", size = 535812, upload-time = "2026-08-07T10:47:13.279Z" }, + { url = "https://files.pythonhosted.org/packages/c9/4c/cf9601c1b4c5f09280acd5d83abdb2e68527a2be8257136eb42304218622/librt-0.15.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:1e47b8ba865d7ede071a91a7163073bbaeb72541f1ef8a07d512c45c7b5007f2", size = 573688, upload-time = "2026-08-07T10:47:14.727Z" }, + { url = "https://files.pythonhosted.org/packages/47/6d/9ac7cbec46189a7625af4b5acbd25f10d827f4141b2002181848c8418923/librt-0.15.0-cp312-cp312-win32.whl", hash = "sha256:a5207ec414d1c4a2a7231b2086970dc036f94293cdf338190984958a013a42f1", size = 106138, upload-time = "2026-08-07T10:47:15.973Z" }, + { url = "https://files.pythonhosted.org/packages/38/d0/2ae99c83be86ce23f925ac1aeeedc777e97f427c4a8d190c70d0a16e9a87/librt-0.15.0-cp312-cp312-win_amd64.whl", hash = "sha256:73b30cfa976659b3917c8f6153bdb0591c6a9ec6583599fd24a689b690622022", size = 126974, upload-time = "2026-08-07T10:47:17.049Z" }, + { url = "https://files.pythonhosted.org/packages/5d/ef/dd24f9635c730b86b87587967dda7516b1845e8b17684603d31607fed598/librt-0.15.0-cp312-cp312-win_arm64.whl", hash = "sha256:a54cf9e0ef47b96af580849db5471142200568ce1e02cbf416addab551369570", size = 112292, upload-time = "2026-08-07T10:47:18.222Z" }, + { url = "https://files.pythonhosted.org/packages/e7/42/467b53a601b406ccd7b97c1fd54b59cb34f9185ad5ce7e9d5c3c4e8961c8/librt-0.15.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:db13ca398005abcbe538deda87b686d9bd08b7001cf40c4c06b444960ae10a26", size = 151029, upload-time = "2026-08-07T10:47:19.312Z" }, + { url = "https://files.pythonhosted.org/packages/3e/e6/36c2299b7a94b84fdd01220d8a777a71be5be0925bb0dbdf71c0a06a34d9/librt-0.15.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:aa1f1995789dca3698bc550aaceb09a51bd5df0a057ff84ff15296cd1975b801", size = 155194, upload-time = "2026-08-07T10:47:20.398Z" }, + { url = "https://files.pythonhosted.org/packages/c9/b6/ed5071f9325845e670bd36012757419767fbf56af77ed483077b9e4db541/librt-0.15.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:55456ea87d8df21808446d03817be2f65e20391c1c615d9187440dff28cd08dc", size = 502568, upload-time = "2026-08-07T10:47:21.652Z" }, + { url = "https://files.pythonhosted.org/packages/7f/81/6450c67c3615d87704bcbc21323fafc69c799b06a044c447529f725d4b01/librt-0.15.0-cp313-cp313-manylinux2014_i686.manylinux_2_17_i686.manylinux_2_28_i686.whl", hash = "sha256:5a86a5a08c2235316bdb359d5dbb6ce0abfca7fac06363103e2c5af571d92f95", size = 496153, upload-time = "2026-08-07T10:47:22.925Z" }, + { url = "https://files.pythonhosted.org/packages/e1/d6/5f52b722bc75076954b3bfd49be15ea362df4d580c6fb315d0f617100d30/librt-0.15.0-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:e56b6a368529bed262da40ce13f8fef590db0479819cca84f16a1f01ac356d0b", size = 513336, upload-time = "2026-08-07T10:47:24.213Z" }, + { url = "https://files.pythonhosted.org/packages/8d/e2/c08fd1d36ce63ea5a12b85c5d37f4550b5f86a692167e41e5a74222607ae/librt-0.15.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:234d8d394721fa0d786af15ebf1f3fb7f3ed82fd1cd0cde45c2f247b5d4281d2", size = 531661, upload-time = "2026-08-07T10:47:25.507Z" }, + { url = "https://files.pythonhosted.org/packages/3f/d8/d9482fcbeb177b9eb87bb3899eeb3b42be690313c652f9e146b1d0681fb2/librt-0.15.0-cp313-cp313-manylinux_2_34_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:d8363d7accb0286ac3a0e633f396e93800dafb8150494505daf9515bbda591f3", size = 524487, upload-time = "2026-08-07T10:47:26.79Z" }, + { url = "https://files.pythonhosted.org/packages/10/cc/075171517b41f861753034fbb151b42cfc83bcc853849f24f5e66fd60ccf/librt-0.15.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:0f0ee3644d951f31055ad07d77d92520e84505dd7a432cc4cd501dd70ee06785", size = 543201, upload-time = "2026-08-07T10:47:27.999Z" }, + { url = "https://files.pythonhosted.org/packages/b0/03/42c2330f37eeb475b6affeedd06518f60035f323af3a839335e3fc9fef2d/librt-0.15.0-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:2cfd1a81a648806e6a7717be4cc4d1bb392fa229752bf8444ba365e381e984d6", size = 546467, upload-time = "2026-08-07T10:47:29.396Z" }, + { url = "https://files.pythonhosted.org/packages/57/1e/1ad4c5638f7e64d8560328bd25c54b409a661bdb6ff254b38ff90744288d/librt-0.15.0-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:a6cd22c9da0d866558e46a041f1cc0c2bbb26b61b137b2347fa834c332e1d101", size = 555139, upload-time = "2026-08-07T10:47:30.815Z" }, + { url = "https://files.pythonhosted.org/packages/49/41/39fa7d15db1204cd1cbe6514680fbdc243adf754a0885061308f43afc013/librt-0.15.0-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:6d5225ef8801e4ea5e482fa9b5dfb891dd9ef6f6d870f1f25d449ca2c70ac218", size = 536050, upload-time = "2026-08-07T10:47:32.222Z" }, + { url = "https://files.pythonhosted.org/packages/1e/88/c6dcf0dd8e26dc0c9a499a2abab8646c86dcaf9ecea9524cb46d3686331a/librt-0.15.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:6d28a05796b99f749bf8794f17ba9ba1612d0076b802e9cfc62c554634e9ce3b", size = 573700, upload-time = "2026-08-07T10:47:33.527Z" }, + { url = "https://files.pythonhosted.org/packages/1b/9b/ab54c71a7918a7c34fa5327fb61390a77446a07a146fbfb1165250a61035/librt-0.15.0-cp313-cp313-pyemscripten_2025_0_wasm32.whl", hash = "sha256:2067ff438048cead9d223ca5675bae2a25e520a7c3e6c1498bf9c6892d22caab", size = 82194, upload-time = "2026-08-07T10:47:34.835Z" }, + { url = "https://files.pythonhosted.org/packages/8d/b2/4f9a243bb892395f3becb80789ade13771701091f9f07ab8230247953ba8/librt-0.15.0-cp313-cp313-win32.whl", hash = "sha256:1cd3b721f24c206398b9e26da3c3a9c011e6e89d06f318ba8ebefc30f1003890", size = 106231, upload-time = "2026-08-07T10:47:36.251Z" }, + { url = "https://files.pythonhosted.org/packages/bf/af/64aff4885a40b93132382f2c314647d722574605416504379184ef3045ea/librt-0.15.0-cp313-cp313-win_amd64.whl", hash = "sha256:f395a4a9a03ac062dbe9a9f82e0c720502e590a38feee6a757bc82e9c63afbd8", size = 126996, upload-time = "2026-08-07T10:47:37.453Z" }, + { url = "https://files.pythonhosted.org/packages/27/83/335bccf6c7cb9028cb0b54aead27d9ece3f01f83bc6baa2abace5da655c1/librt-0.15.0-cp313-cp313-win_arm64.whl", hash = "sha256:0a15cb554761247d84a3ec0cbdf4078d70725384f0e4662c0fa3b26266eb60ad", size = 112188, upload-time = "2026-08-07T10:47:38.729Z" }, + { url = "https://files.pythonhosted.org/packages/a8/93/949053fb462eecc4a9a5ee770a81f4b40be7b79538b245545d4aebc6b58b/librt-0.15.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:f5de7feedc56337a088eb15cd9fafa9938367362221d8cc62c642b7f94821993", size = 149833, upload-time = "2026-08-07T10:47:39.86Z" }, + { url = "https://files.pythonhosted.org/packages/61/ca/8281aa6cd560a3420e4497729f6b704b53be3eeaaef82d5aeadddaf7441f/librt-0.15.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:6c0eb900c0e91f4aebe680845242e614f1864edfd44106380d0752ac29522bf8", size = 154088, upload-time = "2026-08-07T10:47:41.065Z" }, + { url = "https://files.pythonhosted.org/packages/dd/02/1a1662dceaba6a086360891448d5ce9a7d3555976cae59a31a39d744b9c7/librt-0.15.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e8c9a650a188e38bac005048cbe6342e81407782944d01934540ab75e417df21", size = 494215, upload-time = "2026-08-07T10:47:42.388Z" }, + { url = "https://files.pythonhosted.org/packages/69/84/99211619dc656370a3740c33d2b0b6d5a3fb1e73689314f6ed477a397dc4/librt-0.15.0-cp314-cp314-manylinux2014_i686.manylinux_2_17_i686.manylinux_2_28_i686.whl", hash = "sha256:92bfed8deec93df30286b9fe9e3b1dd17329cc076a192b4ee5ec223841d54953", size = 491173, upload-time = "2026-08-07T10:47:43.683Z" }, + { url = "https://files.pythonhosted.org/packages/d4/aa/5448d0b05f4579b635d3899176817ebf561af0e57bacd425b5b1887264c1/librt-0.15.0-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:ec4b19788f835711a2072f9dbe6b03b3bf32ed1f0fb30cf399bdd59d9f0c33fa", size = 505512, upload-time = "2026-08-07T10:47:45.314Z" }, + { url = "https://files.pythonhosted.org/packages/95/82/01940e40b83c43a546c4a3c896cf34ca272a9690899d55914e4827b3dcce/librt-0.15.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d4c7bacb70930f3d0a56f4ecf1be474a1f0d941b01dd73b756f3c256d42cb879", size = 523073, upload-time = "2026-08-07T10:47:46.66Z" }, + { url = "https://files.pythonhosted.org/packages/88/fa/759c0030f3ee371439eb26de34fc745807caf0abb878af7af4b8b7c3dd3d/librt-0.15.0-cp314-cp314-manylinux_2_34_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:3e79f05e4a08b4d880342673312bbc895b56df7765605796f15902eb5367d3ae", size = 515080, upload-time = "2026-08-07T10:47:48.319Z" }, + { url = "https://files.pythonhosted.org/packages/0b/27/894e072228fcb159703c655da69f8cd10dbed489c36e3df7dd032a2483be/librt-0.15.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:a417149c0cba4d50b61e992e5a15e69eaf96746609b461cc4ed168aeef6b79dd", size = 534164, upload-time = "2026-08-07T10:47:49.875Z" }, + { url = "https://files.pythonhosted.org/packages/98/a3/0078e91c1f36f8815db17827de15650b9a3fe56c55fbf998c854b34e40d3/librt-0.15.0-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:da7a94d6a3411f579d72aa3e3bc5fbca7ed4549f3dbd7e5de3aa567333374285", size = 540616, upload-time = "2026-08-07T10:47:51.408Z" }, + { url = "https://files.pythonhosted.org/packages/86/33/81a29b796dd52a45e9ef7974c7732926e8f10f15b8d2be505665979f896d/librt-0.15.0-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:856f743ae607f2c1380eccb566c0038a9fb3eabf0fc2be2704d76d9f73557239", size = 545890, upload-time = "2026-08-07T10:47:52.818Z" }, + { url = "https://files.pythonhosted.org/packages/05/82/8be1baa1350e5d30cfd70ae79d0a6f4dc5862ef47f7bb2808aabc9bb86e5/librt-0.15.0-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:779a6e7c894737e5983e7790a9c78c4000c30e23c9aada08081bdbea53b0fa60", size = 523287, upload-time = "2026-08-07T10:47:54.165Z" }, + { url = "https://files.pythonhosted.org/packages/c6/4f/d1be6a01a35c20ef734e0e44113f87d4af756a9354a89dcfbe3b4f8af5e1/librt-0.15.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:96bb17dbe8bab3c0954fbebfc69ed395599de75b6bbc35e3270a878e15d4dd65", size = 565868, upload-time = "2026-08-07T10:47:55.566Z" }, + { url = "https://files.pythonhosted.org/packages/67/88/649cfa33f5825927b160610f670bdab012a64d627eddb94fa795ea4292fd/librt-0.15.0-cp314-cp314-pyemscripten_2026_0_wasm32.whl", hash = "sha256:7220697efaa6e5348fc3d18ee7f8563d4bfecd9872b37ffb915bfc1d08840622", size = 81619, upload-time = "2026-08-07T10:47:56.886Z" }, + { url = "https://files.pythonhosted.org/packages/22/31/8e88a8d5e48fc8d1a817787fb6811dfff6499acd6c8683dd83934aa6ede0/librt-0.15.0-cp314-cp314-win32.whl", hash = "sha256:f54598964d357b1c5ab77cf5d92f21e598fe0e23cdbe9618480807f81b4eba15", size = 100138, upload-time = "2026-08-07T10:47:58.093Z" }, + { url = "https://files.pythonhosted.org/packages/80/92/20fd6c4b6a1b1a564b076d55cd3d427d8428217d7638dc25a654cc4791d4/librt-0.15.0-cp314-cp314-win_amd64.whl", hash = "sha256:3ff5893a2c23d886aa9ce786de5ac6ddc74aeeaf90743682b74d920e117d2e28", size = 121258, upload-time = "2026-08-07T10:47:59.564Z" }, + { url = "https://files.pythonhosted.org/packages/fc/28/6af430b44d9ebb897b865a3c363b6dcace51357be2347cc0f8f869656a86/librt-0.15.0-cp314-cp314-win_arm64.whl", hash = "sha256:3722a099730704c9a3d70c879fc0f51daec25fe5f1555672d97bc595abeafb95", size = 106467, upload-time = "2026-08-07T10:48:01.097Z" }, + { url = "https://files.pythonhosted.org/packages/7e/aa/b42bb798942ced219f6d63b27e07f91237887a8d0bd0921666db79a13790/librt-0.15.0-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:38c0c7d4b6fc06c3324b3f9162c8391bfc4fd9dde53afe1033ce7edb48d5a714", size = 159523, upload-time = "2026-08-07T10:48:02.442Z" }, + { url = "https://files.pythonhosted.org/packages/75/03/1b53cd4ef904e73b1d828a5f90143bf94a2967d7cfff0b9ccf93e12aa9b4/librt-0.15.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:8b2fdd7ead3c995c37940a790690660d0ca006c302db26cc51933f6766866fc3", size = 161638, upload-time = "2026-08-07T10:48:03.725Z" }, + { url = "https://files.pythonhosted.org/packages/ac/c4/9f9c9fba097d49e9e694c2b4dc331df31884645ecbc58a93b4b5fc69d2c5/librt-0.15.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2fde98cf1fc4bac144ce23c2c4c017b924ba714509ea9334977b0b27050c837d", size = 701795, upload-time = "2026-08-07T10:48:05.135Z" }, + { url = "https://files.pythonhosted.org/packages/4c/05/0966840bda0380c8ae167b9043c6230202941cc90ea29c48e096964c765e/librt-0.15.0-cp314-cp314t-manylinux2014_i686.manylinux_2_17_i686.manylinux_2_28_i686.whl", hash = "sha256:e3b461183c5fa7681b48560f91515f53a953122fb30c71e07abc67d7ddf58c38", size = 682147, upload-time = "2026-08-07T10:48:06.555Z" }, + { url = "https://files.pythonhosted.org/packages/18/af/1c47ca573c30ea47d195aec26133af522fea1104afaace028d7b32247ea8/librt-0.15.0-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:4bbcc257e3babea20a91715c361b24554ec4e8f51aa578568afc230799fe1a19", size = 696397, upload-time = "2026-08-07T10:48:08.03Z" }, + { url = "https://files.pythonhosted.org/packages/2e/0f/1aed6223d4f9f9d1171a8596ff100ea4c3f7699fea7a4ba657c3e60daa6c/librt-0.15.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b845b8d48088fad0cadc84be4b8fda63203be7e9237b71015b3925443c1f35ab", size = 722542, upload-time = "2026-08-07T10:48:09.569Z" }, + { url = "https://files.pythonhosted.org/packages/c6/22/9e3a929aea456c97d69e6ef3884efea56d4807f97399471cc946baebd8af/librt-0.15.0-cp314-cp314t-manylinux_2_34_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:b30e600e8f337b9bd7f39b86d9fdfedc73cc46e3d0f745931a23a234220bb7e2", size = 729709, upload-time = "2026-08-07T10:48:11.129Z" }, + { url = "https://files.pythonhosted.org/packages/e9/1b/c327ef6018e3a9ca0b8e7c5eddeeb331ba8f9b76c24e126d37d0f6d62faf/librt-0.15.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:64b0c8c35aa4c4ed79896359f3e0b285cbe4e610042106500da4811c322cc108", size = 752891, upload-time = "2026-08-07T10:48:12.558Z" }, + { url = "https://files.pythonhosted.org/packages/d7/d1/d5f1ea02c56930087009e39db9b70660a663e76c730b27b925d786718457/librt-0.15.0-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:0da0d94cb802f32a0524653e7201f2cef72d5f700a5407678f5290483d4fcd08", size = 745301, upload-time = "2026-08-07T10:48:14.55Z" }, + { url = "https://files.pythonhosted.org/packages/d9/3c/5f7c585d15ebb2250c73e7c0ee4e9e47be72c65d520c07ddbcdc62037674/librt-0.15.0-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:4a6369168d371207339b1e50d4532b06a7121586141f82599505a3f315751d47", size = 747921, upload-time = "2026-08-07T10:48:16.453Z" }, + { url = "https://files.pythonhosted.org/packages/7f/52/1443a446486eba966bcbca1696b472e4f210320ec42f490a47f48fbf0fdc/librt-0.15.0-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:c434e072557ade9cbc642d052c89d031efe47d5c9614523619d0d74a02378e81", size = 727561, upload-time = "2026-08-07T10:48:18.089Z" }, + { url = "https://files.pythonhosted.org/packages/79/91/2270a9380f11725cf83ce1925a5e32dd1dde2be9bba597f25c10a38644e7/librt-0.15.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:c7eec6a42018bc1d45763b1c162d3d2bf7c3b9a1b0ed30d3e91dcba390efefcc", size = 774417, upload-time = "2026-08-07T10:48:19.611Z" }, + { url = "https://files.pythonhosted.org/packages/9e/3b/f4b1548d4f5b99186737fe27aec238e9823e8d5d23bf4df007c030689dc5/librt-0.15.0-cp314-cp314t-win32.whl", hash = "sha256:6912fa5e635d74529ac7cdb1bdf6ca3af4453da8d1edbe0110ee1cb4ad407ebf", size = 104381, upload-time = "2026-08-07T10:48:21.048Z" }, + { url = "https://files.pythonhosted.org/packages/80/b6/134afad262def1de04c0843c376d02135f1168af43f22e09a52bd8394727/librt-0.15.0-cp314-cp314t-win_amd64.whl", hash = "sha256:8e11699ed745931c395acd3621b07062e0f840efa6935aad87a64ed0995f0915", size = 127034, upload-time = "2026-08-07T10:48:22.561Z" }, + { url = "https://files.pythonhosted.org/packages/99/5f/1b6846b20572bd699c9e9ec321a5f781845bee477df2aa2a43b28bc40119/librt-0.15.0-cp314-cp314t-win_arm64.whl", hash = "sha256:5d2a91724463bfed4f573cd7a9fdc856d2e230d0c0e5a61416a93481dccd8605", size = 110827, upload-time = "2026-08-07T10:48:23.804Z" }, + { url = "https://files.pythonhosted.org/packages/c6/44/4de9f4ddadb009a55c7758eb5736d62534a7daaf27bd71bc50e64b606b06/librt-0.15.0-cp315-cp315-macosx_10_15_x86_64.whl", hash = "sha256:8443e38dcfcfdbcf5add5118c623efd788d65ac2e25756d6251a54a06a4d0aca", size = 149843, upload-time = "2026-08-07T10:48:25.148Z" }, + { url = "https://files.pythonhosted.org/packages/1f/eb/5d9ab71e30119c44094e0275f38b47dd327aea0f843a080396677029d508/librt-0.15.0-cp315-cp315-macosx_11_0_arm64.whl", hash = "sha256:6d15a29033c57490cfe2069097c6fc4049e4e65ffbb749be7dc453b7c4c68965", size = 154510, upload-time = "2026-08-07T10:48:26.485Z" }, + { url = "https://files.pythonhosted.org/packages/d0/9c/8505d1b8f5e8c19587bd03f7429993b3e9ce5c06819d856bfb11d919374c/librt-0.15.0-cp315-cp315-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d2c05c729b589e734c09578bf5964be48a911765484840d017bbc84f49d4c4ad", size = 497543, upload-time = "2026-08-07T10:48:28.045Z" }, + { url = "https://files.pythonhosted.org/packages/1d/9a/3a8390775cb095765aded027ac9c63e7c8ea74e731498607544c6505de0e/librt-0.15.0-cp315-cp315-manylinux2014_i686.manylinux_2_17_i686.manylinux_2_28_i686.whl", hash = "sha256:fa60887537e1d0cd2d9982269d33a709bf54b195cd2b9364fc0a758022af5bd9", size = 480452, upload-time = "2026-08-07T10:48:29.531Z" }, + { url = "https://files.pythonhosted.org/packages/e7/40/258a4a7117ee915d66de5cd9b8ade65a440993161107ce3a686f1859955c/librt-0.15.0-cp315-cp315-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:d8bc24219b24c0af375718942ab75e3544b2763085f40f965be4326734ae8328", size = 507768, upload-time = "2026-08-07T10:48:31.007Z" }, + { url = "https://files.pythonhosted.org/packages/6b/c6/2f4dd296c97a0b85b98894519b279408ec9dd602d4f692b1ea0e25dee670/librt-0.15.0-cp315-cp315-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:86a21a7bd3fe3a419512ef424cc1c020f6771d0b29cfddff36d1635a855e63f0", size = 525122, upload-time = "2026-08-07T10:48:32.7Z" }, + { url = "https://files.pythonhosted.org/packages/49/dd/29eab42be13b2bf0ea8cb227135a45d44693e30a7e8b92871981ff56b82b/librt-0.15.0-cp315-cp315-manylinux_2_34_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:dbab647e88d90b3167b91efe7091e248653688ed4337e4f90907a722c7361bb9", size = 520371, upload-time = "2026-08-07T10:48:34.294Z" }, + { url = "https://files.pythonhosted.org/packages/91/ed/4bad71adeca8fe208b775c2a35417fa5a2584c8f4791daaf89a89450fea1/librt-0.15.0-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:d8edcf6f550e918dca779c069b9e156385c60b406f99fc7641f32c52f7193659", size = 537258, upload-time = "2026-08-07T10:48:35.88Z" }, + { url = "https://files.pythonhosted.org/packages/4c/63/59dba6143fdcc7240c54458b629f3250000a61b8945890fc9efd451b19c5/librt-0.15.0-cp315-cp315-musllinux_1_2_i686.whl", hash = "sha256:8b62076030baa2d8b1501a46bf0e19c27a489aa90671c55665bff7887f7660b0", size = 527432, upload-time = "2026-08-07T10:48:37.466Z" }, + { url = "https://files.pythonhosted.org/packages/ec/21/21a24c6a2327d8362580efebe77286bf47b0f4062ec5ea41766e609d3c7d/librt-0.15.0-cp315-cp315-musllinux_1_2_ppc64le.whl", hash = "sha256:d00d20d1818e82a07a0ee0aa89a98b17ed7916b92441090b683719cb20a59b6d", size = 548108, upload-time = "2026-08-07T10:48:39.384Z" }, + { url = "https://files.pythonhosted.org/packages/5a/6d/fc68c89a7971418b41f9a873623ff935cb864097544c6a2f8ce491c8ef5d/librt-0.15.0-cp315-cp315-musllinux_1_2_riscv64.whl", hash = "sha256:4e6ee93fc3cf848dcbf0cce2eca73d8e7dcd0cc2b6df3a529d57750b30a4c55c", size = 529681, upload-time = "2026-08-07T10:48:41.392Z" }, + { url = "https://files.pythonhosted.org/packages/65/7e/c2d98766124400d722063a630b0fde38a9fc768705d37eecca15c47dc192/librt-0.15.0-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:32896a0af72508ea979e0acb4e4c04cbeeae04938167950d535c83c45597167d", size = 567736, upload-time = "2026-08-07T10:48:43.124Z" }, + { url = "https://files.pythonhosted.org/packages/55/6c/f8c34a95e3a515c6e1c192b89511e7253c89a7760c6b500d57ffdb8d2dc8/librt-0.15.0-cp315-cp315-pyemscripten_2026_5_wasm32.whl", hash = "sha256:ec3ba415afaf951f6951b1dd16d3c8e4f540065fc382d7e70b823a79567ca374", size = 81673, upload-time = "2026-08-07T10:48:44.645Z" }, + { url = "https://files.pythonhosted.org/packages/c9/9e/e23fa8e78679ec45728188650b39e8ff476c83b691c96f749217df3b1b7c/librt-0.15.0-cp315-cp315-win32.whl", hash = "sha256:d2813ba2503764f0450680c533d13df7cff9b49df1411062eded5f67db4195b9", size = 100081, upload-time = "2026-08-07T10:48:46.171Z" }, + { url = "https://files.pythonhosted.org/packages/e1/dc/3eb4c5e297343f0620a55532cd7c8d764d3001fa2159212dadf480464827/librt-0.15.0-cp315-cp315-win_amd64.whl", hash = "sha256:b87d67e33afaf265262f2a66db578284b88ee2e6fcd224579cb5c15518677ad8", size = 121228, upload-time = "2026-08-07T10:48:47.631Z" }, + { url = "https://files.pythonhosted.org/packages/97/70/43abce19f04e49762f8ec834c8fafee13cc40fd6b94a72a24e534febfcd0/librt-0.15.0-cp315-cp315-win_arm64.whl", hash = "sha256:713bd7df21170b982e729e46870f31d6b437bd1a9b4648cffb529bd3c2ec5c4b", size = 106487, upload-time = "2026-08-07T10:48:49.095Z" }, + { url = "https://files.pythonhosted.org/packages/de/15/83f2deddb9368b8951ec8c9477269b5b9b8bd9bbf15e57402d0f38817dca/librt-0.15.0-cp315-cp315t-macosx_10_15_x86_64.whl", hash = "sha256:3de789c82752730f94782a5ee518baf9c05edf85733aeaf73bb6e518755cdf54", size = 159448, upload-time = "2026-08-07T10:48:50.649Z" }, + { url = "https://files.pythonhosted.org/packages/06/bf/043097353f9b3c73b583d07f6b8e552795463f4bfc8caf85e42eee50c26a/librt-0.15.0-cp315-cp315t-macosx_11_0_arm64.whl", hash = "sha256:e0b5deec9a8664eb722c797241970fd4aa1894d25fda36a1ddac0f7407606bd6", size = 161686, upload-time = "2026-08-07T10:48:52.174Z" }, + { url = "https://files.pythonhosted.org/packages/f4/2a/8ae77f9719d42ce71cd708560a3557b38ac3c17a0383e57f87084de45bbe/librt-0.15.0-cp315-cp315t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5563302a8359bc2295bb7084d1a8ed1519df96afb30eb2aa4e0bff7b54228988", size = 710668, upload-time = "2026-08-07T10:48:53.782Z" }, + { url = "https://files.pythonhosted.org/packages/61/34/c0436ea134deb9a0d6da80a396a2739a81cb31e0418f7227239e23140898/librt-0.15.0-cp315-cp315t-manylinux2014_i686.manylinux_2_17_i686.manylinux_2_28_i686.whl", hash = "sha256:22d6263b9d39d7bbb286fa791945646e3218f1be2d693e36fb630f1d0e59cd13", size = 679396, upload-time = "2026-08-07T10:48:55.645Z" }, + { url = "https://files.pythonhosted.org/packages/4a/9f/001e0d99aa9250d5cd5715a9081291a20656083459f9019cda15255329e1/librt-0.15.0-cp315-cp315t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:39ffd14646190c454f0d86e0d256b33f00a87a26ab410e619773b841d0e41416", size = 704313, upload-time = "2026-08-07T10:48:57.46Z" }, + { url = "https://files.pythonhosted.org/packages/2d/53/b34fa9d0ff00f136f4d58ebb4c411ff634baed1eb412bb602a2bc8dcafcb/librt-0.15.0-cp315-cp315t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c47318cd3a61401452de11282242937e3e057c4fd3dbaf601e269d0928a06c0a", size = 729847, upload-time = "2026-08-07T10:48:59.231Z" }, + { url = "https://files.pythonhosted.org/packages/86/ac/fa4d7a424665040e95baf480a6d523446057684b6758624c85338e8a23b2/librt-0.15.0-cp315-cp315t-manylinux_2_34_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a56a1d4f859a82ca5b99fc4b82c9b027b15e3c455c5cd99e7d0719f27bb20b6c", size = 742736, upload-time = "2026-08-07T10:49:01.151Z" }, + { url = "https://files.pythonhosted.org/packages/8a/f1/e17a9bb5de6fb8c3186ed1a7d68d21618b027ac2d3633e03d3b6109c67ae/librt-0.15.0-cp315-cp315t-musllinux_1_2_aarch64.whl", hash = "sha256:077471b3182db4e17c36ae91555f36a4d2c00080b267f749bcad34a478a9a302", size = 763454, upload-time = "2026-08-07T10:49:03.039Z" }, + { url = "https://files.pythonhosted.org/packages/1d/ec/ecd02cd30935b931b9cdbfed6ab5a099c51b280b4e7baa274da80978ed27/librt-0.15.0-cp315-cp315t-musllinux_1_2_i686.whl", hash = "sha256:411ca4d1b905b860ceba7570dd6717a71dedaddcc4b0f77ece710aa41ee11f8d", size = 743296, upload-time = "2026-08-07T10:49:04.941Z" }, + { url = "https://files.pythonhosted.org/packages/e6/b5/b3c2b8353ce820a4854f78d19321344242f89fa71c975b71132ba9bf242a/librt-0.15.0-cp315-cp315t-musllinux_1_2_ppc64le.whl", hash = "sha256:1256589e0b0adb31751d685a68bce29d73407ddf4ef05d4188f49d5dcf9566d9", size = 756217, upload-time = "2026-08-07T10:49:06.825Z" }, + { url = "https://files.pythonhosted.org/packages/3c/52/6cc22542ba59146b05cca2a656f9ff8bb67e38e63d12c3b0cc183d837bf1/librt-0.15.0-cp315-cp315t-musllinux_1_2_riscv64.whl", hash = "sha256:f42b74a53e5f26a0ba0007411a7455b66c67ce4022a39cc1f56fc4efd65bcbab", size = 741934, upload-time = "2026-08-07T10:49:08.839Z" }, + { url = "https://files.pythonhosted.org/packages/40/32/a04b72b1aa86e3be23b2ecff8c1aad2dcc955bd3956d6d26e7e34267e57a/librt-0.15.0-cp315-cp315t-musllinux_1_2_x86_64.whl", hash = "sha256:291bf73caf78b9e88d6fae9bfd693207ff7d832e2fdbe2cf8e746bc13f5f892b", size = 783763, upload-time = "2026-08-07T10:49:10.661Z" }, + { url = "https://files.pythonhosted.org/packages/6c/f0/89eb11dffbe9279ff37144dec786927314502ae0b114f1449dc78c458aab/librt-0.15.0-cp315-cp315t-win32.whl", hash = "sha256:c16d15ee371643ab48dc8248a3e680ebbeca573a13af2c3dd0c985b142d77162", size = 104313, upload-time = "2026-08-07T10:49:12.305Z" }, + { url = "https://files.pythonhosted.org/packages/6d/4a/1f1978c200f563beda63c36adff2d65bbecb81e365e8e69e572f5f70fbc6/librt-0.15.0-cp315-cp315t-win_amd64.whl", hash = "sha256:dbd605739f228912dc49027cb764456b9757750bdc2b6b7773164db7096c6fd1", size = 126889, upload-time = "2026-08-07T10:49:13.881Z" }, + { url = "https://files.pythonhosted.org/packages/38/a6/800800bfed7b1fb10fc3f3d557785c3854e80d3f7a9800d784b176a1fc2d/librt-0.15.0-cp315-cp315t-win_arm64.whl", hash = "sha256:84d244b00604d17df3fc7736c327892d6bba66181254aa4087be807b6c342bdc", size = 110700, upload-time = "2026-08-07T10:49:15.499Z" }, +] + +[[package]] +name = "mypy" +version = "2.3.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "ast-serialize" }, + { name = "librt", marker = "platform_python_implementation != 'PyPy'" }, + { name = "mypy-extensions" }, + { name = "pathspec" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/82/6a/878cc1097d4035f82bd516658d0c528d2a9955bc7b363afcbd0b07fea11b/mypy-2.3.1.tar.gz", hash = "sha256:47c1b1207258513a9d93495f69c8be9de73916186f0e52703e8c461b7a623419", size = 3992554, upload-time = "2026-08-15T03:03:38.549Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a4/be/c624d4241484f37dc62839e177ab607a9b8b3e96f0866544ca99e8e41d51/mypy-2.3.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:94f04929f1c44c35fb0061e912087edaf504acede963a4a7d00680bd089d8531", size = 13936739, upload-time = "2026-08-15T03:03:26.475Z" }, + { url = "https://files.pythonhosted.org/packages/53/84/e3cf72f90dce5960871c82551c8fba6da05fc1018f79be41c047bd126bdd/mypy-2.3.1-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f5d716048611e85ca9eefb2e1baa5d73ede389b5820ded260ea27c757d667af8", size = 14166460, upload-time = "2026-08-15T03:01:50.565Z" }, + { url = "https://files.pythonhosted.org/packages/4a/ff/6b97d58aa0f79a5ab9b472db1f6d6df1b11a51d74d0c08ab3760d3a613ba/mypy-2.3.1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b091a455111214cb5c9d54a57b9618e9a49f9fe2a42e4e1ac86e9d104ed96ce8", size = 15100476, upload-time = "2026-08-15T03:03:12.079Z" }, + { url = "https://files.pythonhosted.org/packages/da/f0/cbb4b7d2ae3ac635f6b4f2d9b04070b8a92edf50da599d3b39e5ed109001/mypy-2.3.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:df12e20c9efd614738c71b390007ecd0181125afc4ccafca04d78a1d2eed2c01", size = 15347826, upload-time = "2026-08-15T03:03:02.856Z" }, + { url = "https://files.pythonhosted.org/packages/5f/10/91dcdc6f8d43fc08e6a06ab1f9732f3abaaf835ac1b2e67b9dff56910855/mypy-2.3.1-cp311-cp311-win_amd64.whl", hash = "sha256:52eaf3a155f35cf80b40220288c861eb45f14a2340c1f6cbfbdb0feff32879d1", size = 11142615, upload-time = "2026-08-15T03:03:36.316Z" }, + { url = "https://files.pythonhosted.org/packages/3d/8a/28d54535bf4b9aa43b2d8918c2ef660378b9f66b23d78dcee052744ae622/mypy-2.3.1-cp311-cp311-win_arm64.whl", hash = "sha256:9b4eacbee8a69836c06eff6d0dd4e134a07c2b047755b30c08625fe214f322c6", size = 10141145, upload-time = "2026-08-15T03:03:07.406Z" }, + { url = "https://files.pythonhosted.org/packages/85/da/d6effc4f808a842d91edc22535dc9e799d2ff6e91449168b7f47a0771f54/mypy-2.3.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:a32bbbb940af990d3be0b8af321c7b6815bb1b3b48142fe7459b9cc5f58959ff", size = 14047547, upload-time = "2026-08-15T03:02:57.707Z" }, + { url = "https://files.pythonhosted.org/packages/e4/e6/478229701dab76f26485fc8ff5d6f241f393da22447400bbc56f6946aebe/mypy-2.3.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ff715e45b2231a8e85de1d163d1b42791e4d7aab8f5145f85fee1b710b735aff", size = 14216515, upload-time = "2026-08-15T03:01:26.496Z" }, + { url = "https://files.pythonhosted.org/packages/8d/fe/7c42327a3b21e84681f691982cbfe43f334a3685f3b683b72c376476c4fa/mypy-2.3.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:858fc57d3d91fa728e33e7ad71def60fc6272694607b306cd3292db53ae39080", size = 15307789, upload-time = "2026-08-15T03:03:31.62Z" }, + { url = "https://files.pythonhosted.org/packages/59/f4/7e597edbe01b5a56fa958ce541302dcaabfed979966f1dffedbea0ea0fc2/mypy-2.3.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:851833db876e7b650f93719c74b7879a08e338979c96054fdfc3bfd90a486355", size = 15548831, upload-time = "2026-08-15T03:03:15.55Z" }, + { url = "https://files.pythonhosted.org/packages/a3/52/cb31e084bc0314a1e384bdd677a4b80e55af04ccac077545e2238b9d320a/mypy-2.3.1-cp312-cp312-win_amd64.whl", hash = "sha256:4c5095a327483591c94e0c8d3ef9e50d4ab1369b541eae007c1f23bc2a41f6bb", size = 11226359, upload-time = "2026-08-15T03:03:29.002Z" }, + { url = "https://files.pythonhosted.org/packages/7a/47/88fcf6217b43fa2da81a8c2611370af18141536a4f0294bbf98b457d456d/mypy-2.3.1-cp312-cp312-win_arm64.whl", hash = "sha256:bbfe022634a2a195406bd469e888d2eaf193b02ba7e607391cd7640374aaae3b", size = 10214707, upload-time = "2026-08-15T03:02:48.807Z" }, + { url = "https://files.pythonhosted.org/packages/de/cf/862010ee800ca9c2bd0c4c0dacf0f092e5411824a09b8f97ad4be8fe250e/mypy-2.3.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:114dff494000f18bd10d5d95d84b8567b26da60279ecbe838131841df20e635d", size = 13964542, upload-time = "2026-08-15T03:02:21.43Z" }, + { url = "https://files.pythonhosted.org/packages/75/5a/3f3a2107b41e3e92e617e25daaee121413b91e9784bea733131ed4fecc5d/mypy-2.3.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c8637731bb5eee3671eb2c3200827aa3564ed8a9309ecee4d1afe77e6d031bdb", size = 14168922, upload-time = "2026-08-15T03:03:00.351Z" }, + { url = "https://files.pythonhosted.org/packages/8b/41/04dc4fe7e63d7820fa4eff272e95157d30cbea921388f3ab3fe77794cd0b/mypy-2.3.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1c80fbc405ed8020f5ff3802dc18cf060197bcdd3fbdd6a26ef2fd34dfdd5226", size = 15244791, upload-time = "2026-08-15T03:02:31.089Z" }, + { url = "https://files.pythonhosted.org/packages/96/fc/c3053b26b9054949285aa868cb6af8c10e7591541cacd79c5dcc06a1fcf9/mypy-2.3.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:84081f538ce27375045c02e3d7f81bd11d853400621ae245d87ce7b6c420ec74", size = 15501627, upload-time = "2026-08-15T03:03:34.128Z" }, + { url = "https://files.pythonhosted.org/packages/70/4e/d77daab008bbc4e5001374d7928f4a260d28f0e6747af444fc4763f7a310/mypy-2.3.1-cp313-cp313-win_amd64.whl", hash = "sha256:e9144ac16fde007096f9563eb2041b4433c2d705c4218edeb79e7e9d01035ee6", size = 11243961, upload-time = "2026-08-15T03:02:11.952Z" }, + { url = "https://files.pythonhosted.org/packages/f0/f8/7eb68c136e4abd30569fe31ef2bfcb7eceae9952cab80017c04cd09f5d0c/mypy-2.3.1-cp313-cp313-win_arm64.whl", hash = "sha256:77ad9529e67dca28e511f5cd5671436584ce91f6d3bac159a353158187b986ac", size = 10213219, upload-time = "2026-08-15T03:02:26.361Z" }, + { url = "https://files.pythonhosted.org/packages/be/c4/42a49d44aeff804edf1b19acce0b49e8bd1a9c57dee9605dd8d980aa43d7/mypy-2.3.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:192abaedf75da1bc0b1cef104927e70ec49c1ef0031cc4825c7ee10a438ed24d", size = 13986778, upload-time = "2026-08-15T03:01:33.69Z" }, + { url = "https://files.pythonhosted.org/packages/45/13/9331fd2dfed7194d66c5304072894a8be3e51e9deda6863c1eceaa35a43d/mypy-2.3.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:bf678dffd16efcda2c15cbd30e9ecc0081388e29ea23687a88e686ed92638dc3", size = 14188467, upload-time = "2026-08-15T03:02:40.554Z" }, + { url = "https://files.pythonhosted.org/packages/78/f7/f4a34edab45667c5465855dc585a20e87978ffa8aee711445b7239d120c6/mypy-2.3.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8e036f06b41630f4c8a1d48f9ac6aa26acc65f8be089973f5519da643318f03f", size = 15225538, upload-time = "2026-08-15T03:03:09.761Z" }, + { url = "https://files.pythonhosted.org/packages/40/05/534b3590757bd05794f73e07f6666c2a77b8597ffed795c94ce570096aa0/mypy-2.3.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:71af9c8a894e862b58e92abb08e53b05a384a1e5e5d6dc7cda59126211a53d82", size = 15480805, upload-time = "2026-08-15T03:01:41.134Z" }, + { url = "https://files.pythonhosted.org/packages/55/da/bdfba852e2562f599624af5bb7d29e36b0b4f526f2b8bac85efe0dd1803d/mypy-2.3.1-cp314-cp314-pyemscripten_2026_0_wasm32.whl", hash = "sha256:3c80cd23d85368bdd9f37d5231dfd97d35bcbf5bf41af96ef3a9b078ad1957f9", size = 7761712, upload-time = "2026-08-15T03:02:36.008Z" }, + { url = "https://files.pythonhosted.org/packages/98/31/60fc64a74cdba4f2a5d642d32317993e479163e1ac7d91b695e5d15e2264/mypy-2.3.1-cp314-cp314-win_amd64.whl", hash = "sha256:4956f34d145e145562a0a0bf367f642bbc85c04ec2baf47ae015947c3169a85d", size = 11423968, upload-time = "2026-08-15T03:02:06.931Z" }, + { url = "https://files.pythonhosted.org/packages/a9/23/eb5950b24cd26ba3b78f87707a275568d633c77dae8e61c9661be6055ca6/mypy-2.3.1-cp314-cp314-win_arm64.whl", hash = "sha256:cfb12e360242d23d91f5e978d94f58ea66acf5804c4fb6f2f794a20d4cb1b595", size = 10399323, upload-time = "2026-08-15T03:02:33.671Z" }, + { url = "https://files.pythonhosted.org/packages/82/c7/f80f4e46c0b9a00eb5f78a79d49dda8bdf56a5230f7257fb33e76be04da7/mypy-2.3.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:e5f1c50bb05b64e2026b52867e8d21106f01313c744a2c4ecc34c90d12e8d6e2", size = 15121308, upload-time = "2026-08-15T03:01:46.053Z" }, + { url = "https://files.pythonhosted.org/packages/5d/74/9b04f17c7074cc5188f02fb63a2ca1d43fedf479e84fe3091c39061a1d7f/mypy-2.3.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:667196b352f4cf304ded4c10f90cfc179263a1acfb3cdcfa984bdfd340d498bc", size = 15536590, upload-time = "2026-08-15T03:01:35.941Z" }, + { url = "https://files.pythonhosted.org/packages/26/04/c837ef6208e567774e2ed1f863f8ba6ec4817b1b6dd426315e5d559b6ec9/mypy-2.3.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b9c53e395c12cad2c6d4b67d5da7c6057638a132d85c08b73646b18f802a0045", size = 16791074, upload-time = "2026-08-15T03:01:31.073Z" }, + { url = "https://files.pythonhosted.org/packages/37/68/48730230afa45192d5bd429a6a2ff24a6f8dedda90fdf2b221792b54518f/mypy-2.3.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:18162b128c3f9c703cd35f5537446900b0d21a2549aa7a95d21380d2ef643fb0", size = 17069183, upload-time = "2026-08-15T03:02:28.566Z" }, + { url = "https://files.pythonhosted.org/packages/1c/ea/ca23fc9c20eeda09a15c9cbcf50015d0e73f409f6ead059e42aa69a608ff/mypy-2.3.1-cp314-cp314t-win_amd64.whl", hash = "sha256:30c0477d4aab7b7f39c8397dc877f2c96b9fe5588ec379f372c56eb63d599f63", size = 12154679, upload-time = "2026-08-15T03:02:04.809Z" }, + { url = "https://files.pythonhosted.org/packages/3b/67/8d982126034990869466f73b8db80dcb2234a7ac39b4dad093e047a79835/mypy-2.3.1-cp314-cp314t-win_arm64.whl", hash = "sha256:6941ab3619377bc3f32ca02876b07d27f216f5201604b664d3937ea0fdd23bb4", size = 10969159, upload-time = "2026-08-15T03:02:38.152Z" }, + { url = "https://files.pythonhosted.org/packages/ee/f7/41e7f2d8117fbc7a7587286162ffe2f688984b69c46ed63cf5f2e4fc3bae/mypy-2.3.1-cp315-cp315-macosx_11_0_arm64.whl", hash = "sha256:6f041a6de52c9217ca125e78ba0a335cb7fd98a1c0580978e49ab2b126f70b57", size = 13990694, upload-time = "2026-08-15T03:03:21.919Z" }, + { url = "https://files.pythonhosted.org/packages/06/85/8f665811a0c8f3bf6fa1d9acd665ec2d97a2bcc453ae68dcd92340941cd6/mypy-2.3.1-cp315-cp315-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5159ae60f5dbc3a498af5ba8365505808ac8031bc63f9e00304ad545d40bdd9b", size = 14203518, upload-time = "2026-08-15T03:01:48.455Z" }, + { url = "https://files.pythonhosted.org/packages/2d/82/91b866c8546b120bff83b73a439d90d2d63ef3aff113599e6b8e4d566848/mypy-2.3.1-cp315-cp315-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:47a8a7a0a7f6f6e63995c0ac36fa0c07b127413fdc81f0439b7f3dccafd33561", size = 15220224, upload-time = "2026-08-15T03:01:23.577Z" }, + { url = "https://files.pythonhosted.org/packages/c8/78/c226c99208ee40de7c768369fa533f933afa003dfdc606ff021450724e91/mypy-2.3.1-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:2329c0501293d4e1f33bc15d04d6304d65a1cdda967ee93a05c1e681a3923133", size = 15501512, upload-time = "2026-08-15T03:02:09.453Z" }, + { url = "https://files.pythonhosted.org/packages/a9/e7/7cfb3f106c393979f4cc37ad6c0586044d50401e3c35b0c003e4f3ba6bc9/mypy-2.3.1-cp315-cp315-pyemscripten_2026_5_wasm32.whl", hash = "sha256:bb26deed807bdb0457cf3e3f1cd7c4a1cf9d66864eaf1b4a61e06805d4c6b1f9", size = 7761913, upload-time = "2026-08-15T03:01:55.65Z" }, + { url = "https://files.pythonhosted.org/packages/99/3c/52affefa273b97939a1f474ae4a349c8718635c15b941112dfab4291b0c1/mypy-2.3.1-cp315-cp315-win_amd64.whl", hash = "sha256:375d7013876a8233b2d05be185bfa09f689696cd999ce8b1cfe6acac5c80e8a3", size = 11422533, upload-time = "2026-08-15T03:03:24.101Z" }, + { url = "https://files.pythonhosted.org/packages/2a/b7/75643e70c72a5b346d8a9b1543c967ea8824df2ee3fb7ccba652c272b7bb/mypy-2.3.1-cp315-cp315-win_arm64.whl", hash = "sha256:586b3612214cceabb3c0f588c97e7d1e535393f06a60e912e994f6b3ace97523", size = 10397931, upload-time = "2026-08-15T03:02:55.265Z" }, + { url = "https://files.pythonhosted.org/packages/10/ce/53be21f2d4adfcd26f63f1184a13ed797015ab463853f117e2e11e4d726f/mypy-2.3.1-cp315-cp315t-macosx_11_0_arm64.whl", hash = "sha256:ef0c6335cda9d807f8193d8ff6204a72bc909fa9882aacbca14f43cdb7188306", size = 15118669, upload-time = "2026-08-15T03:02:51.479Z" }, + { url = "https://files.pythonhosted.org/packages/62/43/20de757cd42989d291a17fad607742c4c74e875ce5cea00e5a5225020ac1/mypy-2.3.1-cp315-cp315t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e598c8c66401d26b150872154a286e6d484cf2789c3bb28a7556806298423021", size = 15545627, upload-time = "2026-08-15T03:03:05.132Z" }, + { url = "https://files.pythonhosted.org/packages/7e/fc/092bdf77ad280eaf501422f0f3b966012b528076cc13e41a774861c907d1/mypy-2.3.1-cp315-cp315t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:eda22fd4efa9dcd39331d1dede9b5b8b8a7fd69af07592e778433da98610d29e", size = 16764157, upload-time = "2026-08-15T03:02:23.958Z" }, + { url = "https://files.pythonhosted.org/packages/94/5c/c94c4d62d909b07f552d0d9356d7acc943825558e602a64822ffa2231536/mypy-2.3.1-cp315-cp315t-musllinux_1_2_x86_64.whl", hash = "sha256:2a0ba2e57847849fb0d1fcdabb32786d223095ed8bc121dfe322bcdb3d9c46bc", size = 17073258, upload-time = "2026-08-15T03:02:14.573Z" }, + { url = "https://files.pythonhosted.org/packages/c0/f7/511a88b89e478053c02d22039bb8f3ce4183efe8fd7a4f0a5910a8bb0a32/mypy-2.3.1-cp315-cp315t-win_amd64.whl", hash = "sha256:3f7e865dd51f235f60a2dbcd8728a1c095f5ca28f095d48a725b84cd935735c4", size = 12135505, upload-time = "2026-08-15T03:02:16.714Z" }, + { url = "https://files.pythonhosted.org/packages/71/bf/02573b56964ecb0f7c644f915f53c325ae15c3faec521c5adf11599a32df/mypy-2.3.1-cp315-cp315t-win_arm64.whl", hash = "sha256:8ad80807dc3ab8ea978b1b2b6e4a657194ace1d4ef03e0e731aff1abd517da29", size = 10962647, upload-time = "2026-08-15T03:01:43.712Z" }, + { url = "https://files.pythonhosted.org/packages/8e/41/9675c7a1e78edecfba0b79e587a52594c56e189368261dc7b3a7fffb9527/mypy-2.3.1-py3-none-any.whl", hash = "sha256:6ed5c7e3419083268e5c9258bd1c1ef91af44a9e89374dbcaf37b775716e72eb", size = 2754338, upload-time = "2026-08-15T03:02:53.4Z" }, +] + +[[package]] +name = "mypy-extensions" +version = "1.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/a2/6e/371856a3fb9d31ca8dac321cda606860fa4548858c0cc45d9d1d4ca2628b/mypy_extensions-1.1.0.tar.gz", hash = "sha256:52e68efc3284861e772bbcd66823fde5ae21fd2fdb51c62a211403730b916558", size = 6343, upload-time = "2025-04-22T14:54:24.164Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/79/7b/2c79738432f5c924bef5071f933bcc9efd0473bac3b4aa584a6f7c1c8df8/mypy_extensions-1.1.0-py3-none-any.whl", hash = "sha256:1be4cccdb0f2482337c4743e60421de3a356cd97508abadd57d47403e94f5505", size = 4963, upload-time = "2025-04-22T14:54:22.983Z" }, +] + +[[package]] +name = "packaging" +version = "26.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/7d/fa/3944b40b07da9ce895c0e6303a5ab7d53da063554f534556b134a54d6093/packaging-26.3.tar.gz", hash = "sha256:94edc256424af38762eb31306eed28beb9f0efc50a8837492c9d6fd6004aed79", size = 313412, upload-time = "2026-08-04T18:15:28.737Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/63/34/ba1c580383c9eada3711951fef0795c80b829a078d72188184bcab9dd527/packaging-26.3-py3-none-any.whl", hash = "sha256:d7193f7c8e4e93f444fde0262bf90af30e16fa0ad0ad44cb553c87339b23cd1c", size = 129956, upload-time = "2026-08-04T18:15:27.159Z" }, +] + +[[package]] +name = "pathspec" +version = "1.1.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/5a/82/42f767fc1c1143d6fd36efb827202a2d997a375e160a71eb2888a925aac1/pathspec-1.1.1.tar.gz", hash = "sha256:17db5ecd524104a120e173814c90367a96a98d07c45b2e10c2f3919fff91bf5a", size = 135180, upload-time = "2026-04-27T01:46:08.907Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f1/d9/7fb5aa316bc299258e68c73ba3bddbc499654a07f151cba08f6153988714/pathspec-1.1.1-py3-none-any.whl", hash = "sha256:a00ce642f577bf7f473932318056212bc4f8bfdf53128c78bbd5af0b9b20b189", size = 57328, upload-time = "2026-04-27T01:46:07.06Z" }, +] + +[[package]] +name = "pyproject-hooks" +version = "1.2.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/e7/82/28175b2414effca1cdac8dc99f76d660e7a4fb0ceefa4b4ab8f5f6742925/pyproject_hooks-1.2.0.tar.gz", hash = "sha256:1e859bd5c40fae9448642dd871adf459e5e2084186e8d2c2a79a824c970da1f8", size = 19228, upload-time = "2024-09-29T09:24:13.293Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/bd/24/12818598c362d7f300f18e74db45963dbcb85150324092410c8b49405e42/pyproject_hooks-1.2.0-py3-none-any.whl", hash = "sha256:9e5c6bfa8dcc30091c74b0cf803c81fdd29d94f01992a7707bc97babb1141913", size = 10216, upload-time = "2024-09-29T09:24:11.978Z" }, +] + +[[package]] +name = "ruff" +version = "0.16.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/00/8f/d8074b1f25e003164087a8bfe79a0f1a3945135764dbb6aaab04103dcaf9/ruff-0.16.4.tar.gz", hash = "sha256:13171aa9d9af2240ee3504e639de73122c67e74036de5ba2e1d01422cd17e3dc", size = 4899731, upload-time = "2026-08-20T17:43:59.196Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ff/80/779895ef584e089d22f2c6df0d0e99a65ec2df0805f1fffd439415b8c1f0/ruff-0.16.4-py3-none-linux_armv6l.whl", hash = "sha256:df4075f71ddac40b9934af60c3ec8a53047dd5a5fdc43224e6e4e8e9a27cb6f7", size = 10006909, upload-time = "2026-08-20T17:43:16.888Z" }, + { url = "https://files.pythonhosted.org/packages/a9/e6/f553199b5e8927a05cb5c422d921fd0656b29ab976e91c44802107c6b0da/ruff-0.16.4-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:0c95538517af68004306b0fb3214ff2f2af67a65092aee77cd9eb86db6656604", size = 10240201, upload-time = "2026-08-20T17:43:19.337Z" }, + { url = "https://files.pythonhosted.org/packages/1c/70/4a6dc4bb34da4dee35e30f09bbd1bfbdd26f33b62fb9b8df31f08a199cd2/ruff-0.16.4-py3-none-macosx_11_0_arm64.whl", hash = "sha256:963f83df8e69e575b64d67dd447ebbc917db41a14bf38d4593a4183e7aaa8255", size = 9835122, upload-time = "2026-08-20T17:43:21.708Z" }, + { url = "https://files.pythonhosted.org/packages/24/12/c6e22d686372c15bcb7af99831f1a1be96df696491babf4f24e4f942c527/ruff-0.16.4-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:32a5057c7ff3f6e6480a48fccfb3a412a690f48a3d03ac5cf08177d6c2da3ade", size = 9977162, upload-time = "2026-08-20T17:43:24.236Z" }, + { url = "https://files.pythonhosted.org/packages/46/49/72b10ec912f5ab5854992eaf7aa7cd36729b6937d9dc4e0fb41b3bf428ec/ruff-0.16.4-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:b3dce8d9b0c57c265b91885a66a567d8ea1372e8eb4e250fa8e5e3f579e99cff", size = 9829789, upload-time = "2026-08-20T17:43:26.966Z" }, + { url = "https://files.pythonhosted.org/packages/fa/80/0f30e32e7f6ee26edc39075502db9d368d788a44a79b55f763eb4ab03796/ruff-0.16.4-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7dc651db49283c69f8e72c834eec4fe5573e4c646856aebece0ce385dceb2a80", size = 10527949, upload-time = "2026-08-20T17:43:29.384Z" }, + { url = "https://files.pythonhosted.org/packages/52/3d/86e8ad3542169e56cac3859a343afdb9df2ad54d35a59ce1e67baee83421/ruff-0.16.4-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3817b87dbcabc92f13b05019257c5b89b5b4d51b5fb20f56fb5235ceb723cd07", size = 11333695, upload-time = "2026-08-20T17:43:31.872Z" }, + { url = "https://files.pythonhosted.org/packages/d0/16/481c29b380c20a0054a8261066665e1b3488e23636c49d0a43e75975b9bb/ruff-0.16.4-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e9fce1499134b2c8c68e5166f95705a5812062bb93aacc5f9873bb1a27084bc7", size = 10727741, upload-time = "2026-08-20T17:43:34.596Z" }, + { url = "https://files.pythonhosted.org/packages/5e/b6/56bc0b8cf45b54b28b3a5e6381c8945d51b5b18adf659454c32295209a31/ruff-0.16.4-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f2d812e482f5a7e02eee26cd73d2a37ebbdf47d795ea63ba1b89110ae93e9fb3", size = 10286522, upload-time = "2026-08-20T17:43:37.288Z" }, + { url = "https://files.pythonhosted.org/packages/e8/8b/b345b4fb110f2fbe2bd31eabd271e5e8b3b7e4ee6c0e02f2dc6be78db000/ruff-0.16.4-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:6baaf984aa7976edf93d3b627fe2d1d22ee94bbca05fa6f90fc76d73924e3454", size = 10584182, upload-time = "2026-08-20T17:43:39.984Z" }, + { url = "https://files.pythonhosted.org/packages/29/e5/827b34041c35f58774a9681a4213994c164fc987800f4dddabcf451da0bf/ruff-0.16.4-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:bdfcf0b28662eb890372d50f92c283bb94e67e7635ed93c7fd533970acff7b2b", size = 10134195, upload-time = "2026-08-20T17:43:42.351Z" }, + { url = "https://files.pythonhosted.org/packages/0f/10/d0bffcdd6729b87afc82ba0ef377173356a7dc8e972f5179968cf2fdf98c/ruff-0.16.4-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:b66b02cb9b04f537643cadf5768e5f98dc461890d530cb67113d71c8c76e605d", size = 9825821, upload-time = "2026-08-20T17:43:44.532Z" }, + { url = "https://files.pythonhosted.org/packages/f5/32/0db2a863b796ca62d83e92a07a3ccf00921b14db02059347576a2fda3d4b/ruff-0.16.4-py3-none-musllinux_1_2_i686.whl", hash = "sha256:8528bf9a4b291a60bf02ea453511e8ce6215bd2b982ee80405b66b008b6c30a0", size = 10267658, upload-time = "2026-08-20T17:43:46.989Z" }, + { url = "https://files.pythonhosted.org/packages/b2/a0/fbdeb59e48c6261f523e56c8f12e9c08fbe693786595cc7e3959207a9232/ruff-0.16.4-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:fbd85d2875fdd67e833213a651f613bbf25303abf6aa822a5121f4531195678d", size = 10697071, upload-time = "2026-08-20T17:43:49.891Z" }, + { url = "https://files.pythonhosted.org/packages/aa/28/0c6dd865859c6d17bc8ccc34cb72b0e02d6c7eb25e8a1e22b5bea681e2c0/ruff-0.16.4-py3-none-win32.whl", hash = "sha256:312769988007aaeb8e189b443ccdd03c0e6374489e053467be6d96518ebff76e", size = 10021687, upload-time = "2026-08-20T17:43:52.281Z" }, + { url = "https://files.pythonhosted.org/packages/a3/03/e724450f621698117f9aa6dd241c94d0274ae96781378dc86745ae29f0e7/ruff-0.16.4-py3-none-win_amd64.whl", hash = "sha256:05d9d27a18c4bcbefada602480ec9e01e0bc949d432e0ced5df77edac195919c", size = 10567657, upload-time = "2026-08-20T17:43:54.78Z" }, + { url = "https://files.pythonhosted.org/packages/0e/fe/da8b9e1347696bb22120b77280ec5ce25d500ca5cb39d5ad6e5c18de19c1/ruff-0.16.4-py3-none-win_arm64.whl", hash = "sha256:a3a61621c9b6f6a89573e938a080e648f1695baa3f58570a3a707bc51ff65a21", size = 10451579, upload-time = "2026-08-20T17:43:57.135Z" }, +] + +[[package]] +name = "setuptools" +version = "84.0.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/6d/44/f5da03a8ef95d369145c5bb53050e7877c9f3d312e128605fd9504829143/setuptools-84.0.0.tar.gz", hash = "sha256:f4695c21257f0d9b537ec2692c941d02ee143b7cc1276941349a546573b2ef73", size = 1168449, upload-time = "2026-08-08T18:27:58.365Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/95/9c/c510029fc6ef33a6275cd2c5d3cecd6613dfd6aa401d57c54f1c18852ccf/setuptools-84.0.0-py3-none-any.whl", hash = "sha256:51a52592b3b99e102b609654876bd65f19f999935166d1352678931132b0c670", size = 818216, upload-time = "2026-08-08T18:27:56.719Z" }, +] + +[[package]] +name = "typing-extensions" +version = "4.16.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f6/cc/6253133b5bb138fc3306cebfbda2c520f545d36b5be2c7255cc528bb45d6/typing_extensions-4.16.0.tar.gz", hash = "sha256:dc983d19a509c94dba722ee6abd33940f7c05a89e243c47e907eb4db6f1a43e5", size = 113555, upload-time = "2026-07-02T08:40:05.92Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/49/d3/b8441a820a491ddfc024b0b0cf0393375b75ea13866d9c66727e54c2fc80/typing_extensions-4.16.0-py3-none-any.whl", hash = "sha256:481caa481374e813c1b176ada14e97f1f67a4539ce9cfeb3f350d78d6370c2e8", size = 45571, upload-time = "2026-07-02T08:40:04.659Z" }, +]