From 954ddf6939e8cf421146696ee5b1731ee378cb51 Mon Sep 17 00:00:00 2001 From: Token Terminator maintenance <33731256+AronAxe@users.noreply.github.com> Date: Mon, 5 Oct 2026 08:10:59 +0200 Subject: [PATCH] fix: restore Sol token counting and serialize profile-isolated TT discovery --- .github/workflows/sol-discovery.yml | 33 ++++ CHANGELOG.md | 7 + Cargo.toml | 2 +- MIGRATION.md | 17 ++ README.md | 35 ++-- SECURITY.md | 10 ++ docs/CONTEXT_ENGINE.md | 6 + docs/HERMES_DISCOVERY.md | 44 +++++ docs/TOKENIZER_COMPAT.md | 38 +++++ docs/releases/v0.11.2.md | 58 +++++++ pyproject.toml | 2 +- src/rtk_hermes_plus/_version.py | 2 +- src/rtk_hermes_plus/cli.py | 8 +- src/rtk_hermes_plus/engine_install.py | 18 +- src/rtk_hermes_plus/hermes_discovery.py | 145 ++++++++++++++++ src/rtk_hermes_plus/token_budget.py | 27 ++- tests/fixtures/gpt61-token-counts.json | 55 ++++++ tests/test_discovery_repair.py | 212 ++++++++++++++++++++++++ tests/test_gpt61_tokenizer.py | 138 +++++++++++++++ wiki/Context-Engine.md | 6 + wiki/Home.md | 4 + wiki/Release-0.11.2.md | 58 +++++++ wiki/Tokenizer-and-Discovery-Repair.md | 46 +++++ wiki/_Footer.md | 2 +- wiki/_Sidebar.md | 3 + 25 files changed, 957 insertions(+), 19 deletions(-) create mode 100644 .github/workflows/sol-discovery.yml create mode 100644 docs/HERMES_DISCOVERY.md create mode 100644 docs/TOKENIZER_COMPAT.md create mode 100644 docs/releases/v0.11.2.md create mode 100644 src/rtk_hermes_plus/hermes_discovery.py create mode 100644 tests/fixtures/gpt61-token-counts.json create mode 100644 tests/test_discovery_repair.py create mode 100644 tests/test_gpt61_tokenizer.py create mode 100644 wiki/Release-0.11.2.md create mode 100644 wiki/Tokenizer-and-Discovery-Repair.md diff --git a/.github/workflows/sol-discovery.yml b/.github/workflows/sol-discovery.yml new file mode 100644 index 0000000..0eb1178 --- /dev/null +++ b/.github/workflows/sol-discovery.yml @@ -0,0 +1,33 @@ +name: Sol and discovery regression +on: + pull_request: + branches: [main] + push: + branches: [main] +permissions: + contents: read +jobs: + regression: + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - uses: actions/checkout@v6 + - uses: actions/checkout@v6 + with: + repository: NousResearch/hermes-agent + ref: 188a1a5efd098f067638032fea6e24c42d06178f + path: upstream-hermes + persist-credentials: false + - uses: actions/setup-python@v6 + with: + python-version: '3.13' + - run: python -m pip install -e '.[dev]' + - name: Actual Sol pipeline and actual host-loader race regressions + env: + TT_HERMES_LOADER: ${{ github.workspace }}/upstream-hermes/plugins/plugin_loader.py + run: python -m pytest tests/test_gpt61_tokenizer.py tests/test_discovery_repair.py --junitxml=sol-discovery-tests.xml + - uses: actions/upload-artifact@v6 + if: always() + with: + name: sol-discovery-verification + path: sol-discovery-tests.xml diff --git a/CHANGELOG.md b/CHANGELOG.md index 95a65b3..e44523c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,12 @@ # Changelog +## 0.11.2 - 2026-10-05 + +- Added empirically validated GPT-6.1-Sol tokenizer compatibility, without overriding upstream mappings or guessing future model IDs. +- Fixed concurrent TT directory discovery and cross-profile module reuse through an explicit, source-guarded installer repair with exact backup. +- Added non-skipping Sol pipeline and real-loader concurrency CI regressions. +- Preserved call scope, protected context, exact recovery and strict final gates. + ## 0.11.1 - 2026-10-04 - Fixed the Hermes live-compression adapter's missing `_coerce_threshold_tokens_cap` diff --git a/Cargo.toml b/Cargo.toml index 8f98d97..166aa39 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "token-terminator" -version = "0.11.1" +version = "0.11.2" edition = "2021" rust-version = "1.78" description = "Rust interoperability helpers for Token Terminator artifact identities and reduction invariants" diff --git a/MIGRATION.md b/MIGRATION.md index 86e2a72..36dec79 100644 --- a/MIGRATION.md +++ b/MIGRATION.md @@ -1,3 +1,20 @@ +# Migration to Token Terminator 0.11.2 + +Install with the managed Hermes interpreter: + +```bash +python -m pip install --upgrade 'git+https://github.com/AronAxe/Token-Terminator.git@v0.11.2' +python -m rtk_hermes_plus.cli install-context-engine +``` + +The explicit engine installer now requests the reviewed host-loader repair; inspect +its `discovery_repair` result before restarting the appropriate idle backend once. +A byte-exact host backup is retained. Unknown host sources are refused. No new keys, +manual Sol encoding override or data migration is required. See +[repair/rollback](docs/HERMES_DISCOVERY.md) and [counter evidence](docs/TOKENIZER_COMPAT.md). + +--- + # Migration to Token Terminator 0.11.1 ## 0.11.0 to 0.11.1 diff --git a/README.md b/README.md index ac82ffa..f0600a1 100644 --- a/README.md +++ b/README.md @@ -12,10 +12,10 @@

CI - Release v0.11.1 + Release v0.11.2 JEV-powered GitHub Wiki - crates.io + crates.io docs.rs Python 3.10–3.13 MIT License @@ -27,6 +27,21 @@ Token Terminator is an agent-runtime optimization layer. It removes token bloat Documentation: see the [GitHub Wiki](https://github.com/AronAxe/Token-Terminator/wiki) for quick start, architecture, configuration, recovery, security, troubleshooting, migration, and release notes. +## Patch v0.11.2: GPT-6.1-Sol and multiplex startup + +**GPT-6.1-Sol now reaches TT/JEV context reduction without a manual tokenizer +override.** A provider-checked local compatibility mapping fills the missing +upstream model-name entry. Exact recovery and strict token/character gates remain. + +**Concurrent TT discovery is repaired during explicit `install-context-engine`.** +For the reviewed affected Hermes loader, the installer makes a narrowly scoped, +backed-up host-file change: initialization is synchronized and profile module +caches are separated. Unknown host implementations are refused, not overwritten. +`install-dashboard` alone does not apply this repair. + +[Release notes](docs/releases/v0.11.2.md) · [Tokenizer validation](docs/TOKENIZER_COMPAT.md) · +[Discovery repair and rollback](docs/HERMES_DISCOVERY.md) + ## Patch v0.11.1: Hermes live-compression compatibility Repairs the missing `_coerce_threshold_tokens_cap` adapter method and live @@ -159,12 +174,12 @@ See [JEV roles and implemented scope](docs/JEV_ROLES.md) and ## Select Token Terminator as the Hermes ContextEngine -Install **v0.11.1** in the Python environment that runs Hermes, then install its +Install **v0.11.2** in the Python environment that runs Hermes, then install its managed adapter. Replace `python` below with that environment's interpreter: ```bash python -m pip install --upgrade \ - 'git+https://github.com/AronAxe/Token-Terminator.git@v0.11.1' + 'git+https://github.com/AronAxe/Token-Terminator.git@v0.11.2' python -m rtk_hermes_plus.cli install-context-engine ``` @@ -322,9 +337,9 @@ The optional working-state block defaults to zero characters, even in `balanced` ## Install: Hermes Agent (turnkey) -Token Terminator 0.11.1 builds on v0.9.0 and replaces the earlier `rtk-hermes-plus` distribution. `token-terminator` and `rtk-hermes-plus` must not coexist because both own the `rtk_hermes_plus` Python import package. +Token Terminator 0.11.2 builds on v0.9.0 and replaces the earlier `rtk-hermes-plus` distribution. `token-terminator` and `rtk-hermes-plus` must not coexist because both own the `rtk_hermes_plus` Python import package. -This is the supported zero-glue installation: the repository already contains the Hermes hooks, slash command, recovery tool, and lifecycle accounting. The commands below pin the `v0.11.1` release tag. +This is the supported zero-glue installation: the repository already contains the Hermes hooks, slash command, recovery tool, and lifecycle accounting. The commands below pin the `v0.11.2` release tag. ### 1. Install RTK when using terminal rewriting @@ -346,7 +361,7 @@ HERMES_PY="$HOME/.hermes/hermes-agent/venv/bin/python" hermes plugins disable rtk-plus "$HERMES_PY" -m pip uninstall -y rtk-hermes-plus token-terminator "$HERMES_PY" -m pip install \ - 'git+https://github.com/AronAxe/Token-Terminator.git@v0.11.1' + 'git+https://github.com/AronAxe/Token-Terminator.git@v0.11.2' ``` Windows example: @@ -355,7 +370,7 @@ Windows example: $HermesPy = "$env:LOCALAPPDATA\hermes\hermes-agent\venv\Scripts\python.exe" hermes plugins disable rtk-plus & $HermesPy -m pip uninstall -y rtk-hermes-plus token-terminator -& $HermesPy -m pip install "git+https://github.com/AronAxe/Token-Terminator.git@v0.11.1" +& $HermesPy -m pip install "git+https://github.com/AronAxe/Token-Terminator.git@v0.11.2" ``` `tiktoken` now ships with Token Terminator and is used automatically for supported OpenAI-family models, including common provider-qualified model IDs. Hugging Face `tokenizers` remains optional when pointing Token Terminator at a local `tokenizer.json`: @@ -387,7 +402,7 @@ Installation and enablement are separate operations. Disabling affects subsequen Rust agent hosts can use the supported `token-terminator` companion crate for the stable cross-language pieces of the Token Terminator contract: ```bash -cargo add token-terminator@0.11.1 +cargo add token-terminator@0.11.2 ``` ```rust @@ -409,7 +424,7 @@ Install the same distribution in the environment that owns your agent loop: ```bash python -m pip install \ - 'git+https://github.com/AronAxe/Token-Terminator.git@v0.11.1' + 'git+https://github.com/AronAxe/Token-Terminator.git@v0.11.2' ``` Then connect your runtime's tool-result and final-request hooks to `Runtime`. The adapter must map equivalent tools to Token Terminator's canonical names (`search_files`, `process`, and optionally `read_file`) and expose `Runtime.tool` to the model for exact recovery. diff --git a/SECURITY.md b/SECURITY.md index 0552eab..54bbf9b 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -180,3 +180,13 @@ silently route a remote dashboard to this machine's localhost. Installation and enable are separate and opt-in. See [DASHBOARD.md](docs/DASHBOARD.md) for limits, accounting qualifications and rollback. The dashboard adds no service credential, background training or publication. + +## v0.11.2 explicit Hermes loader repair + +The engine-install CLI can modify only the reviewed affected plugin_loader.py, +with a normalized source-hash guard, adjacent exact backup, and atomic replacement. +This resolves a race before TT plugin initialization; it is not a hidden import-time +patch. The wrapper targets only TT user ContextEngine loads and separates profile +module identities. Other plugin loads retain their host behavior. Unknown/modified +source is refused. No credentials, config, conversation or vault files are changed. +The library installer API requires explicit repair_discovery=True for this action. diff --git a/docs/CONTEXT_ENGINE.md b/docs/CONTEXT_ENGINE.md index a854e87..23941b4 100644 --- a/docs/CONTEXT_ENGINE.md +++ b/docs/CONTEXT_ENGINE.md @@ -1,5 +1,11 @@ # Token Terminator ContextEngine — v0.11.0 +> **v0.11.2 install change:** The explicit engine installer also repairs the reviewed +> affected Hermes directory loader, with an exact source guard and backup. This is +> a narrow host-file repair for concurrent TT loading, not a replacement Hermes +> runtime. See the repository `docs/HERMES_DISCOVERY.md` before installation. + + **Experimental and explicitly selected.** Available in v0.11.0; ordinary middleware mode remains the default. Installing the managed adapter does not change Hermes core, select a context engine, enable plugins, alter LCM data or diff --git a/docs/HERMES_DISCOVERY.md b/docs/HERMES_DISCOVERY.md new file mode 100644 index 0000000..1f82137 --- /dev/null +++ b/docs/HERMES_DISCOVERY.md @@ -0,0 +1,44 @@ +# Hermes TT discovery repair + +The reviewed Hermes shared directory loader inserts a new module into sys.modules +before running it, but treats a cached module with __file__ as ready. Concurrent +requests can receive that half-module. User engine module names also contain only +the plugin slug, not the profile path. These are initialization and isolation bugs. + +## What 0.11.2 installs + +The explicit `token-terminator install-context-engine` command requests the repair. +It patches only a reviewed loader file with normalized SHA-256 +`402d3fd6f6077c344f15713ac3b793fbccbe0602181bceff7a23690270e3ff35`, checked against +Hermes `188a1a5efd098f067638032fea6e24c42d06178f` and the installed affected source. + +The appended wrapper applies only to token-terminator in the user ContextEngine +namespace. A lock covers the original load until initialization finishes. Other +profiles receive path-derived namespaces; unrelated plugins retain original +behavior. Recursive same-thread loads return unavailable, never a partial module. +Waiting is bounded to ten seconds. Failed imports remain retryable. + +No plugin-only hook can repair a lookup that happens before that plugin starts. +This release therefore makes the host-file change explicit. It is not a general +Hermes plugin-loader rewrite, new runtime dependency, or replacement context engine. + +The original bytes are backed up beside the source as +`plugin_loader.py.tt-before-0.11.2`. The installer refuses unknown source versions, +modified patches, symlinks, mismatching backups and concurrent source changes. +Package import and ordinary middleware do not write host files. The Python +`install_context_engine` API defaults to `repair_discovery=False`; callers that +want the explicit CLI behavior pass `repair_discovery=True`. + +## Verification and rollback + +`tests/test_discovery_repair.py` deterministically holds one initialization open +while another lookup arrives. It also tests 64 parallel calls and profile isolation. +Set TT_HERMES_LOADER to a copy or checkout of the actual reviewed plugin_loader.py +to test its real implementation; tests only modify temporary files. + +After installation, restart the correct idle backend once and verify discovery and +provider-request measurements. Do not delete history, vaults or unrelated plugins. +To roll back, stop the affected backend and restore the adjacent original backup +only after checking that no subsequent Hermes changes would be overwritten. If +Hermes upgrades its loader, review that new implementation rather than forcing this +patch onto it. A successful health check alone is not proof of token reduction. diff --git a/docs/TOKENIZER_COMPAT.md b/docs/TOKENIZER_COMPAT.md new file mode 100644 index 0000000..5cab879 --- /dev/null +++ b/docs/TOKENIZER_COMPAT.md @@ -0,0 +1,38 @@ +# GPT-6.1-Sol tokenizer compatibility + +Upstream tiktoken's model-name lookup did not contain GPT-6.1-Sol at the time of this +repair. TT 0.11.1 consequently returned no token count and its ContextEngine exited +before JEV. Version 0.11.2 adds a local compatibility mapping for that exact model, +not a blanket rule for future GPT versions. The upstream library remains unmodified. + +The supported provider prefixes are bare, openai/, openai:, openai-codex/ and +openrouter/openai/. Cache keys retain the original provider identity. A recognized +upstream tokenizer takes precedence over TT's compatibility mapping. Local explicit +encoding/Hugging Face configuration keeps its existing behavior. + +## Empirical validation + +`tests/fixtures/gpt61-token-counts.json` contains five synthetic public inputs and +actual responses from https://api.openai.com/v1/responses/input_tokens on 2026-10-05. +All content counts matched o200k_base, with a constant six-token message framing: + +| Case | Provider | Local content | Framing | +|---|---:|---:|---:| +| English | 13 | 7 | 6 | +| Multilingual and emoji | 330 | 324 | 6 | +| Code and values | 633 | 627 | 6 | +| JSON records | 610 | 604 | 6 | +| Literal markers | 204 | 198 | 6 | + +This is an empirically validated compatibility mapping, not an official upstream +model declaration or universal proof. No private conversations or generation calls +were involved. The runtime does not call a remote counter or need a new key. + +TT reports `tiktoken:compat:gpt-6.1-sol:o200k_base` (with the normalized model for +qualified routes). Its gate measures complete canonical provider-request JSON; +provider-internal framing, image tokenization and invoices remain outside that +measurement claim. Strict token AND character decrease, protected context and +exact recovery are unchanged. The full pipeline regression must actually reach +JEV and reduce the request, not merely return a numeric count. + +Reference: https://developers.openai.com/api/docs/guides/token-counting diff --git a/docs/releases/v0.11.2.md b/docs/releases/v0.11.2.md new file mode 100644 index 0000000..f94709a --- /dev/null +++ b/docs/releases/v0.11.2.md @@ -0,0 +1,58 @@ +# Token Terminator v0.11.2 — Sol counting and reliable TT discovery + +This repair addresses two blockers left in 0.11.1. It does not change JEV keys, +semantic policies, the evidence vault, Context IR, or the main-call-only boundary. + +## GPT-6.1-Sol + +The exact `gpt-6.1-sol` ID and known OpenAI/Codex/OpenRouter-qualified forms now +resolve to `o200k_base` when the installed tiktoken model-name table lacks the ID. +An upstream mapping still wins, and an explicit operator encoding remains supported. +Unrelated providers, invented versions and sibling names are not inferred. + +Five synthetic inputs were checked against OpenAI's documented input-token counter: +English, multilingual/emoji text, code/numbers, structured JSON and literal special +markers. All matched local content counts plus the same six-token message framing. +The recorded corpus distinguishes o200k_base from cl100k_base. This is empirical +compatibility evidence, not an upstream tiktoken declaration or proof about every +possible input. TT still measures complete canonical request JSON, not hidden +provider framing or billed tokens. No generation requests were used in that check. + +## Concurrent Hermes discovery + +The affected shared loader can return a module before its initialization completes, +and its slug-only cache can reuse one profile's module for another. TT's explicit +`install-context-engine` command now applies a source-hash-guarded repair to the +reviewed affected loader. TT loads are synchronized, recursive discovery cannot +consume a partial module, and each resolved profile path has a distinct namespace. +Other plugins use the original loader. Failed initialization can be retried. + +This is a narrow host-file compatibility patch, not merely a plugin import change: +the race occurs before the plugin can run. An adjacent byte-exact backup is kept. +Unknown or modified host implementations are refused rather than overwritten. +The Python installer API and `install-dashboard` do not patch the host by default. +No changes occur on package import, and configuration/history are not rewritten. + +## Install + +Use the interpreter/environment actually loaded by the managed Hermes launcher: + +```bash +python -m pip install --upgrade 'git+https://github.com/AronAxe/Token-Terminator.git@v0.11.2' +python -m rtk_hermes_plus.cli install-context-engine +``` + +The second command reports `discovery_repair`; for an affected installed host it +must say `repaired` or `already_repaired`. Then restart the appropriate idle backend +through its supervisor once. A package download alone cannot repair a running process. +Existing explicit engine selection and JEV/IR opt-ins remain unchanged. + +Regression checks exercise the actual GPT-6.1-Sol model name without encoding +overrides, JEV selection, strict complete-request reduction and exact recovery. +Discovery checks include controlled overlapping initialization, 64 simultaneous +loads, profile separation, recursive loading, failed-load retry and safe installation. +A dedicated CI job runs those same tests against the actual pinned Hermes loader +source, not an optional skipped module. Full CI results are recorded on the PR. + +The Rust companion is version-aligned only. Existing tags, Terminator artwork and +schematics remain unchanged. See docs/TOKENIZER_COMPAT.md and docs/HERMES_DISCOVERY.md. diff --git a/pyproject.toml b/pyproject.toml index fa38af8..354af5b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "token-terminator" -version = "0.11.1" +version = "0.11.2" description = "Fail-open token reduction, exact recovery, and request compilation for Hermes Agent" readme = "README.md" requires-python = ">=3.10" diff --git a/src/rtk_hermes_plus/_version.py b/src/rtk_hermes_plus/_version.py index fee46bd..e2bd072 100644 --- a/src/rtk_hermes_plus/_version.py +++ b/src/rtk_hermes_plus/_version.py @@ -1 +1 @@ -__version__ = "0.11.1" +__version__ = "0.11.2" diff --git a/src/rtk_hermes_plus/cli.py b/src/rtk_hermes_plus/cli.py index 77421d3..f6154d4 100644 --- a/src/rtk_hermes_plus/cli.py +++ b/src/rtk_hermes_plus/cli.py @@ -129,7 +129,13 @@ def run(args: argparse.Namespace) -> int: if args.command in {"install-context-engine", "install-dashboard"}: from .engine_install import install_context_engine - _emit(install_context_engine(args.hermes_home), as_json=args.json) + _emit( + install_context_engine( + args.hermes_home, + repair_discovery=args.command == "install-context-engine", + ), + as_json=args.json, + ) return 0 config, store, graph, compiler = _runtime() if args.command == "status": diff --git a/src/rtk_hermes_plus/engine_install.py b/src/rtk_hermes_plus/engine_install.py index 6dcb913..5ec0ab7 100644 --- a/src/rtk_hermes_plus/engine_install.py +++ b/src/rtk_hermes_plus/engine_install.py @@ -1,4 +1,4 @@ -"""Install the supported user-directory adapter, without editing Hermes core/config.""" +"""Install the user adapter; explicit CLI installation also repairs the known host loader.""" from __future__ import annotations @@ -26,12 +26,14 @@ def register(ctx): """ -def install_context_engine(home: Path | None = None) -> dict: +def install_context_engine( + home: Path | None = None, *, repair_discovery: bool = False +) -> dict: home = Path(home) if home is not None else _hermes_home() destination = home / "plugins" / "token-terminator" files = { "__init__.py": _SHIM, - "plugin.yaml": f"# Token Terminator managed ContextEngine adapter\nname: token-terminator\nversion: '{__version__}'\ndescription: 'Token Terminator — exact-history ContextEngine and request middleware'\n", + "plugin.yaml": f"# Token Terminator managed ContextEngine adapter\nname: token-terminator\nversion: '{__version__}'\ndescription: 'Token Terminator — exact-history ContextEngine and request middleware'\n", } files.update( { @@ -84,6 +86,15 @@ def install_context_engine(home: Path | None = None) -> dict: managed = text.startswith(marker) if path.is_symlink() or not managed: raise ValueError("refusing to replace an unmanaged plugin file") + # This host bug happens before plugin code can run. Repair only the reviewed + # affected loader, only during an explicit install, with a recoverable backup. + from .hermes_discovery import repair_hermes_discovery + + discovery = ( + repair_hermes_discovery() + if repair_discovery + else {"state": "not_requested", "changed": False} + ) destination.mkdir(parents=True, exist_ok=True) for name, content in files.items(): path = destination / name @@ -97,6 +108,7 @@ def install_context_engine(home: Path | None = None) -> dict: "installed": str(destination), "version": __version__, "config_changed": False, + "discovery_repair": discovery, "next_steps": [ "Enable token-terminator in hermes plugins (general plugin/middleware).", "Choose token-terminator under Provider Plugins > Context Engine.", diff --git a/src/rtk_hermes_plus/hermes_discovery.py b/src/rtk_hermes_plus/hermes_discovery.py new file mode 100644 index 0000000..5891095 --- /dev/null +++ b/src/rtk_hermes_plus/hermes_discovery.py @@ -0,0 +1,145 @@ +"""Install a narrow, backed-up repair for the known Hermes TT discovery race. + +Never runs on import. The affected host publishes half-loaded directory modules; +no hook inside that module can synchronize a reader that arrives before its code +runs. The explicit installer therefore patches the host loader, for TT only. +""" + +from __future__ import annotations + +import hashlib +import importlib.util +import os +import tempfile +from pathlib import Path + +_KNOWN_SOURCE_SHA256 = frozenset( + { + "402d3fd6f6077c344f15713ac3b793fbccbe0602181bceff7a23690270e3ff35", + } +) +_MARKER = "# BEGIN TOKEN TERMINATOR DISCOVERY REPAIR 1" +_PATCH = """ + +# BEGIN TOKEN TERMINATOR DISCOVERY REPAIR 1 +# Narrow compatibility repair: only TT's user-directory ContextEngine loads. +# Installed with an exact-source guard and adjacent original-file backup. +import functools as _tt_functools +import hashlib as _tt_hashlib +import threading as _tt_threading + +_tt_loader_original = load_plugin_module +_tt_loader_guard = _tt_threading.Lock() +_tt_loader_locks = {} +_tt_loader_active = set() + + +@_tt_functools.wraps(_tt_loader_original) +def load_plugin_module(module_name, plugin_dir, *, parents, logger, + synthetic_namespace=None): + if (synthetic_namespace != "_hermes_user_context_engine" + or Path(plugin_dir).name != "token-terminator"): + return _tt_loader_original(module_name, plugin_dir, parents=parents, + logger=logger, + synthetic_namespace=synthetic_namespace) + # The upstream synthetic name contains only the slug. Separate profile homes + # must not reuse the first profile's imported module (or its sibling modules). + identity = str(Path(plugin_dir).resolve()) + suffix = _tt_hashlib.sha256(identity.encode("utf-8")).hexdigest()[:32] + namespace = synthetic_namespace + "_tt_" + suffix + scoped_name = namespace + ".token-terminator" + with _tt_loader_guard: + lock = _tt_loader_locks.setdefault(scoped_name, _tt_threading.RLock()) + if not lock.acquire(timeout=10): + logger.warning("Token Terminator initialization still busy; retry discovery") + return None + try: + # A same-thread recursive discovery must not consume its own half-module. + if scoped_name in _tt_loader_active: + return None + _tt_loader_active.add(scoped_name) + try: + return _tt_loader_original(scoped_name, plugin_dir, parents=parents, + logger=logger, + synthetic_namespace=namespace) + finally: + _tt_loader_active.discard(scoped_name) + finally: + lock.release() +# END TOKEN TERMINATOR DISCOVERY REPAIR 1 +""" + + +def patched_source(source: str) -> str: + """Return the repair only for the reviewed host implementation.""" + normalized = source.replace("\r\n", "\n") + if _MARKER in normalized: + if ( + normalized.endswith(_PATCH) + and hashlib.sha256(normalized[: -len(_PATCH)].encode("utf-8")).hexdigest() + in _KNOWN_SOURCE_SHA256 + ): + return normalized + raise ValueError("unrecognized modification to Hermes discovery repair") + digest = hashlib.sha256(normalized.encode("utf-8")).hexdigest() + if digest not in _KNOWN_SOURCE_SHA256: + raise ValueError( + "Hermes plugin_loader.py is not the reviewed affected version; " + "left unchanged. Review its concurrency handling before selecting TT." + ) + result = normalized + _PATCH + compile(result, "plugin_loader.py", "exec") + return result + + +def repair_hermes_discovery(loader_path: Path | None = None) -> dict: + """Explicit install-time repair. Preserve original bytes for rollback.""" + if loader_path is None: + try: + spec = importlib.util.find_spec("plugins.plugin_loader") + except (ImportError, ValueError, AttributeError): + spec = None + if spec is None or not spec.origin: + return {"state": "hermes_not_installed", "changed": False} + loader_path = Path(spec.origin) + path = Path(loader_path) + if path.is_symlink() or any(p.is_symlink() for p in path.parents): + raise ValueError("refusing a symlinked Hermes loader") + before = path.read_bytes() + source = before.decode("utf-8") + after = patched_source(source).encode("utf-8") + if _MARKER in source: + return {"state": "already_repaired", "changed": False} + backup = path.with_name(path.name + ".tt-before-0.11.2") + if backup.is_symlink(): + raise ValueError("refusing a symlinked Hermes backup") + try: + with backup.open("xb") as handle: + handle.write(before) + except FileExistsError: + if backup.read_bytes() != before: + raise ValueError("existing Hermes loader backup differs; left unchanged") + temp_name = None + try: + with tempfile.NamedTemporaryFile( + dir=path.parent, prefix=".tt-loader-", delete=False + ) as handle: + temp_name = handle.name + handle.write(after) + handle.flush() + os.fsync(handle.fileno()) + os.chmod(temp_name, path.stat().st_mode) + if path.read_bytes() != before: + raise ValueError("Hermes loader changed during repair; retry after review") + os.replace(temp_name, path) + temp_name = None + finally: + if temp_name: + Path(temp_name).unlink(missing_ok=True) + return { + "state": "repaired", + "changed": True, + "path": str(path), + "backup": str(backup), + "restart_required": True, + } diff --git a/src/rtk_hermes_plus/token_budget.py b/src/rtk_hermes_plus/token_budget.py index 887cae3..e0eabfc 100644 --- a/src/rtk_hermes_plus/token_budget.py +++ b/src/rtk_hermes_plus/token_budget.py @@ -9,6 +9,23 @@ _TRUE = frozenset({"1", "true", "yes", "on"}) _FALSE = frozenset({"0", "false", "no", "off"}) _OPENAI_MODEL_PREFIXES = ("gpt-4o", "gpt-4.1", "gpt-5", "o1", "o3", "o4") +# Explicit compatibility entry while upstream tiktoken's model-name table lags. +# See docs/TOKENIZER_COMPAT.md for the measurement boundary and validation. +_MODEL_ENCODING_COMPAT = {"gpt-6.1-sol": "o200k_base"} +_COMPAT_NAMESPACES = frozenset({"", "openai", "openai-codex", "openrouter/openai"}) + + +def _compat_encoding(model: str) -> str: + raw = str(model or "").strip().lower() + if "/" in raw: + namespace, name = raw.rsplit("/", 1) + elif ":" in raw: + namespace, name = raw.rsplit(":", 1) + else: + namespace, name = "", raw + if namespace not in _COMPAT_NAMESPACES: + return "" + return _MODEL_ENCODING_COMPAT.get(name, "") def _env_bool(name: str, default: bool) -> bool: @@ -147,7 +164,7 @@ def _tiktoken_encoding(self, model: str) -> tuple[Any, str] | tuple[None, str]: return None, "" lookup_model = _tokenizer_model_name(model) - cache_key = lookup_model or self.encoding_name or "" + cache_key = str(model or "").strip() or self.encoding_name or "" if cache_key in self._encodings: return self._encodings[cache_key] @@ -167,6 +184,14 @@ def _tiktoken_encoding(self, model: str) -> tuple[Any, str] | tuple[None, str]: except ValueError: encoding = None + compat = _compat_encoding(model) + if encoding is None and compat: + try: + encoding = tiktoken.get_encoding(compat) + label = f"tiktoken:compat:{lookup_model}:{compat}" + except ValueError: + encoding = None + # Recent OpenAI families use o200k_base. Keep the fallback narrowly # scoped rather than silently applying an OpenAI tokenizer to unrelated # model families. diff --git a/tests/fixtures/gpt61-token-counts.json b/tests/fixtures/gpt61-token-counts.json new file mode 100644 index 0000000..13e3da5 --- /dev/null +++ b/tests/fixtures/gpt61-token-counts.json @@ -0,0 +1,55 @@ +{ + "model": "gpt-6.1-sol", + "at": "2026-10-05T05:59:42.996595+00:00", + "endpoint": "https://api.openai.com/v1/responses/input_tokens", + "generation_calls": 0, + "count_calls": 5, + "tiktoken": "0.14.0", + "results": [ + { + "case": 0, + "text": "Token Terminator tokenizer compatibility test.", + "http_status": 200, + "provider_tokens": 13, + "o200k_base": 7, + "cl100k_base": 6, + "framing_delta": 6 + }, + { + "case": 1, + "text": "Hallo wereld. café naïef. 日本語 中文 한국어 مرحبا. 🧪✨ Hallo wereld. café naïef. 日本語 中文 한국어 مرحبا. 🧪✨ Hallo wereld. café naïef. 日本語 中文 한국어 مرحبا. 🧪✨ Hallo wereld. café naïef. 日本語 中文 한국어 مرحبا. 🧪✨ Hallo wereld. café naïef. 日本語 中文 한국어 مرحبا. 🧪✨ Hallo wereld. café naïef. 日本語 中文 한국어 مرحبا. 🧪✨ Hallo wereld. café naïef. 日本語 中文 한국어 مرحبا. 🧪✨ Hallo wereld. café naïef. 日本語 中文 한국어 مرحبا. 🧪✨ Hallo wereld. café naïef. 日本語 中文 한국어 مرحبا. 🧪✨ Hallo wereld. café naïef. 日本語 中文 한국어 مرحبا. 🧪✨ Hallo wereld. café naïef. 日本語 中文 한국어 مرحبا. 🧪✨ Hallo wereld. café naïef. 日本語 中文 한국어 مرحبا. 🧪✨ Hallo wereld. café naïef. 日本語 中文 한국어 مرحبا. 🧪✨ Hallo wereld. café naïef. 日本語 中文 한국어 مرحبا. 🧪✨ Hallo wereld. café naïef. 日本語 中文 한국어 مرحبا. 🧪✨ Hallo wereld. café naïef. 日本語 中文 한국어 مرحبا. 🧪✨ Hallo wereld. café naïef. 日本語 中文 한국어 مرحبا. 🧪✨ ", + "http_status": 200, + "provider_tokens": 330, + "o200k_base": 324, + "cl100k_base": 493, + "framing_delta": 6 + }, + { + "case": 2, + "text": "def f(x: int) -> str:\n return f\"exact={x:.2f}\"\n# 000001.2300 -9e-17\ndef f(x: int) -> str:\n return f\"exact={x:.2f}\"\n# 000001.2300 -9e-17\ndef f(x: int) -> str:\n return f\"exact={x:.2f}\"\n# 000001.2300 -9e-17\ndef f(x: int) -> str:\n return f\"exact={x:.2f}\"\n# 000001.2300 -9e-17\ndef f(x: int) -> str:\n return f\"exact={x:.2f}\"\n# 000001.2300 -9e-17\ndef f(x: int) -> str:\n return f\"exact={x:.2f}\"\n# 000001.2300 -9e-17\ndef f(x: int) -> str:\n return f\"exact={x:.2f}\"\n# 000001.2300 -9e-17\ndef f(x: int) -> str:\n return f\"exact={x:.2f}\"\n# 000001.2300 -9e-17\ndef f(x: int) -> str:\n return f\"exact={x:.2f}\"\n# 000001.2300 -9e-17\ndef f(x: int) -> str:\n return f\"exact={x:.2f}\"\n# 000001.2300 -9e-17\ndef f(x: int) -> str:\n return f\"exact={x:.2f}\"\n# 000001.2300 -9e-17\ndef f(x: int) -> str:\n return f\"exact={x:.2f}\"\n# 000001.2300 -9e-17\ndef f(x: int) -> str:\n return f\"exact={x:.2f}\"\n# 000001.2300 -9e-17\ndef f(x: int) -> str:\n return f\"exact={x:.2f}\"\n# 000001.2300 -9e-17\ndef f(x: int) -> str:\n return f\"exact={x:.2f}\"\n# 000001.2300 -9e-17\ndef f(x: int) -> str:\n return f\"exact={x:.2f}\"\n# 000001.2300 -9e-17\ndef f(x: int) -> str:\n return f\"exact={x:.2f}\"\n# 000001.2300 -9e-17\ndef f(x: int) -> str:\n return f\"exact={x:.2f}\"\n# 000001.2300 -9e-17\ndef f(x: int) -> str:\n return f\"exact={x:.2f}\"\n# 000001.2300 -9e-17\n", + "http_status": 200, + "provider_tokens": 633, + "o200k_base": 627, + "cl100k_base": 627, + "framing_delta": 6 + }, + { + "case": 3, + "text": "[{\"id\":0,\"amount\":\"12.30\",\"flag\":true},{\"id\":1,\"amount\":\"12.30\",\"flag\":false},{\"id\":2,\"amount\":\"12.30\",\"flag\":true},{\"id\":3,\"amount\":\"12.30\",\"flag\":false},{\"id\":4,\"amount\":\"12.30\",\"flag\":true},{\"id\":5,\"amount\":\"12.30\",\"flag\":false},{\"id\":6,\"amount\":\"12.30\",\"flag\":true},{\"id\":7,\"amount\":\"12.30\",\"flag\":false},{\"id\":8,\"amount\":\"12.30\",\"flag\":true},{\"id\":9,\"amount\":\"12.30\",\"flag\":false},{\"id\":10,\"amount\":\"12.30\",\"flag\":true},{\"id\":11,\"amount\":\"12.30\",\"flag\":false},{\"id\":12,\"amount\":\"12.30\",\"flag\":true},{\"id\":13,\"amount\":\"12.30\",\"flag\":false},{\"id\":14,\"amount\":\"12.30\",\"flag\":true},{\"id\":15,\"amount\":\"12.30\",\"flag\":false},{\"id\":16,\"amount\":\"12.30\",\"flag\":true},{\"id\":17,\"amount\":\"12.30\",\"flag\":false},{\"id\":18,\"amount\":\"12.30\",\"flag\":true},{\"id\":19,\"amount\":\"12.30\",\"flag\":false},{\"id\":20,\"amount\":\"12.30\",\"flag\":true},{\"id\":21,\"amount\":\"12.30\",\"flag\":false},{\"id\":22,\"amount\":\"12.30\",\"flag\":true},{\"id\":23,\"amount\":\"12.30\",\"flag\":false},{\"id\":24,\"amount\":\"12.30\",\"flag\":true},{\"id\":25,\"amount\":\"12.30\",\"flag\":false},{\"id\":26,\"amount\":\"12.30\",\"flag\":true},{\"id\":27,\"amount\":\"12.30\",\"flag\":false},{\"id\":28,\"amount\":\"12.30\",\"flag\":true},{\"id\":29,\"amount\":\"12.30\",\"flag\":false},{\"id\":30,\"amount\":\"12.30\",\"flag\":true},{\"id\":31,\"amount\":\"12.30\",\"flag\":false},{\"id\":32,\"amount\":\"12.30\",\"flag\":true},{\"id\":33,\"amount\":\"12.30\",\"flag\":false},{\"id\":34,\"amount\":\"12.30\",\"flag\":true},{\"id\":35,\"amount\":\"12.30\",\"flag\":false},{\"id\":36,\"amount\":\"12.30\",\"flag\":true},{\"id\":37,\"amount\":\"12.30\",\"flag\":false},{\"id\":38,\"amount\":\"12.30\",\"flag\":true},{\"id\":39,\"amount\":\"12.30\",\"flag\":false},{\"id\":40,\"amount\":\"12.30\",\"flag\":true},{\"id\":41,\"amount\":\"12.30\",\"flag\":false},{\"id\":42,\"amount\":\"12.30\",\"flag\":true}]", + "http_status": 200, + "provider_tokens": 610, + "o200k_base": 604, + "cl100k_base": 604, + "framing_delta": 6 + }, + { + "case": 4, + "text": "literal <|endoftext|> <|im_start|> \"quoted\"\n\t\r\nliteral <|endoftext|> <|im_start|> \"quoted\"\n\t\r\nliteral <|endoftext|> <|im_start|> \"quoted\"\n\t\r\nliteral <|endoftext|> <|im_start|> \"quoted\"\n\t\r\nliteral <|endoftext|> <|im_start|> \"quoted\"\n\t\r\nliteral <|endoftext|> <|im_start|> \"quoted\"\n\t\r\nliteral <|endoftext|> <|im_start|> \"quoted\"\n\t\r\nliteral <|endoftext|> <|im_start|> \"quoted\"\n\t\r\nliteral <|endoftext|> <|im_start|> \"quoted\"\n\t\r\nliteral <|endoftext|> <|im_start|> \"quoted\"\n\t\r\nliteral <|endoftext|> <|im_start|> \"quoted\"\n\t\r\n", + "http_status": 200, + "provider_tokens": 204, + "o200k_base": 198, + "cl100k_base": 176, + "framing_delta": 6 + } + ] +} diff --git a/tests/test_discovery_repair.py b/tests/test_discovery_repair.py new file mode 100644 index 0000000..a55f7a1 --- /dev/null +++ b/tests/test_discovery_repair.py @@ -0,0 +1,212 @@ +"""Deterministic concurrent discovery tests; never patch the live Hermes loader.""" + +import concurrent.futures +import hashlib +import logging +import os +import sys +import threading +import types +from pathlib import Path + +import pytest + +from rtk_hermes_plus import hermes_discovery as repair + +# The minimal fixture reproduces the same cache-before-exec ordering. Integration +# CI supplies the actual pinned Hermes source instead through TT_HERMES_LOADER. +_FIXTURE = """import importlib.util +import sys +from pathlib import Path + +def load_plugin_module(module_name, plugin_dir, *, parents, logger, synthetic_namespace=None): + cached = sys.modules.get(module_name) + if cached is not None and getattr(cached, "__file__", None): + return cached + spec = importlib.util.spec_from_file_location(module_name, plugin_dir / "__init__.py") + mod = importlib.util.module_from_spec(spec) + sys.modules[module_name] = mod + try: + spec.loader.exec_module(mod) + except Exception: + sys.modules.pop(module_name, None) + return None + return mod +""" + + +@pytest.fixture +def loader_source(monkeypatch): + source_path = os.environ.get("TT_HERMES_LOADER") + source = Path(source_path).read_text(encoding="utf-8") if source_path else _FIXTURE + if source_path: + # Do not whitelist an arbitrary host in the integration test. + assert ( + hashlib.sha256(source.encode()).hexdigest() in repair._KNOWN_SOURCE_SHA256 + ) + isolation = types.ModuleType("hermes_cli.plugin_isolation") + isolation.in_process_import_refusal = lambda _: None + monkeypatch.setitem(sys.modules, "hermes_cli.plugin_isolation", isolation) + else: + monkeypatch.setattr( + repair, + "_KNOWN_SOURCE_SHA256", + {hashlib.sha256(source.encode()).hexdigest()}, + ) + return source + + +def make_loader(tmp_path, source): + module = types.ModuleType("fixture_host_loader") + module.__file__ = str(tmp_path / "plugin_loader.py") + exec( # noqa: S102 - exact reviewed host source or literal test fixture + compile(repair.patched_source(source), module.__file__, "exec"), module.__dict__ + ) + return module + + +def load(module, directory): + return module.load_plugin_module( + "_hermes_user_context_engine.token-terminator", + directory, + parents=(), + logger=logging.getLogger("discovery-test"), + synthetic_namespace="_hermes_user_context_engine", + ) + + +def plugin(tmp_path, name, body): + directory = tmp_path / name / "token-terminator" + directory.mkdir(parents=True) + (directory / "__init__.py").write_text(body, encoding="utf-8") + return directory + + +def test_concurrent_reader_waits_until_initialization_finishes( + tmp_path, monkeypatch, loader_source +): + module = make_loader(tmp_path, loader_source) + control = types.ModuleType("tt_test_load_control") + control.started, control.release = threading.Event(), threading.Event() + monkeypatch.setitem(sys.modules, control.__name__, control) + directory = plugin( + tmp_path, + "default", + "from tt_test_load_control import started, release\nstarted.set()\nassert release.wait(5)\nREADY = True\n", + ) + with concurrent.futures.ThreadPoolExecutor(max_workers=2) as pool: + first = pool.submit(load, module, directory) + assert control.started.wait(5) + second = pool.submit(load, module, directory) + try: + with pytest.raises(concurrent.futures.TimeoutError): + second.result(timeout=0.05) + finally: + control.release.set() + a, b = first.result(timeout=5), second.result(timeout=5) + assert a is b and a.READY + + +def test_many_simultaneous_requests_return_complete_module(tmp_path, loader_source): + module = make_loader(tmp_path, loader_source) + directory = plugin( + tmp_path, "default", "import time\ntime.sleep(.03)\nREADY = 42\n" + ) + with concurrent.futures.ThreadPoolExecutor(max_workers=16) as pool: + modules = list(pool.map(lambda _: load(module, directory), range(64))) + assert all(m is modules[0] and m.READY == 42 for m in modules) + + +def test_profiles_with_same_slug_use_different_modules(tmp_path, loader_source): + module = make_loader(tmp_path, loader_source) + a = plugin(tmp_path, "default", 'PROFILE = "default"\n') + b = plugin(tmp_path, "brain", 'PROFILE = "brain"\n') + with concurrent.futures.ThreadPoolExecutor(max_workers=2) as pool: + first, second = list(pool.map(lambda d: load(module, d), [a, b])) + assert first.PROFILE == "default" and second.PROFILE == "brain" + assert first is not second + assert first.__name__ != second.__name__ + + +def test_failed_initialization_can_be_retried(tmp_path, loader_source): + module = make_loader(tmp_path, loader_source) + directory = plugin(tmp_path, "default", 'raise RuntimeError("fixture")\n') + assert load(module, directory) is None + (directory / "__init__.py").write_text('READY = "recovered successfully"\n') + assert load(module, directory).READY == "recovered successfully" + + +def test_recursive_discovery_does_not_expose_partial_module( + tmp_path, monkeypatch, loader_source +): + module = make_loader(tmp_path, loader_source) + control = types.ModuleType("tt_test_recursive") + directory = plugin( + tmp_path, + "default", + "from tt_test_recursive import again\nassert again() is None\nREADY = True\n", + ) + control.again = lambda: load(module, directory) + monkeypatch.setitem(sys.modules, control.__name__, control) + assert load(module, directory).READY + + +def test_other_plugins_use_original_loader_unchanged(tmp_path, loader_source): + module = make_loader(tmp_path, loader_source) + seen = [] + module._tt_loader_original = lambda *a, **kw: seen.append((a, kw)) or "original" + assert ( + module.load_plugin_module("other", tmp_path, parents=(), logger=None) + == "original" + ) + assert seen[0][0] == ("other", tmp_path) + + +def test_installer_backup_and_idempotency(tmp_path, loader_source): + path = tmp_path / "plugin_loader.py" + before = loader_source.replace("\n", "\r\n").encode() + path.write_bytes(before) + result = repair.repair_hermes_discovery(path) + assert result["changed"] + assert Path(result["backup"]).read_bytes() == before + installed = path.read_bytes() + assert repair.repair_hermes_discovery(path)["state"] == "already_repaired" + assert path.read_bytes() == installed + assert not list(tmp_path.glob(".tt-loader-*")) + + +def test_unknown_host_is_not_modified(tmp_path): + path = tmp_path / "plugin_loader.py" + path.write_text("unrelated = True\n") + with pytest.raises(ValueError, match="not the reviewed"): + repair.repair_hermes_discovery(path) + assert path.read_text() == "unrelated = True\n" + assert len(list(tmp_path.iterdir())) == 1 + + +def test_modified_repair_is_rejected(loader_source): + patched = repair.patched_source(loader_source) + with pytest.raises(ValueError, match="unrecognized"): + repair.patched_source(patched + "\n# another change\n") + + +def test_changed_host_with_intact_repair_suffix_is_not_approved(loader_source): + patched = repair.patched_source(loader_source) + with pytest.raises(ValueError, match="unrecognized"): + repair.patched_source("# host changed\n" + patched) + + +def test_backup_symlink_is_refused(tmp_path, loader_source): + path = tmp_path / "plugin_loader.py" + path.write_text(loader_source) + target = tmp_path / "other.txt" + target.write_text("unrelated") + backup = path.with_name(path.name + ".tt-before-0.11.2") + try: + backup.symlink_to(target) + except OSError: + pytest.skip("symlinks unavailable on this Windows account") + with pytest.raises(ValueError, match="symlinked Hermes backup"): + repair.repair_hermes_discovery(path) + assert path.read_text() == loader_source + assert target.read_text() == "unrelated" diff --git a/tests/test_gpt61_tokenizer.py b/tests/test_gpt61_tokenizer.py new file mode 100644 index 0000000..10390e4 --- /dev/null +++ b/tests/test_gpt61_tokenizer.py @@ -0,0 +1,138 @@ +"""New-model regression: no override, no substituted model name, real BPE.""" + +import copy +import json +from types import SimpleNamespace + +import pytest +import tiktoken + +from rtk_hermes_plus.token_budget import TokenBudgetAdapter, _compat_encoding + + +@pytest.mark.parametrize( + "model", + [ + "gpt-6.1-sol", + "openai/gpt-6.1-sol", + "openai:gpt-6.1-sol", + "openai-codex/gpt-6.1-sol", + "openrouter/openai/gpt-6.1-sol", + ], +) +def test_sol_request_has_measured_tokens_without_override(monkeypatch, model): + monkeypatch.delenv("TOKEN_TERMINATOR_TOKENIZER_JSON", raising=False) + monkeypatch.delenv("TOKEN_TERMINATOR_TIKTOKEN_ENCODING", raising=False) + monkeypatch.setenv("TOKEN_TERMINATOR_TOKEN_BUDGET", "true") + request = { + "model": model, + "messages": [ + {"role": "user", "content": "Exact 12.30; café; 日本語; 🧪"} + ], + } + before = copy.deepcopy(request) + result = TokenBudgetAdapter().measure_request(request) + canonical = json.dumps( + request, ensure_ascii=False, sort_keys=True, separators=(",", ":") + ) + assert result.tokens == len( + tiktoken.get_encoding("o200k_base").encode(canonical, disallowed_special=()) + ) + assert result.model == model + assert result.available + assert request == before + + +@pytest.mark.parametrize( + "model", + [ + "vendor/gpt-6.1-sol", + "gpt-6.1-sol-FAKE", + "gpt-6.2-sol", + "gpt-60", + "claude-opus", + "unknown", + "gpt-6.1-terra", + ], +) +def test_new_mapping_does_not_guess_other_models(model): + assert _compat_encoding(model) == "" + + +def test_namespace_cache_does_not_leak_compatibility(monkeypatch): + monkeypatch.delenv("TOKEN_TERMINATOR_TIKTOKEN_ENCODING", raising=False) + monkeypatch.delenv("TOKEN_TERMINATOR_TOKENIZER_JSON", raising=False) + adapter = TokenBudgetAdapter() + # Deliberately emulate the upstream version missing this model. + adapter._tiktoken_attempted = True + + def unknown(_): + raise KeyError("missing") + + adapter._tiktoken = SimpleNamespace( + encoding_for_model=unknown, get_encoding=tiktoken.get_encoding + ) + assert adapter.measure_text("x", model="openai/gpt-6.1-sol").available + assert not adapter.measure_text("x", model="other/gpt-6.1-sol").available + + +def test_upstream_mapping_takes_precedence(monkeypatch): + monkeypatch.delenv("TOKEN_TERMINATOR_TOKENIZER_JSON", raising=False) + adapter = TokenBudgetAdapter() + adapter._tiktoken_attempted = True + selected = tiktoken.get_encoding("cl100k_base") + adapter._tiktoken = SimpleNamespace(encoding_for_model=lambda _: selected) + encoding, label = adapter._tiktoken_encoding("gpt-6.1-sol") + assert encoding is selected + assert label == "tiktoken:model:gpt-6.1-sol" + + +def test_sol_context_engine_reaches_selection_and_strict_gate(tmp_path, monkeypatch): + from test_history_engine import low_scores, request, run, setup + + monkeypatch.delenv("TOKEN_TERMINATOR_TIKTOKEN_ENCODING", raising=False) + monkeypatch.delenv("TOKEN_TERMINATOR_TOKENIZER_JSON", raising=False) + runtime, engine = setup(tmp_path) + engine.update_model("gpt-6.1-sol", 1050000) + req = request(engine) + req["model"] = "gpt-6.1-sol" + original = copy.deepcopy(req) + calls = [] + + def score(payload): + calls.append(1) + return low_scores(payload) + + engine.transport = score + final = run(runtime, engine, req) + assert calls, "Sol must not return early as unknown tokenizer before JEV" + assert engine.get_status()["context_engine"]["omitted_messages"] > 0 + assert final["messages"][0] == original["messages"][0] + assert final["messages"][-1] == original["messages"][-1] + aid = final["messages"][1]["content"].split()[2].rstrip(";") + assert engine._history().read("session", aid) == original["messages"][1] + counter = TokenBudgetAdapter() + assert counter.measure_request(final).tokens < counter.measure_request(req).tokens + assert req == original + + +def test_recorded_provider_corpus_matches_local_content_counter(): + from pathlib import Path + + report = json.loads( + (Path(__file__).parent / "fixtures/gpt61-token-counts.json").read_text( + encoding="utf-8" + ) + ) + assert report["model"] == "gpt-6.1-sol" + assert report["generation_calls"] == 0 + assert len(report["results"]) == 5 + encoding = tiktoken.get_encoding("o200k_base") + for row in report["results"]: + assert row["http_status"] == 200 + assert ( + len(encoding.encode(row["text"], disallowed_special=())) + == row["o200k_base"] + ) + assert row["provider_tokens"] - row["o200k_base"] == 6 + assert any(r["o200k_base"] != r["cl100k_base"] for r in report["results"]) diff --git a/wiki/Context-Engine.md b/wiki/Context-Engine.md index f5ebd64..dc88a52 100644 --- a/wiki/Context-Engine.md +++ b/wiki/Context-Engine.md @@ -1,5 +1,11 @@ # Token Terminator ContextEngine · v0.11.0 +> **v0.11.2 install change:** The explicit engine installer also repairs the reviewed +> affected Hermes directory loader, with an exact source guard and backup. This is +> a narrow host-file repair for concurrent TT loading, not a replacement Hermes +> runtime. See the repository `docs/HERMES_DISCOVERY.md` before installation. + + **Experimental, explicitly selected.** TT can replace the selected LCM/built-in context engine through supported Hermes plugin discovery, without modifying Hermes core. Middleware mode remains available. [Quick Start](Quick-Start) contains the diff --git a/wiki/Home.md b/wiki/Home.md index 4081ece..68d342c 100644 --- a/wiki/Home.md +++ b/wiki/Home.md @@ -1,5 +1,9 @@ # Token Terminator Wiki +**Current patch: [v0.11.2](Release-0.11.2)** — GPT-6.1-Sol token lookup and synchronized, profile-isolated TT discovery. + +The explicit engine installer now includes a guarded host-loader repair for affected Hermes versions; see [repair and rollback](Tokenizer-and-Discovery-Repair). + **Keep the evidence. Terminate the redundant tokens.** ## v0.11.0 · Context, learning & observability diff --git a/wiki/Release-0.11.2.md b/wiki/Release-0.11.2.md new file mode 100644 index 0000000..f94709a --- /dev/null +++ b/wiki/Release-0.11.2.md @@ -0,0 +1,58 @@ +# Token Terminator v0.11.2 — Sol counting and reliable TT discovery + +This repair addresses two blockers left in 0.11.1. It does not change JEV keys, +semantic policies, the evidence vault, Context IR, or the main-call-only boundary. + +## GPT-6.1-Sol + +The exact `gpt-6.1-sol` ID and known OpenAI/Codex/OpenRouter-qualified forms now +resolve to `o200k_base` when the installed tiktoken model-name table lacks the ID. +An upstream mapping still wins, and an explicit operator encoding remains supported. +Unrelated providers, invented versions and sibling names are not inferred. + +Five synthetic inputs were checked against OpenAI's documented input-token counter: +English, multilingual/emoji text, code/numbers, structured JSON and literal special +markers. All matched local content counts plus the same six-token message framing. +The recorded corpus distinguishes o200k_base from cl100k_base. This is empirical +compatibility evidence, not an upstream tiktoken declaration or proof about every +possible input. TT still measures complete canonical request JSON, not hidden +provider framing or billed tokens. No generation requests were used in that check. + +## Concurrent Hermes discovery + +The affected shared loader can return a module before its initialization completes, +and its slug-only cache can reuse one profile's module for another. TT's explicit +`install-context-engine` command now applies a source-hash-guarded repair to the +reviewed affected loader. TT loads are synchronized, recursive discovery cannot +consume a partial module, and each resolved profile path has a distinct namespace. +Other plugins use the original loader. Failed initialization can be retried. + +This is a narrow host-file compatibility patch, not merely a plugin import change: +the race occurs before the plugin can run. An adjacent byte-exact backup is kept. +Unknown or modified host implementations are refused rather than overwritten. +The Python installer API and `install-dashboard` do not patch the host by default. +No changes occur on package import, and configuration/history are not rewritten. + +## Install + +Use the interpreter/environment actually loaded by the managed Hermes launcher: + +```bash +python -m pip install --upgrade 'git+https://github.com/AronAxe/Token-Terminator.git@v0.11.2' +python -m rtk_hermes_plus.cli install-context-engine +``` + +The second command reports `discovery_repair`; for an affected installed host it +must say `repaired` or `already_repaired`. Then restart the appropriate idle backend +through its supervisor once. A package download alone cannot repair a running process. +Existing explicit engine selection and JEV/IR opt-ins remain unchanged. + +Regression checks exercise the actual GPT-6.1-Sol model name without encoding +overrides, JEV selection, strict complete-request reduction and exact recovery. +Discovery checks include controlled overlapping initialization, 64 simultaneous +loads, profile separation, recursive loading, failed-load retry and safe installation. +A dedicated CI job runs those same tests against the actual pinned Hermes loader +source, not an optional skipped module. Full CI results are recorded on the PR. + +The Rust companion is version-aligned only. Existing tags, Terminator artwork and +schematics remain unchanged. See docs/TOKENIZER_COMPAT.md and docs/HERMES_DISCOVERY.md. diff --git a/wiki/Tokenizer-and-Discovery-Repair.md b/wiki/Tokenizer-and-Discovery-Repair.md new file mode 100644 index 0000000..9ea044b --- /dev/null +++ b/wiki/Tokenizer-and-Discovery-Repair.md @@ -0,0 +1,46 @@ +# Hermes TT discovery repair + +The reviewed Hermes shared directory loader inserts a new module into sys.modules +before running it, but treats a cached module with __file__ as ready. Concurrent +requests can receive that half-module. User engine module names also contain only +the plugin slug, not the profile path. These are initialization and isolation bugs. + +## What 0.11.2 installs + +The explicit `token-terminator install-context-engine` command requests the repair. +It patches only a reviewed loader file with normalized SHA-256 +`402d3fd6f6077c344f15713ac3b793fbccbe0602181bceff7a23690270e3ff35`, checked against +Hermes `188a1a5efd098f067638032fea6e24c42d06178f` and the installed affected source. + +The appended wrapper applies only to token-terminator in the user ContextEngine +namespace. A lock covers the original load until initialization finishes. Other +profiles receive path-derived namespaces; unrelated plugins retain original +behavior. Recursive same-thread loads return unavailable, never a partial module. +Waiting is bounded to ten seconds. Failed imports remain retryable. + +No plugin-only hook can repair a lookup that happens before that plugin starts. +This release therefore makes the host-file change explicit. It is not a general +Hermes plugin-loader rewrite, new runtime dependency, or replacement context engine. + +The original bytes are backed up beside the source as +`plugin_loader.py.tt-before-0.11.2`. The installer refuses unknown source versions, +modified patches, symlinks, mismatching backups and concurrent source changes. +Package import and ordinary middleware do not write host files. The Python +`install_context_engine` API defaults to `repair_discovery=False`; callers that +want the explicit CLI behavior pass `repair_discovery=True`. + +## Verification and rollback + +`tests/test_discovery_repair.py` deterministically holds one initialization open +while another lookup arrives. It also tests 64 parallel calls and profile isolation. +Set TT_HERMES_LOADER to a copy or checkout of the actual reviewed plugin_loader.py +to test its real implementation; tests only modify temporary files. + +After installation, restart the correct idle backend once and verify discovery and +provider-request measurements. Do not delete history, vaults or unrelated plugins. +To roll back, stop the affected backend and restore the adjacent original backup +only after checking that no subsequent Hermes changes would be overwritten. If +Hermes upgrades its loader, review that new implementation rather than forcing this +patch onto it. A successful health check alone is not proof of token reduction. + +See the repository docs/TOKENIZER_COMPAT.md for the Sol counting evidence. diff --git a/wiki/_Footer.md b/wiki/_Footer.md index 3f0b189..18092fe 100644 --- a/wiki/_Footer.md +++ b/wiki/_Footer.md @@ -1,3 +1,3 @@ -Token Terminator v0.11.0 · [Repository](https://github.com/AronAxe/Token-Terminator) · [Releases](https://github.com/AronAxe/Token-Terminator/releases) · [crates.io](https://crates.io/crates/token-terminator) · MIT +Token Terminator v0.11.2 · [Repository](https://github.com/AronAxe/Token-Terminator) · [Releases](https://github.com/AronAxe/Token-Terminator/releases) · [crates.io](https://crates.io/crates/token-terminator) · MIT Documentation is source-controlled from the repository `wiki/` directory. diff --git a/wiki/_Sidebar.md b/wiki/_Sidebar.md index 5829034..6cfb6ff 100644 --- a/wiki/_Sidebar.md +++ b/wiki/_Sidebar.md @@ -35,3 +35,6 @@ - [Release 0.6.0](Release-0.6.0) - [Release 0.5.2](Release-0.5.2) - [Release 0.5.1](Release-0.5.1) + +- [Release 0.11.2](Release-0.11.2) +- [Tokenizer and discovery repair](Tokenizer-and-Discovery-Repair)