From ffb967c46d5a240880287be8a6856f3dc876bb9b Mon Sep 17 00:00:00 2001 From: Evgeny Kiriyak <224408464+evkir@users.noreply.github.com> Date: Fri, 30 Oct 2026 09:29:04 +0100 Subject: [PATCH 1/4] test: gate the prose that states the measured scope edge The boundary guard holds two counts against the tree and reds when either moves. It does not read the page, so a move is paid for by rewriting the constant while the sentence stating the same two numbers on docs/architecture/typing-scope.md stays behind, green and wrong. Three numbers on that page had drifted that way before anyone looked. This guard runs the other way: the constant is the standard and the prose is what is checked, which is sound only because the neighbour holds that constant against the tree. It reads the constants by importing the neighbour rather than recomputing the crossings, so the two cannot answer from different methods. One sentence states the current edge; five others on the page state historical pairs and carry dates. The pattern matches the undated form only, and a second occurrence of it reds rather than drifting behind the first. The neighbour's own docstring had drifted the same way: it claimed 107 modules against a constant of 118, and spelled its crosser count as a word, which no reader of digits can check. Restated in digits, remeasured today, and gated by the third assertion here. Mutation: 5 mutants, 3 killed, 2 equivalent. Moving either constant kills three tests; dropping the first number from the page kills two; a wrong second number alone kills one. The one-match and whole-pair comparisons survive as equivalents, measured rather than assumed. --- README.md | 2 +- blog/launch-post-draft.md | 2 +- ...test_the_prose_quotes_the_measured_edge.py | 70 +++++++++++++++++++ .../test_typing_scope_boundary_is_declared.py | 8 +-- 4 files changed, 76 insertions(+), 6 deletions(-) create mode 100644 tests/architecture/test_the_prose_quotes_the_measured_edge.py diff --git a/README.md b/README.md index 89643dd..953bd4b 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@ ![Python](https://img.shields.io/badge/python-3.11%20%7C%203.12%20%7C%203.13%20%7C%203.14-blue) ![License](https://img.shields.io/badge/license-Apache_2.0-blue) ![Version](https://img.shields.io/badge/version-v1.7.0-brightgreen) -![Tests](https://img.shields.io/badge/tests-3241%20collected-brightgreen) +![Tests](https://img.shields.io/badge/tests-3244%20collected-brightgreen) ![Mypy](https://img.shields.io/badge/mypy-strict%3A%20118%2F175%20modules-blue) ![LLM](https://img.shields.io/badge/LLM-OpenAI%20%7C%20Anthropic%20%7C%20Ollama-blueviolet) ![Air-Gapped](https://img.shields.io/badge/air--gapped-ready-success) diff --git a/blog/launch-post-draft.md b/blog/launch-post-draft.md index ed7e91f..0a8e6d9 100644 --- a/blog/launch-post-draft.md +++ b/blog/launch-post-draft.md @@ -13,7 +13,7 @@ not exist yet, it says so. CyberAI is a multi-agent offensive-security platform: eight agents (recon, intel, exploit, report, planner, mcp-scan, redteam, web3) run a typed, audited pipeline over a shared knowledge base. -3241 tests collected under the gated selection run before every commit, with the +3244 tests collected under the gated selection run before every commit, with the slow and smoke tests deselected there and run separately, `mypy --strict` clean over 118 of 175 modules, Apache-2.0. diff --git a/tests/architecture/test_the_prose_quotes_the_measured_edge.py b/tests/architecture/test_the_prose_quotes_the_measured_edge.py new file mode 100644 index 0000000..d7c5d65 --- /dev/null +++ b/tests/architecture/test_the_prose_quotes_the_measured_edge.py @@ -0,0 +1,70 @@ +"""The prose beside a measured count must quote that count. + +`test_typing_scope_boundary_is_declared.py` holds two numbers against the +tree and reds when either moves. What it does not do is read the page. A +move therefore reds there, the constant gets rewritten by whoever moved it, +and the sentence in docs/architecture/typing-scope.md that states the same +two numbers stays behind -- green, silent and wrong. Three numbers on that +page drifted exactly that way before anyone noticed. + +The direction of this check is the opposite of the one next door, and that +is deliberate. There the tree is the standard and the prose is not read, so +the test cannot agree with itself. Here the constant is the standard and the +prose is what is checked, which is sound only because the constant is itself +held against the tree by that other test. Remove it and this one measures +nothing. + +Dated counts are out of reach on purpose. A sentence that says when it was +measured is a record, not a claim about today, and pinning one would red on +every release of the checker. +""" + +import importlib.util +import pathlib +import re +import sys + +_ROOT = pathlib.Path(__file__).resolve().parents[2] +_PAGE = _ROOT / "docs" / "architecture" / "typing-scope.md" +_NEIGHBOUR = _ROOT / "tests" / "architecture" / "test_typing_scope_boundary_is_declared.py" + + +def _neighbour_constants() -> tuple[int, int]: + spec = importlib.util.spec_from_file_location("_boundary_guard", _NEIGHBOUR) + assert spec is not None and spec.loader is not None, f"cannot load {_NEIGHBOUR}" + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return int(module._EXPECTED_CROSSERS), int(module._EXPECTED_REACHED) + + +_CURRENT = re.compile( + r"(\d+) import a module outside it at module level,\s*\n" + r"and between them they reach (\d+) such modules" +) + + +def test_the_page_states_the_edge_in_one_undated_sentence() -> None: + """One match, so a second phrasing cannot drift unseen behind the first.""" + found = _CURRENT.findall(_PAGE.read_text(encoding="utf-8")) + assert len(found) == 1, f"the page states the current edge {len(found)} times, not once" + + +def test_the_page_quotes_the_edge_the_neighbour_holds() -> None: + found = _CURRENT.search(_PAGE.read_text(encoding="utf-8")) + assert found, "the page no longer states the edge in the form this guard reads" + quoted = (int(found.group(1)), int(found.group(2))) + assert quoted == _neighbour_constants(), ( + f"the page says {quoted} while the guard beside it holds {_neighbour_constants()}" + ) + + +def test_the_neighbour_prose_quotes_its_own_constants() -> None: + """The guard's own docstring states the counts and drifted away from them.""" + crossers, _ = _neighbour_constants() + head = _NEIGHBOUR.read_text(encoding="utf-8").split('"""')[1] + match = re.search(r"(\d+) of them import a module outside the scope", head) + assert match, "the guard's docstring no longer states its crosser count in digits" + assert int(match.group(1)) == crossers, ( + f"the docstring says {match.group(1)} crossers while the constant holds {crossers}" + ) diff --git a/tests/architecture/test_typing_scope_boundary_is_declared.py b/tests/architecture/test_typing_scope_boundary_is_declared.py index 7c67f0c..8fc3d69 100644 --- a/tests/architecture/test_typing_scope_boundary_is_declared.py +++ b/tests/architecture/test_typing_scope_boundary_is_declared.py @@ -1,10 +1,10 @@ """The typing scope has an edge, and until now nothing said where. -`[tool.mypy] files` resolves to 107 modules and the run reports `Success` on -exactly 107, which reads like a guarantee about those modules and is one only -up to the boundary. Twenty-two of them import a module outside the scope at +`[tool.mypy] files` resolves to 118 modules and the run reports `Success` on +exactly 118, which reads like a guarantee about those modules and is one only +up to the boundary. 16 of them import a module outside the scope at module level. Both numbers were 95 and 19 when this was written and are -restated here on 2026-09-23; the assertions below are what holds them. mypy follows such an import to resolve the name and stays silent about +restated here on 2026-10-30; the assertions below are what holds them. mypy follows such an import to resolve the name and stays silent about what it finds: appending an unannotated function to `cyberai/core/config.py`, which is outside the scope and imported from inside it, changed nothing about the output on a cold cache. A name crossing the edge is therefore typed by a From c26236345064d20c750beb5bb56a67d8c4a290aa Mon Sep 17 00:00:00 2001 From: Evgeny Kiriyak <224408464+evkir@users.noreply.github.com> Date: Fri, 30 Oct 2026 09:35:12 +0100 Subject: [PATCH 2/4] docs: remeasure the two undated counts on the typing scope page Both paragraphs stated a number as current and carried no date, so nothing on the page or in the tree said when they had last been true. Both were stale. The cache-hazard paragraph said a cold scoped run reports Success on 97 modules and a run over the cache left by a wide one gives 239 errors in 54 files. Measured today: 118, then 216 in 44. The qualitative claim beside them held -- all 44 lie outside the scope, checked by resolving [tool.mypy] files against the error lines rather than by reading the count. The partition paragraph said the clean side holds 98 modules. The drift report computes that set as the package minus what fails, and prints 118 today under the same release the sentence names. Both now carry the measurement date, which is what the page already does for the error count beside the ratio and what the guard there requires. Neither is gated: they state what a checker printed on a day, and a test pinning them would red on the next release and teach the reader to delete it. --- docs/architecture/typing-scope.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/architecture/typing-scope.md b/docs/architecture/typing-scope.md index fc75c39..376e32f 100644 --- a/docs/architecture/typing-scope.md +++ b/docs/architecture/typing-scope.md @@ -122,9 +122,9 @@ mypy --strict --python-version 3.11 --ignore-missing-imports cyberai The cache purge is not decoration. A scoped run that reuses a cache left by a wider run re-emits errors for modules outside the scope, and reports them as -if the declared set were dirty. Measured: cold cache gives `Success` on 97 -modules, the same command after a full-package run gives 239 errors in 54 -files, and every one of those files lies outside the scope. +if the declared set were dirty. Measured 2026-10-30: cold cache gives +`Success` on 118 modules, the same command after a full-package run gives +216 errors in 44 files, and every one of those files lies outside the scope. `scripts/typing_scope_drift.py` runs the same partition without the hazard. It reads the flags from `[tool.mypy]` instead of repeating them, so this page @@ -133,9 +133,9 @@ wide run a cache directory of its own instead of purging the shared one. ## What the numbers depend on -The partition moves with the checker. Measured on mypy 1.19.1 the clean side -holds 98 modules; a later release moved it by one module in the other -direction. The dev extra therefore bounds the checker rather than naming a +The partition moves with the checker. Measured 2026-10-30 on mypy 1.19.1 the +clean side holds 118 modules; a later release moved it by one module in the +other direction. The dev extra therefore bounds the checker rather than naming a floor and admitting every future release. The checker is not the only version these counts depend on, and on 2026-09-18 From b39de3049ba46081ff5beeaecfa582623340a893 Mon Sep 17 00:00:00 2001 From: Evgeny Kiriyak <224408464+evkir@users.noreply.github.com> Date: Fri, 30 Oct 2026 09:48:36 +0100 Subject: [PATCH 3/4] docs: name the two guards the edge paragraph leans on The paragraph said the crossing counts are held by 'the test named below' and nothing below named a test. The reference had been broken for as long as it had existed: a reader following it found no name, and no step in the tree noticed, because a sentence pointing at nothing parses the same as one pointing at something. It now names both guards by path and says what each holds, which are different things. One walks the imports and reds when a count moves in the tree. The other reds when this sentence stops quoting what the first one holds -- the drift that rewriting the constant alone used to leave behind. Naming a file on a page is a claim about the tree, so it gets a guard in the same commit: the paths are read off the page and checked for existence, and the page is required to name at least one. Mutation: 3 mutants, 2 killed, 1 equivalent. A path that names no file kills the new test; removing both paths kills it on the non-empty requirement; dropping that requirement while the paths are still there changes nothing and is equivalent only while they are. A first run scored the second of those SURVIVED -- the harness collapsed two edits to one file into a single snapshot and wrote only the later one, so the mutant on disk was half of what it claimed to be. Measured again, singly. --- README.md | 2 +- blog/launch-post-draft.md | 2 +- docs/architecture/typing-scope.md | 11 ++++++++--- .../test_the_prose_quotes_the_measured_edge.py | 14 ++++++++++++++ 4 files changed, 24 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 953bd4b..67b6f95 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@ ![Python](https://img.shields.io/badge/python-3.11%20%7C%203.12%20%7C%203.13%20%7C%203.14-blue) ![License](https://img.shields.io/badge/license-Apache_2.0-blue) ![Version](https://img.shields.io/badge/version-v1.7.0-brightgreen) -![Tests](https://img.shields.io/badge/tests-3244%20collected-brightgreen) +![Tests](https://img.shields.io/badge/tests-3245%20collected-brightgreen) ![Mypy](https://img.shields.io/badge/mypy-strict%3A%20118%2F175%20modules-blue) ![LLM](https://img.shields.io/badge/LLM-OpenAI%20%7C%20Anthropic%20%7C%20Ollama-blueviolet) ![Air-Gapped](https://img.shields.io/badge/air--gapped-ready-success) diff --git a/blog/launch-post-draft.md b/blog/launch-post-draft.md index 0a8e6d9..8e9f389 100644 --- a/blog/launch-post-draft.md +++ b/blog/launch-post-draft.md @@ -13,7 +13,7 @@ not exist yet, it says so. CyberAI is a multi-agent offensive-security platform: eight agents (recon, intel, exploit, report, planner, mcp-scan, redteam, web3) run a typed, audited pipeline over a shared knowledge base. -3244 tests collected under the gated selection run before every commit, with the +3245 tests collected under the gated selection run before every commit, with the slow and smoke tests deselected there and run separately, `mypy --strict` clean over 118 of 175 modules, Apache-2.0. diff --git a/docs/architecture/typing-scope.md b/docs/architecture/typing-scope.md index 376e32f..d338ce8 100644 --- a/docs/architecture/typing-scope.md +++ b/docs/architecture/typing-scope.md @@ -26,9 +26,14 @@ than repeatable as written; the mechanism it showed is unchanged. So a name crossing the boundary is typed by a module nothing checks, and adding a module to the scope costs its import closure rather than its own error count -- `mypy` on three single-error modules alone reports 90 errors across 26 -files. The crossing is measured on every run by the test named below, which -reds when either count moves, so the paragraph cannot drift away from it. Both numbers are measured, not chosen, and they -are measured on the runner: the typecheck job prints them on every run. +files. Two tests hold the pair above, and they hold different things. +`tests/architecture/test_typing_scope_boundary_is_declared.py` walks the +imports and reds when either count moves in the tree. +`tests/architecture/test_the_prose_quotes_the_measured_edge.py` reds when +the sentence above stops quoting what that one holds, which is the drift +that fixing the first test alone used to leave behind. Both numbers are +measured, not chosen, and they are measured on the runner: the typecheck +job prints them on every run. Four modules were carried into the scope rather than found there. The call at `cyberai/cli/bench.py` that reported `Cannot call function of unknown type` diff --git a/tests/architecture/test_the_prose_quotes_the_measured_edge.py b/tests/architecture/test_the_prose_quotes_the_measured_edge.py index d7c5d65..5962d88 100644 --- a/tests/architecture/test_the_prose_quotes_the_measured_edge.py +++ b/tests/architecture/test_the_prose_quotes_the_measured_edge.py @@ -59,6 +59,20 @@ def test_the_page_quotes_the_edge_the_neighbour_holds() -> None: ) +def test_the_page_names_tests_that_exist() -> None: + """The page points the reader at two files by path. + + A renamed or deleted guard leaves the sentence pointing at nothing, and + the reader who follows it finds out before the page does. The earlier + wording said "the test named below" and named none, which is the same + break with nothing to check it against. + """ + named = re.findall(r"`(tests/architecture/\S+?\.py)`", _PAGE.read_text(encoding="utf-8")) + assert named, "the page names no guard for the counts it states" + missing = [path for path in named if not (_ROOT / path).is_file()] + assert not missing, f"the page names guards that do not exist: {missing}" + + def test_the_neighbour_prose_quotes_its_own_constants() -> None: """The guard's own docstring states the counts and drifted away from them.""" crossers, _ = _neighbour_constants() From 1a4318db77ae5e538522a16fe61793be29e7a417 Mon Sep 17 00:00:00 2001 From: Evgeny Kiriyak <224408464+evkir@users.noreply.github.com> Date: Fri, 30 Oct 2026 09:55:29 +0100 Subject: [PATCH 4/4] docs: record the prose guard in the changelog Unreleased entry for the three commits on this branch: the guard that holds the edge paragraph against the constant the neighbouring test keeps, the two counts remeasured and dated, and the two guards now named by path where the page used to point at a test that did not exist. --- CHANGELOG.md | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 64d0b8f..fb532e9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,21 @@ All notable changes to CyberAI are documented here. ### Added +- **The prose that states a measured count is held against that count.** Two + numbers describe where the typing scope ends, and a test walked the imports + and red when either moved in the tree. It never read the page that states + them, so a move was paid for by rewriting the constant while the sentence + quoting it stayed behind -- green, silent and wrong. Three numbers on that + page had drifted exactly that way before anyone looked, and the guard's own + docstring had drifted with them, claiming a scope size the constant beside + it contradicted. The new guard runs the other way round: the constant is the + standard and the prose is what is checked, which is sound only because the + older test holds that constant against the tree. It reads the constants by + importing that test rather than recomputing the crossings, so the two cannot + answer from different methods. Dated sentences stay out of reach on purpose + -- a count that says when it was measured is a record, not a claim about + today. + - **A finding's confidence reaches the machine-readable export.** The field carries what the agents did not fully evidence: an out-of-band attempt with no callback, a redteam report the pipeline could not confirm. Both renderers