From 564bbbade6d63d5214d4e7a2577936a31a260716 Mon Sep 17 00:00:00 2001 From: Andrii Shylenko <14119286+w1ne@users.noreply.github.com> Date: Thu, 13 Aug 2026 10:09:16 +0200 Subject: [PATCH 01/48] test: add first Twin2Silicon repair fixture --- benchmarks/twin2silicon/prepare-oracle.py | 10 ++++++++ .../f103-gpio-clock-001/hidden/oracle.yaml | 8 ++++++ .../f103-gpio-clock-001/hidden/system.yaml | 4 +++ .../f103-gpio-clock-001/public/README.md | 6 +++++ .../public/firmware/Makefile | 13 ++++++++++ .../public/firmware/bench.ld | 12 +++++++++ .../public/firmware/main.c | 25 +++++++++++++++++++ .../public/firmware/startup.c | 19 ++++++++++++++ .../tasks/f103-gpio-clock-001/task.json | 1 + tests/twin2silicon-fixture.sh | 22 ++++++++++++++++ 10 files changed, 120 insertions(+) create mode 100644 benchmarks/twin2silicon/prepare-oracle.py create mode 100644 benchmarks/twin2silicon/tasks/f103-gpio-clock-001/hidden/oracle.yaml create mode 100644 benchmarks/twin2silicon/tasks/f103-gpio-clock-001/hidden/system.yaml create mode 100644 benchmarks/twin2silicon/tasks/f103-gpio-clock-001/public/README.md create mode 100644 benchmarks/twin2silicon/tasks/f103-gpio-clock-001/public/firmware/Makefile create mode 100644 benchmarks/twin2silicon/tasks/f103-gpio-clock-001/public/firmware/bench.ld create mode 100644 benchmarks/twin2silicon/tasks/f103-gpio-clock-001/public/firmware/main.c create mode 100644 benchmarks/twin2silicon/tasks/f103-gpio-clock-001/public/firmware/startup.c create mode 100644 benchmarks/twin2silicon/tasks/f103-gpio-clock-001/task.json create mode 100755 tests/twin2silicon-fixture.sh diff --git a/benchmarks/twin2silicon/prepare-oracle.py b/benchmarks/twin2silicon/prepare-oracle.py new file mode 100644 index 0000000..5e74d40 --- /dev/null +++ b/benchmarks/twin2silicon/prepare-oracle.py @@ -0,0 +1,10 @@ +from pathlib import Path +import sys + +template, system, firmware, output = map(Path, sys.argv[1:]) +text = template.read_text() +text = text.replace('"__SYSTEM__"', f'"{system.resolve()}"') +text = text.replace('"__FIRMWARE__"', f'"{firmware.resolve()}"') +if "__SYSTEM__" in text or "__FIRMWARE__" in text: + raise SystemExit("unresolved oracle placeholder") +output.write_text(text) diff --git a/benchmarks/twin2silicon/tasks/f103-gpio-clock-001/hidden/oracle.yaml b/benchmarks/twin2silicon/tasks/f103-gpio-clock-001/hidden/oracle.yaml new file mode 100644 index 0000000..d331f14 --- /dev/null +++ b/benchmarks/twin2silicon/tasks/f103-gpio-clock-001/hidden/oracle.yaml @@ -0,0 +1,8 @@ +schema_version: "1.0" +inputs: + system: "__SYSTEM__" + firmware: "__FIRMWARE__" +limits: + max_steps: 200000 +assertions: + - uart_contains: "READY" diff --git a/benchmarks/twin2silicon/tasks/f103-gpio-clock-001/hidden/system.yaml b/benchmarks/twin2silicon/tasks/f103-gpio-clock-001/hidden/system.yaml new file mode 100644 index 0000000..df89161 --- /dev/null +++ b/benchmarks/twin2silicon/tasks/f103-gpio-clock-001/hidden/system.yaml @@ -0,0 +1,4 @@ +name: "f103-gpio-clock-001" +chip: "/Volumes/LabWired/worktrees/twin2silicon-core/configs/chips/stm32f103.yaml" +external_devices: [] +board_io: [] diff --git a/benchmarks/twin2silicon/tasks/f103-gpio-clock-001/public/README.md b/benchmarks/twin2silicon/tasks/f103-gpio-clock-001/public/README.md new file mode 100644 index 0000000..b0c09cf --- /dev/null +++ b/benchmarks/twin2silicon/tasks/f103-gpio-clock-001/public/README.md @@ -0,0 +1,6 @@ +# GPIO startup fault + +The STM32F103 firmware must configure PA0 as a push-pull output, drive it high, +and print `READY` on USART1. It builds, but the GPIO requirement does not pass +on the target. Diagnose and repair the firmware. Do not change the linker +script, startup code, or required output string. diff --git a/benchmarks/twin2silicon/tasks/f103-gpio-clock-001/public/firmware/Makefile b/benchmarks/twin2silicon/tasks/f103-gpio-clock-001/public/firmware/Makefile new file mode 100644 index 0000000..bb23966 --- /dev/null +++ b/benchmarks/twin2silicon/tasks/f103-gpio-clock-001/public/firmware/Makefile @@ -0,0 +1,13 @@ +BUILD_DIR := build +CROSS ?= arm-none-eabi- +CC := $(CROSS)gcc +CFLAGS := -mcpu=cortex-m3 -mthumb -ffreestanding -fno-builtin -ffunction-sections -fdata-sections -Os -g -Wall -Wextra -Werror +LDFLAGS := -nostdlib -T bench.ld -Wl,--gc-sections +all: $(BUILD_DIR)/firmware.elf +$(BUILD_DIR): + mkdir -p $@ +$(BUILD_DIR)/firmware.elf: startup.c main.c bench.ld | $(BUILD_DIR) + $(CC) $(CFLAGS) $(LDFLAGS) startup.c main.c -o $@ +clean: + rm -rf $(BUILD_DIR) +.PHONY: all clean diff --git a/benchmarks/twin2silicon/tasks/f103-gpio-clock-001/public/firmware/bench.ld b/benchmarks/twin2silicon/tasks/f103-gpio-clock-001/public/firmware/bench.ld new file mode 100644 index 0000000..686bc00 --- /dev/null +++ b/benchmarks/twin2silicon/tasks/f103-gpio-clock-001/public/firmware/bench.ld @@ -0,0 +1,12 @@ +ENTRY(Reset) +MEMORY { FLASH (rx) : ORIGIN = 0x08000000, LENGTH = 64K + RAM (rwx) : ORIGIN = 0x20000000, LENGTH = 20K } +_estack = ORIGIN(RAM) + LENGTH(RAM); +SECTIONS { + .isr_vector : { KEEP(*(.isr_vector)) } > FLASH + .text : { *(.text*) *(.rodata*) } > FLASH + _sidata = LOADADDR(.data); + .data : { . = ALIGN(4); _sdata = .; *(.data*) . = ALIGN(4); _edata = .; } > RAM AT > FLASH + .bss : { . = ALIGN(4); _sbss = .; *(.bss*) *(COMMON) . = ALIGN(4); _ebss = .; } > RAM + /DISCARD/ : { *(.ARM.exidx*) *(.note.gnu.build-id*) } +} diff --git a/benchmarks/twin2silicon/tasks/f103-gpio-clock-001/public/firmware/main.c b/benchmarks/twin2silicon/tasks/f103-gpio-clock-001/public/firmware/main.c new file mode 100644 index 0000000..32c0d0d --- /dev/null +++ b/benchmarks/twin2silicon/tasks/f103-gpio-clock-001/public/firmware/main.c @@ -0,0 +1,25 @@ +#include +#define REG32(a) (*(volatile uint32_t *)(a)) +#define RCC_APB2ENR REG32(0x40021018u) +#define GPIOA_CRL REG32(0x40010800u) +#define GPIOA_ODR REG32(0x4001080cu) +#define USART1_SR REG32(0x40013800u) +#define USART1_DR REG32(0x40013804u) +#define USART1_CR1 REG32(0x4001380cu) + +static void putc(char c) { + while ((USART1_SR & (1u << 7)) == 0u) {} + USART1_DR = (uint32_t)(uint8_t)c; +} + +int main(void) { + RCC_APB2ENR |= (1u << 14); + USART1_CR1 = (1u << 13) | (1u << 3); + GPIOA_CRL = (GPIOA_CRL & ~0xfu) | 0x3u; + GPIOA_ODR |= 1u; + if ((GPIOA_CRL & 0xfu) == 0x3u && (GPIOA_ODR & 1u) != 0u) { + const char *s = "READY\n"; + while (*s) putc(*s++); + } + for (;;) {} +} diff --git a/benchmarks/twin2silicon/tasks/f103-gpio-clock-001/public/firmware/startup.c b/benchmarks/twin2silicon/tasks/f103-gpio-clock-001/public/firmware/startup.c new file mode 100644 index 0000000..75c1094 --- /dev/null +++ b/benchmarks/twin2silicon/tasks/f103-gpio-clock-001/public/firmware/startup.c @@ -0,0 +1,19 @@ +#include +extern uint32_t _sidata, _sdata, _edata, _sbss, _ebss, _estack; +extern int main(void); +void Default_Handler(void) { for (;;) {} } +__attribute__((used, noreturn)) static void Reset_C(void) { + uint32_t *src = &_sidata, *dst = &_sdata; + while (dst < &_edata) *dst++ = *src++; + for (dst = &_sbss; dst < &_ebss;) *dst++ = 0u; + (void)main(); + for (;;) {} +} +__attribute__((naked, used, noreturn)) void Reset(void) { + __asm volatile("ldr r0, =_estack\nmov sp, r0\nbl Reset_C\nb .\n"); +} +__attribute__((section(".isr_vector"), used)) void (*const g_vectors[16])(void) = { + (void (*)(void))&_estack, Reset, Default_Handler, Default_Handler, + Default_Handler, Default_Handler, Default_Handler, 0, 0, 0, 0, + Default_Handler, Default_Handler, 0, Default_Handler, Default_Handler, +}; diff --git a/benchmarks/twin2silicon/tasks/f103-gpio-clock-001/task.json b/benchmarks/twin2silicon/tasks/f103-gpio-clock-001/task.json new file mode 100644 index 0000000..818e083 --- /dev/null +++ b/benchmarks/twin2silicon/tasks/f103-gpio-clock-001/task.json @@ -0,0 +1 @@ +{"schema_version":"1.0","id":"f103-gpio-clock-001","board":"stm32f103-bluepill","model":"labwired/labwired-fast","budgets":{"wall_time_seconds":1200,"model_tokens":50000,"repair_iterations":6,"simulator_runs":8,"diagnostic_hil_runs":0},"public_dir":"public","hidden_oracle":"hidden/oracle.yaml","hidden_system":"hidden/system.yaml","firmware_elf":"firmware/build/firmware.elf"} diff --git a/tests/twin2silicon-fixture.sh b/tests/twin2silicon-fixture.sh new file mode 100755 index 0000000..0f49197 --- /dev/null +++ b/tests/twin2silicon-fixture.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT="$(cd "$(dirname "$0")/.." && pwd)" +TASK="$ROOT/benchmarks/twin2silicon/tasks/f103-gpio-clock-001" +TMP="$(mktemp -d)" +trap 'rm -rf "$TMP"' EXIT + +cp -R "$TASK/public/." "$TMP/" +make -C "$TMP/firmware" +python3 "$ROOT/benchmarks/twin2silicon/prepare-oracle.py" \ + "$TASK/hidden/oracle.yaml" "$TASK/hidden/system.yaml" \ + "$TMP/firmware/build/firmware.elf" "$TMP/oracle.yaml" + +if "${LABWIRED_CLI:?set LABWIRED_CLI}" test \ + --script "$TMP/oracle.yaml" --output-dir "$TMP/result"; then + echo "FAIL: buggy fixture unexpectedly passed" + exit 1 +fi + +grep -q '"status": "fail"' "$TMP/result/result.json" +echo "ok twin2silicon fixture starts red" From 06d3db68967ebf669ea5bd5116463b96a6069bfe Mon Sep 17 00:00:00 2001 From: Andrii Shylenko <14119286+w1ne@users.noreply.github.com> Date: Fri, 14 Aug 2026 13:28:04 +0200 Subject: [PATCH 02/48] fix(agent): request canonical hosted tool names --- config/opencode.hosted.json | 2 +- tests/hosted-config.sh | 13 ++++++++++++- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/config/opencode.hosted.json b/config/opencode.hosted.json index 01e6892..dbea864 100644 --- a/config/opencode.hosted.json +++ b/config/opencode.hosted.json @@ -3,7 +3,7 @@ "mcp": { "labwired": { "type": "remote", - "url": "https://api.labwired.com/mcp", + "url": "https://api.labwired.com/mcp?toolNames=unprefixed", "enabled": true, "oauth": false, "headers": { diff --git a/tests/hosted-config.sh b/tests/hosted-config.sh index b00e0d2..72c5082 100644 --- a/tests/hosted-config.sh +++ b/tests/hosted-config.sh @@ -18,10 +18,21 @@ test -f "$ROOT/config/opencode.json" || bad "missing opencode.json" python3 - <_. The +# hosted profile strips the raw server prefix, so model-facing names remain +# canonical instead of becoming labwired_labwired_*. +raw_name = "labwired_context" +wire_name = raw_name.removeprefix("labwired_") +model_name = f"labwired_{wire_name}" +assert model_name == raw_name, model_name +assert not model_name.startswith("labwired_labwired_"), model_name # Bearer header is the product auth path — do not open OpenCode MCP OAuth /connect. assert mcp.get("oauth") is False, mcp auth = (mcp.get("headers") or {}).get("Authorization", "") From 83f0622b551a340dfc9503a8225ba813bdbde9b5 Mon Sep 17 00:00:00 2001 From: Andrii Shylenko <14119286+w1ne@users.noreply.github.com> Date: Fri, 14 Aug 2026 13:37:33 +0200 Subject: [PATCH 03/48] test(agent): resolve canonical tools through public launcher --- package.json | 1 + tests/all.sh | 1 + tests/public-tool-names.sh | 152 +++++++++++++++++++++++++++++++++++++ 3 files changed, 154 insertions(+) create mode 100644 tests/public-tool-names.sh diff --git a/package.json b/package.json index 2b9a648..234e913 100644 --- a/package.json +++ b/package.json @@ -148,6 +148,7 @@ "test:develop:release": "LABWIRED_ACCEPTANCE_REQUIRE_COMPLETE=1 bash tests/develop-acceptance-smoke.sh", "test:dispatcher": "bash tests/dispatcher.sh", "test:agent-lifecycle": "bash tests/agent-lifecycle.sh", + "test:tool-names": "bash tests/public-tool-names.sh", "test:public-install-safety": "bash tests/public-install-safety.sh", "test:install": "bash tests/install-smoke.sh", "test:llm": "bash tests/llm-deepinfra.sh" diff --git a/tests/all.sh b/tests/all.sh index ca01fe4..bb85f83 100755 --- a/tests/all.sh +++ b/tests/all.sh @@ -26,6 +26,7 @@ run "skills-verify-all" "$ROOT/tests/skills-verify-all.sh" run "develop-skill" "$ROOT/tests/develop-skill.sh" run "develop-acceptance-smoke" "$ROOT/tests/develop-acceptance-smoke.sh" run "hosted-config" "$ROOT/tests/hosted-config.sh" +run "public-tool-names" "$ROOT/tests/public-tool-names.sh" run "hosted-auth-probe" "$ROOT/tests/hosted-auth-probe.sh" run "agents-tool-search" "$ROOT/tests/agents-tool-search.sh" run "desktop-session" "$ROOT/tests/desktop-session.sh" diff --git a/tests/public-tool-names.sh b/tests/public-tool-names.sh new file mode 100644 index 0000000..4b7e4e7 --- /dev/null +++ b/tests/public-tool-names.sh @@ -0,0 +1,152 @@ +#!/usr/bin/env bash +# Resolve the real OpenCode registry through the public LabWired launcher, with +# both MCP and model traffic confined to one stdlib-only localhost fixture. +set -euo pipefail +ROOT="$(cd "$(dirname "$0")/.." && pwd)" +TMP="$(mktemp -d)" +cleanup() { + if [[ -n "${SERVER_PID:-}" ]]; then + kill "$SERVER_PID" 2>/dev/null || true + wait "$SERVER_PID" 2>/dev/null || true + fi + rm -rf "$TMP" +} +trap cleanup EXIT + +PORT_FILE="$TMP/port" +CAPTURE_FILE="$TMP/model-request.json" +python3 - "$PORT_FILE" "$CAPTURE_FILE" <<'PY' & +import json, sys +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path +from urllib.parse import urlparse + +port_file, capture_file = map(Path, sys.argv[1:]) +class Handler(BaseHTTPRequestHandler): + def log_message(self, *_): + pass + + def do_GET(self): + if urlparse(self.path).path != "/v1/models": + self.send_response(404) + self.end_headers() + return + data = json.dumps({"object": "list", "data": [{"id": "labwired-default", "object": "model"}]}).encode() + self.send_response(200) + self.send_header("content-type", "application/json") + self.send_header("content-length", str(len(data))) + self.end_headers() + self.wfile.write(data) + + def do_POST(self): + length = int(self.headers.get("content-length", "0")) + body = json.loads(self.rfile.read(length) or b"{}") + parsed = urlparse(self.path) + if parsed.path == "/v1/chat/completions": + capture_file.write_text(json.dumps(body)) + payload = { + "id": "offline", "object": "chat.completion", "created": 0, "model": "offline", + "choices": [{"index": 0, "message": {"role": "assistant", "content": "done"}, "finish_reason": "stop"}], + "usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2}, + } + else: + self.send_response(404) + self.end_headers() + return + data = json.dumps(payload).encode() + self.send_response(200) + self.send_header("content-type", "application/json") + self.send_header("content-length", str(len(data))) + self.end_headers() + self.wfile.write(data) + +server = ThreadingHTTPServer(("127.0.0.1", 0), Handler) +port_file.write_text(str(server.server_port)) +server.serve_forever() +PY +SERVER_PID=$! +for _ in $(seq 1 100); do + [[ -s "$PORT_FILE" ]] && break + sleep 0.05 +done +[[ -s "$PORT_FILE" ]] || { echo "FAIL offline fixture did not start" >&2; exit 1; } +PORT="$(cat "$PORT_FILE")" + +mkdir -p "$TMP/home" "$TMP/labwired" "$TMP/config" "$TMP/project" "$TMP/bin" +PORT="$PORT" TMP="$TMP" python3 - <<'PY' +import json, os +from pathlib import Path + +tmp = Path(os.environ["TMP"]) +port = os.environ["PORT"] +(tmp / "project/opencode.json").write_text(json.dumps({ + "$schema": "https://opencode.ai/config.json", + "provider": {"offline": { + "npm": "@ai-sdk/openai-compatible", + "name": "Offline", + "options": {"baseURL": f"http://127.0.0.1:{port}/v1", "apiKey": "offline"}, + "models": {"default": {"name": "Offline"}}, + }}, + "model": "offline/default", +})) +(tmp / "bin/npx").write_text("#!/bin/sh\nexec python3 \"$FAKE_MCP\"\n") +(tmp / "bin/npx").chmod(0o755) +(tmp / "fake-mcp.py").write_text(r'''import json, sys + +names = ["context", "compile", "run", "verify"] +for line in sys.stdin: + try: + request = json.loads(line) + except Exception: + continue + method = request.get("method") + if method == "initialize": + result = { + "protocolVersion": "2025-06-18", + "capabilities": {"tools": {}}, + "serverInfo": {"name": "offline-labwired", "version": "1"}, + } + elif method == "tools/list": + result = {"tools": [ + {"name": name, "description": name, "inputSchema": {"type": "object", "properties": {}}} + for name in names + ]} + elif method == "ping": + result = {} + else: + continue + print(json.dumps({"jsonrpc": "2.0", "id": request.get("id"), "result": result}), flush=True) +''') +PY + +if ! ( + cd "$TMP/project" + HOME="$TMP/home" \ + LABWIRED_HOME="$TMP/labwired" \ + OPENCODE_CONFIG_DIR="$TMP/config" \ + OPENCODE_CONFIG="$TMP/project/opencode.json" \ + FAKE_MCP="$TMP/fake-mcp.py" \ + PATH="$TMP/bin:$PATH" \ + LABWIRED_SKIP_LOGIN=1 \ + "$ROOT/bin/labwired-agent" agent run --model offline/default --format json "Reply done without calling tools." \ + >"$TMP/run.out" 2>"$TMP/run.err" +); then + cat "$TMP/run.out" >&2 + cat "$TMP/run.err" >&2 + echo "FAIL public launcher exited before model capture" >&2 + exit 1 +fi + +[[ -s "$CAPTURE_FILE" ]] || { cat "$TMP/run.err" >&2; echo "FAIL model request not captured" >&2; exit 1; } +CAPTURE_FILE="$CAPTURE_FILE" python3 - <<'PY' +import json, os +body = json.load(open(os.environ["CAPTURE_FILE"])) +names = [tool["function"]["name"] for tool in body.get("tools", []) if tool.get("type") == "function"] +expected = ["labwired_context", "labwired_compile", "labwired_run", "labwired_verify"] +for name in expected: + assert names.count(name) == 1, (name, names) +assert not any(name.startswith("labwired_labwired_") for name in names), names +print("ok public launcher sends canonical MCP tool names to the model provider") +PY + +echo "public-tool-names PASS" From 4cbf6eb464f4d47a1803f71d13a73f59fe82e625 Mon Sep 17 00:00:00 2001 From: Andrii Shylenko <14119286+w1ne@users.noreply.github.com> Date: Sat, 15 Aug 2026 03:15:33 +0200 Subject: [PATCH 04/48] docs: design ESP32-S3 HIL benchmark --- ...2026-08-15-esp32s3-hil-benchmark-design.md | 105 ++++++++++++++++++ 1 file changed, 105 insertions(+) create mode 100644 docs/superpowers/specs/2026-08-15-esp32s3-hil-benchmark-design.md diff --git a/docs/superpowers/specs/2026-08-15-esp32s3-hil-benchmark-design.md b/docs/superpowers/specs/2026-08-15-esp32s3-hil-benchmark-design.md new file mode 100644 index 0000000..518d3b0 --- /dev/null +++ b/docs/superpowers/specs/2026-08-15-esp32s3-hil-benchmark-design.md @@ -0,0 +1,105 @@ +# ESP32-S3 HIL Benchmark Design + +**Date:** 2026-08-15 + +## Objective + +Build the first end-to-end Twin2Silicon hardware-in-the-loop repair benchmark on the connected ESP32-S3. A model run is successful only when its candidate firmware builds, flashes to the intended board, emits a run-specific UART nonce, and leaves the physical GPIO peripheral in the hidden oracle's expected state as observed through USB-JTAG. + +This specification covers one reusable evaluator and one task, `esp32s3-gpio-hil-001`. A larger task suite, scheduled lab service, and multi-model leaderboard are separate follow-on work. + +## Task + +The public task is an ESP-IDF C firmware project built through PlatformIO. It contains one realistic GPIO configuration defect and a concise repair prompt. The model may modify only the copied public workspace. The task's hidden descriptor contains: + +- the board profile and PlatformIO environment; +- the expected USB-JTAG serial identity, supplied by evaluator configuration rather than model context; +- the UART device-selection rule and baud rate; +- the flash artifact location; +- a GPIO register read plan with address, mask, and expected masked value; +- build, flash, UART, JTAG, wall-time, token, and iteration limits. + +The evaluator injects a cryptographically random run nonce into a generated header before the model starts. Correct firmware prints `LABWIRED_READY:`. A fixed string or output from an earlier run cannot satisfy the UART oracle. + +## Architecture + +The implementation consists of four focused units: + +1. **Task runner.** Creates a run directory and isolated public workspace, injects the nonce, invokes the existing public LabWired Agent, enforces model budgets, and records provider usage. +2. **Build and flash adapter.** Runs pinned PlatformIO commands, identifies the produced firmware, flashes only after board identity validation, and preserves command logs and exit metadata. +3. **UART/JTAG HIL evaluator.** Acquires an exclusive board lock, validates the expected Espressif USB-JTAG adapter, captures UART until the nonce or timeout, halts the target with Espressif OpenOCD, reads the hidden GPIO register plan, and evaluates masked values. +4. **Evidence writer.** Produces one canonical result manifest plus immutable raw logs and SHA-256 hashes. + +The runner orchestrates these units in this order: + +```text +prepare workspace + -> model repair + -> clean build + -> acquire board lock + -> validate board identity + -> start UART capture + -> flash and reset + -> observe nonce + -> halt and read GPIO registers + -> evaluate oracle + -> release board lock + -> write evidence +``` + +Simulator scoring remains a separate field. It may be `not_supported` for this first ESP-IDF task and cannot substitute for the physical HIL result. + +## Hardware Safety and Identity + +The evaluator is destructive to the board's installed firmware, as explicitly authorized. It must not operate on an ambiguous target. + +- The expected USB-JTAG serial is an explicit evaluator input. The currently connected S3 reports `9C:CC:01:D0:98:E0`, but the repository does not treat that machine-specific value as a universal default. +- Zero or multiple matching adapters produces `infrastructure_error`; the evaluator does not flash. +- UART selection must resolve to exactly one device associated with the target profile. A CLI override is allowed for laboratory setup and is recorded in redacted form in the manifest. +- A filesystem lock keyed by board identity prevents concurrent flash/JTAG runs. Lock acquisition has a timeout and records contention as infrastructure failure. +- Every subprocess has a hard timeout and process-group cleanup. Signal handling releases ports, OpenOCD, and the board lock. + +## Result Contract + +The canonical `run.json` records: + +- schema, run, task, harness, and model identifiers; +- model request count, fresh/cached/output/reasoning tokens, final context size, latency, and estimated provider cost; +- configured budgets and whether each was respected; +- `model_status`, `compile_status`, `simulator_status`, `hardware_status`, and `infrastructure_status` as independent fields; +- UART nonce result and JTAG register assertions without exposing hidden expectations to the model; +- termination reason and normalized failure category; +- hashes for the candidate source tree, firmware, oracle descriptor, raw logs, and evaluator result. + +`hardware_status` is `pass` only when the clean build, flash, nonce, and every required register assertion pass. Missing tools, missing hardware, ambiguous identity, port contention, and transport failure are `infrastructure_error`, never model failures. A candidate that builds and flashes but misses the nonce or register oracle is `fail`. + +The runner exits zero only for a valid completed evaluation, whether the model passes or fails. It uses a distinct nonzero exit for invalid or incomplete infrastructure runs so batch aggregation cannot silently count them. + +## Evidence and Reproducibility + +Each run directory contains the public prompt, initial and final source hashes, exact harness revision, sanitized tool versions, model/provider identity, budgets, command timing, raw build/flash/UART/OpenOCD logs, parsed register observations, `run.json`, and `cost.json`. + +Secrets, bearer tokens, full environment dumps, and hidden oracle values are not copied into model-visible files. Published results may include the oracle after the evaluation set is retired; active hidden tasks publish only schema and aggregate assertion outcomes. + +Pricing is versioned with source URL, effective date, and fresh-input, cached-input, and output rates. Cost remains an estimate unless reconciled against a provider invoice. + +## Testing + +Offline tests drive the real orchestration boundary with fixture executables for PlatformIO, serial capture, and OpenOCD. They cover: + +- a complete pass; +- compilation and flash failures; +- absent, incorrect, and stale UART nonces; +- incorrect GPIO masked values; +- absent, wrong, and ambiguous board identity; +- UART, JTAG, and lock timeouts; +- subprocess interruption and cleanup; +- correct separation of model failure and infrastructure error; +- stable JSON schema, evidence hashes, and cost arithmetic. + +Tests are written before production behavior and must demonstrate the expected red failure before implementation. The live acceptance test is opt-in, names the target serial and UART explicitly, overwrites the board firmware, and is excluded from ordinary CI. Acceptance requires one fresh physical run that produces the nonce and passing GPIO register evidence. + +## Non-Goals + +This first increment does not add external voltage or waveform instrumentation, continuous unattended lab scheduling, automatic firmware restoration, more than one task, or GPT/Claude/Grok comparison runs. Those additions build on the evaluator contract after the connected ESP32-S3 path is proven. + From d0ee2ca3be358e6faec64489535da92916dcf383 Mon Sep 17 00:00:00 2001 From: Andrii Shylenko <14119286+w1ne@users.noreply.github.com> Date: Sat, 15 Aug 2026 12:07:34 +0200 Subject: [PATCH 05/48] docs: plan ESP32-S3 HIL benchmark --- .../plans/2026-08-15-esp32s3-hil-benchmark.md | 402 ++++++++++++++++++ 1 file changed, 402 insertions(+) create mode 100644 docs/superpowers/plans/2026-08-15-esp32s3-hil-benchmark.md diff --git a/docs/superpowers/plans/2026-08-15-esp32s3-hil-benchmark.md b/docs/superpowers/plans/2026-08-15-esp32s3-hil-benchmark.md new file mode 100644 index 0000000..8c86cda --- /dev/null +++ b/docs/superpowers/plans/2026-08-15-esp32s3-hil-benchmark.md @@ -0,0 +1,402 @@ +# ESP32-S3 HIL Benchmark Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Add one reproducible ESP-IDF repair task and an automated evaluator that scores a connected ESP32-S3 using both a nonce-tagged USB-serial observation and hidden GPIO register assertions read through USB-JTAG. + +**Architecture:** Keep model execution separate from physical evaluation. A Python standard-library benchmark package owns typed results, bounded subprocesses, board locking, UART/JTAG evidence, and manifest generation; a small CLI composes those units. PlatformIO supplies the pinned ESP-IDF build/flash path, while fixture executables make every orchestration branch testable without hardware. + +**Tech Stack:** Python 3 standard library, PlatformIO with Espressif32/ESP-IDF, Espressif OpenOCD, macOS USB serial, Bash test registration, JSON task/oracle manifests. + +--- + +## File Map + +- Create `benchmarks/twin2silicon/hil/__init__.py`: package marker and public result types. +- Create `benchmarks/twin2silicon/hil/process.py`: timeout-bounded subprocess execution and captured command evidence. +- Create `benchmarks/twin2silicon/hil/esp32s3.py`: board identity validation, lock lifecycle, UART capture, flash, OpenOCD reads, and register evaluation. +- Create `benchmarks/twin2silicon/hil/results.py`: canonical status/result dataclasses, hashing, and atomic JSON output. +- Create `benchmarks/twin2silicon/run_hil.py`: CLI that loads the task, prepares a run, optionally invokes the Agent, and evaluates the candidate. +- Create `benchmarks/twin2silicon/tasks/esp32s3-gpio-hil-001/task.json`: public metadata and budgets. +- Create `benchmarks/twin2silicon/tasks/esp32s3-gpio-hil-001/hidden/hil-oracle.json`: GPIO register assertions and transport settings. +- Create `benchmarks/twin2silicon/tasks/esp32s3-gpio-hil-001/public/README.md`: repair prompt. +- Create `benchmarks/twin2silicon/tasks/esp32s3-gpio-hil-001/public/firmware/platformio.ini`: pinned ESP-IDF project configuration. +- Create `benchmarks/twin2silicon/tasks/esp32s3-gpio-hil-001/public/firmware/sdkconfig.defaults`: USB Serial/JTAG console configuration. +- Create `benchmarks/twin2silicon/tasks/esp32s3-gpio-hil-001/public/firmware/src/main.c`: deliberately faulty GPIO firmware. +- Create `tests/twin2silicon-hil.py`: offline unit/integration tests with fixture executables and pseudo-terminals. +- Create `tests/twin2silicon-hil.sh`: shell entrypoint for the offline suite. +- Create `tests/twin2silicon-hil-live.sh`: explicit destructive live acceptance test. +- Modify `tests/all.sh`: register only the offline HIL suite. +- Modify `package.json`: add `test:twin2silicon-hil` and `test:twin2silicon-hil:live` commands. +- Modify `docs/TESTING.md`: document offline and destructive live commands. + +### Task 1: Define result and process contracts + +**Files:** +- Create: `benchmarks/twin2silicon/hil/__init__.py` +- Create: `benchmarks/twin2silicon/hil/results.py` +- Create: `benchmarks/twin2silicon/hil/process.py` +- Create: `tests/twin2silicon-hil.py` + +- [ ] **Step 1: Write failing tests for statuses, atomic evidence, hashing, timeout, and process-group cleanup** + +Add tests that import `CommandResult`, `RunResult`, `sha256_file`, `write_json_atomic`, and `run_command`. Assert that: + +```python +def test_run_result_keeps_infrastructure_separate_from_candidate_failure(self): + result = RunResult.infrastructure_error("board_identity", "wrong adapter") + self.assertEqual(result.hardware_status, "not_run") + self.assertEqual(result.infrastructure_status, "error") + self.assertEqual(result.failure_category, "board_identity") + +def test_run_command_times_out_and_captures_evidence(self): + result = run_command( + [sys.executable, "-c", "import time; time.sleep(10)"], + cwd=self.tmp, + timeout_seconds=0.1, + stdout_path=self.tmp / "stdout.log", + stderr_path=self.tmp / "stderr.log", + ) + self.assertTrue(result.timed_out) + self.assertIsNotNone(result.duration_seconds) + self.assertNotEqual(result.returncode, 0) +``` + +- [ ] **Step 2: Run the tests and verify RED** + +Run: `python3 tests/twin2silicon-hil.py -k 'run_result or run_command'` + +Expected: import failure because `benchmarks.twin2silicon.hil` does not exist. + +- [ ] **Step 3: Implement minimal typed results and bounded command execution** + +Use frozen dataclasses and literal string statuses. `run_command()` must use `subprocess.Popen(..., start_new_session=True)`, `communicate(timeout=...)`, then `os.killpg(process.pid, signal.SIGTERM)` followed by a bounded SIGKILL fallback. It returns command, sanitized cwd, return code, timeout flag, start/end UTC timestamps, duration, and log paths. `write_json_atomic()` writes a sibling temporary file, fsyncs it, and replaces the destination. + +The constructors must encode these exact distinctions: + +```python +@classmethod +def infrastructure_error(cls, category: str, detail: str) -> "RunResult": + return cls( + model_status="not_run", + compile_status="not_run", + simulator_status="not_supported", + hardware_status="not_run", + infrastructure_status="error", + failure_category=category, + detail=detail, + ) +``` + +- [ ] **Step 4: Run focused tests and verify GREEN** + +Run: `python3 tests/twin2silicon-hil.py -k 'run_result or run_command or atomic or sha256'` + +Expected: all selected tests pass and the timeout test completes in under two seconds. + +- [ ] **Step 5: Commit** + +```bash +git add benchmarks/twin2silicon/hil tests/twin2silicon-hil.py +git commit -m "feat(bench): add bounded HIL result primitives" +``` + +### Task 2: Add the ESP32-S3 task fixture + +**Files:** +- Create: `benchmarks/twin2silicon/tasks/esp32s3-gpio-hil-001/task.json` +- Create: `benchmarks/twin2silicon/tasks/esp32s3-gpio-hil-001/hidden/hil-oracle.json` +- Create: `benchmarks/twin2silicon/tasks/esp32s3-gpio-hil-001/public/README.md` +- Create: `benchmarks/twin2silicon/tasks/esp32s3-gpio-hil-001/public/firmware/platformio.ini` +- Create: `benchmarks/twin2silicon/tasks/esp32s3-gpio-hil-001/public/firmware/sdkconfig.defaults` +- Create: `benchmarks/twin2silicon/tasks/esp32s3-gpio-hil-001/public/firmware/src/main.c` +- Test: `tests/twin2silicon-hil.py` + +- [ ] **Step 1: Write a failing fixture-contract test** + +Load both JSON files and assert schema `1.0`, task id `esp32s3-gpio-hil-001`, board `esp32-s3-devkitc-1`, framework `espidf`, a 50,000-token cap, zero diagnostic HIL calls for the model, UART prefix `LABWIRED_READY:`, and exactly these GPIO2 assertions: + +```json +[ + {"name":"gpio2_output_enabled","address":"0x60004020","mask":"0x00000004","expected":"0x00000004"}, + {"name":"gpio2_output_high","address":"0x60004004","mask":"0x00000004","expected":"0x00000004"} +] +``` + +Also assert that the public tree contains no expected register values or JTAG commands, and that `main.c` initially calls `gpio_set_direction(TEST_GPIO, GPIO_MODE_INPUT)`. + +- [ ] **Step 2: Run the fixture test and verify RED** + +Run: `python3 tests/twin2silicon-hil.py -k fixture_contract` + +Expected: failure because the task directory is absent. + +- [ ] **Step 3: Create the minimal ESP-IDF fixture** + +Pin PlatformIO's `espressif32` platform to the installed, lockable version selected by `pio pkg list`; set `board = esp32-s3-devkitc-1`, `framework = espidf`, `monitor_speed = 115200`, and `board_upload.flash_size = 4MB`. Configure USB Serial/JTAG as the primary console in `sdkconfig.defaults`. + +In `main.c`, define `TEST_GPIO GPIO_NUM_2`, include generated `run_nonce.h`, set the initial deliberate defect to input mode, call `gpio_set_level(TEST_GPIO, 1)`, print `LABWIRED_READY:%s\n`, flush stdout, and remain alive. The intended one-line repair is `GPIO_MODE_OUTPUT`. + +- [ ] **Step 4: Verify the fixture starts red semantically and builds** + +Run: + +```bash +python3 tests/twin2silicon-hil.py -k fixture_contract +pio run -d benchmarks/twin2silicon/tasks/esp32s3-gpio-hil-001/public/firmware +``` + +Expected: contract passes and PlatformIO reports `SUCCESS`; the source still configures GPIO2 as input. + +- [ ] **Step 5: Commit** + +```bash +git add benchmarks/twin2silicon/tasks/esp32s3-gpio-hil-001 tests/twin2silicon-hil.py +git commit -m "test(bench): add ESP32-S3 GPIO repair fixture" +``` + +### Task 3: Implement board identity, locking, UART, and JTAG evaluation + +**Files:** +- Create: `benchmarks/twin2silicon/hil/esp32s3.py` +- Modify: `tests/twin2silicon-hil.py` + +- [ ] **Step 1: Write failing offline hardware-adapter tests** + +Use temporary executable fixtures and `pty.openpty()` to cover: + +- exactly one configured JTAG serial succeeds; +- absent, wrong, or duplicate serials return `board_identity` infrastructure errors before flash; +- a second evaluator cannot acquire the same board lock; +- UART accepts only `LABWIRED_READY:` and rejects absent, wrong, and stale nonces; +- flash nonzero exit is a candidate failure only after identity succeeds; +- OpenOCD text is parsed only for explicitly named `@@REG
` records; +- masked register mismatch returns hardware `fail`; +- all assertions matching returns hardware `pass`; +- timeouts close the pseudo-terminal, terminate children, and release the lock. + +- [ ] **Step 2: Run the adapter tests and verify RED** + +Run: `python3 tests/twin2silicon-hil.py -k 'identity or lock or uart or jtag or register or flash'` + +Expected: import failure for `hil.esp32s3`. + +- [ ] **Step 3: Implement the adapter with dependency-injected commands** + +Define `Esp32S3Config`, `RegisterAssertion`, `BoardLock`, `validate_identity()`, `capture_uart_nonce()`, `flash_firmware()`, `read_registers()`, and `evaluate_registers()`. Commands come from the hidden descriptor or explicit CLI overrides; no shell interpolation is permitted. + +Identity validation consumes a command returning one serial per line and requires `matches == [expected_serial]`. OpenOCD receives: + +```text +adapter serial ; adapter speed 4000; init; reset run; sleep 750; +halt; echo "@@REG gpio2_output_enabled 0x60004020"; mdw 0x60004020 1; +echo "@@REG gpio2_output_high 0x60004004"; mdw 0x60004004 1; exit +``` + +The parser pairs each marker only with the immediately following OpenOCD memory word line and rejects missing, duplicate, or unrequested observations. + +- [ ] **Step 4: Run adapter tests and verify GREEN** + +Run: `python3 tests/twin2silicon-hil.py -k 'identity or lock or uart or jtag or register or flash'` + +Expected: all selected tests pass without accessing `/dev` or a network. + +- [ ] **Step 5: Commit** + +```bash +git add benchmarks/twin2silicon/hil/esp32s3.py tests/twin2silicon-hil.py +git commit -m "feat(bench): evaluate ESP32-S3 UART and JTAG evidence" +``` + +### Task 4: Compose the benchmark CLI and evidence manifest + +**Files:** +- Create: `benchmarks/twin2silicon/run_hil.py` +- Modify: `tests/twin2silicon-hil.py` + +- [ ] **Step 1: Write failing end-to-end fixture tests** + +Run the CLI against fake identity, PlatformIO, UART, and OpenOCD endpoints. Assert: + +- `--evaluate-only` copies a candidate, injects a 128-bit nonce header, and never exposes `hidden/` beneath the workspace; +- complete evidence yields exit 0 with hardware `pass`; +- a valid model failure also yields exit 0 with hardware `fail`; +- infrastructure failure yields exit 2 and is excluded from valid aggregate counts; +- manifest paths are relative, environment values are allowlisted, and secret-looking values never appear; +- every raw artifact listed in `run.json` has a matching SHA-256; +- rerunning into an existing run directory is refused; +- model budgets and provider usage supplied through `--usage-json` are preserved and cost arithmetic is exact. + +- [ ] **Step 2: Run CLI tests and verify RED** + +Run: `python3 tests/twin2silicon-hil.py -k 'cli or manifest or secret or cost'` + +Expected: failure because `run_hil.py` is absent. + +- [ ] **Step 3: Implement the CLI** + +Support these explicit modes and arguments: + +```text +run_hil.py TASK --run-dir DIR --evaluate-only --candidate DIR +run_hil.py TASK --run-dir DIR --agent-bin PATH --model MODEL + --jtag-serial SERIAL --uart-device DEVICE --openocd PATH + --usage-json FILE +``` + +The Agent mode copies `public/`, injects `include/run_nonce.h`, invokes `labwired-agent agent run` in the workspace with the public README as prompt, and then evaluates the resulting candidate. `--evaluate-only` skips model invocation but uses the same build/HIL/evidence path. Require explicit `--jtag-serial` and `--uart-device` for live operation; tests may inject fixture commands. + +Write `run.json` atomically after every phase so interrupted runs remain diagnosable, then finalize hashes only after all logs close. Use exit 0 for completed pass/fail and exit 2 for invalid infrastructure. + +- [ ] **Step 4: Run all offline Python tests and verify GREEN** + +Run: `python3 tests/twin2silicon-hil.py` + +Expected: all tests pass; no test opens a real serial device, invokes real PlatformIO, or contacts a model API. + +- [ ] **Step 5: Commit** + +```bash +git add benchmarks/twin2silicon/run_hil.py tests/twin2silicon-hil.py +git commit -m "feat(bench): orchestrate reproducible ESP32-S3 HIL runs" +``` + +### Task 5: Register tests and operator documentation + +**Files:** +- Create: `tests/twin2silicon-hil.sh` +- Create: `tests/twin2silicon-hil-live.sh` +- Modify: `tests/all.sh` +- Modify: `package.json` +- Modify: `docs/TESTING.md` + +- [ ] **Step 1: Write the shell entrypoints and registration assertions first** + +`tests/twin2silicon-hil.sh` runs `python3 tests/twin2silicon-hil.py`. `tests/twin2silicon-hil-live.sh` refuses to run unless `LABWIRED_HIL_DESTRUCTIVE=1`, `ESP32S3_SERIAL`, and `ESP32S3_UART` are set, then calls `run_hil.py --evaluate-only` with the known-good repaired fixture copy. + +Add static assertions that ordinary `tests/all.sh` includes only the offline script and never the live script. + +- [ ] **Step 2: Run registration assertions and verify RED** + +Run: `bash tests/twin2silicon-hil.sh` + +Expected: failure until package/test registration assertions are satisfied. + +- [ ] **Step 3: Register the offline lane and document the destructive lane** + +Add to `tests/all.sh`: + +```bash +run "twin2silicon-hil" "$ROOT/tests/twin2silicon-hil.sh" +``` + +Add package scripts: + +```json +"test:twin2silicon-hil": "bash tests/twin2silicon-hil.sh", +"test:twin2silicon-hil:live": "bash tests/twin2silicon-hil-live.sh" +``` + +Document the exact live command, destructive warning, required board serial/UART variables, result semantics, and evidence directory. + +- [ ] **Step 4: Verify offline registration and syntax** + +Run: + +```bash +bash -n tests/twin2silicon-hil.sh tests/twin2silicon-hil-live.sh +npm run test:twin2silicon-hil +``` + +Expected: syntax clean and all offline HIL tests pass. + +- [ ] **Step 5: Commit** + +```bash +git add tests/twin2silicon-hil.sh tests/twin2silicon-hil-live.sh tests/all.sh package.json docs/TESTING.md +git commit -m "test(bench): register ESP32-S3 HIL benchmark lanes" +``` + +### Task 6: Run the destructive connected-board acceptance test + +**Files:** +- Runtime artifacts only: `out/twin2silicon/esp32s3-gpio-hil-001-/` + +- [ ] **Step 1: Verify prerequisites without modifying the board** + +Run: + +```bash +pio --version +/private/tmp/openocd-esp32/bin/openocd --version +system_profiler SPUSBDataType | grep -A20 'USB JTAG/serial debug unit' +ls -l /dev/cu.usbmodem* +``` + +Expected: PlatformIO and Espressif OpenOCD are available, JTAG serial `9C:CC:01:D0:98:E0` is present exactly once, and the chosen S3 UART device exists. + +- [ ] **Step 2: Prove the live gate refuses implicit destructive execution** + +Run: `bash tests/twin2silicon-hil-live.sh` + +Expected: nonzero exit explaining `LABWIRED_HIL_DESTRUCTIVE=1` is required; no flash command runs. + +- [ ] **Step 3: Run a fresh known-good physical acceptance** + +Copy the public fixture to a temporary candidate, change only `GPIO_MODE_INPUT` to `GPIO_MODE_OUTPUT`, then run: + +```bash +LABWIRED_HIL_DESTRUCTIVE=1 \ +ESP32S3_SERIAL='9C:CC:01:D0:98:E0' \ +ESP32S3_UART='/dev/cu.usbmodem11101' \ +bash tests/twin2silicon-hil-live.sh +``` + +Expected: clean build and flash succeed, current nonce is observed, both GPIO2 masked assertions pass, `hardware_status` is `pass`, and evidence hashes validate. + +- [ ] **Step 4: Run the complete offline regression suite** + +Run: + +```bash +npm run test:twin2silicon-hil +bash tests/twin2silicon-fixture.sh +git diff --check +``` + +Expected: both benchmark suites pass and the diff check is clean. If the legacy fixture requires `LABWIRED_CLI`, provide the same simulator binary used by its existing documented invocation. + +- [ ] **Step 5: Record acceptance evidence without committing runtime output** + +Confirm `out/` remains untracked. Report the run directory, manifest link, firmware hash, nonce assertion, JTAG observations, elapsed time, and whether any infrastructure retry occurred. + +### Task 7: Final review and branch handoff + +**Files:** +- Review all files changed since design commit `4cbf6eb`. + +- [ ] **Step 1: Review requirements against the approved design** + +Check that model execution and evaluation are separated, hidden values never enter the workspace, missing hardware is not a model failure, board identity is explicit, subprocesses and locks are bounded, evidence is hash-addressed, and the live test is absent from ordinary CI. + +- [ ] **Step 2: Run final verification from a clean process** + +Run: + +```bash +npm run test:twin2silicon-hil +python3 -m py_compile benchmarks/twin2silicon/run_hil.py benchmarks/twin2silicon/hil/*.py tests/twin2silicon-hil.py +git diff --check 4cbf6eb..HEAD +git status --short +``` + +Expected: tests and compilation pass, diff is clean, and only the pre-existing untracked `out/` remains. + +- [ ] **Step 3: Request code review and address only verified findings** + +Review the implementation for specification compliance first and code quality second. Reproduce every blocking finding with a focused failing test before changing production behavior. + +- [ ] **Step 4: Prepare PR-only handoff** + +Push the feature branch and open a draft PR. Do not merge or deploy; the user previously requested PR-only delivery for repository changes. + From ee630fa8dcbf83bf6c2ba0d4bf0a2afe3c06dd5e Mon Sep 17 00:00:00 2001 From: Andrii Shylenko <14119286+w1ne@users.noreply.github.com> Date: Sat, 15 Aug 2026 12:10:40 +0200 Subject: [PATCH 06/48] feat(bench): add bounded HIL result primitives --- benchmarks/twin2silicon/hil/__init__.py | 2 + benchmarks/twin2silicon/hil/process.py | 66 +++++++++++ benchmarks/twin2silicon/hil/results.py | 84 ++++++++++++++ tests/twin2silicon-hil.py | 141 ++++++++++++++++++++++++ 4 files changed, 293 insertions(+) create mode 100644 benchmarks/twin2silicon/hil/__init__.py create mode 100644 benchmarks/twin2silicon/hil/process.py create mode 100644 benchmarks/twin2silicon/hil/results.py create mode 100644 tests/twin2silicon-hil.py diff --git a/benchmarks/twin2silicon/hil/__init__.py b/benchmarks/twin2silicon/hil/__init__.py new file mode 100644 index 0000000..f0ac38f --- /dev/null +++ b/benchmarks/twin2silicon/hil/__init__.py @@ -0,0 +1,2 @@ +"""Reusable hardware-in-the-loop benchmark primitives.""" + diff --git a/benchmarks/twin2silicon/hil/process.py b/benchmarks/twin2silicon/hil/process.py new file mode 100644 index 0000000..393a3ea --- /dev/null +++ b/benchmarks/twin2silicon/hil/process.py @@ -0,0 +1,66 @@ +"""Bounded subprocess execution with persistent evidence.""" + +from datetime import datetime, timezone +import os +from pathlib import Path +import signal +import subprocess +import time +from typing import Sequence, Union + +from .results import CommandResult, PathLike + + +def _utc_now() -> str: + return datetime.now(timezone.utc).isoformat().replace("+00:00", "Z") + + +def run_command( + command: Sequence[Union[str, os.PathLike[str]]], + *, + cwd: PathLike, + stdout_path: PathLike, + stderr_path: PathLike, + timeout_seconds: float, +) -> CommandResult: + normalized_command = tuple(os.fspath(part) for part in command) + normalized_cwd = str(Path(cwd).resolve()) + normalized_stdout = str(Path(stdout_path).resolve()) + normalized_stderr = str(Path(stderr_path).resolve()) + Path(normalized_stdout).parent.mkdir(parents=True, exist_ok=True) + Path(normalized_stderr).parent.mkdir(parents=True, exist_ok=True) + + started_at = _utc_now() + started_monotonic = time.monotonic() + timed_out = False + with open(normalized_stdout, "wb") as stdout, open(normalized_stderr, "wb") as stderr: + process = subprocess.Popen( + normalized_command, + cwd=normalized_cwd, + stdout=stdout, + stderr=stderr, + start_new_session=True, + ) + try: + process.communicate(timeout=timeout_seconds) + except subprocess.TimeoutExpired: + timed_out = True + os.killpg(process.pid, signal.SIGTERM) + try: + process.communicate(timeout=0.5) + except subprocess.TimeoutExpired: + os.killpg(process.pid, signal.SIGKILL) + process.communicate(timeout=0.5) + + ended_at = _utc_now() + return CommandResult( + command=normalized_command, + cwd=normalized_cwd, + return_code=process.returncode, + timed_out=timed_out, + started_at_utc=started_at, + ended_at_utc=ended_at, + duration_seconds=time.monotonic() - started_monotonic, + stdout_path=normalized_stdout, + stderr_path=normalized_stderr, + ) diff --git a/benchmarks/twin2silicon/hil/results.py b/benchmarks/twin2silicon/hil/results.py new file mode 100644 index 0000000..257b808 --- /dev/null +++ b/benchmarks/twin2silicon/hil/results.py @@ -0,0 +1,84 @@ +"""Result contracts and durable result-file helpers.""" + +from dataclasses import dataclass +import hashlib +import json +import os +from pathlib import Path +import tempfile +from typing import Any, Optional, Union + + +PathLike = Union[str, os.PathLike[str]] + + +@dataclass(frozen=True) +class CommandResult: + command: tuple[str, ...] + cwd: str + return_code: int + timed_out: bool + started_at_utc: str + ended_at_utc: str + duration_seconds: float + stdout_path: str + stderr_path: str + + +@dataclass(frozen=True) +class RunResult: + model_status: str + compile_status: str + simulator_status: str + hardware_status: str + infrastructure_status: str + failure_category: Optional[str] = None + failure_detail: Optional[str] = None + + @classmethod + def infrastructure_error(cls, category: str, detail: str) -> "RunResult": + return cls( + model_status="not_run", + compile_status="not_run", + simulator_status="not_supported", + hardware_status="not_run", + infrastructure_status="error", + failure_category=category, + failure_detail=detail, + ) + + +def sha256_file(path: PathLike) -> str: + digest = hashlib.sha256() + with open(path, "rb") as source: + for chunk in iter(lambda: source.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def write_json_atomic(path: PathLike, value: Any) -> None: + destination = Path(path) + destination.parent.mkdir(parents=True, exist_ok=True) + temporary_path: Optional[str] = None + try: + with tempfile.NamedTemporaryFile( + mode="w", + encoding="utf-8", + dir=destination.parent, + prefix=f".{destination.name}.", + suffix=".tmp", + delete=False, + ) as temporary: + temporary_path = temporary.name + json.dump(value, temporary, indent=2, sort_keys=True) + temporary.write("\n") + temporary.flush() + os.fsync(temporary.fileno()) + os.replace(temporary_path, destination) + temporary_path = None + finally: + if temporary_path is not None: + try: + os.unlink(temporary_path) + except FileNotFoundError: + pass diff --git a/tests/twin2silicon-hil.py b/tests/twin2silicon-hil.py new file mode 100644 index 0000000..7bb1265 --- /dev/null +++ b/tests/twin2silicon-hil.py @@ -0,0 +1,141 @@ +#!/usr/bin/env python3 +import hashlib +import json +from pathlib import Path +import sys +import tempfile +import textwrap +import time +import unittest + + +REPOSITORY_ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(REPOSITORY_ROOT)) + +from benchmarks.twin2silicon.hil.process import run_command +from benchmarks.twin2silicon.hil.results import ( + CommandResult, + RunResult, + sha256_file, + write_json_atomic, +) + + +class ResultContractTests(unittest.TestCase): + def test_run_result_infrastructure_error_marks_execution_not_run(self): + result = RunResult.infrastructure_error("board_identity", "wrong adapter") + + self.assertEqual(result.model_status, "not_run") + self.assertEqual(result.compile_status, "not_run") + self.assertEqual(result.simulator_status, "not_supported") + self.assertEqual(result.hardware_status, "not_run") + self.assertEqual(result.infrastructure_status, "error") + self.assertEqual(result.failure_category, "board_identity") + self.assertEqual(result.failure_detail, "wrong adapter") + + def test_sha256_file_streams_file_contents(self): + with tempfile.TemporaryDirectory() as directory: + source = Path(directory) / "firmware.bin" + contents = (b"LabWired\x00" * 10000) + b"tail" + source.write_bytes(contents) + + self.assertEqual(sha256_file(source), hashlib.sha256(contents).hexdigest()) + + def test_write_json_atomic_replaces_destination_with_json(self): + with tempfile.TemporaryDirectory() as directory: + destination = Path(directory) / "result.json" + destination.write_text("stale", encoding="utf-8") + + write_json_atomic(destination, {"status": "ok", "count": 2}) + + self.assertEqual( + json.loads(destination.read_text(encoding="utf-8")), + {"status": "ok", "count": 2}, + ) + self.assertEqual(list(Path(directory).iterdir()), [destination]) + + +class ProcessContractTests(unittest.TestCase): + def test_run_command_timeout_captures_evidence_and_is_bounded(self): + with tempfile.TemporaryDirectory() as directory: + evidence = Path(directory) + started = time.monotonic() + result = run_command( + [ + sys.executable, + "-c", + "import sys,time; print('stdout evidence', flush=True); " + "print('stderr evidence', file=sys.stderr, flush=True); time.sleep(30)", + ], + cwd=evidence, + stdout_path=evidence / "stdout.log", + stderr_path=evidence / "stderr.log", + timeout_seconds=0.1, + ) + elapsed = time.monotonic() - started + + self.assertIsInstance(result, CommandResult) + self.assertTrue(result.timed_out) + self.assertNotEqual(result.return_code, 0) + self.assertGreater(result.duration_seconds, 0) + self.assertLess(elapsed, 2) + self.assertEqual((evidence / "stdout.log").read_text(), "stdout evidence\n") + self.assertEqual((evidence / "stderr.log").read_text(), "stderr evidence\n") + self.assertEqual(result.cwd, str(evidence.resolve())) + self.assertTrue(result.started_at_utc.endswith("Z")) + self.assertTrue(result.ended_at_utc.endswith("Z")) + + def test_run_command_timeout_terminates_process_group(self): + with tempfile.TemporaryDirectory() as directory: + evidence = Path(directory) + child_ready = evidence / "child-ready" + child_terminated = evidence / "child-terminated" + script = textwrap.dedent( + f""" + import pathlib, signal, subprocess, sys, time + child = ''' + import pathlib, signal, time + ready = pathlib.Path({str(child_ready)!r}) + terminated = pathlib.Path({str(child_terminated)!r}) + def stop(signum, frame): + terminated.write_text("terminated") + raise SystemExit(0) + signal.signal(signal.SIGTERM, stop) + ready.write_text("ready") + while True: time.sleep(1) + ''' + subprocess.Popen([sys.executable, "-c", child]) + while not pathlib.Path({str(child_ready)!r}).exists(): + pass + def stop(signum, frame): + while not pathlib.Path({str(child_terminated)!r}).exists(): + pass + raise SystemExit(0) + signal.signal(signal.SIGTERM, stop) + print("child synchronized", flush=True) + while True: time.sleep(1) + """ + ) + + result = run_command( + [sys.executable, "-c", script], + cwd=evidence, + stdout_path=evidence / "group.stdout.log", + stderr_path=evidence / "group.stderr.log", + timeout_seconds=0.2, + ) + + self.assertTrue(result.timed_out) + self.assertEqual((evidence / "group.stdout.log").read_text(), "child synchronized\n") + self.assertEqual(child_terminated.read_text(), "terminated") + + +if __name__ == "__main__": + if "-k" in sys.argv: + pattern_index = sys.argv.index("-k") + 1 + if pattern_index < len(sys.argv) and " or " in sys.argv[pattern_index]: + patterns = sys.argv.pop(pattern_index).split(" or ") + sys.argv.pop(pattern_index - 1) + for pattern in patterns: + sys.argv.extend(("-k", pattern)) + unittest.main() From ef5cbd77b8df5f2590294757b115d73b8a3b66f7 Mon Sep 17 00:00:00 2001 From: Andrii Shylenko <14119286+w1ne@users.noreply.github.com> Date: Sat, 15 Aug 2026 12:12:53 +0200 Subject: [PATCH 07/48] fix(bench): tighten HIL result contracts --- benchmarks/twin2silicon/hil/__init__.py | 1 - benchmarks/twin2silicon/hil/process.py | 2 +- benchmarks/twin2silicon/hil/results.py | 23 ++++++++++++++--------- tests/twin2silicon-hil.py | 20 ++++++++++++++++++-- 4 files changed, 33 insertions(+), 13 deletions(-) diff --git a/benchmarks/twin2silicon/hil/__init__.py b/benchmarks/twin2silicon/hil/__init__.py index f0ac38f..7411ae8 100644 --- a/benchmarks/twin2silicon/hil/__init__.py +++ b/benchmarks/twin2silicon/hil/__init__.py @@ -1,2 +1 @@ """Reusable hardware-in-the-loop benchmark primitives.""" - diff --git a/benchmarks/twin2silicon/hil/process.py b/benchmarks/twin2silicon/hil/process.py index 393a3ea..46efc50 100644 --- a/benchmarks/twin2silicon/hil/process.py +++ b/benchmarks/twin2silicon/hil/process.py @@ -56,7 +56,7 @@ def run_command( return CommandResult( command=normalized_command, cwd=normalized_cwd, - return_code=process.returncode, + returncode=process.returncode, timed_out=timed_out, started_at_utc=started_at, ended_at_utc=ended_at, diff --git a/benchmarks/twin2silicon/hil/results.py b/benchmarks/twin2silicon/hil/results.py index 257b808..f00de64 100644 --- a/benchmarks/twin2silicon/hil/results.py +++ b/benchmarks/twin2silicon/hil/results.py @@ -6,17 +6,22 @@ import os from pathlib import Path import tempfile -from typing import Any, Optional, Union +from typing import Any, Literal, Optional, Union PathLike = Union[str, os.PathLike[str]] +ModelStatus = Literal["pass", "fail", "not_run"] +CompileStatus = Literal["pass", "fail", "not_run"] +SimulatorStatus = Literal["pass", "fail", "not_run", "not_supported"] +HardwareStatus = Literal["pass", "fail", "not_run"] +InfrastructureStatus = Literal["ok", "error"] @dataclass(frozen=True) class CommandResult: command: tuple[str, ...] cwd: str - return_code: int + returncode: int timed_out: bool started_at_utc: str ended_at_utc: str @@ -27,13 +32,13 @@ class CommandResult: @dataclass(frozen=True) class RunResult: - model_status: str - compile_status: str - simulator_status: str - hardware_status: str - infrastructure_status: str + model_status: ModelStatus + compile_status: CompileStatus + simulator_status: SimulatorStatus + hardware_status: HardwareStatus + infrastructure_status: InfrastructureStatus failure_category: Optional[str] = None - failure_detail: Optional[str] = None + detail: Optional[str] = None @classmethod def infrastructure_error(cls, category: str, detail: str) -> "RunResult": @@ -44,7 +49,7 @@ def infrastructure_error(cls, category: str, detail: str) -> "RunResult": hardware_status="not_run", infrastructure_status="error", failure_category=category, - failure_detail=detail, + detail=detail, ) diff --git a/tests/twin2silicon-hil.py b/tests/twin2silicon-hil.py index 7bb1265..7663a51 100644 --- a/tests/twin2silicon-hil.py +++ b/tests/twin2silicon-hil.py @@ -6,6 +6,7 @@ import tempfile import textwrap import time +from typing import get_args, get_origin, get_type_hints, Literal import unittest @@ -31,7 +32,22 @@ def test_run_result_infrastructure_error_marks_execution_not_run(self): self.assertEqual(result.hardware_status, "not_run") self.assertEqual(result.infrastructure_status, "error") self.assertEqual(result.failure_category, "board_identity") - self.assertEqual(result.failure_detail, "wrong adapter") + self.assertEqual(result.detail, "wrong adapter") + + def test_run_result_status_fields_use_explicit_literal_contracts(self): + hints = get_type_hints(RunResult) + expected_choices = { + "model_status": ("pass", "fail", "not_run"), + "compile_status": ("pass", "fail", "not_run"), + "simulator_status": ("pass", "fail", "not_run", "not_supported"), + "hardware_status": ("pass", "fail", "not_run"), + "infrastructure_status": ("ok", "error"), + } + + for field, choices in expected_choices.items(): + with self.subTest(field=field): + self.assertIs(get_origin(hints[field]), Literal) + self.assertEqual(get_args(hints[field]), choices) def test_sha256_file_streams_file_contents(self): with tempfile.TemporaryDirectory() as directory: @@ -76,7 +92,7 @@ def test_run_command_timeout_captures_evidence_and_is_bounded(self): self.assertIsInstance(result, CommandResult) self.assertTrue(result.timed_out) - self.assertNotEqual(result.return_code, 0) + self.assertNotEqual(result.returncode, 0) self.assertGreater(result.duration_seconds, 0) self.assertLess(elapsed, 2) self.assertEqual((evidence / "stdout.log").read_text(), "stdout evidence\n") From 5c2619efab2adaebdf727d71cb776e4afbdd50ae Mon Sep 17 00:00:00 2001 From: Andrii Shylenko <14119286+w1ne@users.noreply.github.com> Date: Sat, 15 Aug 2026 12:16:20 +0200 Subject: [PATCH 08/48] fix(bench): reap timed-out process groups --- benchmarks/twin2silicon/hil/process.py | 40 ++++++++++++++++++--- tests/twin2silicon-hil.py | 48 ++++++++++++++++++++++++++ 2 files changed, 84 insertions(+), 4 deletions(-) diff --git a/benchmarks/twin2silicon/hil/process.py b/benchmarks/twin2silicon/hil/process.py index 46efc50..0540b03 100644 --- a/benchmarks/twin2silicon/hil/process.py +++ b/benchmarks/twin2silicon/hil/process.py @@ -15,6 +15,26 @@ def _utc_now() -> str: return datetime.now(timezone.utc).isoformat().replace("+00:00", "Z") +def _process_group_exists(process_group_id: int) -> bool: + try: + os.killpg(process_group_id, 0) + except ProcessLookupError: + return False + except PermissionError: + return True + return True + + +def _wait_for_process_group_exit(process_group_id: int, timeout_seconds: float) -> bool: + deadline = time.monotonic() + timeout_seconds + while _process_group_exists(process_group_id): + remaining = deadline - time.monotonic() + if remaining <= 0: + return False + time.sleep(min(0.01, remaining)) + return True + + def run_command( command: Sequence[Union[str, os.PathLike[str]]], *, @@ -45,12 +65,24 @@ def run_command( process.communicate(timeout=timeout_seconds) except subprocess.TimeoutExpired: timed_out = True - os.killpg(process.pid, signal.SIGTERM) try: - process.communicate(timeout=0.5) + os.killpg(process.pid, signal.SIGTERM) + except (PermissionError, ProcessLookupError): + pass + if not _wait_for_process_group_exit(process.pid, 0.5): + try: + os.killpg(process.pid, signal.SIGKILL) + except (PermissionError, ProcessLookupError): + pass + try: + process.wait(timeout=0.5) except subprocess.TimeoutExpired: - os.killpg(process.pid, signal.SIGKILL) - process.communicate(timeout=0.5) + try: + os.killpg(process.pid, signal.SIGKILL) + except (PermissionError, ProcessLookupError): + pass + process.wait() + _wait_for_process_group_exit(process.pid, 0.5) ended_at = _utc_now() return CommandResult( diff --git a/tests/twin2silicon-hil.py b/tests/twin2silicon-hil.py index 7663a51..2ddd32c 100644 --- a/tests/twin2silicon-hil.py +++ b/tests/twin2silicon-hil.py @@ -1,7 +1,9 @@ #!/usr/bin/env python3 import hashlib import json +import os from pathlib import Path +import signal import sys import tempfile import textwrap @@ -145,6 +147,52 @@ def stop(signum, frame): self.assertEqual((evidence / "group.stdout.log").read_text(), "child synchronized\n") self.assertEqual(child_terminated.read_text(), "terminated") + def test_run_command_timeout_kills_descendant_that_ignores_sigterm(self): + with tempfile.TemporaryDirectory() as directory: + evidence = Path(directory) + leader_pid_path = evidence / "leader-pid" + child_pid_path = evidence / "ignoring-child-pid" + script = textwrap.dedent( + f""" + import os, pathlib, signal, subprocess, sys, time + pathlib.Path({str(leader_pid_path)!r}).write_text(str(os.getpid())) + child = ''' + import os, pathlib, signal, time + signal.signal(signal.SIGTERM, signal.SIG_IGN) + pathlib.Path({str(child_pid_path)!r}).write_text(str(os.getpid())) + while True: time.sleep(1) + ''' + subprocess.Popen([sys.executable, "-c", child]) + while not pathlib.Path({str(child_pid_path)!r}).exists(): + pass + print("ignoring child synchronized", flush=True) + while True: time.sleep(1) + """ + ) + + try: + result = run_command( + [sys.executable, "-c", script], + cwd=evidence, + stdout_path=evidence / "ignoring.stdout.log", + stderr_path=evidence / "ignoring.stderr.log", + timeout_seconds=0.2, + ) + leader_pid = int(leader_pid_path.read_text()) + child_pid = int(child_pid_path.read_text()) + + self.assertTrue(result.timed_out) + with self.assertRaises(ProcessLookupError): + os.killpg(leader_pid, 0) + with self.assertRaises(ProcessLookupError): + os.kill(child_pid, 0) + finally: + if leader_pid_path.exists(): + try: + os.killpg(int(leader_pid_path.read_text()), signal.SIGKILL) + except (PermissionError, ProcessLookupError): + pass + if __name__ == "__main__": if "-k" in sys.argv: From 78c34f1958af90b9b53d7d6167fb29fd7b78f6b2 Mon Sep 17 00:00:00 2001 From: Andrii Shylenko <14119286+w1ne@users.noreply.github.com> Date: Sat, 15 Aug 2026 12:18:02 +0200 Subject: [PATCH 09/48] fix(bench): bound timed-out process reaping --- benchmarks/twin2silicon/hil/process.py | 9 ++++-- benchmarks/twin2silicon/hil/results.py | 1 + tests/twin2silicon-hil.py | 45 ++++++++++++++++++++++++++ 3 files changed, 53 insertions(+), 2 deletions(-) diff --git a/benchmarks/twin2silicon/hil/process.py b/benchmarks/twin2silicon/hil/process.py index 0540b03..7054719 100644 --- a/benchmarks/twin2silicon/hil/process.py +++ b/benchmarks/twin2silicon/hil/process.py @@ -53,6 +53,7 @@ def run_command( started_at = _utc_now() started_monotonic = time.monotonic() timed_out = False + cleanup_error = None with open(normalized_stdout, "wb") as stdout, open(normalized_stderr, "wb") as stderr: process = subprocess.Popen( normalized_command, @@ -81,18 +82,22 @@ def run_command( os.killpg(process.pid, signal.SIGKILL) except (PermissionError, ProcessLookupError): pass - process.wait() + try: + process.wait(timeout=0.5) + except subprocess.TimeoutExpired: + cleanup_error = "process_group_did_not_exit" _wait_for_process_group_exit(process.pid, 0.5) ended_at = _utc_now() return CommandResult( command=normalized_command, cwd=normalized_cwd, - returncode=process.returncode, + returncode=process.returncode if process.returncode is not None else -signal.SIGKILL, timed_out=timed_out, started_at_utc=started_at, ended_at_utc=ended_at, duration_seconds=time.monotonic() - started_monotonic, stdout_path=normalized_stdout, stderr_path=normalized_stderr, + cleanup_error=cleanup_error, ) diff --git a/benchmarks/twin2silicon/hil/results.py b/benchmarks/twin2silicon/hil/results.py index f00de64..f5fe117 100644 --- a/benchmarks/twin2silicon/hil/results.py +++ b/benchmarks/twin2silicon/hil/results.py @@ -28,6 +28,7 @@ class CommandResult: duration_seconds: float stdout_path: str stderr_path: str + cleanup_error: Optional[str] = None @dataclass(frozen=True) diff --git a/tests/twin2silicon-hil.py b/tests/twin2silicon-hil.py index 2ddd32c..d5a35fe 100644 --- a/tests/twin2silicon-hil.py +++ b/tests/twin2silicon-hil.py @@ -4,18 +4,21 @@ import os from pathlib import Path import signal +import subprocess import sys import tempfile import textwrap import time from typing import get_args, get_origin, get_type_hints, Literal import unittest +from unittest import mock REPOSITORY_ROOT = Path(__file__).resolve().parents[1] sys.path.insert(0, str(REPOSITORY_ROOT)) from benchmarks.twin2silicon.hil.process import run_command +from benchmarks.twin2silicon.hil import process as process_module from benchmarks.twin2silicon.hil.results import ( CommandResult, RunResult, @@ -102,6 +105,7 @@ def test_run_command_timeout_captures_evidence_and_is_bounded(self): self.assertEqual(result.cwd, str(evidence.resolve())) self.assertTrue(result.started_at_utc.endswith("Z")) self.assertTrue(result.ended_at_utc.endswith("Z")) + self.assertIsNone(result.cleanup_error) def test_run_command_timeout_terminates_process_group(self): with tempfile.TemporaryDirectory() as directory: @@ -193,6 +197,47 @@ def test_run_command_timeout_kills_descendant_that_ignores_sigterm(self): except (PermissionError, ProcessLookupError): pass + def test_run_command_reports_cleanup_error_when_leader_cannot_be_reaped(self): + class UnreapableProcess: + pid = 424242 + returncode = None + + def __init__(self): + self.wait_timeouts = [] + + def communicate(self, timeout): + raise subprocess.TimeoutExpired(("stuck-tool",), timeout) + + def wait(self, timeout=None): + self.wait_timeouts.append(timeout) + if timeout is None: + raise AssertionError("run_command used an unbounded wait") + raise subprocess.TimeoutExpired(("stuck-tool",), timeout) + + def fake_killpg(process_group_id, signal_number): + if signal_number == 0: + raise ProcessLookupError + + with tempfile.TemporaryDirectory() as directory: + evidence = Path(directory) + fake_process = UnreapableProcess() + started = time.monotonic() + with mock.patch.object(process_module.subprocess, "Popen", return_value=fake_process), mock.patch.object( + process_module.os, "killpg", side_effect=fake_killpg + ): + result = run_command( + ["stuck-tool"], + cwd=evidence, + stdout_path=evidence / "stuck.stdout.log", + stderr_path=evidence / "stuck.stderr.log", + timeout_seconds=0.01, + ) + + self.assertLess(time.monotonic() - started, 1) + self.assertEqual(result.cleanup_error, "process_group_did_not_exit") + self.assertTrue(fake_process.wait_timeouts) + self.assertNotIn(None, fake_process.wait_timeouts) + if __name__ == "__main__": if "-k" in sys.argv: From 970e7f2b6e8bf5819d4588651520e705abb1e142 Mon Sep 17 00:00:00 2001 From: Andrii Shylenko <14119286+w1ne@users.noreply.github.com> Date: Sat, 15 Aug 2026 12:19:28 +0200 Subject: [PATCH 10/48] fix(bench): report lingering process groups --- benchmarks/twin2silicon/hil/process.py | 4 +++- tests/twin2silicon-hil.py | 28 ++++++++++++++++++++++++++ 2 files changed, 31 insertions(+), 1 deletion(-) diff --git a/benchmarks/twin2silicon/hil/process.py b/benchmarks/twin2silicon/hil/process.py index 7054719..63a1c2a 100644 --- a/benchmarks/twin2silicon/hil/process.py +++ b/benchmarks/twin2silicon/hil/process.py @@ -86,7 +86,9 @@ def run_command( process.wait(timeout=0.5) except subprocess.TimeoutExpired: cleanup_error = "process_group_did_not_exit" - _wait_for_process_group_exit(process.pid, 0.5) + group_exited = _wait_for_process_group_exit(process.pid, 0.5) + if not group_exited and cleanup_error is None: + cleanup_error = "process_group_did_not_exit" ended_at = _utc_now() return CommandResult( diff --git a/tests/twin2silicon-hil.py b/tests/twin2silicon-hil.py index d5a35fe..895fbe7 100644 --- a/tests/twin2silicon-hil.py +++ b/tests/twin2silicon-hil.py @@ -238,6 +238,34 @@ def fake_killpg(process_group_id, signal_number): self.assertTrue(fake_process.wait_timeouts) self.assertNotIn(None, fake_process.wait_timeouts) + def test_run_command_reports_cleanup_error_when_descendant_remains_after_reap(self): + class ReapedLeader: + pid = 424243 + returncode = -signal.SIGTERM + + def communicate(self, timeout): + raise subprocess.TimeoutExpired(("stuck-descendant",), timeout) + + def wait(self, timeout): + return self.returncode + + with tempfile.TemporaryDirectory() as directory: + evidence = Path(directory) + started = time.monotonic() + with mock.patch.object(process_module.subprocess, "Popen", return_value=ReapedLeader()), mock.patch.object( + process_module.os, "killpg" + ), mock.patch.object(process_module, "_process_group_exists", return_value=True): + result = run_command( + ["stuck-descendant"], + cwd=evidence, + stdout_path=evidence / "descendant.stdout.log", + stderr_path=evidence / "descendant.stderr.log", + timeout_seconds=0.01, + ) + + self.assertLess(time.monotonic() - started, 2) + self.assertEqual(result.cleanup_error, "process_group_did_not_exit") + if __name__ == "__main__": if "-k" in sys.argv: From a0714397c72e70309ab82c81246a8d78b9419905 Mon Sep 17 00:00:00 2001 From: Andrii Shylenko <14119286+w1ne@users.noreply.github.com> Date: Sat, 15 Aug 2026 12:38:27 +0200 Subject: [PATCH 11/48] test(bench): add ESP32-S3 GPIO repair fixture --- .../hidden/hil-oracle.json | 41 +++++++++++ .../esp32s3-gpio-hil-001/public/README.md | 11 +++ .../public/firmware/include/run_nonce.h | 3 + .../public/firmware/platformio.ini | 6 ++ .../public/firmware/sdkconfig.defaults | 8 +++ .../public/firmware/src/main.c | 22 ++++++ .../tasks/esp32s3-gpio-hil-001/task.json | 17 +++++ tests/twin2silicon-hil.py | 72 +++++++++++++++++++ 8 files changed, 180 insertions(+) create mode 100644 benchmarks/twin2silicon/tasks/esp32s3-gpio-hil-001/hidden/hil-oracle.json create mode 100644 benchmarks/twin2silicon/tasks/esp32s3-gpio-hil-001/public/README.md create mode 100644 benchmarks/twin2silicon/tasks/esp32s3-gpio-hil-001/public/firmware/include/run_nonce.h create mode 100644 benchmarks/twin2silicon/tasks/esp32s3-gpio-hil-001/public/firmware/platformio.ini create mode 100644 benchmarks/twin2silicon/tasks/esp32s3-gpio-hil-001/public/firmware/sdkconfig.defaults create mode 100644 benchmarks/twin2silicon/tasks/esp32s3-gpio-hil-001/public/firmware/src/main.c create mode 100644 benchmarks/twin2silicon/tasks/esp32s3-gpio-hil-001/task.json diff --git a/benchmarks/twin2silicon/tasks/esp32s3-gpio-hil-001/hidden/hil-oracle.json b/benchmarks/twin2silicon/tasks/esp32s3-gpio-hil-001/hidden/hil-oracle.json new file mode 100644 index 0000000..d1f98ab --- /dev/null +++ b/benchmarks/twin2silicon/tasks/esp32s3-gpio-hil-001/hidden/hil-oracle.json @@ -0,0 +1,41 @@ +{ + "schema_version": "1.0", + "platformio": { + "project_dir": "public/firmware", + "environment": "esp32s3" + }, + "flash": { + "target": "upload", + "artifact": ".pio/build/esp32s3/firmware.bin", + "timeout_seconds": 120 + }, + "identity": { + "command": ["__LABWIRED_IDENTITY_RUNNER__"], + "expected_board": "esp32-s3-devkitc-1", + "timeout_seconds": 10 + }, + "uart": { + "ready_prefix": "LABWIRED_READY:", + "baud": 115200, + "timeout_seconds": 30 + }, + "openocd": { + "board_config": "esp32s3-builtin.cfg", + "startup_timeout_seconds": 20, + "command_timeout_seconds": 10 + }, + "register_assertions": [ + { + "name": "gpio2_output_enabled", + "address": "0x60004020", + "mask": "0x00000004", + "expected": "0x00000004" + }, + { + "name": "gpio2_output_high", + "address": "0x60004004", + "mask": "0x00000004", + "expected": "0x00000004" + } + ] +} diff --git a/benchmarks/twin2silicon/tasks/esp32s3-gpio-hil-001/public/README.md b/benchmarks/twin2silicon/tasks/esp32s3-gpio-hil-001/public/README.md new file mode 100644 index 0000000..4f42d8a --- /dev/null +++ b/benchmarks/twin2silicon/tasks/esp32s3-gpio-hil-001/public/README.md @@ -0,0 +1,11 @@ +# ESP32-S3 GPIO repair + +Repair the ESP-IDF firmware in `firmware/` so GPIO 2 is driven high when the +firmware reports readiness. Keep the readiness message and nonce behavior +intact. The evaluator builds the PlatformIO project, flashes an ESP32-S3 +DevKitC-1, observes its console output, and validates the resulting hardware +state. + +The checked-in nonce header makes the project build standalone. During an +evaluation run, the harness replaces its placeholder value with a unique run +nonce. diff --git a/benchmarks/twin2silicon/tasks/esp32s3-gpio-hil-001/public/firmware/include/run_nonce.h b/benchmarks/twin2silicon/tasks/esp32s3-gpio-hil-001/public/firmware/include/run_nonce.h new file mode 100644 index 0000000..e877c7e --- /dev/null +++ b/benchmarks/twin2silicon/tasks/esp32s3-gpio-hil-001/public/firmware/include/run_nonce.h @@ -0,0 +1,3 @@ +#pragma once + +#define LABWIRED_RUN_NONCE "standalone-placeholder" diff --git a/benchmarks/twin2silicon/tasks/esp32s3-gpio-hil-001/public/firmware/platformio.ini b/benchmarks/twin2silicon/tasks/esp32s3-gpio-hil-001/public/firmware/platformio.ini new file mode 100644 index 0000000..2697a6e --- /dev/null +++ b/benchmarks/twin2silicon/tasks/esp32s3-gpio-hil-001/public/firmware/platformio.ini @@ -0,0 +1,6 @@ +[env:esp32s3] +platform = platformio/espressif32@7.0.1 +board = esp32-s3-devkitc-1 +framework = espidf +monitor_speed = 115200 +board_build.flash_size = 4MB diff --git a/benchmarks/twin2silicon/tasks/esp32s3-gpio-hil-001/public/firmware/sdkconfig.defaults b/benchmarks/twin2silicon/tasks/esp32s3-gpio-hil-001/public/firmware/sdkconfig.defaults new file mode 100644 index 0000000..e7a97b5 --- /dev/null +++ b/benchmarks/twin2silicon/tasks/esp32s3-gpio-hil-001/public/firmware/sdkconfig.defaults @@ -0,0 +1,8 @@ +CONFIG_ESP_CONSOLE_USB_SERIAL_JTAG=y +# CONFIG_ESP_CONSOLE_UART_DEFAULT is not set +# CONFIG_ESP_CONSOLE_UART_CUSTOM is not set +# CONFIG_ESP_CONSOLE_UART_NONE is not set +# CONFIG_ESP_CONSOLE_UART is not set +CONFIG_ESP_CONSOLE_SECONDARY_NONE=y +# CONFIG_ESP_CONSOLE_SECONDARY_USB_SERIAL_JTAG is not set +CONFIG_ESPTOOLPY_FLASHSIZE_4MB=y diff --git a/benchmarks/twin2silicon/tasks/esp32s3-gpio-hil-001/public/firmware/src/main.c b/benchmarks/twin2silicon/tasks/esp32s3-gpio-hil-001/public/firmware/src/main.c new file mode 100644 index 0000000..f0524ee --- /dev/null +++ b/benchmarks/twin2silicon/tasks/esp32s3-gpio-hil-001/public/firmware/src/main.c @@ -0,0 +1,22 @@ +#include + +#include "driver/gpio.h" +#include "esp_err.h" +#include "freertos/FreeRTOS.h" +#include "freertos/task.h" +#include "run_nonce.h" + +#define TEST_GPIO GPIO_NUM_2 + +void app_main(void) +{ + ESP_ERROR_CHECK(gpio_set_direction(TEST_GPIO, GPIO_MODE_INPUT)); + ESP_ERROR_CHECK(gpio_set_level(TEST_GPIO, 1)); + + printf("LABWIRED_READY:%s\n", LABWIRED_RUN_NONCE); + fflush(stdout); + + for (;;) { + vTaskDelay(pdMS_TO_TICKS(1000)); + } +} diff --git a/benchmarks/twin2silicon/tasks/esp32s3-gpio-hil-001/task.json b/benchmarks/twin2silicon/tasks/esp32s3-gpio-hil-001/task.json new file mode 100644 index 0000000..cd9f888 --- /dev/null +++ b/benchmarks/twin2silicon/tasks/esp32s3-gpio-hil-001/task.json @@ -0,0 +1,17 @@ +{ + "schema_version": "1.0", + "id": "esp32s3-gpio-hil-001", + "board": "esp32-s3-devkitc-1", + "framework": "espidf", + "model": "labwired/labwired-fast", + "budgets": { + "wall_time_seconds": 1200, + "model_tokens": 50000, + "repair_iterations": 6, + "simulator_runs": 0, + "diagnostic_hil_runs": 0 + }, + "public_dir": "public", + "hidden_oracle": "hidden/hil-oracle.json", + "firmware_elf": "firmware/.pio/build/esp32s3/firmware.elf" +} diff --git a/tests/twin2silicon-hil.py b/tests/twin2silicon-hil.py index 895fbe7..c2e83f4 100644 --- a/tests/twin2silicon-hil.py +++ b/tests/twin2silicon-hil.py @@ -27,6 +27,78 @@ ) +class FixtureContractTests(unittest.TestCase): + def test_esp32s3_gpio_hil_fixture_contract(self): + task_root = ( + REPOSITORY_ROOT + / "benchmarks" + / "twin2silicon" + / "tasks" + / "esp32s3-gpio-hil-001" + ) + self.assertTrue(task_root.is_dir(), f"missing fixture: {task_root}") + task = json.loads((task_root / "task.json").read_text(encoding="utf-8")) + oracle = json.loads( + (task_root / task["hidden_oracle"]).read_text(encoding="utf-8") + ) + + self.assertEqual(task["schema_version"], "1.0") + self.assertEqual(task["id"], "esp32s3-gpio-hil-001") + self.assertEqual(task["board"], "esp32-s3-devkitc-1") + self.assertEqual(task["framework"], "espidf") + self.assertEqual(task["budgets"]["model_tokens"], 50000) + self.assertEqual(task["budgets"]["diagnostic_hil_runs"], 0) + self.assertEqual(oracle["schema_version"], "1.0") + self.assertEqual(oracle["uart"]["ready_prefix"], "LABWIRED_READY:") + self.assertEqual( + oracle["register_assertions"], + [ + { + "name": "gpio2_output_enabled", + "address": "0x60004020", + "mask": "0x00000004", + "expected": "0x00000004", + }, + { + "name": "gpio2_output_high", + "address": "0x60004004", + "mask": "0x00000004", + "expected": "0x00000004", + }, + ], + ) + + public_files = sorted( + path for path in (task_root / task["public_dir"]).rglob("*") if path.is_file() + ) + expected_public_files = { + "README.md", + "firmware/include/run_nonce.h", + "firmware/platformio.ini", + "firmware/sdkconfig.defaults", + "firmware/src/main.c", + } + self.assertEqual( + {str(path.relative_to(task_root / task["public_dir"])) for path in public_files}, + expected_public_files, + ) + public_text = "\n".join(path.read_text(encoding="utf-8") for path in public_files) + for hidden_detail in ( + "0x60004020", + "0x60004004", + "0x00000004", + "esp32s3-builtin.cfg", + "openocd", + "mdw", + ): + with self.subTest(hidden_detail=hidden_detail): + self.assertNotIn(hidden_detail, public_text.lower()) + main_source = (task_root / "public" / "firmware" / "src" / "main.c").read_text( + encoding="utf-8" + ) + self.assertIn("gpio_set_direction(TEST_GPIO, GPIO_MODE_INPUT)", main_source) + + class ResultContractTests(unittest.TestCase): def test_run_result_infrastructure_error_marks_execution_not_run(self): result = RunResult.infrastructure_error("board_identity", "wrong adapter") From cf8932b4e8191e09e5cd26d4be1e51572cc7f3b6 Mon Sep 17 00:00:00 2001 From: Andrii Shylenko <14119286+w1ne@users.noreply.github.com> Date: Sat, 15 Aug 2026 12:41:14 +0200 Subject: [PATCH 12/48] fix(bench): correct ESP-IDF console symbol --- .../esp32s3-gpio-hil-001/public/firmware/sdkconfig.defaults | 2 +- tests/twin2silicon-hil.py | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/benchmarks/twin2silicon/tasks/esp32s3-gpio-hil-001/public/firmware/sdkconfig.defaults b/benchmarks/twin2silicon/tasks/esp32s3-gpio-hil-001/public/firmware/sdkconfig.defaults index e7a97b5..4f55897 100644 --- a/benchmarks/twin2silicon/tasks/esp32s3-gpio-hil-001/public/firmware/sdkconfig.defaults +++ b/benchmarks/twin2silicon/tasks/esp32s3-gpio-hil-001/public/firmware/sdkconfig.defaults @@ -1,7 +1,7 @@ CONFIG_ESP_CONSOLE_USB_SERIAL_JTAG=y # CONFIG_ESP_CONSOLE_UART_DEFAULT is not set # CONFIG_ESP_CONSOLE_UART_CUSTOM is not set -# CONFIG_ESP_CONSOLE_UART_NONE is not set +# CONFIG_ESP_CONSOLE_NONE is not set # CONFIG_ESP_CONSOLE_UART is not set CONFIG_ESP_CONSOLE_SECONDARY_NONE=y # CONFIG_ESP_CONSOLE_SECONDARY_USB_SERIAL_JTAG is not set diff --git a/tests/twin2silicon-hil.py b/tests/twin2silicon-hil.py index c2e83f4..fb308b4 100644 --- a/tests/twin2silicon-hil.py +++ b/tests/twin2silicon-hil.py @@ -97,6 +97,11 @@ def test_esp32s3_gpio_hil_fixture_contract(self): encoding="utf-8" ) self.assertIn("gpio_set_direction(TEST_GPIO, GPIO_MODE_INPUT)", main_source) + sdkconfig_defaults = ( + task_root / "public" / "firmware" / "sdkconfig.defaults" + ).read_text(encoding="utf-8") + self.assertIn("# CONFIG_ESP_CONSOLE_NONE is not set", sdkconfig_defaults) + self.assertNotIn("CONFIG_ESP_CONSOLE_UART_NONE", sdkconfig_defaults) class ResultContractTests(unittest.TestCase): From 3f2d0ab374af7d192e7c510e96aeb7964a94f719 Mon Sep 17 00:00:00 2001 From: Andrii Shylenko <14119286+w1ne@users.noreply.github.com> Date: Sat, 15 Aug 2026 12:55:57 +0200 Subject: [PATCH 13/48] feat(bench): evaluate ESP32-S3 UART and JTAG evidence --- benchmarks/twin2silicon/hil/esp32s3.py | 356 +++++++++++++++++++++++++ tests/twin2silicon-hil.py | 272 +++++++++++++++++++ 2 files changed, 628 insertions(+) create mode 100644 benchmarks/twin2silicon/hil/esp32s3.py diff --git a/benchmarks/twin2silicon/hil/esp32s3.py b/benchmarks/twin2silicon/hil/esp32s3.py new file mode 100644 index 0000000..b5941f3 --- /dev/null +++ b/benchmarks/twin2silicon/hil/esp32s3.py @@ -0,0 +1,356 @@ +"""Offline-testable ESP32-S3 HIL evidence collection primitives.""" + +from dataclasses import dataclass +import fcntl +import hashlib +import json +import math +import os +from pathlib import Path +import re +import select +import termios +import time +import tty +from typing import Callable, Mapping, Optional, Sequence + +from .process import run_command +from .results import CommandResult, PathLike + + +_NAME = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$") +_SERIAL = re.compile(r"^[A-Za-z0-9_.:/+-]+$") +_MARKER = re.compile(r"^@@REG ([A-Za-z_][A-Za-z0-9_]*) (0x[0-9A-Fa-f]{8})$") +_VALUE = re.compile(r"^(0x[0-9A-Fa-f]{8}): (0x[0-9A-Fa-f]{8})$") + + +def _uint32(value: object, field: str) -> int: + if isinstance(value, bool): + raise TypeError(f"{field} must be an integer or hexadecimal string") + if isinstance(value, str): + if re.fullmatch(r"0x[0-9A-Fa-f]+", value) is None: + raise ValueError(f"invalid hexadecimal {field}") + parsed = int(value, 16) + elif isinstance(value, int): + parsed = value + else: + raise TypeError(f"{field} must be an integer or hexadecimal string") + if parsed < 0 or parsed > 0xFFFFFFFF: + raise ValueError(f"{field} is outside uint32 bounds") + return parsed + + +@dataclass(frozen=True) +class RegisterAssertion: + name: str + address: int + mask: int + expected: int + + def __post_init__(self) -> None: + if not _NAME.fullmatch(self.name): + raise ValueError("register assertion name is unsafe") + for field in ("address", "mask", "expected"): + value = getattr(self, field) + if isinstance(value, bool) or not isinstance(value, int) or not 0 <= value <= 0xFFFFFFFF: + raise ValueError(f"{field} is outside uint32 bounds") + if self.address % 4: + raise ValueError("register address must be 32-bit aligned") + if self.expected & ~self.mask: + raise ValueError("expected value contains bits outside mask") + + @classmethod + def from_json(cls, record: Mapping[str, object]) -> "RegisterAssertion": + name = record.get("name") + if not isinstance(name, str): + raise TypeError("register assertion name must be a string") + return cls(name, _uint32(record.get("address"), "address"), + _uint32(record.get("mask"), "mask"), + _uint32(record.get("expected"), "expected")) + + +@dataclass(frozen=True) +class Esp32S3Config: + uart_device: str + uart_baud: int + uart_timeout_seconds: float + jtag_serial: str + openocd_config: str + assertions: tuple[RegisterAssertion, ...] + + @classmethod + def from_oracle(cls, oracle: Mapping[str, object]) -> "Esp32S3Config": + uart = oracle.get("uart") + jtag = oracle.get("jtag", oracle.get("openocd")) + records = oracle.get("register_assertions") + if not isinstance(uart, Mapping) or not isinstance(jtag, Mapping) or not isinstance(records, list): + raise TypeError("oracle uart, jtag/openocd, and register_assertions are required") + assertions = tuple(RegisterAssertion.from_json(record) for record in records) + if not assertions: + raise ValueError("at least one register assertion is required") + names = [item.name for item in assertions] + addresses = [item.address for item in assertions] + if len(names) != len(set(names)) or len(addresses) != len(set(addresses)): + raise ValueError("register assertion names and addresses must be unique") + baud = _positive_int(uart.get("baud"), "uart baud") + timeout = _positive_float(uart.get("timeout_seconds"), "uart timeout") + device = uart.get("device", "") + serial = jtag.get("serial", "") + config = jtag.get("config", jtag.get("board_config", "")) + if not all(isinstance(value, str) for value in (device, serial, config)): + raise TypeError("device, serial, and config must be strings") + if not device or not serial or not config: + raise ValueError("device, serial, and config must not be empty") + return cls(device, baud, timeout, serial, config, assertions) + + +def _positive_int(value: object, field: str) -> int: + if isinstance(value, bool) or not isinstance(value, int) or value <= 0: + raise ValueError(f"{field} must be positive") + return value + + +def _positive_float(value: object, field: str) -> float: + if (isinstance(value, bool) or not isinstance(value, (int, float)) + or not math.isfinite(value) or value <= 0): + raise ValueError(f"{field} must be positive") + return float(value) + + +class BoardLockTimeout(TimeoutError): + pass + + +class BoardLock: + def __init__(self, directory: PathLike, identity: str, *, timeout_seconds: float, + poll_interval_seconds: float = 0.01) -> None: + if (not identity or not math.isfinite(timeout_seconds) or timeout_seconds < 0 + or not math.isfinite(poll_interval_seconds) or poll_interval_seconds <= 0): + raise ValueError("invalid board lock parameters") + safe = re.sub(r"[^A-Za-z0-9_.-]", "_", identity)[:48] or "board" + digest = hashlib.sha256(identity.encode()).hexdigest()[:12] + self.path = Path(directory) / f"{safe}-{digest}.lock" + self.identity = identity + self.timeout_seconds = timeout_seconds + self.poll_interval_seconds = poll_interval_seconds + self._file = None + + def acquire(self) -> "BoardLock": + self.path.parent.mkdir(parents=True, exist_ok=True) + held = open(self.path, "a+", encoding="utf-8") + deadline = time.monotonic() + self.timeout_seconds + while True: + try: + fcntl.flock(held.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB) + break + except BlockingIOError: + remaining = deadline - time.monotonic() + if remaining <= 0: + held.close() + raise BoardLockTimeout(f"board {self.identity!r} is locked") + time.sleep(min(self.poll_interval_seconds, remaining)) + self._file = held + try: + held.seek(0) + held.truncate() + json.dump({"identity": self.identity, "pid": os.getpid(), "acquired_monotonic": time.monotonic()}, held) + held.flush() + os.fsync(held.fileno()) + except BaseException: + self.release() + raise + return self + + def release(self) -> None: + if self._file is not None: + held, self._file = self._file, None + try: + fcntl.flock(held.fileno(), fcntl.LOCK_UN) + finally: + held.close() + + def __enter__(self) -> "BoardLock": + return self.acquire() + + def __exit__(self, exc_type, exc, traceback) -> None: + self.release() + + +@dataclass(frozen=True) +class PhaseResult: + status: str + category: Optional[str] = None + detail: Optional[str] = None + command_result: Optional[CommandResult] = None + + +Runner = Callable[..., CommandResult] + + +def validate_identity(command: Sequence[os.PathLike[str] | str], expected_serial: str, *, + cwd: PathLike, evidence_dir: PathLike, timeout_seconds: float, + runner: Runner = run_command) -> PhaseResult: + evidence = Path(evidence_dir) + try: + result = runner(command, cwd=cwd, stdout_path=evidence / "identity.stdout.log", + stderr_path=evidence / "identity.stderr.log", timeout_seconds=timeout_seconds) + except OSError as error: + return PhaseResult("infrastructure_error", "board_identity", str(error)) + if result.cleanup_error or result.timed_out or result.returncode != 0: + return PhaseResult("infrastructure_error", "board_identity", "identity command failed", result) + try: + lines = [line.strip() for line in Path(result.stdout_path).read_text(encoding="utf-8").splitlines() + if line.strip()] + except OSError as error: + return PhaseResult("infrastructure_error", "board_identity", str(error), result) + if lines != [expected_serial]: + return PhaseResult("infrastructure_error", "board_identity", + f"expected exactly [{expected_serial!r}], observed {lines!r}", result) + return PhaseResult("pass", command_result=result) + + +def flash_firmware(command: Sequence[os.PathLike[str] | str], *, cwd: PathLike, + evidence_dir: PathLike, timeout_seconds: float, identity_validated: bool, + runner: Runner = run_command) -> PhaseResult: + if not identity_validated: + raise ValueError("board identity must be validated before flashing") + evidence = Path(evidence_dir) + try: + result = runner(command, cwd=cwd, stdout_path=evidence / "flash.stdout.log", + stderr_path=evidence / "flash.stderr.log", timeout_seconds=timeout_seconds) + except OSError as error: + return PhaseResult("infrastructure_error", "flash_infrastructure", str(error)) + if result.cleanup_error or result.timed_out: + return PhaseResult("infrastructure_error", "flash_infrastructure", "flash did not exit cleanly", result) + if result.returncode: + return PhaseResult("hardware_fail", "flash", "candidate firmware flash failed", result) + return PhaseResult("pass", command_result=result) + + +@dataclass(frozen=True) +class UartResult: + matched: bool + bytes_captured: int + timed_out: bool + + +def capture_uart_nonce(device: PathLike, baud: int, nonce: str, timeout_seconds: float, + log: PathLike, *, max_bytes: int = 65536) -> UartResult: + fd = os.open(device, os.O_RDWR | os.O_NOCTTY | os.O_NONBLOCK) + try: + speeds = {9600: termios.B9600, 115200: termios.B115200} + if baud not in speeds: + raise ValueError(f"unsupported UART baud: {baud}") + if not math.isfinite(timeout_seconds) or timeout_seconds < 0 or max_bytes <= 0: + raise ValueError("invalid UART bounds") + attrs = termios.tcgetattr(fd) + tty.setraw(fd, termios.TCSANOW) + attrs = termios.tcgetattr(fd) + attrs[4] = attrs[5] = speeds[baud] + attrs[2] = (attrs[2] & ~(termios.CSIZE | termios.PARENB | termios.CSTOPB)) | termios.CS8 | termios.CLOCAL | termios.CREAD + termios.tcsetattr(fd, termios.TCSANOW, attrs) + deadline = time.monotonic() + timeout_seconds + captured = bytearray() + pending = bytearray() + matched = False + expected = f"LABWIRED_READY:{nonce}".encode() + while not matched and len(captured) < max_bytes: + remaining = deadline - time.monotonic() + if remaining <= 0: + break + readable, _, _ = select.select([fd], [], [], remaining) + if not readable: + break + try: + chunk = os.read(fd, min(4096, max_bytes - len(captured))) + except BlockingIOError: + continue + if not chunk: + continue + captured.extend(chunk) + pending.extend(chunk) + while b"\n" in pending: + line, _, remainder = pending.partition(b"\n") + pending = bytearray(remainder) + if line.endswith(b"\r"): + line = line[:-1] + if line == expected: + matched = True + break + Path(log).parent.mkdir(parents=True, exist_ok=True) + Path(log).write_bytes(captured) + return UartResult(matched, len(captured), not matched) + finally: + os.close(fd) + + +def build_openocd_command(executable: str, config: str, adapter_serial: str, + assertions: Sequence[RegisterAssertion]) -> list[str]: + if not _SERIAL.fullmatch(adapter_serial): + raise ValueError("unsafe adapter serial") + if any(character in config for character in "\r\n\x00"): + raise ValueError("unsafe OpenOCD config") + commands = [f"adapter serial {adapter_serial}", "adapter speed 4000", "init", "reset run", + "sleep 750", "halt"] + for assertion in assertions: + commands.extend((f"echo @@REG {assertion.name} 0x{assertion.address:08x}", + f"mdw 0x{assertion.address:08x}")) + commands.append("exit") + return [executable, "-f", config, "-c", "; ".join(commands)] + + +def parse_openocd_registers(text: str, requested: Sequence[RegisterAssertion]) -> dict[str, int]: + by_name = {item.name: item for item in requested} + if len(by_name) != len(requested): + raise ValueError("requested assertion names are not unique") + observed: dict[str, int] = {} + lines = text.splitlines() + index = 0 + while index < len(lines): + stripped = lines[index].strip() + marker = _MARKER.fullmatch(stripped) + if marker is None: + if (stripped.startswith("@@REG") or _VALUE.fullmatch(stripped) + or stripped.lower().startswith("error:")): + raise ValueError("unpaired or malformed register observation") + index += 1 + continue + name, address_text = marker.groups() + assertion = by_name.get(name) + if assertion is None or name in observed or int(address_text, 16) != assertion.address: + raise ValueError("invalid or duplicate register marker") + if index + 1 >= len(lines): + raise ValueError("register marker has no observation") + value_line = _VALUE.fullmatch(lines[index + 1].strip()) + if value_line is None or int(value_line.group(1), 16) != assertion.address: + raise ValueError("register observation is malformed or has wrong address") + observed[name] = int(value_line.group(2), 16) + index += 2 + if set(observed) != set(by_name): + raise ValueError("missing requested register observations") + return observed + + +@dataclass(frozen=True) +class RegisterObservation: + name: str + address: int + value: int + mask: int + expected: int + passed: bool + + +@dataclass(frozen=True) +class RegisterEvaluation: + status: str + observations: tuple[RegisterObservation, ...] + + +def evaluate_registers(observed: Mapping[str, int], assertions: Sequence[RegisterAssertion]) -> RegisterEvaluation: + if set(observed) != {item.name for item in assertions}: + raise ValueError("observed registers do not exactly match assertions") + results = tuple(RegisterObservation(item.name, item.address, observed[item.name], item.mask, + item.expected, (observed[item.name] & item.mask) == item.expected) + for item in assertions) + return RegisterEvaluation("pass" if all(item.passed for item in results) else "hardware_fail", results) diff --git a/tests/twin2silicon-hil.py b/tests/twin2silicon-hil.py index fb308b4..af07908 100644 --- a/tests/twin2silicon-hil.py +++ b/tests/twin2silicon-hil.py @@ -3,12 +3,15 @@ import json import os from pathlib import Path +import pty import signal import subprocess import sys import tempfile import textwrap +import threading import time +import tty from typing import get_args, get_origin, get_type_hints, Literal import unittest from unittest import mock @@ -25,6 +28,25 @@ sha256_file, write_json_atomic, ) +from benchmarks.twin2silicon.hil.esp32s3 import ( + BoardLock, + BoardLockTimeout, + Esp32S3Config, + RegisterAssertion, + build_openocd_command, + capture_uart_nonce, + evaluate_registers, + flash_firmware, + parse_openocd_registers, + validate_identity, +) + + +def executable_fixture(directory, body): + path = Path(directory) / "fixture.py" + path.write_text("#!/usr/bin/env python3\n" + body, encoding="utf-8") + path.chmod(0o755) + return path class FixtureContractTests(unittest.TestCase): @@ -344,6 +366,256 @@ def wait(self, timeout): self.assertEqual(result.cleanup_error, "process_group_did_not_exit") +class Esp32S3ConfigTests(unittest.TestCase): + def test_parses_valid_oracle_and_hex_register_values(self): + config = Esp32S3Config.from_oracle( + { + "uart": {"device": "/dev/cu.board", "baud": 115200, "timeout_seconds": 1}, + "jtag": {"serial": "JTAG-1", "config": "board/esp32s3.cfg"}, + "register_assertions": [ + {"name": "gpio", "address": "0x60004020", "mask": "0x4", "expected": "0x4"} + ], + } + ) + self.assertEqual(config.assertions[0].address, 0x60004020) + self.assertEqual(config.assertions[0].mask, 4) + + def test_rejects_invalid_bounds_alignment_duplicates_and_names(self): + good = {"name": "gpio", "address": "0x60004020", "mask": "0x4", "expected": "0x4"} + bad = [ + {**good, "address": "-1"}, + {**good, "address": "0x60004021"}, + {**good, "mask": "0x100000000"}, + {**good, "expected": "xyz"}, + {**good, "expected": "4"}, + {**good, "mask": 1.5}, + {**good, "name": "gpio; shutdown"}, + ] + for record in bad: + with self.subTest(record=record), self.assertRaises((TypeError, ValueError)): + RegisterAssertion.from_json(record) + with self.assertRaises(ValueError): + Esp32S3Config.from_oracle({ + "uart": {"device": "x", "baud": 115200, "timeout_seconds": 1}, + "jtag": {"serial": "s", "config": "c"}, + "register_assertions": [good, {**good, "address": "0x60004024"}], + }) + with self.assertRaises(ValueError): + Esp32S3Config.from_oracle({ + "uart": {"device": "x", "baud": 115200, "timeout_seconds": 1}, + "jtag": {"serial": "s", "config": "c"}, + "register_assertions": [good, {**good, "name": "gpio2"}], + }) + + def test_rejects_empty_configuration_assertions_and_nonfinite_timeout(self): + for timeout in (float("nan"), float("inf")): + with self.subTest(timeout=timeout), self.assertRaises(ValueError): + Esp32S3Config.from_oracle({ + "uart": {"device": "device", "baud": 115200, "timeout_seconds": timeout}, + "jtag": {"serial": "serial", "config": "config"}, + "register_assertions": [{"name": "gpio", "address": "0x4", "mask": "0x4", "expected": "0x4"}], + }) + with self.assertRaises(ValueError): + Esp32S3Config.from_oracle({ + "uart": {"device": "", "baud": 115200, "timeout_seconds": 1}, + "jtag": {"serial": "", "config": ""}, + "register_assertions": [], + }) + + +class BoardIdentityAndFlashTests(unittest.TestCase): + def test_exactly_one_configured_serial_succeeds(self): + with tempfile.TemporaryDirectory() as directory: + tool = executable_fixture(directory, "print(' JTAG-1 ')\n") + result = validate_identity([tool], "JTAG-1", cwd=directory, evidence_dir=directory, timeout_seconds=1) + self.assertEqual((result.status, result.category), ("pass", None)) + + def test_absent_wrong_and_duplicate_serials_are_infrastructure_before_flash(self): + for output in ("", "OTHER\\n", "JTAG-1\\nJTAG-1\\n"): + with self.subTest(output=output), tempfile.TemporaryDirectory() as directory: + marker = Path(directory) / "flashed" + tool = executable_fixture(directory, f"print({output!r}, end='')\n") + result = validate_identity([tool], "JTAG-1", cwd=directory, evidence_dir=directory, timeout_seconds=1) + self.assertEqual((result.status, result.category), ("infrastructure_error", "board_identity")) + self.assertFalse(marker.exists()) + + def test_identity_tool_timeout_nonzero_and_cleanup_are_infrastructure(self): + with tempfile.TemporaryDirectory() as directory: + slow = executable_fixture(directory, "import time; time.sleep(30)\n") + result = validate_identity([slow], "JTAG-1", cwd=directory, evidence_dir=directory, timeout_seconds=.05) + self.assertEqual(result.status, "infrastructure_error") + with tempfile.TemporaryDirectory() as directory: + failed = executable_fixture(directory, "raise SystemExit(3)\n") + result = validate_identity([failed], "JTAG-1", cwd=directory, evidence_dir=directory, timeout_seconds=1) + self.assertEqual((result.status, result.category), ("infrastructure_error", "board_identity")) + fake = CommandResult(("x",), "/", 0, False, "", "", 0, "/tmp/o", "/tmp/e", "cleanup") + with tempfile.TemporaryDirectory() as directory: + result = validate_identity(["x"], "JTAG-1", cwd=directory, evidence_dir=directory, + timeout_seconds=1, runner=lambda *a, **k: fake) + self.assertEqual((result.status, result.category), ("infrastructure_error", "board_identity")) + + def test_command_launch_errors_are_infrastructure(self): + def unavailable(*args, **kwargs): + raise FileNotFoundError("tool missing") + with tempfile.TemporaryDirectory() as directory: + identity = validate_identity(["missing"], "JTAG-1", cwd=directory, evidence_dir=directory, + timeout_seconds=1, runner=unavailable) + flash = flash_firmware(["missing"], cwd=directory, evidence_dir=directory, + timeout_seconds=1, identity_validated=True, runner=unavailable) + self.assertEqual(identity.status, "infrastructure_error") + self.assertEqual(flash.status, "infrastructure_error") + + def test_flash_classifies_nonzero_as_hardware_and_timeout_cleanup_as_infrastructure(self): + def result(code=0, timed_out=False, cleanup=None): + return CommandResult(("flash",), "/", code, timed_out, "", "", 0, "/tmp/o", "/tmp/e", cleanup) + with tempfile.TemporaryDirectory() as directory: + failed = flash_firmware(["flash"], cwd=directory, evidence_dir=directory, timeout_seconds=1, + identity_validated=True, runner=lambda *a, **k: result(2)) + timeout = flash_firmware(["flash"], cwd=directory, evidence_dir=directory, timeout_seconds=1, + identity_validated=True, runner=lambda *a, **k: result(-15, True)) + cleanup = flash_firmware(["flash"], cwd=directory, evidence_dir=directory, timeout_seconds=1, + identity_validated=True, runner=lambda *a, **k: result(0, False, "stuck")) + self.assertEqual((failed.status, failed.category), ("hardware_fail", "flash")) + self.assertEqual(timeout.status, "infrastructure_error") + self.assertEqual(cleanup.status, "infrastructure_error") + with tempfile.TemporaryDirectory() as directory, self.assertRaises(ValueError): + flash_firmware(["flash"], cwd=directory, evidence_dir=directory, timeout_seconds=1, + identity_validated=False) + + +class BoardLockTests(unittest.TestCase): + def test_lock_is_identity_keyed_bounded_and_released_normally(self): + with tempfile.TemporaryDirectory() as directory: + first = BoardLock(directory, "usb/serial:one", timeout_seconds=.1) + with first: + self.assertIn("usb_serial_one", first.path.name) + started = time.monotonic() + with self.assertRaises(BoardLockTimeout): + with BoardLock(directory, "usb/serial:one", timeout_seconds=.05): + pass + self.assertLess(time.monotonic() - started, .5) + with BoardLock(directory, "usb/serial:one", timeout_seconds=.1): + pass + + def test_lock_releases_after_exception_and_stale_metadata_does_not_claim_lock(self): + with tempfile.TemporaryDirectory() as directory: + lock = BoardLock(directory, "JTAG-1", timeout_seconds=.1) + lock.path.parent.mkdir(parents=True, exist_ok=True) + lock.path.write_text('{"pid": 999999, "identity": "stale"}') + with self.assertRaises(RuntimeError): + with lock: + metadata = json.loads(lock.path.read_text()) + self.assertEqual(metadata["identity"], "JTAG-1") + raise RuntimeError("candidate failed") + with BoardLock(directory, "JTAG-1", timeout_seconds=.1): + pass + + def test_lock_releases_if_metadata_persistence_fails_and_rejects_nonfinite_timeout(self): + with tempfile.TemporaryDirectory() as directory: + with mock.patch("benchmarks.twin2silicon.hil.esp32s3.os.fsync", side_effect=OSError("disk")): + with self.assertRaises(OSError): + with BoardLock(directory, "JTAG-1", timeout_seconds=.1): + pass + with BoardLock(directory, "JTAG-1", timeout_seconds=.1): + pass + for timeout in (float("nan"), float("inf")): + with self.subTest(timeout=timeout), self.assertRaises(ValueError): + BoardLock(directory, "JTAG-1", timeout_seconds=timeout) + + +class UartNonceTests(unittest.TestCase): + def _capture(self, chunks, nonce="current", timeout=.2): + master, slave = pty.openpty() + device = os.ttyname(slave) + tty.setraw(slave) + with tempfile.TemporaryDirectory() as directory: + log = Path(directory) / "uart.log" + started = threading.Event() + def writer(): + started.wait() + for chunk in chunks: + os.write(master, chunk) + thread = threading.Thread(target=writer) + thread.start() + started.set() + try: + result = capture_uart_nonce(device, 115200, nonce, timeout, log, max_bytes=64) + finally: + thread.join(1) + os.close(master) + os.close(slave) + return result, log.read_bytes() + + def test_accepts_only_exact_current_nonce_as_complete_line_and_logs_raw_bytes(self): + result, raw = self._capture([b"boot\r\nLABWIRED_READY:current\r", b"\n"]) + self.assertTrue(result.matched) + self.assertEqual(raw, b"boot\r\nLABWIRED_READY:current\r\n") + + def test_rejects_absent_wrong_stale_or_incomplete_nonce_with_bounded_evidence(self): + for chunks in ([b"boot\n"], [b"LABWIRED_READY:wrong\n"], + [b"LABWIRED_READY:stale\n"], [b"LABWIRED_READY:current"]): + with self.subTest(chunks=chunks): + started = time.monotonic() + result, raw = self._capture(chunks, timeout=.05) + self.assertFalse(result.matched) + self.assertLess(time.monotonic() - started, .5) + self.assertLessEqual(len(raw), 64) + + def test_rejects_unsupported_baud_and_closes_opened_fd(self): + master, slave = pty.openpty() + device = os.ttyname(slave) + os.close(slave) + real_close = os.close + closed = [] + with tempfile.TemporaryDirectory() as directory, mock.patch("benchmarks.twin2silicon.hil.esp32s3.os.close", side_effect=lambda fd: (closed.append(fd), real_close(fd))[1]): + with self.assertRaises(ValueError): + capture_uart_nonce(device, 12345, "n", .01, Path(directory) / "log") + real_close(master) + self.assertTrue(closed) + + +class OpenOcdEvidenceTests(unittest.TestCase): + def setUp(self): + self.assertions = ( + RegisterAssertion("enable", 0x60004020, 4, 4), + RegisterAssertion("high", 0x60004004, 4, 4), + ) + + def test_command_is_argv_and_requests_marked_records_at_fixed_speed(self): + command = build_openocd_command("openocd", "board.cfg", "JTAG-1", self.assertions) + self.assertEqual(command[:3], ["openocd", "-f", "board.cfg"]) + script = command[command.index("-c") + 1] + for fragment in ("adapter serial JTAG-1", "adapter speed 4000", "reset run", "sleep 750", "halt", + "echo @@REG enable 0x60004020", "mdw 0x60004020", "exit"): + self.assertIn(fragment, script) + self.assertNotIn(";", command[:-1]) + + def test_parser_accepts_only_immediately_paired_canonical_requested_records(self): + text = "noise\n@@REG enable 0x60004020\n0x60004020: 0x00000004\n@@REG high 0x60004004\n0x60004004: 0x00000004\n" + self.assertEqual(parse_openocd_registers(text, self.assertions), {"enable": 4, "high": 4}) + invalid = [ + text.replace("0x60004020: 0x00000004", "noise\n0x60004020: 0x00000004"), + text + "@@REG enable 0x60004020\n0x60004020: 0x00000004\n", + text.replace("enable", "other"), + text.replace("@@REG enable 0x60004020", "@@REG enable 0x60004024"), + text.replace("0x60004020: 0x00000004", "60004020 = 4"), + text.replace("@@REG enable 0x60004020", "@@REG enable not-an-address"), + text + "@@REG malformed\n", + text + "Error: target not halted\n", + text.split("@@REG high")[0], + ] + for evidence in invalid: + with self.subTest(evidence=evidence), self.assertRaises(ValueError): + parse_openocd_registers(evidence, self.assertions) + + def test_masked_mismatch_fails_and_all_assertions_pass(self): + passing = evaluate_registers({"enable": 0x104, "high": 4}, self.assertions) + failing = evaluate_registers({"enable": 0, "high": 4}, self.assertions) + self.assertEqual(passing.status, "pass") + self.assertEqual(failing.status, "hardware_fail") + self.assertFalse(failing.observations[0].passed) + + if __name__ == "__main__": if "-k" in sys.argv: pattern_index = sys.argv.index("-k") + 1 From 03cb978f2555b68d2ae4d613351221d357314054 Mon Sep 17 00:00:00 2001 From: Andrii Shylenko <14119286+w1ne@users.noreply.github.com> Date: Sat, 15 Aug 2026 13:01:09 +0200 Subject: [PATCH 14/48] fix(bench): run bounded ESP32-S3 register checks --- benchmarks/twin2silicon/hil/esp32s3.py | 133 ++++++++++++++++---- tests/twin2silicon-hil.py | 164 ++++++++++++++++++++++--- 2 files changed, 258 insertions(+), 39 deletions(-) diff --git a/benchmarks/twin2silicon/hil/esp32s3.py b/benchmarks/twin2silicon/hil/esp32s3.py index b5941f3..16217d0 100644 --- a/benchmarks/twin2silicon/hil/esp32s3.py +++ b/benchmarks/twin2silicon/hil/esp32s3.py @@ -12,7 +12,7 @@ import termios import time import tty -from typing import Callable, Mapping, Optional, Sequence +from typing import Callable, Literal, Mapping, Optional, Sequence from .process import run_command from .results import CommandResult, PathLike @@ -71,20 +71,36 @@ def from_json(cls, record: Mapping[str, object]) -> "RegisterAssertion": @dataclass(frozen=True) class Esp32S3Config: - uart_device: str uart_baud: int + uart_ready_prefix: str uart_timeout_seconds: float - jtag_serial: str - openocd_config: str + identity_command: tuple[str, ...] + identity_expected_board: str + identity_timeout_seconds: float + flash_target: str + flash_artifact: str + flash_timeout_seconds: float + openocd_board_config: str + openocd_startup_timeout_seconds: float + openocd_command_timeout_seconds: float + platformio_project_dir: Optional[str] + platformio_environment: Optional[str] assertions: tuple[RegisterAssertion, ...] @classmethod def from_oracle(cls, oracle: Mapping[str, object]) -> "Esp32S3Config": uart = oracle.get("uart") - jtag = oracle.get("jtag", oracle.get("openocd")) + identity = oracle.get("identity") + flash = oracle.get("flash") + openocd = oracle.get("openocd") + platformio = oracle.get("platformio") records = oracle.get("register_assertions") - if not isinstance(uart, Mapping) or not isinstance(jtag, Mapping) or not isinstance(records, list): - raise TypeError("oracle uart, jtag/openocd, and register_assertions are required") + if (not isinstance(uart, Mapping) or not isinstance(identity, Mapping) + or not isinstance(flash, Mapping) or not isinstance(openocd, Mapping) + or not isinstance(records, list)): + raise TypeError("oracle phase mappings and register_assertions are required") + if platformio is not None and not isinstance(platformio, Mapping): + raise TypeError("platformio must be a mapping") assertions = tuple(RegisterAssertion.from_json(record) for record in records) if not assertions: raise ValueError("at least one register assertion is required") @@ -93,15 +109,31 @@ def from_oracle(cls, oracle: Mapping[str, object]) -> "Esp32S3Config": if len(names) != len(set(names)) or len(addresses) != len(set(addresses)): raise ValueError("register assertion names and addresses must be unique") baud = _positive_int(uart.get("baud"), "uart baud") - timeout = _positive_float(uart.get("timeout_seconds"), "uart timeout") - device = uart.get("device", "") - serial = jtag.get("serial", "") - config = jtag.get("config", jtag.get("board_config", "")) - if not all(isinstance(value, str) for value in (device, serial, config)): - raise TypeError("device, serial, and config must be strings") - if not device or not serial or not config: - raise ValueError("device, serial, and config must not be empty") - return cls(device, baud, timeout, serial, config, assertions) + command = identity.get("command") + if not isinstance(command, list) or not command: + raise ValueError("identity command must be a nonempty list") + normalized_command = tuple(_safe_string(item, "identity command item") for item in command) + project_dir = environment = None + if platformio is not None: + project_dir = _safe_string(platformio.get("project_dir"), "platformio project_dir") + environment = _safe_string(platformio.get("environment"), "platformio environment") + return cls( + baud, + _safe_string(uart.get("ready_prefix"), "uart ready_prefix"), + _nonnegative_float(uart.get("timeout_seconds"), "uart timeout"), + normalized_command, + _safe_string(identity.get("expected_board"), "identity expected_board"), + _nonnegative_float(identity.get("timeout_seconds"), "identity timeout"), + _safe_string(flash.get("target"), "flash target"), + _safe_string(flash.get("artifact"), "flash artifact"), + _nonnegative_float(flash.get("timeout_seconds"), "flash timeout"), + _safe_string(openocd.get("board_config"), "openocd board_config"), + _nonnegative_float(openocd.get("startup_timeout_seconds"), "openocd startup timeout"), + _nonnegative_float(openocd.get("command_timeout_seconds"), "openocd command timeout"), + project_dir, + environment, + assertions, + ) def _positive_int(value: object, field: str) -> int: @@ -110,13 +142,21 @@ def _positive_int(value: object, field: str) -> int: return value -def _positive_float(value: object, field: str) -> float: +def _nonnegative_float(value: object, field: str) -> float: if (isinstance(value, bool) or not isinstance(value, (int, float)) - or not math.isfinite(value) or value <= 0): - raise ValueError(f"{field} must be positive") + or not math.isfinite(value) or value < 0): + raise ValueError(f"{field} must be finite and nonnegative") return float(value) +def _safe_string(value: object, field: str) -> str: + if not isinstance(value, str): + raise TypeError(f"{field} must be a string") + if not value or any(character in value for character in "\r\n\x00"): + raise ValueError(f"{field} must be a nonempty safe string") + return value + + class BoardLockTimeout(TimeoutError): pass @@ -178,7 +218,7 @@ def __exit__(self, exc_type, exc, traceback) -> None: @dataclass(frozen=True) class PhaseResult: - status: str + status: Literal["pass", "hardware_fail", "infrastructure_error"] category: Optional[str] = None detail: Optional[str] = None command_result: Optional[CommandResult] = None @@ -286,6 +326,8 @@ def capture_uart_nonce(device: PathLike, baud: int, nonce: str, timeout_seconds: def build_openocd_command(executable: str, config: str, adapter_serial: str, assertions: Sequence[RegisterAssertion]) -> list[str]: + if not assertions: + raise ValueError("at least one register assertion is required") if not _SERIAL.fullmatch(adapter_serial): raise ValueError("unsafe adapter serial") if any(character in config for character in "\r\n\x00"): @@ -293,13 +335,15 @@ def build_openocd_command(executable: str, config: str, adapter_serial: str, commands = [f"adapter serial {adapter_serial}", "adapter speed 4000", "init", "reset run", "sleep 750", "halt"] for assertion in assertions: - commands.extend((f"echo @@REG {assertion.name} 0x{assertion.address:08x}", - f"mdw 0x{assertion.address:08x}")) + commands.extend((f'echo "@@REG {assertion.name} 0x{assertion.address:08x}"', + f"mdw 0x{assertion.address:08x} 1")) commands.append("exit") return [executable, "-f", config, "-c", "; ".join(commands)] def parse_openocd_registers(text: str, requested: Sequence[RegisterAssertion]) -> dict[str, int]: + if not requested: + raise ValueError("at least one register assertion is required") by_name = {item.name: item for item in requested} if len(by_name) != len(requested): raise ValueError("requested assertion names are not unique") @@ -343,14 +387,57 @@ class RegisterObservation: @dataclass(frozen=True) class RegisterEvaluation: - status: str + status: Literal["pass", "hardware_fail"] observations: tuple[RegisterObservation, ...] def evaluate_registers(observed: Mapping[str, int], assertions: Sequence[RegisterAssertion]) -> RegisterEvaluation: + if not assertions: + raise ValueError("at least one register assertion is required") if set(observed) != {item.name for item in assertions}: raise ValueError("observed registers do not exactly match assertions") results = tuple(RegisterObservation(item.name, item.address, observed[item.name], item.mask, item.expected, (observed[item.name] & item.mask) == item.expected) for item in assertions) return RegisterEvaluation("pass" if all(item.passed for item in results) else "hardware_fail", results) + + +@dataclass(frozen=True) +class OpenOcdResult: + status: Literal["pass", "hardware_fail", "infrastructure_error"] + category: Optional[Literal["openocd"]] + detail: Optional[str] + observed: Optional[dict[str, int]] + evaluation: Optional[RegisterEvaluation] + command_result: Optional[CommandResult] + + +def read_registers(executable: str, config: str, adapter_serial: str, + assertions: Sequence[RegisterAssertion], *, cwd: PathLike, + evidence_dir: PathLike, timeout_seconds: float, + runner: Runner = run_command) -> OpenOcdResult: + if not math.isfinite(timeout_seconds) or timeout_seconds < 0: + raise ValueError("OpenOCD timeout must be finite and nonnegative") + command = build_openocd_command(executable, config, adapter_serial, assertions) + evidence = Path(evidence_dir) + try: + result = runner(command, cwd=cwd, stdout_path=evidence / "openocd.stdout.log", + stderr_path=evidence / "openocd.stderr.log", timeout_seconds=timeout_seconds) + except OSError as error: + return OpenOcdResult("infrastructure_error", "openocd", str(error), None, None, None) + if result.cleanup_error: + return OpenOcdResult("infrastructure_error", "openocd", result.cleanup_error, + None, None, result) + if result.timed_out: + return OpenOcdResult("infrastructure_error", "openocd", "OpenOCD timed out", + None, None, result) + if result.returncode: + return OpenOcdResult("infrastructure_error", "openocd", + f"OpenOCD exited {result.returncode}", None, None, result) + try: + transcript = Path(result.stderr_path).read_text(encoding="utf-8") + observed = parse_openocd_registers(transcript, assertions) + evaluation = evaluate_registers(observed, assertions) + except (OSError, UnicodeError, ValueError) as error: + return OpenOcdResult("infrastructure_error", "openocd", str(error), None, None, result) + return OpenOcdResult(evaluation.status, None, None, observed, evaluation, result) diff --git a/tests/twin2silicon-hil.py b/tests/twin2silicon-hil.py index af07908..997fde5 100644 --- a/tests/twin2silicon-hil.py +++ b/tests/twin2silicon-hil.py @@ -38,6 +38,7 @@ evaluate_registers, flash_firmware, parse_openocd_registers, + read_registers, validate_identity, ) @@ -367,11 +368,30 @@ def wait(self, timeout): class Esp32S3ConfigTests(unittest.TestCase): + def test_parses_shipped_oracle_exactly(self): + oracle_path = (REPOSITORY_ROOT / "benchmarks/twin2silicon/tasks/esp32s3-gpio-hil-001/hidden/hil-oracle.json") + config = Esp32S3Config.from_oracle(json.loads(oracle_path.read_text())) + self.assertEqual(config.uart_ready_prefix, "LABWIRED_READY:") + self.assertEqual((config.uart_baud, config.uart_timeout_seconds), (115200, 30)) + self.assertEqual(config.identity_command, ("__LABWIRED_IDENTITY_RUNNER__",)) + self.assertEqual((config.identity_expected_board, config.identity_timeout_seconds), + ("esp32-s3-devkitc-1", 10)) + self.assertEqual((config.flash_target, config.flash_artifact, config.flash_timeout_seconds), + ("upload", ".pio/build/esp32s3/firmware.bin", 120)) + self.assertEqual((config.openocd_board_config, config.openocd_startup_timeout_seconds, + config.openocd_command_timeout_seconds), ("esp32s3-builtin.cfg", 20, 10)) + self.assertEqual((config.platformio_project_dir, config.platformio_environment), + ("public/firmware", "esp32s3")) + self.assertEqual(len(config.assertions), 2) + def test_parses_valid_oracle_and_hex_register_values(self): config = Esp32S3Config.from_oracle( { - "uart": {"device": "/dev/cu.board", "baud": 115200, "timeout_seconds": 1}, - "jtag": {"serial": "JTAG-1", "config": "board/esp32s3.cfg"}, + "uart": {"ready_prefix": "READY:", "baud": 115200, "timeout_seconds": 1}, + "identity": {"command": ["identity"], "expected_board": "board", "timeout_seconds": 0}, + "flash": {"target": "upload", "artifact": "firmware.bin", "timeout_seconds": 0}, + "openocd": {"board_config": "board.cfg", "startup_timeout_seconds": 0, + "command_timeout_seconds": 0}, "register_assertions": [ {"name": "gpio", "address": "0x60004020", "mask": "0x4", "expected": "0x4"} ], @@ -396,14 +416,18 @@ def test_rejects_invalid_bounds_alignment_duplicates_and_names(self): RegisterAssertion.from_json(record) with self.assertRaises(ValueError): Esp32S3Config.from_oracle({ - "uart": {"device": "x", "baud": 115200, "timeout_seconds": 1}, - "jtag": {"serial": "s", "config": "c"}, + "uart": {"ready_prefix": "READY:", "baud": 115200, "timeout_seconds": 1}, + "identity": {"command": ["id"], "expected_board": "board", "timeout_seconds": 0}, + "flash": {"target": "upload", "artifact": "fw", "timeout_seconds": 0}, + "openocd": {"board_config": "cfg", "startup_timeout_seconds": 0, "command_timeout_seconds": 0}, "register_assertions": [good, {**good, "address": "0x60004024"}], }) with self.assertRaises(ValueError): Esp32S3Config.from_oracle({ - "uart": {"device": "x", "baud": 115200, "timeout_seconds": 1}, - "jtag": {"serial": "s", "config": "c"}, + "uart": {"ready_prefix": "READY:", "baud": 115200, "timeout_seconds": 1}, + "identity": {"command": ["id"], "expected_board": "board", "timeout_seconds": 0}, + "flash": {"target": "upload", "artifact": "fw", "timeout_seconds": 0}, + "openocd": {"board_config": "cfg", "startup_timeout_seconds": 0, "command_timeout_seconds": 0}, "register_assertions": [good, {**good, "name": "gpio2"}], }) @@ -411,14 +435,18 @@ def test_rejects_empty_configuration_assertions_and_nonfinite_timeout(self): for timeout in (float("nan"), float("inf")): with self.subTest(timeout=timeout), self.assertRaises(ValueError): Esp32S3Config.from_oracle({ - "uart": {"device": "device", "baud": 115200, "timeout_seconds": timeout}, - "jtag": {"serial": "serial", "config": "config"}, + "uart": {"ready_prefix": "READY:", "baud": 115200, "timeout_seconds": timeout}, + "identity": {"command": ["id"], "expected_board": "board", "timeout_seconds": 0}, + "flash": {"target": "upload", "artifact": "fw", "timeout_seconds": 0}, + "openocd": {"board_config": "cfg", "startup_timeout_seconds": 0, "command_timeout_seconds": 0}, "register_assertions": [{"name": "gpio", "address": "0x4", "mask": "0x4", "expected": "0x4"}], }) with self.assertRaises(ValueError): Esp32S3Config.from_oracle({ - "uart": {"device": "", "baud": 115200, "timeout_seconds": 1}, - "jtag": {"serial": "", "config": ""}, + "uart": {"ready_prefix": "", "baud": 115200, "timeout_seconds": 1}, + "identity": {"command": [], "expected_board": "", "timeout_seconds": 0}, + "flash": {"target": "", "artifact": "", "timeout_seconds": 0}, + "openocd": {"board_config": "", "startup_timeout_seconds": 0, "command_timeout_seconds": 0}, "register_assertions": [], }) @@ -437,8 +465,26 @@ def test_absent_wrong_and_duplicate_serials_are_infrastructure_before_flash(self tool = executable_fixture(directory, f"print({output!r}, end='')\n") result = validate_identity([tool], "JTAG-1", cwd=directory, evidence_dir=directory, timeout_seconds=1) self.assertEqual((result.status, result.category), ("infrastructure_error", "board_identity")) + flash = executable_fixture(directory, f"from pathlib import Path; Path({str(marker)!r}).write_text('flashed')\n") + with self.assertRaises(ValueError): + flash_firmware([flash], cwd=directory, evidence_dir=directory, timeout_seconds=1, + identity_validated=False) self.assertFalse(marker.exists()) + def test_identity_success_then_flash_executes_in_order(self): + with tempfile.TemporaryDirectory() as directory: + marker = Path(directory) / "flashed" + identity_tool = executable_fixture(directory, "print('JTAG-1')\n") + identity = validate_identity([identity_tool], "JTAG-1", cwd=directory, + evidence_dir=directory, timeout_seconds=1) + flash_tool = Path(directory) / "flash.py" + flash_tool.write_text("#!/usr/bin/env python3\nfrom pathlib import Path\nPath(%r).write_text('flashed')\n" % str(marker)) + flash_tool.chmod(0o755) + flashed = flash_firmware([flash_tool], cwd=directory, evidence_dir=directory, + timeout_seconds=1, identity_validated=identity.status == "pass") + self.assertEqual(flashed.status, "pass") + self.assertEqual(marker.read_text(), "flashed") + def test_identity_tool_timeout_nonzero_and_cleanup_are_infrastructure(self): with tempfile.TemporaryDirectory() as directory: slow = executable_fixture(directory, "import time; time.sleep(30)\n") @@ -573,6 +619,24 @@ def test_rejects_unsupported_baud_and_closes_opened_fd(self): real_close(master) self.assertTrue(closed) + def test_timeout_closes_the_opened_uart_fd(self): + master, slave = pty.openpty() + device = os.ttyname(slave) + opened = [] + real_open = os.open + with tempfile.TemporaryDirectory() as directory, mock.patch( + "benchmarks.twin2silicon.hil.esp32s3.os.open", + side_effect=lambda *args, **kwargs: (lambda fd: (opened.append(fd), fd)[1])(real_open(*args, **kwargs)), + ): + result = capture_uart_nonce(device, 115200, "never", .01, Path(directory) / "uart.log") + os.close(master) + os.close(slave) + self.assertFalse(result.matched) + self.assertEqual(len(opened), 1) + with self.assertRaises(OSError) as error: + os.fstat(opened[0]) + self.assertEqual(error.exception.errno, 9) + class OpenOcdEvidenceTests(unittest.TestCase): def setUp(self): @@ -583,12 +647,18 @@ def setUp(self): def test_command_is_argv_and_requests_marked_records_at_fixed_speed(self): command = build_openocd_command("openocd", "board.cfg", "JTAG-1", self.assertions) - self.assertEqual(command[:3], ["openocd", "-f", "board.cfg"]) - script = command[command.index("-c") + 1] - for fragment in ("adapter serial JTAG-1", "adapter speed 4000", "reset run", "sleep 750", "halt", - "echo @@REG enable 0x60004020", "mdw 0x60004020", "exit"): - self.assertIn(fragment, script) - self.assertNotIn(";", command[:-1]) + self.assertEqual(command, ["openocd", "-f", "board.cfg", "-c", + 'adapter serial JTAG-1; adapter speed 4000; init; reset run; sleep 750; halt; ' + 'echo "@@REG enable 0x60004020"; mdw 0x60004020 1; ' + 'echo "@@REG high 0x60004004"; mdw 0x60004004 1; exit']) + + def test_empty_assertions_are_rejected_by_all_register_paths(self): + with self.assertRaises(ValueError): + build_openocd_command("openocd", "board.cfg", "serial", ()) + with self.assertRaises(ValueError): + parse_openocd_registers("", ()) + with self.assertRaises(ValueError): + evaluate_registers({}, ()) def test_parser_accepts_only_immediately_paired_canonical_requested_records(self): text = "noise\n@@REG enable 0x60004020\n0x60004020: 0x00000004\n@@REG high 0x60004004\n0x60004004: 0x00000004\n" @@ -616,6 +686,68 @@ def test_masked_mismatch_fails_and_all_assertions_pass(self): self.assertFalse(failing.observations[0].passed) +class OpenOcdExecutionTests(unittest.TestCase): + def setUp(self): + self.assertions = (RegisterAssertion("gpio", 0x60004020, 4, 4),) + + def _run(self, directory, body, timeout=1): + tool = executable_fixture(directory, body) + return read_registers(tool, "board.cfg", "JTAG-1", self.assertions, cwd=directory, + evidence_dir=directory, timeout_seconds=timeout) + + def test_reads_openocd_stderr_and_returns_typed_evaluation(self): + with tempfile.TemporaryDirectory() as directory: + result = self._run(directory, "import sys\nprint('@@REG gpio 0x60004020', file=sys.stderr)\nprint('0x60004020: 0x00000004', file=sys.stderr)\n") + self.assertEqual((result.status, result.category), ("pass", None)) + self.assertEqual(result.observed, {"gpio": 4}) + self.assertEqual(result.evaluation.status, "pass") + self.assertEqual(result.command_result.returncode, 0) + + def test_classifies_nonzero_timeout_cleanup_launch_and_parse_as_infrastructure(self): + with tempfile.TemporaryDirectory() as directory: + nonzero = self._run(directory, "raise SystemExit(2)\n") + timeout = self._run(directory, "import time; time.sleep(30)\n", timeout=.05) + malformed = self._run(directory, "import sys; print('@@REG gpio bad', file=sys.stderr)\n") + fake = CommandResult(("x",), "/", 0, False, "", "", 0, "/tmp/o", "/tmp/e", "stuck") + cleanup = read_registers("x", "c", "s", self.assertions, cwd=directory, + evidence_dir=directory, timeout_seconds=1, runner=lambda *a, **k: fake) + launch = read_registers("x", "c", "s", self.assertions, cwd=directory, + evidence_dir=directory, timeout_seconds=1, + runner=lambda *a, **k: (_ for _ in ()).throw(FileNotFoundError("missing"))) + for result in (nonzero, timeout, malformed, cleanup, launch): + with self.subTest(result=result): + self.assertEqual((result.status, result.category), ("infrastructure_error", "openocd")) + + def test_timeout_terminates_openocd_process_group(self): + with tempfile.TemporaryDirectory() as directory: + terminated = Path(directory) / "child-terminated" + ready = Path(directory) / "child-ready" + body = textwrap.dedent(f""" + import pathlib, signal, subprocess, sys, time + child = ''' + import pathlib, signal, time + terminated = pathlib.Path({str(terminated)!r}) + def stop(signum, frame): + terminated.write_text("terminated") + raise SystemExit(0) + signal.signal(signal.SIGTERM, stop) + pathlib.Path({str(ready)!r}).write_text("ready") + while True: time.sleep(1) + ''' + subprocess.Popen([sys.executable, "-c", child]) + while not pathlib.Path({str(ready)!r}).exists(): pass + def stop(signum, frame): + while not pathlib.Path({str(terminated)!r}).exists(): pass + raise SystemExit(0) + signal.signal(signal.SIGTERM, stop) + print("ready", flush=True) + while True: time.sleep(1) + """) + result = self._run(directory, body, timeout=.5) + self.assertEqual(result.status, "infrastructure_error") + self.assertEqual(terminated.read_text(), "terminated") + + if __name__ == "__main__": if "-k" in sys.argv: pattern_index = sys.argv.index("-k") + 1 From e8be3a1ae1bb93684d8c1f74708f1964141413f9 Mon Sep 17 00:00:00 2001 From: Andrii Shylenko <14119286+w1ne@users.noreply.github.com> Date: Sat, 15 Aug 2026 13:05:24 +0200 Subject: [PATCH 15/48] fix(bench): accept Espressif register evidence --- benchmarks/twin2silicon/hil/esp32s3.py | 19 ++++++++++++++--- tests/twin2silicon-hil.py | 29 ++++++++++++++++++++++++-- 2 files changed, 43 insertions(+), 5 deletions(-) diff --git a/benchmarks/twin2silicon/hil/esp32s3.py b/benchmarks/twin2silicon/hil/esp32s3.py index 16217d0..944b464 100644 --- a/benchmarks/twin2silicon/hil/esp32s3.py +++ b/benchmarks/twin2silicon/hil/esp32s3.py @@ -21,7 +21,7 @@ _NAME = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$") _SERIAL = re.compile(r"^[A-Za-z0-9_.:/+-]+$") _MARKER = re.compile(r"^@@REG ([A-Za-z_][A-Za-z0-9_]*) (0x[0-9A-Fa-f]{8})$") -_VALUE = re.compile(r"^(0x[0-9A-Fa-f]{8}): (0x[0-9A-Fa-f]{8})$") +_VALUE = re.compile(r"^(0x[0-9A-Fa-f]{8}): ((?:0x)?[0-9A-Fa-f]{8})$") def _uint32(value: object, field: str) -> int: @@ -241,7 +241,7 @@ def validate_identity(command: Sequence[os.PathLike[str] | str], expected_serial try: lines = [line.strip() for line in Path(result.stdout_path).read_text(encoding="utf-8").splitlines() if line.strip()] - except OSError as error: + except (OSError, UnicodeError) as error: return PhaseResult("infrastructure_error", "board_identity", str(error), result) if lines != [expected_serial]: return PhaseResult("infrastructure_error", "board_identity", @@ -272,6 +272,7 @@ class UartResult: matched: bool bytes_captured: int timed_out: bool + termination_reason: Literal["matched", "timeout", "max_bytes"] def capture_uart_nonce(device: PathLike, baud: int, nonce: str, timeout_seconds: float, @@ -319,7 +320,14 @@ def capture_uart_nonce(device: PathLike, baud: int, nonce: str, timeout_seconds: break Path(log).parent.mkdir(parents=True, exist_ok=True) Path(log).write_bytes(captured) - return UartResult(matched, len(captured), not matched) + reason: Literal["matched", "timeout", "max_bytes"] + if matched: + reason = "matched" + elif len(captured) >= max_bytes: + reason = "max_bytes" + else: + reason = "timeout" + return UartResult(matched, len(captured), reason == "timeout", reason) finally: os.close(fd) @@ -396,6 +404,11 @@ def evaluate_registers(observed: Mapping[str, int], assertions: Sequence[Registe raise ValueError("at least one register assertion is required") if set(observed) != {item.name for item in assertions}: raise ValueError("observed registers do not exactly match assertions") + for name, value in observed.items(): + if isinstance(value, bool) or not isinstance(value, int): + raise TypeError(f"observed register {name} must be an integer") + if value < 0 or value > 0xFFFFFFFF: + raise ValueError(f"observed register {name} is outside uint32 bounds") results = tuple(RegisterObservation(item.name, item.address, observed[item.name], item.mask, item.expected, (observed[item.name] & item.mask) == item.expected) for item in assertions) diff --git a/tests/twin2silicon-hil.py b/tests/twin2silicon-hil.py index 997fde5..f9b8ab4 100644 --- a/tests/twin2silicon-hil.py +++ b/tests/twin2silicon-hil.py @@ -500,6 +500,13 @@ def test_identity_tool_timeout_nonzero_and_cleanup_are_infrastructure(self): timeout_seconds=1, runner=lambda *a, **k: fake) self.assertEqual((result.status, result.category), ("infrastructure_error", "board_identity")) + def test_non_utf8_identity_output_is_infrastructure(self): + with tempfile.TemporaryDirectory() as directory: + tool = executable_fixture(directory, "import sys; sys.stdout.buffer.write(b'\\xff\\xfe')\n") + result = validate_identity([tool], "JTAG-1", cwd=directory, evidence_dir=directory, + timeout_seconds=1) + self.assertEqual((result.status, result.category), ("infrastructure_error", "board_identity")) + def test_command_launch_errors_are_infrastructure(self): def unavailable(*args, **kwargs): raise FileNotFoundError("tool missing") @@ -570,7 +577,7 @@ def test_lock_releases_if_metadata_persistence_fails_and_rejects_nonfinite_timeo class UartNonceTests(unittest.TestCase): - def _capture(self, chunks, nonce="current", timeout=.2): + def _capture(self, chunks, nonce="current", timeout=.2, max_bytes=64): master, slave = pty.openpty() device = os.ttyname(slave) tty.setraw(slave) @@ -585,7 +592,7 @@ def writer(): thread.start() started.set() try: - result = capture_uart_nonce(device, 115200, nonce, timeout, log, max_bytes=64) + result = capture_uart_nonce(device, 115200, nonce, timeout, log, max_bytes=max_bytes) finally: thread.join(1) os.close(master) @@ -632,11 +639,20 @@ def test_timeout_closes_the_opened_uart_fd(self): os.close(master) os.close(slave) self.assertFalse(result.matched) + self.assertTrue(result.timed_out) + self.assertEqual(result.termination_reason, "timeout") self.assertEqual(len(opened), 1) with self.assertRaises(OSError) as error: os.fstat(opened[0]) self.assertEqual(error.exception.errno, 9) + def test_max_bytes_exhaustion_is_not_reported_as_timeout(self): + result, raw = self._capture([b"1234567890"], timeout=1, max_bytes=10) + self.assertFalse(result.matched) + self.assertFalse(result.timed_out) + self.assertEqual(result.termination_reason, "max_bytes") + self.assertEqual(raw, b"1234567890") + class OpenOcdEvidenceTests(unittest.TestCase): def setUp(self): @@ -678,6 +694,10 @@ def test_parser_accepts_only_immediately_paired_canonical_requested_records(self with self.subTest(evidence=evidence), self.assertRaises(ValueError): parse_openocd_registers(evidence, self.assertions) + def test_parser_accepts_real_espressif_bare_eight_digit_mdw_value(self): + text = "@@REG enable 0x60004020\n0x60004020: 00000004\n@@REG high 0x60004004\n0x60004004: 00000004\n" + self.assertEqual(parse_openocd_registers(text, self.assertions), {"enable": 4, "high": 4}) + def test_masked_mismatch_fails_and_all_assertions_pass(self): passing = evaluate_registers({"enable": 0x104, "high": 4}, self.assertions) failing = evaluate_registers({"enable": 0, "high": 4}, self.assertions) @@ -685,6 +705,11 @@ def test_masked_mismatch_fails_and_all_assertions_pass(self): self.assertEqual(failing.status, "hardware_fail") self.assertFalse(failing.observations[0].passed) + def test_evaluation_rejects_non_uint32_observed_values(self): + for value in (True, -1, 0x100000000, 1.5, "4"): + with self.subTest(value=value), self.assertRaises((TypeError, ValueError)): + evaluate_registers({"enable": value, "high": 4}, self.assertions) + class OpenOcdExecutionTests(unittest.TestCase): def setUp(self): From 1db308faaa5de5a9e2e495e4fc9c2890e7fca524 Mon Sep 17 00:00:00 2001 From: Andrii Shylenko <14119286+w1ne@users.noreply.github.com> Date: Sat, 15 Aug 2026 13:11:48 +0200 Subject: [PATCH 16/48] feat(bench): orchestrate reproducible ESP32-S3 HIL runs --- benchmarks/twin2silicon/hil/process.py | 4 +- benchmarks/twin2silicon/run_hil.py | 322 +++++++++++++++++++++++++ tests/twin2silicon-hil.py | 87 +++++++ 3 files changed, 412 insertions(+), 1 deletion(-) create mode 100644 benchmarks/twin2silicon/run_hil.py diff --git a/benchmarks/twin2silicon/hil/process.py b/benchmarks/twin2silicon/hil/process.py index 63a1c2a..0d0ee15 100644 --- a/benchmarks/twin2silicon/hil/process.py +++ b/benchmarks/twin2silicon/hil/process.py @@ -6,7 +6,7 @@ import signal import subprocess import time -from typing import Sequence, Union +from typing import Mapping, Optional, Sequence, Union from .results import CommandResult, PathLike @@ -42,6 +42,7 @@ def run_command( stdout_path: PathLike, stderr_path: PathLike, timeout_seconds: float, + env: Optional[Mapping[str, str]] = None, ) -> CommandResult: normalized_command = tuple(os.fspath(part) for part in command) normalized_cwd = str(Path(cwd).resolve()) @@ -61,6 +62,7 @@ def run_command( stdout=stdout, stderr=stderr, start_new_session=True, + env=env, ) try: process.communicate(timeout=timeout_seconds) diff --git a/benchmarks/twin2silicon/run_hil.py b/benchmarks/twin2silicon/run_hil.py new file mode 100644 index 0000000..570067d --- /dev/null +++ b/benchmarks/twin2silicon/run_hil.py @@ -0,0 +1,322 @@ +#!/usr/bin/env python3 +"""Run an isolated, reproducible ESP32-S3 hardware-in-loop evaluation. + +Fixture commands are argv arrays supplied with ``--identity-command-json``; +shell command strings are deliberately unsupported. +""" + +from __future__ import annotations + +import argparse +from datetime import datetime, timezone +import hashlib +import json +import math +import os +from pathlib import Path +import secrets +import shutil +import sys +import threading +from typing import Any, Mapping + +if __package__ in (None, ""): + sys.path.insert(0, str(Path(__file__).resolve().parents[2])) + +from benchmarks.twin2silicon.hil.esp32s3 import ( + BoardLock, BoardLockTimeout, Esp32S3Config, capture_uart_nonce, + flash_firmware, read_registers, validate_identity, +) +from benchmarks.twin2silicon.hil.process import run_command +from benchmarks.twin2silicon.hil.results import sha256_file, write_json_atomic + + +ROOT = Path(__file__).resolve().parent +TASKS = ROOT / "tasks" +HARNESS_REVISION = "twin2silicon-hil-1" + + +def _number(value: object, field: str, *, integer: bool = False) -> int | float: + valid = isinstance(value, int if integer else (int, float)) and not isinstance(value, bool) + finite = integer or (valid and math.isfinite(value)) + if not valid or value < 0 or not finite: + raise ValueError(f"{field} must be a nonnegative finite {'integer' if integer else 'number'}") + return int(value) if integer else float(value) + + +def parse_usage(path: Path) -> dict[str, Any]: + raw = json.loads(path.read_text(encoding="utf-8")) + if not isinstance(raw, Mapping) or not isinstance(raw.get("tokens"), Mapping): + raise ValueError("usage must contain an object and tokens object") + if not isinstance(raw.get("rates_usd_per_million"), Mapping): + raise ValueError("usage must contain rates_usd_per_million") + tokens = {name: _number(raw["tokens"].get(name), f"tokens.{name}", integer=True) + for name in ("fresh_input", "cached_input", "output", "reasoning")} + rates = {name: _number(raw["rates_usd_per_million"].get(name), f"rates.{name}") + for name in ("fresh_input", "cached_input", "output")} + for name in ("provider", "model", "price_source", "price_date"): + if not isinstance(raw.get(name), str) or not raw[name]: + raise ValueError(f"{name} must be a nonempty string") + result = { + "schema_version": "1.0", "requests": _number(raw.get("requests"), "requests", integer=True), + "tokens": tokens, "final_context_tokens": _number(raw.get("final_context_tokens"), "final_context_tokens", integer=True), + "latency_seconds": _number(raw.get("latency_seconds"), "latency_seconds"), + "provider": raw["provider"], "model": raw["model"], "rates_usd_per_million": rates, + "price_source": raw["price_source"], "price_date": raw["price_date"], + } + result["cost_usd"] = (tokens["fresh_input"] * rates["fresh_input"] + + tokens["cached_input"] * rates["cached_input"] + + tokens["output"] * rates["output"]) / 1_000_000 + return result + + +def _tree_hash(root: Path) -> str: + digest = hashlib.sha256() + for path in sorted(item for item in root.rglob("*") if item.is_file()): + digest.update(path.relative_to(root).as_posix().encode()) + digest.update(b"\0") + with path.open("rb") as source: + for chunk in iter(lambda: source.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def _safe_tree(source: Path, field: str) -> Path: + source = source.resolve(strict=True) + if not source.is_dir(): + raise ValueError(f"{field} is not a directory") + for item in source.rglob("*"): + if item.is_symlink(): + raise ValueError(f"{field} contains a symlink: {item.relative_to(source)}") + return source + + +def _resolve_task(value: str) -> Path: + supplied = Path(value) + candidate = supplied if supplied.is_absolute() or len(supplied.parts) > 1 else TASKS / supplied + resolved = candidate.resolve(strict=True) + if resolved != TASKS.resolve() and TASKS.resolve() not in resolved.parents: + raise ValueError("task must be beneath the benchmark tasks directory") + if not resolved.is_dir() or (resolved / "task.json").is_symlink(): + raise ValueError("invalid task directory") + return resolved + + +def _copy_public(source: Path, destination: Path) -> None: + _safe_tree(source, "candidate/public workspace") + shutil.copytree(source, destination, symlinks=False) + + +def _command_record(result: Any) -> dict[str, Any]: + return {"returncode": result.returncode, "timed_out": result.timed_out, + "duration_seconds": result.duration_seconds, "cleanup_error": result.cleanup_error} + + +def parser() -> argparse.ArgumentParser: + value = argparse.ArgumentParser(description=__doc__) + value.add_argument("task", help="task id, or path beneath benchmarks/twin2silicon/tasks") + value.add_argument("--run-dir", required=True, type=Path) + mode = value.add_mutually_exclusive_group(required=True) + mode.add_argument("--evaluate-only", action="store_true") + mode.add_argument("--agent-bin", type=Path) + value.add_argument("--candidate", type=Path) + value.add_argument("--model") + value.add_argument("--jtag-serial", required=True) + value.add_argument("--uart-device", required=True) + value.add_argument("--openocd", required=True) + value.add_argument("--platformio", default="pio") + value.add_argument("--identity-command-json", help="fixture identity argv as a JSON array (never a shell string)") + value.add_argument("--usage-json", type=Path) + return value + + +def main(argv: list[str] | None = None) -> int: + args = parser().parse_args(argv) + if args.evaluate_only != bool(args.candidate) or (args.agent_bin and not args.model): + parser().error("evaluate-only requires --candidate; agent mode requires --model") + run_dir = args.run_dir.absolute() + if run_dir.exists(): + print("run directory already exists", file=sys.stderr) + return 2 + # Reserve the path before resolving tools or executing any command. + run_dir.mkdir(parents=True) + manifest: dict[str, Any] = { + "schema_version": "1.0", "run": {"id": run_dir.name, "started_at_utc": datetime.now(timezone.utc).isoformat().replace("+00:00", "Z")}, + "task": {}, "harness_revision": HARNESS_REVISION, "model": None, "provider": None, + "requests": None, "tokens": {"fresh_input": None, "cached_input": None, "output": None, "reasoning": None}, + "final_context_tokens": None, "latency_seconds": None, "cost_usd": None, + "configured_budgets": {}, "budget_validity": "not_available", + "model_status": "not_run", "compile_status": "not_run", "simulator_status": "not_supported", + "hardware_status": "not_run", "infrastructure_status": "ok", "uart": None, + "register_assertions": [], "termination": "running", "failure_category": None, + "hashes": {}, "artifacts": [], "environment": {}, "phases": [], + } + def persist() -> None: + write_json_atomic(run_dir / "run.json", manifest) + persist() + try: + task_root = _resolve_task(args.task) + task = json.loads((task_root / "task.json").read_text(encoding="utf-8")) + if task.get("schema_version") != "1.0": + raise ValueError("unsupported task schema") + public = (task_root / task["public_dir"]).resolve(strict=True) + oracle_path = (task_root / task["hidden_oracle"]).resolve(strict=True) + if task_root not in public.parents or task_root not in oracle_path.parents: + raise ValueError("task path escapes task directory") + config = Esp32S3Config.from_oracle(json.loads(oracle_path.read_text(encoding="utf-8"))) + source = _safe_tree(args.candidate, "candidate") if args.evaluate_only else _safe_tree(public, "public") + workspace = run_dir / "workspace" + _copy_public(source, workspace) + nonce = secrets.token_hex(16) + nonce_header = workspace / "firmware/include/run_nonce.h" + nonce_header.parent.mkdir(parents=True, exist_ok=True) + nonce_header.write_text(f'#pragma once\n#define LABWIRED_RUN_NONCE "{nonce}"\n', encoding="utf-8") + manifest["task"] = {"id": task["id"], "schema_version": task["schema_version"]} + manifest["configured_budgets"] = task.get("budgets", {}) + manifest["hashes"]["oracle_descriptor"] = sha256_file(oracle_path) + manifest["hashes"]["source_initial"] = _tree_hash(workspace) + manifest["phases"].append("prepared") + persist() + + if args.usage_json: + usage = parse_usage(args.usage_json) + write_json_atomic(run_dir / "cost.json", usage) + for name in ("requests", "tokens", "final_context_tokens", "latency_seconds", "provider", "model", "cost_usd"): + manifest[name] = usage[name] + manifest["budget_validity"] = "valid" if usage["tokens"]["fresh_input"] + usage["tokens"]["cached_input"] + usage["tokens"]["output"] <= task.get("budgets", {}).get("model_tokens", math.inf) else "exceeded" + elif args.agent_bin: + manifest["model"] = args.model + + if args.agent_bin: + prompt = (workspace / "README.md").read_text(encoding="utf-8") + clean_env = {name: os.environ[name] for name in ("PATH", "LANG", "LC_ALL", "TMPDIR") if name in os.environ} + agent = run_command([args.agent_bin, "agent", "run", "--model", args.model, prompt], cwd=workspace, + stdout_path=run_dir / "agent.stdout.log", stderr_path=run_dir / "agent.stderr.log", + timeout_seconds=float(task["budgets"]["wall_time_seconds"]), env=clean_env) + manifest["phases"].append({"agent": _command_record(agent)}) + if agent.timed_out or agent.cleanup_error: + raise RuntimeError("agent process did not terminate cleanly") + manifest["model_status"] = "pass" if agent.returncode == 0 else "fail" + if agent.returncode != 0: + manifest["termination"], manifest["failure_category"] = "completed", "model" + manifest["hashes"]["source_final"] = _tree_hash(workspace) + return _finalize(run_dir, manifest) + else: + manifest["model_status"] = "not_run" + manifest["hashes"]["source_final"] = _tree_hash(workspace) + persist() + + firmware = workspace / "firmware" + build_command = [args.platformio, "run", "--project-dir", str(firmware), "--environment", config.platformio_environment or "esp32s3"] + clean = run_command(build_command + ["--target", "clean"], cwd=workspace, + stdout_path=run_dir / "clean.stdout.log", stderr_path=run_dir / "clean.stderr.log", + timeout_seconds=float(task["budgets"]["wall_time_seconds"])) + manifest["phases"].append({"clean": _command_record(clean)}) + if clean.timed_out or clean.cleanup_error: + raise RuntimeError("clean process did not terminate cleanly") + if clean.returncode: + manifest["compile_status"] = "fail" + manifest["termination"], manifest["failure_category"] = "completed", "compile" + return _finalize(run_dir, manifest) + persist() + build = run_command(build_command, cwd=workspace, stdout_path=run_dir / "build.stdout.log", + stderr_path=run_dir / "build.stderr.log", timeout_seconds=float(task["budgets"]["wall_time_seconds"])) + manifest["phases"].append({"build": _command_record(build)}) + if build.timed_out or build.cleanup_error: + raise RuntimeError("build process did not terminate cleanly") + if build.returncode: + manifest["compile_status"] = "fail" + manifest["hardware_status"] = "not_run" + manifest["termination"], manifest["failure_category"] = "completed", "compile" + return _finalize(run_dir, manifest) + manifest["compile_status"] = "pass" + artifact = firmware / config.flash_artifact + if not artifact.is_file(): + raise RuntimeError("successful build produced no firmware artifact") + manifest["hashes"]["firmware"] = sha256_file(artifact) + persist() + + if args.identity_command_json: + identity_command = json.loads(args.identity_command_json) + if not isinstance(identity_command, list) or not identity_command or not all(isinstance(x, str) for x in identity_command): + raise ValueError("identity command JSON must be a nonempty string array") + else: + identity_command = list(config.identity_command) + uart_result: dict[str, Any] = {} + uart_error: list[BaseException] = [] + def capture() -> None: + try: + uart_result["value"] = capture_uart_nonce(args.uart_device, config.uart_baud, nonce, + config.uart_timeout_seconds, run_dir / "uart.raw.log") + except BaseException as error: + uart_error.append(error) + with BoardLock(run_dir.parent / ".board-locks", args.jtag_serial, timeout_seconds=config.identity_timeout_seconds): + identity = validate_identity(identity_command, args.jtag_serial, cwd=workspace, evidence_dir=run_dir, + timeout_seconds=config.identity_timeout_seconds) + manifest["phases"].append("identity") + if identity.status != "pass": + raise RuntimeError(identity.detail or "board identity failed") + persist() + uart_thread = threading.Thread(target=capture, name="hil-uart", daemon=True) + uart_thread.start() + flash_command = [args.platformio, "run", "--project-dir", str(firmware), "--environment", + config.platformio_environment or "esp32s3", "--target", config.flash_target] + flashed = flash_firmware(flash_command, cwd=workspace, evidence_dir=run_dir, + timeout_seconds=config.flash_timeout_seconds, identity_validated=True) + uart_thread.join(config.uart_timeout_seconds + 1) + if uart_thread.is_alive() or uart_error: + raise RuntimeError(f"UART capture failed: {uart_error[0] if uart_error else 'cleanup timeout'}") + uart = uart_result["value"] + manifest["uart"] = {"matched": uart.matched, "termination": uart.termination_reason, + "bytes_captured": uart.bytes_captured} + if flashed.status == "infrastructure_error": + raise RuntimeError(flashed.detail or "flash infrastructure failure") + if flashed.status == "hardware_fail" or not uart.matched: + manifest["hardware_status"] = "fail" + manifest["termination"], manifest["failure_category"] = "completed", flashed.category or "uart_nonce" + return _finalize(run_dir, manifest) + registers = read_registers(args.openocd, config.openocd_board_config, args.jtag_serial, + config.assertions, cwd=workspace, evidence_dir=run_dir, + timeout_seconds=config.openocd_command_timeout_seconds) + if registers.status == "infrastructure_error": + raise RuntimeError(registers.detail or "OpenOCD infrastructure failure") + manifest["register_assertions"] = [ + {"name": item.name, "passed": item.passed, "observed_masked": f"0x{item.value & item.mask:08x}"} + for item in registers.evaluation.observations + ] + manifest["hardware_status"] = "pass" if registers.status == "pass" else "fail" + manifest["termination"] = "completed" + manifest["failure_category"] = None if registers.status == "pass" else "register_mismatch" + return _finalize(run_dir, manifest) + except (KeyboardInterrupt, SystemExit) as error: + manifest["infrastructure_status"] = "error" + manifest["termination"] = "interrupted" + manifest["failure_category"] = "interrupt" + manifest["detail"] = type(error).__name__ + _finalize(run_dir, manifest) + return 2 + except (OSError, ValueError, TypeError, KeyError, json.JSONDecodeError, RuntimeError, BoardLockTimeout) as error: + manifest["infrastructure_status"] = "error" + manifest["termination"] = "invalid" + manifest["failure_category"] = "infrastructure" + manifest["detail"] = str(error) + _finalize(run_dir, manifest) + print(str(error), file=sys.stderr) + return 2 + + +def _finalize(run_dir: Path, manifest: dict[str, Any]) -> int: + artifacts: list[str] = [] + hashes = manifest.setdefault("hashes", {}) + for path in sorted(item for item in run_dir.rglob("*") if item.is_file() and item.name != "run.json"): + relative = path.relative_to(run_dir).as_posix() + artifacts.append(relative) + hashes[f"artifact:{relative}"] = sha256_file(path) + manifest["artifacts"] = artifacts + manifest["run"]["ended_at_utc"] = datetime.now(timezone.utc).isoformat().replace("+00:00", "Z") + write_json_atomic(run_dir / "run.json", manifest) + return 2 if manifest["infrastructure_status"] == "error" else 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/twin2silicon-hil.py b/tests/twin2silicon-hil.py index f9b8ab4..a5b9112 100644 --- a/tests/twin2silicon-hil.py +++ b/tests/twin2silicon-hil.py @@ -5,6 +5,7 @@ from pathlib import Path import pty import signal +import shutil import subprocess import sys import tempfile @@ -773,6 +774,92 @@ def stop(signum, frame): self.assertEqual(terminated.read_text(), "terminated") +class HilOrchestrationTests(unittest.TestCase): + def _run_cli(self, *arguments, env=None): + return subprocess.run( + [sys.executable, str(REPOSITORY_ROOT / "benchmarks/twin2silicon/run_hil.py"), *map(str, arguments)], + cwd=REPOSITORY_ROOT, text=True, capture_output=True, env=env, + ) + + def test_evaluate_only_prepares_isolated_workspace_and_records_compile_failure(self): + task = REPOSITORY_ROOT / "benchmarks/twin2silicon/tasks/esp32s3-gpio-hil-001" + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + candidate = root / "candidate" + shutil.copytree(task / "public", candidate) + invocations = root / "pio-invocations" + tool_dir = root / "pio" + tool_dir.mkdir() + platformio = executable_fixture(tool_dir, textwrap.dedent(f""" + import sys + from pathlib import Path + with Path({str(invocations)!r}).open('a') as output: + output.write(json.dumps(sys.argv[1:]) + '\\n') + raise SystemExit(0 if 'clean' in sys.argv else 1) + """).replace("import sys\n", "import json, sys\n")) + identity = executable_fixture(root, "raise AssertionError('identity must not run')\n") + run_dir = root / "run" + result = self._run_cli( + task, "--run-dir", run_dir, "--evaluate-only", "--candidate", candidate, + "--jtag-serial", "JTAG-1", "--uart-device", "/dev/null", + "--openocd", identity, "--platformio", platformio, + "--identity-command-json", json.dumps([str(identity)]), + ) + self.assertEqual(result.returncode, 0, result.stderr) + manifest = json.loads((run_dir / "run.json").read_text()) + self.assertEqual((manifest["compile_status"], manifest["hardware_status"]), ("fail", "not_run")) + pio_commands = [json.loads(line) for line in invocations.read_text().splitlines()] + self.assertEqual(len(pio_commands), 2) + self.assertEqual(pio_commands[0][-2:], ["--target", "clean"]) + nonce_header = (run_dir / "workspace/firmware/include/run_nonce.h").read_text() + nonce = nonce_header.split('"')[1] + self.assertRegex(nonce, r"^[0-9a-f]{32}$") + self.assertNotIn("hidden", {path.name for path in (run_dir / "workspace").rglob("*")}) + self.assertNotEqual(manifest["hashes"]["source_initial"], "") + self.assertEqual(manifest["hashes"]["source_initial"], manifest["hashes"]["source_final"]) + self.assertTrue(all(not Path(path).is_absolute() for path in manifest["artifacts"])) + self.assertEqual(self._run_cli( + task, "--run-dir", run_dir, "--evaluate-only", "--candidate", candidate, + "--jtag-serial", "JTAG-1", "--uart-device", "/dev/null", "--openocd", identity, + ).returncode, 2) + + def test_usage_cost_is_exact_and_rejects_boolean_numbers(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + usage = root / "usage.json" + usage.write_text(json.dumps({ + "requests": 2, "tokens": {"fresh_input": 100, "cached_input": 200, + "output": 300, "reasoning": 40}, "final_context_tokens": 55, + "latency_seconds": 1.25, "provider": "fixture", "model": "m", + "rates_usd_per_million": {"fresh_input": 10, "cached_input": 1, "output": 20}, + "price_source": "fixture", "price_date": "2026-01-01" + })) + sys.path.insert(0, str(REPOSITORY_ROOT / "benchmarks/twin2silicon")) + import run_hil + cost = run_hil.parse_usage(usage) + self.assertEqual(cost["cost_usd"], 0.0072) + usage.write_text(usage.read_text().replace('"requests": 2', '"requests": true')) + with self.assertRaises(ValueError): + run_hil.parse_usage(usage) + + def test_candidate_symlink_escape_is_rejected_before_commands(self): + task = REPOSITORY_ROOT / "benchmarks/twin2silicon/tasks/esp32s3-gpio-hil-001" + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + candidate = root / "candidate" + candidate.mkdir() + (candidate / "escape").symlink_to(task / "hidden") + marker = root / "ran" + tool = executable_fixture(root, f"Path({str(marker)!r}).write_text('ran')\n") + result = self._run_cli( + task, "--run-dir", root / "run", "--evaluate-only", "--candidate", candidate, + "--jtag-serial", "J", "--uart-device", "/dev/null", "--openocd", tool, + "--platformio", tool, + ) + self.assertEqual(result.returncode, 2) + self.assertFalse(marker.exists()) + + if __name__ == "__main__": if "-k" in sys.argv: pattern_index = sys.argv.index("-k") + 1 From 5964eb94f5266e52bf0269dfee4f09c86c4d205d Mon Sep 17 00:00:00 2001 From: Andrii Shylenko <14119286+w1ne@users.noreply.github.com> Date: Sat, 15 Aug 2026 13:24:36 +0200 Subject: [PATCH 17/48] fix(bench): harden HIL orchestration evidence --- benchmarks/twin2silicon/hil/esp32s3.py | 16 +- benchmarks/twin2silicon/hil/process.py | 17 ++ benchmarks/twin2silicon/run_hil.py | 150 ++++++++++++--- tests/twin2silicon-hil.py | 242 ++++++++++++++++++++++++- 4 files changed, 391 insertions(+), 34 deletions(-) diff --git a/benchmarks/twin2silicon/hil/esp32s3.py b/benchmarks/twin2silicon/hil/esp32s3.py index 944b464..a9c81bd 100644 --- a/benchmarks/twin2silicon/hil/esp32s3.py +++ b/benchmarks/twin2silicon/hil/esp32s3.py @@ -13,6 +13,7 @@ import time import tty from typing import Callable, Literal, Mapping, Optional, Sequence +import threading from .process import run_command from .results import CommandResult, PathLike @@ -272,11 +273,12 @@ class UartResult: matched: bool bytes_captured: int timed_out: bool - termination_reason: Literal["matched", "timeout", "max_bytes"] + termination_reason: Literal["matched", "timeout", "max_bytes", "cancelled"] def capture_uart_nonce(device: PathLike, baud: int, nonce: str, timeout_seconds: float, - log: PathLike, *, max_bytes: int = 65536) -> UartResult: + log: PathLike, *, max_bytes: int = 65536, + cancel_event: Optional[threading.Event] = None) -> UartResult: fd = os.open(device, os.O_RDWR | os.O_NOCTTY | os.O_NONBLOCK) try: speeds = {9600: termios.B9600, 115200: termios.B115200} @@ -296,11 +298,15 @@ def capture_uart_nonce(device: PathLike, baud: int, nonce: str, timeout_seconds: matched = False expected = f"LABWIRED_READY:{nonce}".encode() while not matched and len(captured) < max_bytes: + if cancel_event is not None and cancel_event.is_set(): + break remaining = deadline - time.monotonic() if remaining <= 0: break - readable, _, _ = select.select([fd], [], [], remaining) + readable, _, _ = select.select([fd], [], [], min(remaining, 0.05) if cancel_event else remaining) if not readable: + if cancel_event is not None: + continue break try: chunk = os.read(fd, min(4096, max_bytes - len(captured))) @@ -320,9 +326,11 @@ def capture_uart_nonce(device: PathLike, baud: int, nonce: str, timeout_seconds: break Path(log).parent.mkdir(parents=True, exist_ok=True) Path(log).write_bytes(captured) - reason: Literal["matched", "timeout", "max_bytes"] + reason: Literal["matched", "timeout", "max_bytes", "cancelled"] if matched: reason = "matched" + elif cancel_event is not None and cancel_event.is_set(): + reason = "cancelled" elif len(captured) >= max_bytes: reason = "max_bytes" else: diff --git a/benchmarks/twin2silicon/hil/process.py b/benchmarks/twin2silicon/hil/process.py index 0d0ee15..f90dc8c 100644 --- a/benchmarks/twin2silicon/hil/process.py +++ b/benchmarks/twin2silicon/hil/process.py @@ -66,6 +66,23 @@ def run_command( ) try: process.communicate(timeout=timeout_seconds) + except (KeyboardInterrupt, SystemExit) as interruption: + try: + os.killpg(process.pid, signal.SIGTERM) + except (PermissionError, ProcessLookupError): + pass + try: + process.wait(timeout=0.5) + except subprocess.TimeoutExpired: + try: + os.killpg(process.pid, signal.SIGKILL) + except (PermissionError, ProcessLookupError): + pass + try: + process.wait(timeout=0.5) + except subprocess.TimeoutExpired: + pass + raise interruption except subprocess.TimeoutExpired: timed_out = True try: diff --git a/benchmarks/twin2silicon/run_hil.py b/benchmarks/twin2silicon/run_hil.py index 570067d..15cf402 100644 --- a/benchmarks/twin2silicon/run_hil.py +++ b/benchmarks/twin2silicon/run_hil.py @@ -8,6 +8,7 @@ from __future__ import annotations import argparse +from dataclasses import replace from datetime import datetime, timezone import hashlib import json @@ -46,15 +47,23 @@ def _number(value: object, field: str, *, integer: bool = False) -> int | float: def parse_usage(path: Path) -> dict[str, Any]: raw = json.loads(path.read_text(encoding="utf-8")) + root_keys = {"schema_version", "requests", "tokens", "final_context_tokens", "latency_seconds", + "provider", "model", "rates_usd_per_million", "price_source", "price_effective_date"} + if not isinstance(raw, Mapping) or set(raw) != root_keys or raw.get("schema_version") != "1.0": + raise ValueError("usage schema must be exactly version 1.0") if not isinstance(raw, Mapping) or not isinstance(raw.get("tokens"), Mapping): raise ValueError("usage must contain an object and tokens object") if not isinstance(raw.get("rates_usd_per_million"), Mapping): raise ValueError("usage must contain rates_usd_per_million") + if set(raw["tokens"]) != {"fresh_input", "cached_input", "output", "reasoning"}: + raise ValueError("usage token keys are not exact") + if set(raw["rates_usd_per_million"]) != {"fresh_input", "cached_input", "output"}: + raise ValueError("usage rate keys are not exact") tokens = {name: _number(raw["tokens"].get(name), f"tokens.{name}", integer=True) for name in ("fresh_input", "cached_input", "output", "reasoning")} rates = {name: _number(raw["rates_usd_per_million"].get(name), f"rates.{name}") for name in ("fresh_input", "cached_input", "output")} - for name in ("provider", "model", "price_source", "price_date"): + for name in ("provider", "model", "price_source", "price_effective_date"): if not isinstance(raw.get(name), str) or not raw[name]: raise ValueError(f"{name} must be a nonempty string") result = { @@ -62,7 +71,7 @@ def parse_usage(path: Path) -> dict[str, Any]: "tokens": tokens, "final_context_tokens": _number(raw.get("final_context_tokens"), "final_context_tokens", integer=True), "latency_seconds": _number(raw.get("latency_seconds"), "latency_seconds"), "provider": raw["provider"], "model": raw["model"], "rates_usd_per_million": rates, - "price_source": raw["price_source"], "price_date": raw["price_date"], + "price_source": raw["price_source"], "price_effective_date": raw["price_effective_date"], } result["cost_usd"] = (tokens["fresh_input"] * rates["fresh_input"] + tokens["cached_input"] * rates["cached_input"] @@ -107,9 +116,39 @@ def _copy_public(source: Path, destination: Path) -> None: shutil.copytree(source, destination, symlinks=False) -def _command_record(result: Any) -> dict[str, Any]: - return {"returncode": result.returncode, "timed_out": result.timed_out, - "duration_seconds": result.duration_seconds, "cleanup_error": result.cleanup_error} +def _command_record(result: Any, run_dir: Path) -> dict[str, Any]: + def relative(value: str) -> str: + try: + return Path(value).resolve().relative_to(run_dir).as_posix() + except ValueError: + return "workspace" if Path(value).name == "workspace" else Path(value).name + return {"argv": [Path(item).name if Path(item).is_absolute() else item for item in result.command], + "cwd": relative(result.cwd), "stdout": relative(result.stdout_path), + "stderr": relative(result.stderr_path), "started_at_utc": result.started_at_utc, + "ended_at_utc": result.ended_at_utc, "returncode": result.returncode, + "timed_out": result.timed_out, "duration_seconds": result.duration_seconds, + "cleanup_error": result.cleanup_error} + + +def _tool_version(name: str, executable: os.PathLike[str] | str, workspace: Path, + run_dir: Path, env: Mapping[str, str] | None = None) -> dict[str, str]: + record = {"executable": Path(executable).name, "version": "not_available"} + try: + result = run_command([executable, "--version"], cwd=workspace, + stdout_path=run_dir / f"version.{name}.stdout.log", + stderr_path=run_dir / f"version.{name}.stderr.log", timeout_seconds=2, env=env) + if result.returncode == 0 and not result.timed_out and not result.cleanup_error: + text = Path(result.stdout_path).read_text(encoding="utf-8", errors="replace").strip() + record["version"] = " ".join(text.split())[:200] or "unknown" + except OSError: + pass + return record + + +def _agent_environment() -> dict[str, str]: + allowed = ("PATH", "HOME", "LABWIRED_HOME", "XDG_CONFIG_HOME", "TMPDIR", "LANG", "LC_ALL", + "LABWIRED_ACCESS_TOKEN", "LABWIRED_PROJECT", "LABWIRED_MODEL_URL", "LABWIRED_MODEL_KEY") + return {name: os.environ[name] for name in allowed if name in os.environ} def parser() -> argparse.ArgumentParser: @@ -127,6 +166,10 @@ def parser() -> argparse.ArgumentParser: value.add_argument("--platformio", default="pio") value.add_argument("--identity-command-json", help="fixture identity argv as a JSON array (never a shell string)") value.add_argument("--usage-json", type=Path) + value.add_argument("--fixture-uart-timeout-seconds", type=float, + help="offline fixture only: shorten (never extend) the oracle UART timeout") + value.add_argument("--fixture-identity-timeout-seconds", type=float, + help="offline fixture only: shorten (never extend) identity/lock timeout") return value @@ -145,7 +188,7 @@ def main(argv: list[str] | None = None) -> int: "task": {}, "harness_revision": HARNESS_REVISION, "model": None, "provider": None, "requests": None, "tokens": {"fresh_input": None, "cached_input": None, "output": None, "reasoning": None}, "final_context_tokens": None, "latency_seconds": None, "cost_usd": None, - "configured_budgets": {}, "budget_validity": "not_available", + "configured_budgets": {}, "budget_validity": {}, "tool_versions": {}, "model_status": "not_run", "compile_status": "not_run", "simulator_status": "not_supported", "hardware_status": "not_run", "infrastructure_status": "ok", "uart": None, "register_assertions": [], "termination": "running", "failure_category": None, @@ -153,6 +196,12 @@ def main(argv: list[str] | None = None) -> int: } def persist() -> None: write_json_atomic(run_dir / "run.json", manifest) + def record_phase(name: str, result: Any = None, **details: Any) -> None: + phase = {"name": name, **details} + if result is not None: + phase["command"] = _command_record(result, run_dir) + manifest["phases"].append(phase) + persist() persist() try: task_root = _resolve_task(args.task) @@ -164,6 +213,16 @@ def persist() -> None: if task_root not in public.parents or task_root not in oracle_path.parents: raise ValueError("task path escapes task directory") config = Esp32S3Config.from_oracle(json.loads(oracle_path.read_text(encoding="utf-8"))) + if args.fixture_uart_timeout_seconds is not None: + fixture_timeout = args.fixture_uart_timeout_seconds + if not math.isfinite(fixture_timeout) or fixture_timeout < 0 or fixture_timeout > config.uart_timeout_seconds: + raise ValueError("fixture UART timeout must be finite, nonnegative, and no larger than the oracle timeout") + config = replace(config, uart_timeout_seconds=fixture_timeout) + if args.fixture_identity_timeout_seconds is not None: + fixture_timeout = args.fixture_identity_timeout_seconds + if not math.isfinite(fixture_timeout) or fixture_timeout < 0 or fixture_timeout > config.identity_timeout_seconds: + raise ValueError("fixture identity timeout must be finite, nonnegative, and no larger than oracle timeout") + config = replace(config, identity_timeout_seconds=fixture_timeout) source = _safe_tree(args.candidate, "candidate") if args.evaluate_only else _safe_tree(public, "public") workspace = run_dir / "workspace" _copy_public(source, workspace) @@ -173,9 +232,22 @@ def persist() -> None: nonce_header.write_text(f'#pragma once\n#define LABWIRED_RUN_NONCE "{nonce}"\n', encoding="utf-8") manifest["task"] = {"id": task["id"], "schema_version": task["schema_version"]} manifest["configured_budgets"] = task.get("budgets", {}) + manifest["budget_validity"] = { + name: {"configured": task.get("budgets", {}).get(name), "observed": None, "within_budget": None} + for name in ("wall_time_seconds", "model_tokens", "repair_iterations") + } + manifest["environment"] = {"jtag_serial_sha256": hashlib.sha256(args.jtag_serial.encode()).hexdigest(), + "uart_device": Path(args.uart_device).name} manifest["hashes"]["oracle_descriptor"] = sha256_file(oracle_path) manifest["hashes"]["source_initial"] = _tree_hash(workspace) - manifest["phases"].append("prepared") + record_phase("prepared") + manifest["tool_versions"] = { + "platformio": _tool_version("platformio", args.platformio, workspace, run_dir), + "openocd": _tool_version("openocd", args.openocd, workspace, run_dir), + } + if args.agent_bin: + manifest["tool_versions"]["agent"] = _tool_version( + "agent", args.agent_bin, workspace, run_dir, _agent_environment()) persist() if args.usage_json: @@ -183,17 +255,23 @@ def persist() -> None: write_json_atomic(run_dir / "cost.json", usage) for name in ("requests", "tokens", "final_context_tokens", "latency_seconds", "provider", "model", "cost_usd"): manifest[name] = usage[name] - manifest["budget_validity"] = "valid" if usage["tokens"]["fresh_input"] + usage["tokens"]["cached_input"] + usage["tokens"]["output"] <= task.get("budgets", {}).get("model_tokens", math.inf) else "exceeded" + observed_tokens = usage["tokens"]["fresh_input"] + usage["tokens"]["cached_input"] + usage["tokens"]["output"] + manifest["budget_validity"]["model_tokens"].update( + observed=observed_tokens, + within_budget=observed_tokens <= task.get("budgets", {}).get("model_tokens", math.inf)) + manifest["budget_validity"]["wall_time_seconds"].update( + observed=usage["latency_seconds"], + within_budget=usage["latency_seconds"] <= task.get("budgets", {}).get("wall_time_seconds", math.inf)) elif args.agent_bin: manifest["model"] = args.model if args.agent_bin: prompt = (workspace / "README.md").read_text(encoding="utf-8") - clean_env = {name: os.environ[name] for name in ("PATH", "LANG", "LC_ALL", "TMPDIR") if name in os.environ} + clean_env = _agent_environment() agent = run_command([args.agent_bin, "agent", "run", "--model", args.model, prompt], cwd=workspace, stdout_path=run_dir / "agent.stdout.log", stderr_path=run_dir / "agent.stderr.log", timeout_seconds=float(task["budgets"]["wall_time_seconds"]), env=clean_env) - manifest["phases"].append({"agent": _command_record(agent)}) + record_phase("agent", agent) if agent.timed_out or agent.cleanup_error: raise RuntimeError("agent process did not terminate cleanly") manifest["model_status"] = "pass" if agent.returncode == 0 else "fail" @@ -211,17 +289,14 @@ def persist() -> None: clean = run_command(build_command + ["--target", "clean"], cwd=workspace, stdout_path=run_dir / "clean.stdout.log", stderr_path=run_dir / "clean.stderr.log", timeout_seconds=float(task["budgets"]["wall_time_seconds"])) - manifest["phases"].append({"clean": _command_record(clean)}) + record_phase("clean", clean) if clean.timed_out or clean.cleanup_error: raise RuntimeError("clean process did not terminate cleanly") if clean.returncode: - manifest["compile_status"] = "fail" - manifest["termination"], manifest["failure_category"] = "completed", "compile" - return _finalize(run_dir, manifest) - persist() + raise RuntimeError("clean command failed") build = run_command(build_command, cwd=workspace, stdout_path=run_dir / "build.stdout.log", stderr_path=run_dir / "build.stderr.log", timeout_seconds=float(task["budgets"]["wall_time_seconds"])) - manifest["phases"].append({"build": _command_record(build)}) + record_phase("build", build) if build.timed_out or build.cleanup_error: raise RuntimeError("build process did not terminate cleanly") if build.returncode: @@ -244,31 +319,41 @@ def persist() -> None: identity_command = list(config.identity_command) uart_result: dict[str, Any] = {} uart_error: list[BaseException] = [] - def capture() -> None: - try: - uart_result["value"] = capture_uart_nonce(args.uart_device, config.uart_baud, nonce, - config.uart_timeout_seconds, run_dir / "uart.raw.log") - except BaseException as error: - uart_error.append(error) with BoardLock(run_dir.parent / ".board-locks", args.jtag_serial, timeout_seconds=config.identity_timeout_seconds): identity = validate_identity(identity_command, args.jtag_serial, cwd=workspace, evidence_dir=run_dir, timeout_seconds=config.identity_timeout_seconds) - manifest["phases"].append("identity") + record_phase("identity", identity.command_result, status=identity.status) if identity.status != "pass": raise RuntimeError(identity.detail or "board identity failed") persist() - uart_thread = threading.Thread(target=capture, name="hil-uart", daemon=True) + cancel_uart = threading.Event() + def capture_cancellable() -> None: + try: + uart_result["value"] = capture_uart_nonce(args.uart_device, config.uart_baud, nonce, + config.uart_timeout_seconds, run_dir / "uart.raw.log", + cancel_event=cancel_uart) + except BaseException as error: + uart_error.append(error) + uart_thread = threading.Thread(target=capture_cancellable, name="hil-uart", daemon=False) uart_thread.start() - flash_command = [args.platformio, "run", "--project-dir", str(firmware), "--environment", - config.platformio_environment or "esp32s3", "--target", config.flash_target] - flashed = flash_firmware(flash_command, cwd=workspace, evidence_dir=run_dir, - timeout_seconds=config.flash_timeout_seconds, identity_validated=True) - uart_thread.join(config.uart_timeout_seconds + 1) + try: + flash_command = [args.platformio, "run", "--project-dir", str(firmware), "--environment", + config.platformio_environment or "esp32s3", "--target", config.flash_target] + flashed = flash_firmware(flash_command, cwd=workspace, evidence_dir=run_dir, + timeout_seconds=config.flash_timeout_seconds, identity_validated=True) + record_phase("flash", flashed.command_result, status=flashed.status, category=flashed.category) + if flashed.status == "pass": + uart_thread.join(config.uart_timeout_seconds + 0.5) + finally: + cancel_uart.set() + uart_thread.join(1) if uart_thread.is_alive() or uart_error: raise RuntimeError(f"UART capture failed: {uart_error[0] if uart_error else 'cleanup timeout'}") uart = uart_result["value"] manifest["uart"] = {"matched": uart.matched, "termination": uart.termination_reason, "bytes_captured": uart.bytes_captured} + record_phase("uart", matched=uart.matched, termination=uart.termination_reason, + bytes_captured=uart.bytes_captured) if flashed.status == "infrastructure_error": raise RuntimeError(flashed.detail or "flash infrastructure failure") if flashed.status == "hardware_fail" or not uart.matched: @@ -279,12 +364,15 @@ def capture() -> None: config.assertions, cwd=workspace, evidence_dir=run_dir, timeout_seconds=config.openocd_command_timeout_seconds) if registers.status == "infrastructure_error": + record_phase("register", registers.command_result, status=registers.status) raise RuntimeError(registers.detail or "OpenOCD infrastructure failure") manifest["register_assertions"] = [ {"name": item.name, "passed": item.passed, "observed_masked": f"0x{item.value & item.mask:08x}"} for item in registers.evaluation.observations ] manifest["hardware_status"] = "pass" if registers.status == "pass" else "fail" + record_phase("register", registers.command_result, status=registers.status, + assertions=manifest["register_assertions"]) manifest["termination"] = "completed" manifest["failure_category"] = None if registers.status == "pass" else "register_mismatch" return _finalize(run_dir, manifest) @@ -306,6 +394,10 @@ def capture() -> None: def _finalize(run_dir: Path, manifest: dict[str, Any]) -> int: + result = {name: manifest.get(name) for name in ( + "model_status", "compile_status", "simulator_status", "hardware_status", + "infrastructure_status", "termination", "failure_category", "uart", "register_assertions")} + write_json_atomic(run_dir / "result.json", result) artifacts: list[str] = [] hashes = manifest.setdefault("hashes", {}) for path in sorted(item for item in run_dir.rglob("*") if item.is_file() and item.name != "run.json"): diff --git a/tests/twin2silicon-hil.py b/tests/twin2silicon-hil.py index a5b9112..cc27d80 100644 --- a/tests/twin2silicon-hil.py +++ b/tests/twin2silicon-hil.py @@ -1,5 +1,6 @@ #!/usr/bin/env python3 import hashlib +import contextlib import json import os from pathlib import Path @@ -178,6 +179,25 @@ def test_write_json_atomic_replaces_destination_with_json(self): class ProcessContractTests(unittest.TestCase): + def test_run_command_interrupt_terminates_process_group_before_reraising(self): + with tempfile.TemporaryDirectory() as directory: + evidence = Path(directory) + pid_path = evidence / "pid" + timer = threading.Timer(.15, lambda: os.kill(os.getpid(), signal.SIGINT)) + timer.start() + try: + with self.assertRaises(KeyboardInterrupt): + run_command([sys.executable, "-c", f"import os,pathlib,time; pathlib.Path({str(pid_path)!r}).write_text(str(os.getpid())); time.sleep(30)"], + cwd=evidence, stdout_path=evidence / "o", stderr_path=evidence / "e", timeout_seconds=30) + child = int(pid_path.read_text()) + with self.assertRaises(ProcessLookupError): + os.kill(child, 0) + finally: + timer.cancel() + if pid_path.exists(): + try: os.kill(int(pid_path.read_text()), signal.SIGKILL) + except ProcessLookupError: pass + def test_run_command_timeout_captures_evidence_and_is_bounded(self): with tempfile.TemporaryDirectory() as directory: evidence = Path(directory) @@ -615,6 +635,28 @@ def test_rejects_absent_wrong_stale_or_incomplete_nonce_with_bounded_evidence(se self.assertLess(time.monotonic() - started, .5) self.assertLessEqual(len(raw), 64) + def test_cancellation_stops_capture_and_closes_stable_log(self): + master, slave = pty.openpty() + device = os.ttyname(slave) + tty.setraw(slave) + with tempfile.TemporaryDirectory() as directory: + log = Path(directory) / "uart.log" + cancel = threading.Event() + result = [] + thread = threading.Thread(target=lambda: result.append( + capture_uart_nonce(device, 115200, "nonce", 30, log, cancel_event=cancel))) + thread.start() + time.sleep(.05) + cancel.set() + thread.join(.5) + self.assertFalse(thread.is_alive()) + self.assertEqual(result[0].termination_reason, "cancelled") + before = log.read_bytes() + time.sleep(.05) + self.assertEqual(log.read_bytes(), before) + os.close(master) + os.close(slave) + def test_rejects_unsupported_baud_and_closes_opened_fd(self): master, slave = pty.openpty() device = os.ttyname(slave) @@ -781,6 +823,10 @@ def _run_cli(self, *arguments, env=None): cwd=REPOSITORY_ROOT, text=True, capture_output=True, env=env, ) + def _short_task(self): + source = REPOSITORY_ROOT / "benchmarks/twin2silicon/tasks/esp32s3-gpio-hil-001" + return source + def test_evaluate_only_prepares_isolated_workspace_and_records_compile_failure(self): task = REPOSITORY_ROOT / "benchmarks/twin2silicon/tasks/esp32s3-gpio-hil-001" with tempfile.TemporaryDirectory() as directory: @@ -793,6 +839,7 @@ def test_evaluate_only_prepares_isolated_workspace_and_records_compile_failure(s platformio = executable_fixture(tool_dir, textwrap.dedent(f""" import sys from pathlib import Path + if '--version' in sys.argv: print('pio fixture 1'); raise SystemExit(0) with Path({str(invocations)!r}).open('a') as output: output.write(json.dumps(sys.argv[1:]) + '\\n') raise SystemExit(0 if 'clean' in sys.argv else 1) @@ -832,7 +879,7 @@ def test_usage_cost_is_exact_and_rejects_boolean_numbers(self): "output": 300, "reasoning": 40}, "final_context_tokens": 55, "latency_seconds": 1.25, "provider": "fixture", "model": "m", "rates_usd_per_million": {"fresh_input": 10, "cached_input": 1, "output": 20}, - "price_source": "fixture", "price_date": "2026-01-01" + "schema_version": "1.0", "price_source": "fixture", "price_effective_date": "2026-01-01" })) sys.path.insert(0, str(REPOSITORY_ROOT / "benchmarks/twin2silicon")) import run_hil @@ -842,6 +889,22 @@ def test_usage_cost_is_exact_and_rejects_boolean_numbers(self): with self.assertRaises(ValueError): run_hil.parse_usage(usage) + def test_usage_schema_rejects_missing_extra_and_legacy_price_date(self): + sys.path.insert(0, str(REPOSITORY_ROOT / "benchmarks/twin2silicon")) + import run_hil + base = {"schema_version": "1.0", "requests": 1, + "tokens": {"fresh_input": 1, "cached_input": 2, "output": 3, "reasoning": 4}, + "final_context_tokens": 1, "latency_seconds": 1, "provider": "p", "model": "m", + "rates_usd_per_million": {"fresh_input": 1, "cached_input": 1, "output": 1}, + "price_source": "s", "price_effective_date": "2026-01-01"} + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "usage.json" + for invalid in ({**base, "extra": 1}, {key: value for key, value in base.items() if key != "requests"}, + {**base, "price_date": "legacy"}): + path.write_text(json.dumps(invalid)) + with self.assertRaises(ValueError): + run_hil.parse_usage(path) + def test_candidate_symlink_escape_is_rejected_before_commands(self): task = REPOSITORY_ROOT / "benchmarks/twin2silicon/tasks/esp32s3-gpio-hil-001" with tempfile.TemporaryDirectory() as directory: @@ -859,6 +922,183 @@ def test_candidate_symlink_escape_is_rejected_before_commands(self): self.assertEqual(result.returncode, 2) self.assertFalse(marker.exists()) + def test_agent_mode_repairs_workspace_with_allowlisted_home_and_no_secret_evidence(self): + task = REPOSITORY_ROOT / "benchmarks/twin2silicon/tasks/esp32s3-gpio-hil-001" + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + invocation = root / "agent-invocation.json" + agent_dir = root / "agent" + agent_dir.mkdir() + agent = executable_fixture(agent_dir, textwrap.dedent(f""" + import json, os, pathlib, sys + if '--version' in sys.argv: print('agent fixture 1'); raise SystemExit(0) + assert os.environ['HOME'] == 'HOME_SENTINEL' + assert os.environ['LABWIRED_HOME'] == 'LABWIRED_HOME_SENTINEL' + assert 'hidden' not in ' '.join(sys.argv).lower() + source = pathlib.Path('firmware/src/main.c') + source.write_text(source.read_text().replace('GPIO_MODE_INPUT', 'GPIO_MODE_OUTPUT')) + pathlib.Path({str(invocation)!r}).write_text(json.dumps({{'argv': sys.argv[1:], 'cwd': pathlib.Path.cwd().name}})) + """)) + pio_dir = root / "pio" + pio_dir.mkdir() + pio = executable_fixture(pio_dir, "import sys\nif '--version' in sys.argv: print('pio fixture 1'); raise SystemExit(0)\nraise SystemExit(0 if 'clean' in sys.argv else 1)\n") + env = {**os.environ, "HOME": "HOME_SENTINEL", "LABWIRED_HOME": "LABWIRED_HOME_SENTINEL", + "LABWIRED_ACCESS_TOKEN": "SECRET_TOKEN_SENTINEL"} + run_dir = root / "run" + result = self._run_cli(task, "--run-dir", run_dir, "--agent-bin", agent, "--model", "fixture/model", + "--jtag-serial", "J", "--uart-device", "/dev/null", "--openocd", pio, + "--platformio", pio, env=env) + self.assertEqual(result.returncode, 0, result.stderr) + call = json.loads(invocation.read_text()) + self.assertEqual(call["argv"][:4], ["agent", "run", "--model", "fixture/model"]) + self.assertEqual(call["cwd"], "workspace") + manifest_text = (run_dir / "run.json").read_text() + self.assertNotIn("SECRET_TOKEN_SENTINEL", manifest_text) + manifest = json.loads(manifest_text) + self.assertEqual(manifest["model_status"], "pass") + self.assertNotEqual(manifest["hashes"]["source_initial"], manifest["hashes"]["source_final"]) + + def test_cli_physical_pass_uses_pty_and_records_ordered_evidence(self): + task = self._short_task() + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + order = root / "order.log" + marker = root / "flash" + pio_dir = root / "pio"; pio_dir.mkdir() + pio = executable_fixture(pio_dir, textwrap.dedent(f""" + import pathlib, sys, time + if '--version' in sys.argv: print('pio fixture 1'); raise SystemExit(0) + order = pathlib.Path({str(order)!r}) + target = sys.argv[sys.argv.index('--target') + 1] if '--target' in sys.argv else 'build' + with order.open('a') as out: out.write(target + '\\n') + project = pathlib.Path(sys.argv[sys.argv.index('--project-dir') + 1]) + if target == 'build': + artifact = project / '.pio/build/esp32s3/firmware.bin'; artifact.parent.mkdir(parents=True); artifact.write_bytes(b'fw') + if target == 'upload': pathlib.Path({str(marker)!r}).write_text('flash'); time.sleep(.1) + """)) + identity_dir = root / "identity"; identity_dir.mkdir() + identity = executable_fixture(identity_dir, f"from pathlib import Path\nwith Path({str(order)!r}).open('a') as out: out.write('identity\\n')\nprint('JTAG-1')\n") + openocd_dir = root / "openocd"; openocd_dir.mkdir() + openocd = executable_fixture(openocd_dir, textwrap.dedent(f""" + import pathlib, sys + if '--version' in sys.argv: print('openocd fixture 1'); raise SystemExit(0) + with pathlib.Path({str(order)!r}).open('a') as out: out.write('openocd\\n') + print('@@REG gpio2_output_enabled 0x60004020', file=sys.stderr) + print('0x60004020: 00000004', file=sys.stderr) + print('@@REG gpio2_output_high 0x60004004', file=sys.stderr) + print('0x60004004: 00000004', file=sys.stderr) + """)) + master, slave = pty.openpty(); tty.setraw(slave) + run_dir = root / "run" + def writer(): + deadline = time.monotonic() + 2 + while not marker.exists() and time.monotonic() < deadline: time.sleep(.005) + nonce = (run_dir / "workspace/firmware/include/run_nonce.h").read_text().split('"')[1] + os.write(master, f"LABWIRED_READY:{nonce}\n".encode()) + thread = threading.Thread(target=writer); thread.start() + try: + result = self._run_cli(task, "--run-dir", run_dir, "--evaluate-only", "--candidate", task / "public", + "--jtag-serial", "JTAG-1", "--uart-device", os.ttyname(slave), + "--openocd", openocd, "--platformio", pio, + "--identity-command-json", json.dumps([str(identity)])) + finally: + thread.join(2); os.close(master); os.close(slave) + self.assertEqual(result.returncode, 0, result.stderr) + manifest = json.loads((run_dir / "run.json").read_text()) + self.assertEqual(manifest["hardware_status"], "pass") + self.assertEqual(order.read_text().splitlines(), ["clean", "build", "identity", "upload", "openocd"]) + self.assertEqual([phase["name"] for phase in manifest["phases"]], + ["prepared", "clean", "build", "identity", "flash", "uart", "register"]) + self.assertEqual(set(manifest["tool_versions"]), {"platformio", "openocd"}) + self.assertTrue(all("executable" in item and "version" in item + for item in manifest["tool_versions"].values())) + for artifact in manifest["artifacts"]: + path = run_dir / artifact + self.assertTrue(path.is_file()) + self.assertEqual(manifest["hashes"][f"artifact:{artifact}"], sha256_file(path)) + + def test_clean_nonzero_is_infrastructure_and_never_touches_identity(self): + task = REPOSITORY_ROOT / "benchmarks/twin2silicon/tasks/esp32s3-gpio-hil-001" + with tempfile.TemporaryDirectory() as directory: + root = Path(directory); marker = root / "identity-ran" + pio_dir = root / "pio"; pio_dir.mkdir() + pio = executable_fixture(pio_dir, "import sys\nif '--version' in sys.argv: print('v'); raise SystemExit(0)\nraise SystemExit(7)\n") + identity_dir = root / "identity"; identity_dir.mkdir() + identity = executable_fixture(identity_dir, f"from pathlib import Path; Path({str(marker)!r}).write_text('ran')\n") + openocd_dir = root / "openocd"; openocd_dir.mkdir() + openocd = executable_fixture(openocd_dir, "print('v')\n") + run_dir = root / "run" + result = self._run_cli(task, "--run-dir", run_dir, "--evaluate-only", "--candidate", task / "public", + "--jtag-serial", "J", "--uart-device", "/dev/null", "--openocd", openocd, + "--platformio", pio, "--identity-command-json", json.dumps([str(identity)])) + self.assertEqual(result.returncode, 2) + self.assertFalse(marker.exists()) + manifest = json.loads((run_dir / "run.json").read_text()) + self.assertEqual((manifest["compile_status"], manifest["infrastructure_status"]), ("not_run", "error")) + self.assertEqual([phase["name"] for phase in manifest["phases"]], ["prepared", "clean"]) + + def test_cli_classifies_candidate_and_infrastructure_physical_failures(self): + task = self._short_task() + cases = ( + ("flash", 9, True, 4, 0, "fail", "ok"), + ("nonce", 0, False, 4, 0, "fail", "ok"), + ("register", 0, True, 0, 0, "fail", "ok"), + ("identity", 0, True, 4, 0, "not_run", "error"), + ("lock", 0, True, 4, 0, "not_run", "error"), + ("openocd", 0, True, 4, 7, "not_run", "error"), + ) + for name, flash_code, send_nonce, register_value, openocd_code, hardware, infrastructure in cases: + with self.subTest(name=name), tempfile.TemporaryDirectory() as directory: + root = Path(directory); marker = root / "flash" + pio_dir = root / "pio"; pio_dir.mkdir() + pio = executable_fixture(pio_dir, textwrap.dedent(f""" + import pathlib, sys, time + if '--version' in sys.argv: print('v'); raise SystemExit(0) + target = sys.argv[sys.argv.index('--target') + 1] if '--target' in sys.argv else 'build' + project = pathlib.Path(sys.argv[sys.argv.index('--project-dir') + 1]) + if target == 'build': + artifact = project / '.pio/build/esp32s3/firmware.bin'; artifact.parent.mkdir(parents=True); artifact.write_bytes(b'fw') + if target == 'upload': pathlib.Path({str(marker)!r}).write_text('x'); time.sleep(.05); raise SystemExit({flash_code}) + """)) + identity_dir = root / "identity"; identity_dir.mkdir() + identity = executable_fixture(identity_dir, f"print({('OTHER' if name == 'identity' else 'JTAG-1')!r})\n") + openocd_dir = root / "openocd"; openocd_dir.mkdir() + openocd = executable_fixture(openocd_dir, textwrap.dedent(f""" + import sys + if '--version' in sys.argv: print('v'); raise SystemExit(0) + if {openocd_code}: raise SystemExit({openocd_code}) + print('@@REG gpio2_output_enabled 0x60004020', file=sys.stderr); print('0x60004020: {register_value:08x}', file=sys.stderr) + print('@@REG gpio2_output_high 0x60004004', file=sys.stderr); print('0x60004004: {register_value:08x}', file=sys.stderr) + """)) + master, slave = pty.openpty(); tty.setraw(slave); run_dir = root / "run" + def writer(): + deadline = time.monotonic() + 2 + while not marker.exists() and time.monotonic() < deadline: time.sleep(.005) + if send_nonce and marker.exists(): + nonce = (run_dir / "workspace/firmware/include/run_nonce.h").read_text().split('"')[1] + os.write(master, f"LABWIRED_READY:{nonce}\n".encode()) + thread = threading.Thread(target=writer); thread.start() + try: + lock = (BoardLock(run_dir.parent / ".board-locks", "JTAG-1", timeout_seconds=.1) + if name == "lock" else contextlib.nullcontext()) + with lock: + cli = [task, "--run-dir", run_dir, "--evaluate-only", "--candidate", task / "public", + "--jtag-serial", "JTAG-1", "--uart-device", os.ttyname(slave), + "--openocd", openocd, "--platformio", pio, + "--fixture-uart-timeout-seconds", ".3", + "--identity-command-json", json.dumps([str(identity)])] + if name == "lock": + cli += ["--fixture-identity-timeout-seconds", ".3"] + result = self._run_cli(*cli) + finally: + thread.join(2); os.close(master); os.close(slave) + manifest = json.loads((run_dir / "run.json").read_text()) + self.assertEqual(result.returncode, 2 if infrastructure == "error" else 0, result.stderr) + self.assertEqual((manifest["hardware_status"], manifest["infrastructure_status"]), + (hardware, infrastructure)) + with BoardLock(run_dir.parent / ".board-locks", "JTAG-1", timeout_seconds=.1): + pass + if __name__ == "__main__": if "-k" in sys.argv: From 98e5df1045dc8f5ff056054c08bd9bf269ba5658 Mon Sep 17 00:00:00 2001 From: Andrii Shylenko <14119286+w1ne@users.noreply.github.com> Date: Sat, 15 Aug 2026 13:25:55 +0200 Subject: [PATCH 18/48] fix(bench): synchronize UART readiness before flash --- benchmarks/twin2silicon/hil/esp32s3.py | 5 ++++- benchmarks/twin2silicon/run_hil.py | 7 ++++++- tests/twin2silicon-hil.py | 6 ++++-- 3 files changed, 14 insertions(+), 4 deletions(-) diff --git a/benchmarks/twin2silicon/hil/esp32s3.py b/benchmarks/twin2silicon/hil/esp32s3.py index a9c81bd..e8784b2 100644 --- a/benchmarks/twin2silicon/hil/esp32s3.py +++ b/benchmarks/twin2silicon/hil/esp32s3.py @@ -278,7 +278,8 @@ class UartResult: def capture_uart_nonce(device: PathLike, baud: int, nonce: str, timeout_seconds: float, log: PathLike, *, max_bytes: int = 65536, - cancel_event: Optional[threading.Event] = None) -> UartResult: + cancel_event: Optional[threading.Event] = None, + started_event: Optional[threading.Event] = None) -> UartResult: fd = os.open(device, os.O_RDWR | os.O_NOCTTY | os.O_NONBLOCK) try: speeds = {9600: termios.B9600, 115200: termios.B115200} @@ -292,6 +293,8 @@ def capture_uart_nonce(device: PathLike, baud: int, nonce: str, timeout_seconds: attrs[4] = attrs[5] = speeds[baud] attrs[2] = (attrs[2] & ~(termios.CSIZE | termios.PARENB | termios.CSTOPB)) | termios.CS8 | termios.CLOCAL | termios.CREAD termios.tcsetattr(fd, termios.TCSANOW, attrs) + if started_event is not None: + started_event.set() deadline = time.monotonic() + timeout_seconds captured = bytearray() pending = bytearray() diff --git a/benchmarks/twin2silicon/run_hil.py b/benchmarks/twin2silicon/run_hil.py index 15cf402..73e7e17 100644 --- a/benchmarks/twin2silicon/run_hil.py +++ b/benchmarks/twin2silicon/run_hil.py @@ -327,15 +327,20 @@ def record_phase(name: str, result: Any = None, **details: Any) -> None: raise RuntimeError(identity.detail or "board identity failed") persist() cancel_uart = threading.Event() + started_uart = threading.Event() def capture_cancellable() -> None: try: uart_result["value"] = capture_uart_nonce(args.uart_device, config.uart_baud, nonce, config.uart_timeout_seconds, run_dir / "uart.raw.log", - cancel_event=cancel_uart) + cancel_event=cancel_uart, started_event=started_uart) except BaseException as error: uart_error.append(error) uart_thread = threading.Thread(target=capture_cancellable, name="hil-uart", daemon=False) uart_thread.start() + if not started_uart.wait(1): + cancel_uart.set() + uart_thread.join(1) + raise RuntimeError(f"UART capture failed to start: {uart_error[0] if uart_error else 'startup timeout'}") try: flash_command = [args.platformio, "run", "--project-dir", str(firmware), "--environment", config.platformio_environment or "esp32s3", "--target", config.flash_target] diff --git a/tests/twin2silicon-hil.py b/tests/twin2silicon-hil.py index cc27d80..e99947f 100644 --- a/tests/twin2silicon-hil.py +++ b/tests/twin2silicon-hil.py @@ -642,11 +642,13 @@ def test_cancellation_stops_capture_and_closes_stable_log(self): with tempfile.TemporaryDirectory() as directory: log = Path(directory) / "uart.log" cancel = threading.Event() + started = threading.Event() result = [] thread = threading.Thread(target=lambda: result.append( - capture_uart_nonce(device, 115200, "nonce", 30, log, cancel_event=cancel))) + capture_uart_nonce(device, 115200, "nonce", 30, log, cancel_event=cancel, + started_event=started))) thread.start() - time.sleep(.05) + self.assertTrue(started.wait(.5)) cancel.set() thread.join(.5) self.assertFalse(thread.is_alive()) From ccc8ca0031448e610c2d6b2702eb1c34aa7be918 Mon Sep 17 00:00:00 2001 From: Andrii Shylenko <14119286+w1ne@users.noreply.github.com> Date: Sat, 15 Aug 2026 13:35:31 +0200 Subject: [PATCH 19/48] fix(bench): redact and account HIL run evidence --- benchmarks/twin2silicon/hil/process.py | 11 ++-- benchmarks/twin2silicon/run_hil.py | 90 ++++++++++++++++++-------- tests/twin2silicon-hil.py | 57 +++++++++++++++- 3 files changed, 125 insertions(+), 33 deletions(-) diff --git a/benchmarks/twin2silicon/hil/process.py b/benchmarks/twin2silicon/hil/process.py index f90dc8c..acff61b 100644 --- a/benchmarks/twin2silicon/hil/process.py +++ b/benchmarks/twin2silicon/hil/process.py @@ -74,14 +74,17 @@ def run_command( try: process.wait(timeout=0.5) except subprocess.TimeoutExpired: + pass + if _process_group_exists(process.pid): try: os.killpg(process.pid, signal.SIGKILL) except (PermissionError, ProcessLookupError): pass - try: - process.wait(timeout=0.5) - except subprocess.TimeoutExpired: - pass + try: + process.wait(timeout=0.5) + except subprocess.TimeoutExpired: + pass + _wait_for_process_group_exit(process.pid, 1.0) raise interruption except subprocess.TimeoutExpired: timed_out = True diff --git a/benchmarks/twin2silicon/run_hil.py b/benchmarks/twin2silicon/run_hil.py index 73e7e17..38f05d2 100644 --- a/benchmarks/twin2silicon/run_hil.py +++ b/benchmarks/twin2silicon/run_hil.py @@ -19,6 +19,7 @@ import shutil import sys import threading +import time from typing import Any, Mapping if __package__ in (None, ""): @@ -116,13 +117,18 @@ def _copy_public(source: Path, destination: Path) -> None: shutil.copytree(source, destination, symlinks=False) -def _command_record(result: Any, run_dir: Path) -> dict[str, Any]: +def _command_record(result: Any, run_dir: Path, redactions: tuple[str, ...] = ()) -> dict[str, Any]: + def sanitize(value: str) -> str: + for secret in redactions: + if secret: + value = value.replace(secret, "") + return value def relative(value: str) -> str: try: return Path(value).resolve().relative_to(run_dir).as_posix() except ValueError: return "workspace" if Path(value).name == "workspace" else Path(value).name - return {"argv": [Path(item).name if Path(item).is_absolute() else item for item in result.command], + return {"argv": [sanitize(Path(item).name if Path(item).is_absolute() else item) for item in result.command], "cwd": relative(result.cwd), "stdout": relative(result.stdout_path), "stderr": relative(result.stderr_path), "started_at_utc": result.started_at_utc, "ended_at_utc": result.ended_at_utc, "returncode": result.returncode, @@ -174,6 +180,7 @@ def parser() -> argparse.ArgumentParser: def main(argv: list[str] | None = None) -> int: + run_started_monotonic = time.monotonic() args = parser().parse_args(argv) if args.evaluate_only != bool(args.candidate) or (args.agent_bin and not args.model): parser().error("evaluate-only requires --candidate; agent mode requires --model") @@ -196,10 +203,12 @@ def main(argv: list[str] | None = None) -> int: } def persist() -> None: write_json_atomic(run_dir / "run.json", manifest) + redactions = tuple(filter(None, (args.jtag_serial, args.uart_device, + *(os.environ.get(name, "") for name in ("LABWIRED_ACCESS_TOKEN", "LABWIRED_MODEL_KEY"))))) def record_phase(name: str, result: Any = None, **details: Any) -> None: phase = {"name": name, **details} if result is not None: - phase["command"] = _command_record(result, run_dir) + phase["command"] = _command_record(result, run_dir, redactions) manifest["phases"].append(phase) persist() persist() @@ -233,9 +242,13 @@ def record_phase(name: str, result: Any = None, **details: Any) -> None: manifest["task"] = {"id": task["id"], "schema_version": task["schema_version"]} manifest["configured_budgets"] = task.get("budgets", {}) manifest["budget_validity"] = { - name: {"configured": task.get("budgets", {}).get(name), "observed": None, "within_budget": None} - for name in ("wall_time_seconds", "model_tokens", "repair_iterations") + name: {"configured": configured, "observed": None, "within_budget": None} + for name, configured in task.get("budgets", {}).items() } + for name in ("simulator_runs", "diagnostic_hil_runs"): + if name in manifest["budget_validity"]: + configured = manifest["budget_validity"][name]["configured"] + manifest["budget_validity"][name].update(observed=0, within_budget=0 <= configured) manifest["environment"] = {"jtag_serial_sha256": hashlib.sha256(args.jtag_serial.encode()).hexdigest(), "uart_device": Path(args.uart_device).name} manifest["hashes"]["oracle_descriptor"] = sha256_file(oracle_path) @@ -255,13 +268,10 @@ def record_phase(name: str, result: Any = None, **details: Any) -> None: write_json_atomic(run_dir / "cost.json", usage) for name in ("requests", "tokens", "final_context_tokens", "latency_seconds", "provider", "model", "cost_usd"): manifest[name] = usage[name] - observed_tokens = usage["tokens"]["fresh_input"] + usage["tokens"]["cached_input"] + usage["tokens"]["output"] + observed_tokens = usage["final_context_tokens"] manifest["budget_validity"]["model_tokens"].update( observed=observed_tokens, within_budget=observed_tokens <= task.get("budgets", {}).get("model_tokens", math.inf)) - manifest["budget_validity"]["wall_time_seconds"].update( - observed=usage["latency_seconds"], - within_budget=usage["latency_seconds"] <= task.get("budgets", {}).get("wall_time_seconds", math.inf)) elif args.agent_bin: manifest["model"] = args.model @@ -271,14 +281,17 @@ def record_phase(name: str, result: Any = None, **details: Any) -> None: agent = run_command([args.agent_bin, "agent", "run", "--model", args.model, prompt], cwd=workspace, stdout_path=run_dir / "agent.stdout.log", stderr_path=run_dir / "agent.stderr.log", timeout_seconds=float(task["budgets"]["wall_time_seconds"]), env=clean_env) - record_phase("agent", agent) if agent.timed_out or agent.cleanup_error: + manifest["infrastructure_status"] = "error" + record_phase("agent", agent) raise RuntimeError("agent process did not terminate cleanly") manifest["model_status"] = "pass" if agent.returncode == 0 else "fail" if agent.returncode != 0: manifest["termination"], manifest["failure_category"] = "completed", "model" + record_phase("agent", agent) manifest["hashes"]["source_final"] = _tree_hash(workspace) - return _finalize(run_dir, manifest) + return _finalize(run_dir, manifest, run_started_monotonic) + record_phase("agent", agent) else: manifest["model_status"] = "not_run" manifest["hashes"]["source_final"] = _tree_hash(workspace) @@ -289,22 +302,29 @@ def record_phase(name: str, result: Any = None, **details: Any) -> None: clean = run_command(build_command + ["--target", "clean"], cwd=workspace, stdout_path=run_dir / "clean.stdout.log", stderr_path=run_dir / "clean.stderr.log", timeout_seconds=float(task["budgets"]["wall_time_seconds"])) - record_phase("clean", clean) if clean.timed_out or clean.cleanup_error: + manifest["infrastructure_status"] = "error" + record_phase("clean", clean) raise RuntimeError("clean process did not terminate cleanly") if clean.returncode: + manifest["infrastructure_status"] = "error" + record_phase("clean", clean) raise RuntimeError("clean command failed") + record_phase("clean", clean) build = run_command(build_command, cwd=workspace, stdout_path=run_dir / "build.stdout.log", stderr_path=run_dir / "build.stderr.log", timeout_seconds=float(task["budgets"]["wall_time_seconds"])) - record_phase("build", build) if build.timed_out or build.cleanup_error: + manifest["infrastructure_status"] = "error" + record_phase("build", build) raise RuntimeError("build process did not terminate cleanly") if build.returncode: manifest["compile_status"] = "fail" manifest["hardware_status"] = "not_run" manifest["termination"], manifest["failure_category"] = "completed", "compile" - return _finalize(run_dir, manifest) + record_phase("build", build) + return _finalize(run_dir, manifest, run_started_monotonic) manifest["compile_status"] = "pass" + record_phase("build", build) artifact = firmware / config.flash_artifact if not artifact.is_file(): raise RuntimeError("successful build produced no firmware artifact") @@ -322,9 +342,11 @@ def record_phase(name: str, result: Any = None, **details: Any) -> None: with BoardLock(run_dir.parent / ".board-locks", args.jtag_serial, timeout_seconds=config.identity_timeout_seconds): identity = validate_identity(identity_command, args.jtag_serial, cwd=workspace, evidence_dir=run_dir, timeout_seconds=config.identity_timeout_seconds) - record_phase("identity", identity.command_result, status=identity.status) if identity.status != "pass": + manifest["infrastructure_status"] = "error" + record_phase("identity", identity.command_result, status=identity.status) raise RuntimeError(identity.detail or "board identity failed") + record_phase("identity", identity.command_result, status=identity.status) persist() cancel_uart = threading.Event() started_uart = threading.Event() @@ -336,27 +358,34 @@ def capture_cancellable() -> None: except BaseException as error: uart_error.append(error) uart_thread = threading.Thread(target=capture_cancellable, name="hil-uart", daemon=False) - uart_thread.start() - if not started_uart.wait(1): - cancel_uart.set() - uart_thread.join(1) - raise RuntimeError(f"UART capture failed to start: {uart_error[0] if uart_error else 'startup timeout'}") + uart_thread_started = False try: + uart_thread.start() + uart_thread_started = True + if not started_uart.wait(1): + raise RuntimeError(f"UART capture failed to start: {uart_error[0] if uart_error else 'startup timeout'}") flash_command = [args.platformio, "run", "--project-dir", str(firmware), "--environment", config.platformio_environment or "esp32s3", "--target", config.flash_target] flashed = flash_firmware(flash_command, cwd=workspace, evidence_dir=run_dir, timeout_seconds=config.flash_timeout_seconds, identity_validated=True) + if flashed.status == "infrastructure_error": + manifest["infrastructure_status"] = "error" + elif flashed.status == "hardware_fail": + manifest["hardware_status"] = "fail" record_phase("flash", flashed.command_result, status=flashed.status, category=flashed.category) if flashed.status == "pass": uart_thread.join(config.uart_timeout_seconds + 0.5) finally: cancel_uart.set() - uart_thread.join(1) + if uart_thread_started: + uart_thread.join(1) if uart_thread.is_alive() or uart_error: raise RuntimeError(f"UART capture failed: {uart_error[0] if uart_error else 'cleanup timeout'}") uart = uart_result["value"] manifest["uart"] = {"matched": uart.matched, "termination": uart.termination_reason, "bytes_captured": uart.bytes_captured} + if not uart.matched and flashed.status != "infrastructure_error": + manifest["hardware_status"] = "fail" record_phase("uart", matched=uart.matched, termination=uart.termination_reason, bytes_captured=uart.bytes_captured) if flashed.status == "infrastructure_error": @@ -364,11 +393,12 @@ def capture_cancellable() -> None: if flashed.status == "hardware_fail" or not uart.matched: manifest["hardware_status"] = "fail" manifest["termination"], manifest["failure_category"] = "completed", flashed.category or "uart_nonce" - return _finalize(run_dir, manifest) + return _finalize(run_dir, manifest, run_started_monotonic) registers = read_registers(args.openocd, config.openocd_board_config, args.jtag_serial, config.assertions, cwd=workspace, evidence_dir=run_dir, timeout_seconds=config.openocd_command_timeout_seconds) if registers.status == "infrastructure_error": + manifest["infrastructure_status"] = "error" record_phase("register", registers.command_result, status=registers.status) raise RuntimeError(registers.detail or "OpenOCD infrastructure failure") manifest["register_assertions"] = [ @@ -376,29 +406,33 @@ def capture_cancellable() -> None: for item in registers.evaluation.observations ] manifest["hardware_status"] = "pass" if registers.status == "pass" else "fail" - record_phase("register", registers.command_result, status=registers.status, - assertions=manifest["register_assertions"]) manifest["termination"] = "completed" manifest["failure_category"] = None if registers.status == "pass" else "register_mismatch" - return _finalize(run_dir, manifest) + record_phase("register", registers.command_result, status=registers.status, + assertions=manifest["register_assertions"]) + return _finalize(run_dir, manifest, run_started_monotonic) except (KeyboardInterrupt, SystemExit) as error: manifest["infrastructure_status"] = "error" manifest["termination"] = "interrupted" manifest["failure_category"] = "interrupt" manifest["detail"] = type(error).__name__ - _finalize(run_dir, manifest) + _finalize(run_dir, manifest, run_started_monotonic) return 2 except (OSError, ValueError, TypeError, KeyError, json.JSONDecodeError, RuntimeError, BoardLockTimeout) as error: manifest["infrastructure_status"] = "error" manifest["termination"] = "invalid" manifest["failure_category"] = "infrastructure" manifest["detail"] = str(error) - _finalize(run_dir, manifest) + _finalize(run_dir, manifest, run_started_monotonic) print(str(error), file=sys.stderr) return 2 -def _finalize(run_dir: Path, manifest: dict[str, Any]) -> int: +def _finalize(run_dir: Path, manifest: dict[str, Any], started_monotonic: float) -> int: + elapsed = time.monotonic() - started_monotonic + wall = manifest.get("budget_validity", {}).get("wall_time_seconds") + if wall is not None: + wall.update(observed=elapsed, within_budget=elapsed <= wall["configured"]) result = {name: manifest.get(name) for name in ( "model_status", "compile_status", "simulator_status", "hardware_status", "infrastructure_status", "termination", "failure_category", "uart", "register_assertions")} diff --git a/tests/twin2silicon-hil.py b/tests/twin2silicon-hil.py index e99947f..814b248 100644 --- a/tests/twin2silicon-hil.py +++ b/tests/twin2silicon-hil.py @@ -198,6 +198,35 @@ def test_run_command_interrupt_terminates_process_group_before_reraising(self): try: os.kill(int(pid_path.read_text()), signal.SIGKILL) except ProcessLookupError: pass + def test_run_command_interrupt_kills_sigterm_ignoring_descendant_after_leader_exits(self): + with tempfile.TemporaryDirectory() as directory: + evidence = Path(directory); child_path = evidence / "child" + script = textwrap.dedent(f""" + import pathlib, signal, subprocess, sys, time + child = ''' + import os, pathlib, signal, time + signal.signal(signal.SIGTERM, signal.SIG_IGN) + pathlib.Path({str(child_path)!r}).write_text(str(os.getpid())) + while True: time.sleep(1) + ''' + subprocess.Popen([sys.executable, '-c', child]) + while not pathlib.Path({str(child_path)!r}).exists(): pass + signal.signal(signal.SIGTERM, lambda *_: (_ for _ in ()).throw(SystemExit(0))) + while True: time.sleep(1) + """) + timer = threading.Timer(.15, lambda: os.kill(os.getpid(), signal.SIGINT)); timer.start() + try: + with self.assertRaises(KeyboardInterrupt): + run_command([sys.executable, "-c", script], cwd=evidence, + stdout_path=evidence / "o", stderr_path=evidence / "e", timeout_seconds=30) + with self.assertRaises(ProcessLookupError): + os.kill(int(child_path.read_text()), 0) + finally: + timer.cancel() + if child_path.exists(): + try: os.kill(int(child_path.read_text()), signal.SIGKILL) + except ProcessLookupError: pass + def test_run_command_timeout_captures_evidence_and_is_bounded(self): with tempfile.TemporaryDirectory() as directory: evidence = Path(directory) @@ -907,6 +936,20 @@ def test_usage_schema_rejects_missing_extra_and_legacy_price_date(self): with self.assertRaises(ValueError): run_hil.parse_usage(path) + def test_finalizer_uses_total_monotonic_wall_time_not_provider_latency(self): + sys.path.insert(0, str(REPOSITORY_ROOT / "benchmarks/twin2silicon")) + import run_hil + with tempfile.TemporaryDirectory() as directory: + manifest = {"model_status": "not_run", "compile_status": "not_run", + "simulator_status": "not_supported", "hardware_status": "not_run", + "infrastructure_status": "ok", "termination": "completed", "failure_category": None, + "uart": None, "register_assertions": [], "hashes": {}, "run": {}, "latency_seconds": .001, + "budget_validity": {"wall_time_seconds": {"configured": .01, "observed": None, "within_budget": None}}} + run_hil._finalize(Path(directory), manifest, time.monotonic() - .05) + wall = manifest["budget_validity"]["wall_time_seconds"] + self.assertGreaterEqual(wall["observed"], .05) + self.assertFalse(wall["within_budget"]) + def test_candidate_symlink_escape_is_rejected_before_commands(self): task = REPOSITORY_ROOT / "benchmarks/twin2silicon/tasks/esp32s3-gpio-hil-001" with tempfile.TemporaryDirectory() as directory: @@ -946,10 +989,16 @@ def test_agent_mode_repairs_workspace_with_allowlisted_home_and_no_secret_eviden pio = executable_fixture(pio_dir, "import sys\nif '--version' in sys.argv: print('pio fixture 1'); raise SystemExit(0)\nraise SystemExit(0 if 'clean' in sys.argv else 1)\n") env = {**os.environ, "HOME": "HOME_SENTINEL", "LABWIRED_HOME": "LABWIRED_HOME_SENTINEL", "LABWIRED_ACCESS_TOKEN": "SECRET_TOKEN_SENTINEL"} + usage = root / "usage.json" + usage.write_text(json.dumps({"schema_version": "1.0", "requests": 1, + "tokens": {"fresh_input": 1000, "cached_input": 2000, "output": 3000, "reasoning": 4}, + "final_context_tokens": 55, "latency_seconds": .01, "provider": "fixture", "model": "fixture/model", + "rates_usd_per_million": {"fresh_input": 1, "cached_input": 1, "output": 1}, + "price_source": "fixture", "price_effective_date": "2026-01-01"})) run_dir = root / "run" result = self._run_cli(task, "--run-dir", run_dir, "--agent-bin", agent, "--model", "fixture/model", "--jtag-serial", "J", "--uart-device", "/dev/null", "--openocd", pio, - "--platformio", pio, env=env) + "--platformio", pio, "--usage-json", usage, env=env) self.assertEqual(result.returncode, 0, result.stderr) call = json.loads(invocation.read_text()) self.assertEqual(call["argv"][:4], ["agent", "run", "--model", "fixture/model"]) @@ -958,6 +1007,7 @@ def test_agent_mode_repairs_workspace_with_allowlisted_home_and_no_secret_eviden self.assertNotIn("SECRET_TOKEN_SENTINEL", manifest_text) manifest = json.loads(manifest_text) self.assertEqual(manifest["model_status"], "pass") + self.assertEqual(manifest["budget_validity"]["model_tokens"]["observed"], 55) self.assertNotEqual(manifest["hashes"]["source_initial"], manifest["hashes"]["source_final"]) def test_cli_physical_pass_uses_pty_and_records_ordered_evidence(self): @@ -1009,6 +1059,11 @@ def writer(): manifest = json.loads((run_dir / "run.json").read_text()) self.assertEqual(manifest["hardware_status"], "pass") self.assertEqual(order.read_text().splitlines(), ["clean", "build", "identity", "upload", "openocd"]) + self.assertNotIn("JTAG-1", json.dumps(manifest)) + self.assertEqual(set(manifest["budget_validity"]), + {"wall_time_seconds", "model_tokens", "repair_iterations", "simulator_runs", "diagnostic_hil_runs"}) + self.assertEqual(manifest["budget_validity"]["simulator_runs"]["observed"], 0) + self.assertEqual(manifest["budget_validity"]["diagnostic_hil_runs"]["observed"], 0) self.assertEqual([phase["name"] for phase in manifest["phases"]], ["prepared", "clean", "build", "identity", "flash", "uart", "register"]) self.assertEqual(set(manifest["tool_versions"]), {"platformio", "openocd"}) From bed71ba545774e301edbb09535d2fb82f5f3b7c8 Mon Sep 17 00:00:00 2001 From: Andrii Shylenko <14119286+w1ne@users.noreply.github.com> Date: Sat, 15 Aug 2026 13:41:09 +0200 Subject: [PATCH 20/48] fix(bench): isolate UART capture lifecycle --- benchmarks/twin2silicon/run_hil.py | 116 ++++++++++++++++++++--------- tests/twin2silicon-hil.py | 76 ++++++++++++++++++- 2 files changed, 156 insertions(+), 36 deletions(-) diff --git a/benchmarks/twin2silicon/run_hil.py b/benchmarks/twin2silicon/run_hil.py index 38f05d2..5682a13 100644 --- a/benchmarks/twin2silicon/run_hil.py +++ b/benchmarks/twin2silicon/run_hil.py @@ -13,13 +13,14 @@ import hashlib import json import math +import multiprocessing import os from pathlib import Path import secrets import shutil import sys -import threading import time +import queue from typing import Any, Mapping if __package__ in (None, ""): @@ -38,6 +39,38 @@ HARNESS_REVISION = "twin2silicon-hil-1" +class UartWorkerFatal(BaseException): + pass + + +def _uart_process_worker(result_queue: Any, ready_event: Any, cancel_event: Any, + device: str, baud: int, nonce: str, timeout_seconds: float, + log_path: str, mode: str) -> None: + if mode == "startup-hang": + while True: + time.sleep(1) + try: + result = capture_uart_nonce(device, baud, nonce, timeout_seconds, log_path, + cancel_event=cancel_event, started_event=ready_event) + result_queue.put(("result", {"matched": result.matched, "bytes_captured": result.bytes_captured, + "timed_out": result.timed_out, "termination_reason": result.termination_reason})) + except BaseException as error: + result_queue.put(("error", f"{type(error).__name__}: {error}")) + ready_event.set() + + +def _stop_uart_process(process: Any, cancel_event: Any) -> bool: + cancel_event.set() + process.join(.25) + if process.is_alive(): + process.terminate() + process.join(.5) + if process.is_alive(): + process.kill() + process.join(.5) + return not process.is_alive() + + def _number(value: object, field: str, *, integer: bool = False) -> int | float: valid = isinstance(value, int if integer else (int, float)) and not isinstance(value, bool) finite = integer or (valid and math.isfinite(value)) @@ -176,6 +209,8 @@ def parser() -> argparse.ArgumentParser: help="offline fixture only: shorten (never extend) the oracle UART timeout") value.add_argument("--fixture-identity-timeout-seconds", type=float, help="offline fixture only: shorten (never extend) identity/lock timeout") + value.add_argument("--fixture-uart-worker-mode", choices=("normal", "startup-hang"), default="normal", + help="offline fixture only: exercise UART worker startup cleanup") return value @@ -337,8 +372,6 @@ def record_phase(name: str, result: Any = None, **details: Any) -> None: raise ValueError("identity command JSON must be a nonempty string array") else: identity_command = list(config.identity_command) - uart_result: dict[str, Any] = {} - uart_error: list[BaseException] = [] with BoardLock(run_dir.parent / ".board-locks", args.jtag_serial, timeout_seconds=config.identity_timeout_seconds): identity = validate_identity(identity_command, args.jtag_serial, cwd=workspace, evidence_dir=run_dir, timeout_seconds=config.identity_timeout_seconds) @@ -348,22 +381,19 @@ def record_phase(name: str, result: Any = None, **details: Any) -> None: raise RuntimeError(identity.detail or "board identity failed") record_phase("identity", identity.command_result, status=identity.status) persist() - cancel_uart = threading.Event() - started_uart = threading.Event() - def capture_cancellable() -> None: - try: - uart_result["value"] = capture_uart_nonce(args.uart_device, config.uart_baud, nonce, - config.uart_timeout_seconds, run_dir / "uart.raw.log", - cancel_event=cancel_uart, started_event=started_uart) - except BaseException as error: - uart_error.append(error) - uart_thread = threading.Thread(target=capture_cancellable, name="hil-uart", daemon=False) - uart_thread_started = False + context = multiprocessing.get_context("spawn") + cancel_uart = context.Event() + started_uart = context.Event() + uart_queue = context.Queue() + uart_process = context.Process(target=_uart_process_worker, name="hil-uart", + args=(uart_queue, started_uart, cancel_uart, args.uart_device, config.uart_baud, nonce, + config.uart_timeout_seconds, str(run_dir / "uart.raw.log"), args.fixture_uart_worker_mode)) + uart_process_started = False try: - uart_thread.start() - uart_thread_started = True - if not started_uart.wait(1): - raise RuntimeError(f"UART capture failed to start: {uart_error[0] if uart_error else 'startup timeout'}") + uart_process.start() + uart_process_started = True + if not started_uart.wait(2): + raise RuntimeError("UART capture failed to start: startup timeout") flash_command = [args.platformio, "run", "--project-dir", str(firmware), "--environment", config.platformio_environment or "esp32s3", "--target", config.flash_target] flashed = flash_firmware(flash_command, cwd=workspace, evidence_dir=run_dir, @@ -374,23 +404,32 @@ def capture_cancellable() -> None: manifest["hardware_status"] = "fail" record_phase("flash", flashed.command_result, status=flashed.status, category=flashed.category) if flashed.status == "pass": - uart_thread.join(config.uart_timeout_seconds + 0.5) + uart_process.join(config.uart_timeout_seconds + 0.5) finally: - cancel_uart.set() - if uart_thread_started: - uart_thread.join(1) - if uart_thread.is_alive() or uart_error: - raise RuntimeError(f"UART capture failed: {uart_error[0] if uart_error else 'cleanup timeout'}") - uart = uart_result["value"] - manifest["uart"] = {"matched": uart.matched, "termination": uart.termination_reason, - "bytes_captured": uart.bytes_captured} - if not uart.matched and flashed.status != "infrastructure_error": + uart_stopped = not uart_process_started or _stop_uart_process(uart_process, cancel_uart) + if sys.exc_info()[0] is not None: + uart_queue.close() + uart_queue.join_thread() + if not uart_stopped: + raise UartWorkerFatal("UART worker could not be stopped") + try: + uart_kind, uart_payload = uart_queue.get(timeout=.2) + except queue.Empty: + raise RuntimeError("UART capture produced no result") + finally: + uart_queue.close() + uart_queue.join_thread() + if uart_kind == "error": + raise RuntimeError(f"UART capture failed: {uart_payload}") + manifest["uart"] = {"matched": uart_payload["matched"], "termination": uart_payload["termination_reason"], + "bytes_captured": uart_payload["bytes_captured"]} + if not uart_payload["matched"] and flashed.status != "infrastructure_error": manifest["hardware_status"] = "fail" - record_phase("uart", matched=uart.matched, termination=uart.termination_reason, - bytes_captured=uart.bytes_captured) + record_phase("uart", matched=uart_payload["matched"], termination=uart_payload["termination_reason"], + bytes_captured=uart_payload["bytes_captured"]) if flashed.status == "infrastructure_error": raise RuntimeError(flashed.detail or "flash infrastructure failure") - if flashed.status == "hardware_fail" or not uart.matched: + if flashed.status == "hardware_fail" or not uart_payload["matched"]: manifest["hardware_status"] = "fail" manifest["termination"], manifest["failure_category"] = "completed", flashed.category or "uart_nonce" return _finalize(run_dir, manifest, run_started_monotonic) @@ -411,6 +450,13 @@ def capture_cancellable() -> None: record_phase("register", registers.command_result, status=registers.status, assertions=manifest["register_assertions"]) return _finalize(run_dir, manifest, run_started_monotonic) + except UartWorkerFatal as error: + manifest["infrastructure_status"] = "error" + manifest["termination"] = "invalid" + manifest["failure_category"] = "uart_cleanup" + manifest["detail"] = str(error) + persist() + return 2 except (KeyboardInterrupt, SystemExit) as error: manifest["infrastructure_status"] = "error" manifest["termination"] = "interrupted" @@ -429,10 +475,6 @@ def capture_cancellable() -> None: def _finalize(run_dir: Path, manifest: dict[str, Any], started_monotonic: float) -> int: - elapsed = time.monotonic() - started_monotonic - wall = manifest.get("budget_validity", {}).get("wall_time_seconds") - if wall is not None: - wall.update(observed=elapsed, within_budget=elapsed <= wall["configured"]) result = {name: manifest.get(name) for name in ( "model_status", "compile_status", "simulator_status", "hardware_status", "infrastructure_status", "termination", "failure_category", "uart", "register_assertions")} @@ -445,6 +487,10 @@ def _finalize(run_dir: Path, manifest: dict[str, Any], started_monotonic: float) hashes[f"artifact:{relative}"] = sha256_file(path) manifest["artifacts"] = artifacts manifest["run"]["ended_at_utc"] = datetime.now(timezone.utc).isoformat().replace("+00:00", "Z") + elapsed = time.monotonic() - started_monotonic + wall = manifest.get("budget_validity", {}).get("wall_time_seconds") + if wall is not None: + wall.update(observed=elapsed, within_budget=elapsed <= wall["configured"]) write_json_atomic(run_dir / "run.json", manifest) return 2 if manifest["infrastructure_status"] == "error" else 0 diff --git a/tests/twin2silicon-hil.py b/tests/twin2silicon-hil.py index 814b248..918be65 100644 --- a/tests/twin2silicon-hil.py +++ b/tests/twin2silicon-hil.py @@ -940,12 +940,18 @@ def test_finalizer_uses_total_monotonic_wall_time_not_provider_latency(self): sys.path.insert(0, str(REPOSITORY_ROOT / "benchmarks/twin2silicon")) import run_hil with tempfile.TemporaryDirectory() as directory: + artifact = Path(directory) / "slow.log"; artifact.write_text("evidence") manifest = {"model_status": "not_run", "compile_status": "not_run", "simulator_status": "not_supported", "hardware_status": "not_run", "infrastructure_status": "ok", "termination": "completed", "failure_category": None, "uart": None, "register_assertions": [], "hashes": {}, "run": {}, "latency_seconds": .001, "budget_validity": {"wall_time_seconds": {"configured": .01, "observed": None, "within_budget": None}}} - run_hil._finalize(Path(directory), manifest, time.monotonic() - .05) + real_hash = run_hil.sha256_file + def slow_hash(path): + time.sleep(.05) + return real_hash(path) + with mock.patch.object(run_hil, "sha256_file", side_effect=slow_hash): + run_hil._finalize(Path(directory), manifest, time.monotonic()) wall = manifest["budget_validity"]["wall_time_seconds"] self.assertGreaterEqual(wall["observed"], .05) self.assertFalse(wall["within_budget"]) @@ -1094,6 +1100,74 @@ def test_clean_nonzero_is_infrastructure_and_never_touches_identity(self): self.assertEqual((manifest["compile_status"], manifest["infrastructure_status"]), ("not_run", "error")) self.assertEqual([phase["name"] for phase in manifest["phases"]], ["prepared", "clean"]) + def test_uart_startup_hang_is_killed_before_bounded_finalization(self): + task = REPOSITORY_ROOT / "benchmarks/twin2silicon/tasks/esp32s3-gpio-hil-001" + with tempfile.TemporaryDirectory() as directory: + root = Path(directory); pio_dir = root / "pio"; pio_dir.mkdir() + pio = executable_fixture(pio_dir, """import pathlib,sys +if '--version' in sys.argv: print('v'); raise SystemExit(0) +if '--target' not in sys.argv: + p=pathlib.Path(sys.argv[sys.argv.index('--project-dir')+1])/'.pio/build/esp32s3/firmware.bin'; p.parent.mkdir(parents=True); p.write_bytes(b'fw') +""") + identity_dir = root / "id"; identity_dir.mkdir() + identity = executable_fixture(identity_dir, "print('JTAG-HANG')\n") + run_dir = root / "run"; started = time.monotonic() + result = self._run_cli(task, "--run-dir", run_dir, "--evaluate-only", "--candidate", task / "public", + "--jtag-serial", "JTAG-HANG", "--uart-device", "/dev/null", "--openocd", pio, + "--platformio", pio, "--identity-command-json", json.dumps([str(identity)]), + "--fixture-uart-worker-mode", "startup-hang", "--fixture-identity-timeout-seconds", ".3") + self.assertEqual(result.returncode, 2) + self.assertLess(time.monotonic() - started, 4) + manifest_path = run_dir / "run.json"; manifest = json.loads(manifest_path.read_text()) + self.assertEqual(manifest["infrastructure_status"], "error") + before = {path: path.stat().st_mtime_ns for path in run_dir.rglob("*") if path.is_file()} + time.sleep(.2) + self.assertEqual(before, {path: path.stat().st_mtime_ns for path in run_dir.rglob("*") if path.is_file()}) + with BoardLock(run_dir.parent / ".board-locks", "JTAG-HANG", timeout_seconds=.1): pass + + def test_cli_sigint_cleans_uart_flash_descendants_and_finalizes_stable_evidence(self): + task = REPOSITORY_ROOT / "benchmarks/twin2silicon/tasks/esp32s3-gpio-hil-001" + with tempfile.TemporaryDirectory() as directory: + root = Path(directory); marker = root / "flash-pids.json" + pio_dir = root / "pio"; pio_dir.mkdir() + pio = executable_fixture(pio_dir, textwrap.dedent(f""" + import json, os, pathlib, signal, subprocess, sys, time + if '--version' in sys.argv: print('v'); raise SystemExit(0) + project = pathlib.Path(sys.argv[sys.argv.index('--project-dir') + 1]) + target = sys.argv[sys.argv.index('--target') + 1] if '--target' in sys.argv else 'build' + if target == 'build': + p=project/'.pio/build/esp32s3/firmware.bin'; p.parent.mkdir(parents=True); p.write_bytes(b'fw') + if target == 'upload': + child=subprocess.Popen([sys.executable, '-c', 'import signal,time; signal.signal(signal.SIGTERM, signal.SIG_IGN); time.sleep(30)']) + pathlib.Path({str(marker)!r}).write_text(json.dumps({{'leader': os.getpid(), 'child': child.pid}})) + time.sleep(30) + """)) + identity_dir = root / "id"; identity_dir.mkdir() + identity = executable_fixture(identity_dir, "print('JTAG-INT')\n") + master, slave = pty.openpty(); tty.setraw(slave); run_dir = root / "run" + command = [sys.executable, str(REPOSITORY_ROOT / "benchmarks/twin2silicon/run_hil.py"), str(task), + "--run-dir", str(run_dir), "--evaluate-only", "--candidate", str(task / "public"), + "--jtag-serial", "JTAG-INT", "--uart-device", os.ttyname(slave), "--openocd", str(pio), + "--platformio", str(pio), "--identity-command-json", json.dumps([str(identity)])] + process = subprocess.Popen(command, cwd=REPOSITORY_ROOT, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + deadline = time.monotonic() + 8 + while not marker.exists() and time.monotonic() < deadline: time.sleep(.01) + self.assertTrue(marker.exists(), "flash did not become active") + process.send_signal(signal.SIGINT) + stdout, stderr = process.communicate(timeout=6) + os.close(master); os.close(slave) + self.assertEqual(process.returncode, 2, stderr.decode()) + manifest = json.loads((run_dir / "run.json").read_text()) + self.assertEqual(manifest["termination"], "interrupted") + for pid in json.loads(marker.read_text()).values(): + with self.assertRaises(ProcessLookupError): os.kill(pid, 0) + with BoardLock(run_dir.parent / ".board-locks", "JTAG-INT", timeout_seconds=.1): pass + before = {p: (p.stat().st_mtime_ns, sha256_file(p)) for p in run_dir.rglob('*') if p.is_file()} + time.sleep(.2) + self.assertEqual(before, {p: (p.stat().st_mtime_ns, sha256_file(p)) for p in run_dir.rglob('*') if p.is_file()}) + for artifact in manifest["artifacts"]: + self.assertEqual(manifest["hashes"][f"artifact:{artifact}"], sha256_file(run_dir / artifact)) + def test_cli_classifies_candidate_and_infrastructure_physical_failures(self): task = self._short_task() cases = ( From 448229c793149edcd0d04c59970a565a30f219af Mon Sep 17 00:00:00 2001 From: Andrii Shylenko <14119286+w1ne@users.noreply.github.com> Date: Sat, 15 Aug 2026 13:53:56 +0200 Subject: [PATCH 21/48] fix(bench): secure HIL process and evidence boundaries --- benchmarks/twin2silicon/hil/process.py | 45 ++++ benchmarks/twin2silicon/run_hil.py | 279 ++++++++++++++++++------- tests/twin2silicon-hil.py | 113 +++++++++- 3 files changed, 363 insertions(+), 74 deletions(-) diff --git a/benchmarks/twin2silicon/hil/process.py b/benchmarks/twin2silicon/hil/process.py index acff61b..c842224 100644 --- a/benchmarks/twin2silicon/hil/process.py +++ b/benchmarks/twin2silicon/hil/process.py @@ -5,6 +5,7 @@ from pathlib import Path import signal import subprocess +import stat import time from typing import Mapping, Optional, Sequence, Union @@ -43,6 +44,7 @@ def run_command( stderr_path: PathLike, timeout_seconds: float, env: Optional[Mapping[str, str]] = None, + redact_values: Sequence[Union[str, bytes]] = (), ) -> CommandResult: normalized_command = tuple(os.fspath(part) for part in command) normalized_cwd = str(Path(cwd).resolve()) @@ -85,6 +87,7 @@ def run_command( except subprocess.TimeoutExpired: pass _wait_for_process_group_exit(process.pid, 1.0) + _redact_logs((normalized_stdout, normalized_stderr), redact_values) raise interruption except subprocess.TimeoutExpired: timed_out = True @@ -111,7 +114,22 @@ def run_command( group_exited = _wait_for_process_group_exit(process.pid, 0.5) if not group_exited and cleanup_error is None: cleanup_error = "process_group_did_not_exit" + else: + if _process_group_exists(process.pid): + cleanup_error = "unexpected_descendant_processes" + try: + os.killpg(process.pid, signal.SIGTERM) + except (PermissionError, ProcessLookupError): + pass + if not _wait_for_process_group_exit(process.pid, 0.5): + try: + os.killpg(process.pid, signal.SIGKILL) + except (PermissionError, ProcessLookupError): + pass + if not _wait_for_process_group_exit(process.pid, 1.0): + cleanup_error = "process_group_did_not_exit" + _redact_logs((normalized_stdout, normalized_stderr), redact_values) ended_at = _utc_now() return CommandResult( command=normalized_command, @@ -125,3 +143,30 @@ def run_command( stderr_path=normalized_stderr, cleanup_error=cleanup_error, ) + + +def _redact_logs(paths: Sequence[str], values: Sequence[Union[str, bytes]]) -> None: + needles = tuple(value.encode() if isinstance(value, str) else value for value in values if value) + if not needles: + return + for path in paths: + try: + fd = os.open(path, os.O_RDWR | getattr(os, "O_NOFOLLOW", 0)) + try: + info = os.fstat(fd) + if not stat.S_ISREG(info.st_mode): + raise OSError("evidence log is not a regular file") + contents = bytearray() + while chunk := os.read(fd, 1024 * 1024): + contents.extend(chunk) + redacted = contents + for needle in needles: + redacted = redacted.replace(needle, b"[REDACTED]") + if redacted != contents: + os.lseek(fd, 0, os.SEEK_SET) + os.ftruncate(fd, 0) + os.write(fd, redacted) + finally: + os.close(fd) + except FileNotFoundError: + pass diff --git a/benchmarks/twin2silicon/run_hil.py b/benchmarks/twin2silicon/run_hil.py index 5682a13..38b8bfd 100644 --- a/benchmarks/twin2silicon/run_hil.py +++ b/benchmarks/twin2silicon/run_hil.py @@ -13,14 +13,16 @@ import hashlib import json import math -import multiprocessing import os from pathlib import Path +import stat import secrets -import shutil import sys import time -import queue +import signal +import subprocess +import threading +from functools import partial from typing import Any, Mapping if __package__ in (None, ""): @@ -43,32 +45,47 @@ class UartWorkerFatal(BaseException): pass -def _uart_process_worker(result_queue: Any, ready_event: Any, cancel_event: Any, - device: str, baud: int, nonce: str, timeout_seconds: float, - log_path: str, mode: str) -> None: - if mode == "startup-hang": +class _FileEvent: + def __init__(self, path: Path) -> None: + self.path = path + + def set(self) -> None: + self.path.write_text("ready", encoding="utf-8") + + +def _uart_subprocess_worker(spec_path: Path) -> int: + spec = json.loads(spec_path.read_text(encoding="utf-8")) + if spec["mode"] == "startup-hang": + signal.signal(signal.SIGTERM, signal.SIG_IGN) while True: time.sleep(1) + cancel_event = threading.Event() + signal.signal(signal.SIGTERM, lambda *_: cancel_event.set()) try: - result = capture_uart_nonce(device, baud, nonce, timeout_seconds, log_path, - cancel_event=cancel_event, started_event=ready_event) - result_queue.put(("result", {"matched": result.matched, "bytes_captured": result.bytes_captured, - "timed_out": result.timed_out, "termination_reason": result.termination_reason})) + result = capture_uart_nonce(spec["device"], spec["baud"], spec["nonce"], spec["timeout_seconds"], + spec["log_path"], cancel_event=cancel_event, + started_event=_FileEvent(Path(spec["ready_path"]))) + write_json_atomic(spec["result_path"], {"kind": "result", "matched": result.matched, + "bytes_captured": result.bytes_captured, "timed_out": result.timed_out, + "termination_reason": result.termination_reason}) except BaseException as error: - result_queue.put(("error", f"{type(error).__name__}: {error}")) - ready_event.set() + write_json_atomic(spec["result_path"], {"kind": "error", "detail": f"{type(error).__name__}: {error}"}) + Path(spec["ready_path"]).write_text("error", encoding="utf-8") + return 0 -def _stop_uart_process(process: Any, cancel_event: Any) -> bool: - cancel_event.set() - process.join(.25) - if process.is_alive(): - process.terminate() - process.join(.5) - if process.is_alive(): - process.kill() - process.join(.5) - return not process.is_alive() +def _stop_uart_process(process: subprocess.Popen[Any]) -> bool: + if process.poll() is None: + try: os.killpg(process.pid, signal.SIGTERM) + except ProcessLookupError: pass + try: process.wait(timeout=.5) + except subprocess.TimeoutExpired: pass + if process.poll() is None: + try: os.killpg(process.pid, signal.SIGKILL) + except ProcessLookupError: pass + try: process.wait(timeout=.5) + except subprocess.TimeoutExpired: pass + return process.poll() is not None def _number(value: object, field: str, *, integer: bool = False) -> int | float: @@ -115,25 +132,58 @@ def parse_usage(path: Path) -> dict[str, Any]: def _tree_hash(root: Path) -> str: digest = hashlib.sha256() - for path in sorted(item for item in root.rglob("*") if item.is_file()): + for path in _safe_files(root): digest.update(path.relative_to(root).as_posix().encode()) digest.update(b"\0") - with path.open("rb") as source: - for chunk in iter(lambda: source.read(1024 * 1024), b""): - digest.update(chunk) + digest.update(bytes.fromhex(_safe_hash(path))) return digest.hexdigest() def _safe_tree(source: Path, field: str) -> Path: + if stat.S_ISLNK(os.lstat(source).st_mode): + raise ValueError(f"{field} root is a symlink") source = source.resolve(strict=True) if not source.is_dir(): raise ValueError(f"{field} is not a directory") - for item in source.rglob("*"): - if item.is_symlink(): - raise ValueError(f"{field} contains a symlink: {item.relative_to(source)}") + _safe_files(source) return source +def _safe_files(root: Path) -> list[Path]: + root = root.absolute() + files: list[Path] = [] + def visit(directory: Path) -> None: + with os.scandir(directory) as entries: + for entry in entries: + info = entry.stat(follow_symlinks=False) + path = directory / entry.name + if stat.S_ISLNK(info.st_mode): + raise ValueError(f"unsafe symlink: {path.relative_to(root)}") + if stat.S_ISDIR(info.st_mode): + visit(path) + elif stat.S_ISREG(info.st_mode): + files.append(path) + else: + raise ValueError(f"unsafe non-regular file: {path.relative_to(root)}") + visit(root) + return sorted(files) + + +def _safe_hash(path: Path) -> str: + flags = os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0) + fd = os.open(path, flags) + try: + before = os.lstat(path); opened = os.fstat(fd) + if not stat.S_ISREG(opened.st_mode) or (before.st_dev, before.st_ino) != (opened.st_dev, opened.st_ino): + raise ValueError(f"unsafe changed artifact: {path.name}") + digest = hashlib.sha256() + while chunk := os.read(fd, 1024 * 1024): + digest.update(chunk) + return digest.hexdigest() + finally: + os.close(fd) + + def _resolve_task(value: str) -> Path: supplied = Path(value) candidate = supplied if supplied.is_absolute() or len(supplied.parts) > 1 else TASKS / supplied @@ -146,8 +196,21 @@ def _resolve_task(value: str) -> Path: def _copy_public(source: Path, destination: Path) -> None: - _safe_tree(source, "candidate/public workspace") - shutil.copytree(source, destination, symlinks=False) + source = _safe_tree(source, "candidate/public workspace") + destination.mkdir() + for path in _safe_files(source): + target = destination / path.relative_to(source) + target.parent.mkdir(parents=True, exist_ok=True) + source_fd = os.open(path, os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0)) + try: + target_fd = os.open(target, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600) + try: + while chunk := os.read(source_fd, 1024 * 1024): + os.write(target_fd, chunk) + finally: + os.close(target_fd) + finally: + os.close(source_fd) def _command_record(result: Any, run_dir: Path, redactions: tuple[str, ...] = ()) -> dict[str, Any]: @@ -170,12 +233,14 @@ def relative(value: str) -> str: def _tool_version(name: str, executable: os.PathLike[str] | str, workspace: Path, - run_dir: Path, env: Mapping[str, str] | None = None) -> dict[str, str]: + run_dir: Path, env: Mapping[str, str] | None = None, + redactions: tuple[str, ...] = (), timeout_seconds: float = 2) -> dict[str, str]: record = {"executable": Path(executable).name, "version": "not_available"} try: result = run_command([executable, "--version"], cwd=workspace, stdout_path=run_dir / f"version.{name}.stdout.log", - stderr_path=run_dir / f"version.{name}.stderr.log", timeout_seconds=2, env=env) + stderr_path=run_dir / f"version.{name}.stderr.log", timeout_seconds=timeout_seconds, env=env, + redact_values=redactions) if result.returncode == 0 and not result.timed_out and not result.cleanup_error: text = Path(result.stdout_path).read_text(encoding="utf-8", errors="replace").strip() record["version"] = " ".join(text.split())[:200] or "unknown" @@ -190,6 +255,26 @@ def _agent_environment() -> dict[str, str]: return {name: os.environ[name] for name in allowed if name in os.environ} +def _tool_environment() -> dict[str, str]: + allowed = ("PATH", "HOME", "PLATFORMIO_CORE_DIR", "TMPDIR", "LANG", "LC_ALL") + return {name: os.environ[name] for name in allowed if name in os.environ} + + +def _reject_secret_files(root: Path, secret_values: tuple[str, ...]) -> None: + needles = tuple(value.encode() for value in secret_values if value) + for path in _safe_files(root): + flags = os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0) + fd = os.open(path, flags) + try: + contents = bytearray() + while chunk := os.read(fd, 1024 * 1024): + contents.extend(chunk) + if any(needle in contents for needle in needles): + raise ValueError(f"credential value found in workspace artifact: {path.relative_to(root)}") + finally: + os.close(fd) + + def parser() -> argparse.ArgumentParser: value = argparse.ArgumentParser(description=__doc__) value.add_argument("task", help="task id, or path beneath benchmarks/twin2silicon/tasks") @@ -211,6 +296,8 @@ def parser() -> argparse.ArgumentParser: help="offline fixture only: shorten (never extend) identity/lock timeout") value.add_argument("--fixture-uart-worker-mode", choices=("normal", "startup-hang"), default="normal", help="offline fixture only: exercise UART worker startup cleanup") + value.add_argument("--fixture-wall-time-seconds", type=float, + help="offline fixture only: shorten (never extend) overall wall budget") return value @@ -238,8 +325,20 @@ def main(argv: list[str] | None = None) -> int: } def persist() -> None: write_json_atomic(run_dir / "run.json", manifest) + overall_deadline: float | None = None + def bounded_timeout(cap: float) -> float: + if overall_deadline is None: + return cap + remaining = overall_deadline - time.monotonic() + if remaining <= 0: + raise RuntimeError("overall wall-time budget exhausted") + return min(cap, remaining) redactions = tuple(filter(None, (args.jtag_serial, args.uart_device, *(os.environ.get(name, "") for name in ("LABWIRED_ACCESS_TOKEN", "LABWIRED_MODEL_KEY"))))) + credential_values = tuple(filter(None, (os.environ.get(name, "") for name in ( + "LABWIRED_ACCESS_TOKEN", "LABWIRED_PROJECT", "LABWIRED_MODEL_URL", "LABWIRED_MODEL_KEY")))) + tool_env = _tool_environment() + tool_runner = partial(run_command, env=tool_env, redact_values=credential_values) def record_phase(name: str, result: Any = None, **details: Any) -> None: phase = {"name": name, **details} if result is not None: @@ -250,6 +349,14 @@ def record_phase(name: str, result: Any = None, **details: Any) -> None: try: task_root = _resolve_task(args.task) task = json.loads((task_root / "task.json").read_text(encoding="utf-8")) + wall_cap = float(task["budgets"]["wall_time_seconds"]) + if args.fixture_wall_time_seconds is not None: + if (not math.isfinite(args.fixture_wall_time_seconds) or args.fixture_wall_time_seconds < 0 + or args.fixture_wall_time_seconds > wall_cap): + raise ValueError("fixture wall timeout must be finite, nonnegative, and no larger than configured") + wall_cap = args.fixture_wall_time_seconds + task["budgets"]["wall_time_seconds"] = wall_cap + overall_deadline = time.monotonic() + wall_cap if task.get("schema_version") != "1.0": raise ValueError("unsupported task schema") public = (task_root / task["public_dir"]).resolve(strict=True) @@ -290,12 +397,15 @@ def record_phase(name: str, result: Any = None, **details: Any) -> None: manifest["hashes"]["source_initial"] = _tree_hash(workspace) record_phase("prepared") manifest["tool_versions"] = { - "platformio": _tool_version("platformio", args.platformio, workspace, run_dir), - "openocd": _tool_version("openocd", args.openocd, workspace, run_dir), + "platformio": _tool_version("platformio", args.platformio, workspace, run_dir, tool_env, credential_values, + bounded_timeout(2)), + "openocd": _tool_version("openocd", args.openocd, workspace, run_dir, tool_env, credential_values, + bounded_timeout(2)), } if args.agent_bin: manifest["tool_versions"]["agent"] = _tool_version( - "agent", args.agent_bin, workspace, run_dir, _agent_environment()) + "agent", args.agent_bin, workspace, run_dir, _agent_environment(), credential_values, + bounded_timeout(2)) persist() if args.usage_json: @@ -315,7 +425,8 @@ def record_phase(name: str, result: Any = None, **details: Any) -> None: clean_env = _agent_environment() agent = run_command([args.agent_bin, "agent", "run", "--model", args.model, prompt], cwd=workspace, stdout_path=run_dir / "agent.stdout.log", stderr_path=run_dir / "agent.stderr.log", - timeout_seconds=float(task["budgets"]["wall_time_seconds"]), env=clean_env) + timeout_seconds=bounded_timeout(float(task["budgets"]["wall_time_seconds"])), env=clean_env, + redact_values=credential_values) if agent.timed_out or agent.cleanup_error: manifest["infrastructure_status"] = "error" record_phase("agent", agent) @@ -327,6 +438,7 @@ def record_phase(name: str, result: Any = None, **details: Any) -> None: manifest["hashes"]["source_final"] = _tree_hash(workspace) return _finalize(run_dir, manifest, run_started_monotonic) record_phase("agent", agent) + _reject_secret_files(workspace, credential_values) else: manifest["model_status"] = "not_run" manifest["hashes"]["source_final"] = _tree_hash(workspace) @@ -334,9 +446,11 @@ def record_phase(name: str, result: Any = None, **details: Any) -> None: firmware = workspace / "firmware" build_command = [args.platformio, "run", "--project-dir", str(firmware), "--environment", config.platformio_environment or "esp32s3"] + _safe_files(workspace) clean = run_command(build_command + ["--target", "clean"], cwd=workspace, stdout_path=run_dir / "clean.stdout.log", stderr_path=run_dir / "clean.stderr.log", - timeout_seconds=float(task["budgets"]["wall_time_seconds"])) + timeout_seconds=bounded_timeout(float(task["budgets"]["wall_time_seconds"])), env=tool_env, + redact_values=credential_values) if clean.timed_out or clean.cleanup_error: manifest["infrastructure_status"] = "error" record_phase("clean", clean) @@ -347,7 +461,8 @@ def record_phase(name: str, result: Any = None, **details: Any) -> None: raise RuntimeError("clean command failed") record_phase("clean", clean) build = run_command(build_command, cwd=workspace, stdout_path=run_dir / "build.stdout.log", - stderr_path=run_dir / "build.stderr.log", timeout_seconds=float(task["budgets"]["wall_time_seconds"])) + stderr_path=run_dir / "build.stderr.log", timeout_seconds=bounded_timeout(float(task["budgets"]["wall_time_seconds"])), + env=tool_env, redact_values=credential_values) if build.timed_out or build.cleanup_error: manifest["infrastructure_status"] = "error" record_phase("build", build) @@ -372,55 +487,67 @@ def record_phase(name: str, result: Any = None, **details: Any) -> None: raise ValueError("identity command JSON must be a nonempty string array") else: identity_command = list(config.identity_command) - with BoardLock(run_dir.parent / ".board-locks", args.jtag_serial, timeout_seconds=config.identity_timeout_seconds): + with BoardLock(run_dir.parent / ".board-locks", args.jtag_serial, + timeout_seconds=bounded_timeout(config.identity_timeout_seconds)): identity = validate_identity(identity_command, args.jtag_serial, cwd=workspace, evidence_dir=run_dir, - timeout_seconds=config.identity_timeout_seconds) + timeout_seconds=bounded_timeout(config.identity_timeout_seconds), runner=tool_runner) if identity.status != "pass": manifest["infrastructure_status"] = "error" record_phase("identity", identity.command_result, status=identity.status) raise RuntimeError(identity.detail or "board identity failed") record_phase("identity", identity.command_result, status=identity.status) persist() - context = multiprocessing.get_context("spawn") - cancel_uart = context.Event() - started_uart = context.Event() - uart_queue = context.Queue() - uart_process = context.Process(target=_uart_process_worker, name="hil-uart", - args=(uart_queue, started_uart, cancel_uart, args.uart_device, config.uart_baud, nonce, - config.uart_timeout_seconds, str(run_dir / "uart.raw.log"), args.fixture_uart_worker_mode)) + worker_spec = run_dir / ".uart-worker.json" + worker_ready = run_dir / ".uart-worker.ready" + worker_result = run_dir / ".uart-worker.result.json" + uart_timeout = bounded_timeout(config.uart_timeout_seconds) + write_json_atomic(worker_spec, {"device": args.uart_device, "baud": config.uart_baud, + "nonce": nonce, "timeout_seconds": uart_timeout, + "log_path": str(run_dir / "uart.raw.log"), "ready_path": str(worker_ready), + "result_path": str(worker_result), "mode": args.fixture_uart_worker_mode}) + worker_stdout = open(run_dir / "uart-worker.stdout.log", "wb") + worker_stderr = open(run_dir / "uart-worker.stderr.log", "wb") + uart_process: subprocess.Popen[Any] | None = None uart_process_started = False try: - uart_process.start() + uart_process = subprocess.Popen([sys.executable, str(Path(__file__).resolve()), + "--_uart-worker", str(worker_spec)], + cwd=workspace, stdout=worker_stdout, stderr=worker_stderr, + env=tool_env, start_new_session=True) uart_process_started = True - if not started_uart.wait(2): + ready_deadline = time.monotonic() + bounded_timeout(2) + while not worker_ready.exists() and uart_process.poll() is None and time.monotonic() < ready_deadline: + time.sleep(.01) + if not worker_ready.exists(): raise RuntimeError("UART capture failed to start: startup timeout") flash_command = [args.platformio, "run", "--project-dir", str(firmware), "--environment", config.platformio_environment or "esp32s3", "--target", config.flash_target] flashed = flash_firmware(flash_command, cwd=workspace, evidence_dir=run_dir, - timeout_seconds=config.flash_timeout_seconds, identity_validated=True) + timeout_seconds=bounded_timeout(config.flash_timeout_seconds), identity_validated=True, + runner=tool_runner) if flashed.status == "infrastructure_error": manifest["infrastructure_status"] = "error" elif flashed.status == "hardware_fail": manifest["hardware_status"] = "fail" record_phase("flash", flashed.command_result, status=flashed.status, category=flashed.category) if flashed.status == "pass": - uart_process.join(config.uart_timeout_seconds + 0.5) + try: uart_process.wait(timeout=bounded_timeout(uart_timeout + 0.5)) + except subprocess.TimeoutExpired: pass finally: - uart_stopped = not uart_process_started or _stop_uart_process(uart_process, cancel_uart) - if sys.exc_info()[0] is not None: - uart_queue.close() - uart_queue.join_thread() + uart_stopped = not uart_process_started or _stop_uart_process(uart_process) + worker_stdout.close() + worker_stderr.close() if not uart_stopped: raise UartWorkerFatal("UART worker could not be stopped") - try: - uart_kind, uart_payload = uart_queue.get(timeout=.2) - except queue.Empty: + uart_payload = (json.loads(worker_result.read_text(encoding="utf-8")) + if worker_result.exists() else None) + for control_path in (worker_spec, worker_ready, worker_result): + try: control_path.unlink() + except FileNotFoundError: pass + if uart_payload is None: raise RuntimeError("UART capture produced no result") - finally: - uart_queue.close() - uart_queue.join_thread() - if uart_kind == "error": - raise RuntimeError(f"UART capture failed: {uart_payload}") + if uart_payload["kind"] == "error": + raise RuntimeError(f"UART capture failed: {uart_payload['detail']}") manifest["uart"] = {"matched": uart_payload["matched"], "termination": uart_payload["termination_reason"], "bytes_captured": uart_payload["bytes_captured"]} if not uart_payload["matched"] and flashed.status != "infrastructure_error": @@ -435,7 +562,7 @@ def record_phase(name: str, result: Any = None, **details: Any) -> None: return _finalize(run_dir, manifest, run_started_monotonic) registers = read_registers(args.openocd, config.openocd_board_config, args.jtag_serial, config.assertions, cwd=workspace, evidence_dir=run_dir, - timeout_seconds=config.openocd_command_timeout_seconds) + timeout_seconds=bounded_timeout(config.openocd_command_timeout_seconds), runner=tool_runner) if registers.status == "infrastructure_error": manifest["infrastructure_status"] = "error" record_phase("register", registers.command_result, status=registers.status) @@ -462,29 +589,39 @@ def record_phase(name: str, result: Any = None, **details: Any) -> None: manifest["termination"] = "interrupted" manifest["failure_category"] = "interrupt" manifest["detail"] = type(error).__name__ - _finalize(run_dir, manifest, run_started_monotonic) + try: + _finalize(run_dir, manifest, run_started_monotonic) + except (OSError, ValueError): + persist() return 2 except (OSError, ValueError, TypeError, KeyError, json.JSONDecodeError, RuntimeError, BoardLockTimeout) as error: manifest["infrastructure_status"] = "error" manifest["termination"] = "invalid" manifest["failure_category"] = "infrastructure" manifest["detail"] = str(error) - _finalize(run_dir, manifest, run_started_monotonic) + try: + _finalize(run_dir, manifest, run_started_monotonic) + except (OSError, ValueError) as final_error: + manifest["detail"] = f"{error}; evidence rejected: {final_error}" + persist() print(str(error), file=sys.stderr) return 2 def _finalize(run_dir: Path, manifest: dict[str, Any], started_monotonic: float) -> int: + credential_values = tuple(filter(None, (os.environ.get(name, "") for name in ( + "LABWIRED_ACCESS_TOKEN", "LABWIRED_PROJECT", "LABWIRED_MODEL_URL", "LABWIRED_MODEL_KEY")))) + _reject_secret_files(run_dir, credential_values) result = {name: manifest.get(name) for name in ( "model_status", "compile_status", "simulator_status", "hardware_status", "infrastructure_status", "termination", "failure_category", "uart", "register_assertions")} write_json_atomic(run_dir / "result.json", result) artifacts: list[str] = [] hashes = manifest.setdefault("hashes", {}) - for path in sorted(item for item in run_dir.rglob("*") if item.is_file() and item.name != "run.json"): + for path in (item for item in _safe_files(run_dir) if item.name != "run.json"): relative = path.relative_to(run_dir).as_posix() artifacts.append(relative) - hashes[f"artifact:{relative}"] = sha256_file(path) + hashes[f"artifact:{relative}"] = _safe_hash(path) manifest["artifacts"] = artifacts manifest["run"]["ended_at_utc"] = datetime.now(timezone.utc).isoformat().replace("+00:00", "Z") elapsed = time.monotonic() - started_monotonic @@ -496,4 +633,6 @@ def _finalize(run_dir: Path, manifest: dict[str, Any], started_monotonic: float) if __name__ == "__main__": + if len(sys.argv) == 3 and sys.argv[1] == "--_uart-worker": + raise SystemExit(_uart_subprocess_worker(Path(sys.argv[2]))) raise SystemExit(main()) diff --git a/tests/twin2silicon-hil.py b/tests/twin2silicon-hil.py index 918be65..263d2d9 100644 --- a/tests/twin2silicon-hil.py +++ b/tests/twin2silicon-hil.py @@ -179,6 +179,30 @@ def test_write_json_atomic_replaces_destination_with_json(self): class ProcessContractTests(unittest.TestCase): + def test_successful_leader_cannot_leave_mutating_descendant(self): + with tempfile.TemporaryDirectory() as directory: + evidence = Path(directory); child_path = evidence / "child"; mutation = evidence / "mutation" + script = textwrap.dedent(f""" + import pathlib, subprocess, sys + child = ''' + import os, pathlib, time + pathlib.Path({str(child_path)!r}).write_text(str(os.getpid())) + time.sleep(.4) + pathlib.Path({str(mutation)!r}).write_text('late') + time.sleep(30) + ''' + subprocess.Popen([sys.executable, '-c', child]) + while not pathlib.Path({str(child_path)!r}).exists(): pass + print('leader done') + """) + result = run_command([sys.executable, "-c", script], cwd=evidence, + stdout_path=evidence / "o", stderr_path=evidence / "e", timeout_seconds=2) + self.assertEqual(result.returncode, 0) + self.assertEqual(result.cleanup_error, "unexpected_descendant_processes") + with self.assertRaises(ProcessLookupError): os.kill(int(child_path.read_text()), 0) + time.sleep(.5) + self.assertFalse(mutation.exists()) + def test_run_command_interrupt_terminates_process_group_before_reraising(self): with tempfile.TemporaryDirectory() as directory: evidence = Path(directory) @@ -198,6 +222,18 @@ def test_run_command_interrupt_terminates_process_group_before_reraising(self): try: os.kill(int(pid_path.read_text()), signal.SIGKILL) except ProcessLookupError: pass + def test_run_command_redacts_exact_bytes_before_interrupt_reraises(self): + with tempfile.TemporaryDirectory() as directory: + evidence = Path(directory); output = evidence / "o" + timer = threading.Timer(.15, lambda: os.kill(os.getpid(), signal.SIGINT)); timer.start() + try: + with self.assertRaises(KeyboardInterrupt): + run_command([sys.executable, "-c", "import time; print('TOKEN_SENTINEL', flush=True); time.sleep(30)"], + cwd=evidence, stdout_path=output, stderr_path=evidence / "e", timeout_seconds=30, + redact_values=(b"TOKEN_SENTINEL",)) + finally: timer.cancel() + self.assertEqual(output.read_text(), "[REDACTED]\n") + def test_run_command_interrupt_kills_sigterm_ignoring_descendant_after_leader_exits(self): with tempfile.TemporaryDirectory() as directory: evidence = Path(directory); child_path = evidence / "child" @@ -946,11 +982,11 @@ def test_finalizer_uses_total_monotonic_wall_time_not_provider_latency(self): "infrastructure_status": "ok", "termination": "completed", "failure_category": None, "uart": None, "register_assertions": [], "hashes": {}, "run": {}, "latency_seconds": .001, "budget_validity": {"wall_time_seconds": {"configured": .01, "observed": None, "within_budget": None}}} - real_hash = run_hil.sha256_file + real_hash = run_hil._safe_hash def slow_hash(path): time.sleep(.05) return real_hash(path) - with mock.patch.object(run_hil, "sha256_file", side_effect=slow_hash): + with mock.patch.object(run_hil, "_safe_hash", side_effect=slow_hash): run_hil._finalize(Path(directory), manifest, time.monotonic()) wall = manifest["budget_validity"]["wall_time_seconds"] self.assertGreaterEqual(wall["observed"], .05) @@ -985,6 +1021,7 @@ def test_agent_mode_repairs_workspace_with_allowlisted_home_and_no_secret_eviden if '--version' in sys.argv: print('agent fixture 1'); raise SystemExit(0) assert os.environ['HOME'] == 'HOME_SENTINEL' assert os.environ['LABWIRED_HOME'] == 'LABWIRED_HOME_SENTINEL' + print(os.environ['LABWIRED_ACCESS_TOKEN'], os.environ['LABWIRED_MODEL_KEY']) assert 'hidden' not in ' '.join(sys.argv).lower() source = pathlib.Path('firmware/src/main.c') source.write_text(source.read_text().replace('GPIO_MODE_INPUT', 'GPIO_MODE_OUTPUT')) @@ -992,9 +1029,10 @@ def test_agent_mode_repairs_workspace_with_allowlisted_home_and_no_secret_eviden """)) pio_dir = root / "pio" pio_dir.mkdir() - pio = executable_fixture(pio_dir, "import sys\nif '--version' in sys.argv: print('pio fixture 1'); raise SystemExit(0)\nraise SystemExit(0 if 'clean' in sys.argv else 1)\n") + pio = executable_fixture(pio_dir, "import os,sys\nprint(os.environ.get('LABWIRED_ACCESS_TOKEN','ABSENT'), os.environ.get('HOST_SECRET','ABSENT'))\nif '--version' in sys.argv: print('pio fixture 1'); raise SystemExit(0)\nraise SystemExit(0 if 'clean' in sys.argv else 1)\n") env = {**os.environ, "HOME": "HOME_SENTINEL", "LABWIRED_HOME": "LABWIRED_HOME_SENTINEL", - "LABWIRED_ACCESS_TOKEN": "SECRET_TOKEN_SENTINEL"} + "LABWIRED_ACCESS_TOKEN": "SECRET_TOKEN_SENTINEL", "LABWIRED_MODEL_KEY": "MODEL_KEY_SENTINEL", + "HOST_SECRET": "HOST_SECRET_SENTINEL"} usage = root / "usage.json" usage.write_text(json.dumps({"schema_version": "1.0", "requests": 1, "tokens": {"fresh_input": 1000, "cached_input": 2000, "output": 3000, "reasoning": 4}, @@ -1011,11 +1049,56 @@ def test_agent_mode_repairs_workspace_with_allowlisted_home_and_no_secret_eviden self.assertEqual(call["cwd"], "workspace") manifest_text = (run_dir / "run.json").read_text() self.assertNotIn("SECRET_TOKEN_SENTINEL", manifest_text) + for path in run_dir.rglob("*"): + if path.is_file(): + contents = path.read_bytes() + for sentinel in (b"SECRET_TOKEN_SENTINEL", b"MODEL_KEY_SENTINEL", b"HOST_SECRET_SENTINEL"): + self.assertNotIn(sentinel, contents) manifest = json.loads(manifest_text) self.assertEqual(manifest["model_status"], "pass") self.assertEqual(manifest["budget_validity"]["model_tokens"]["observed"], 55) self.assertNotEqual(manifest["hashes"]["source_initial"], manifest["hashes"]["source_final"]) + def test_agent_symlink_and_fifo_are_rejected_without_reading_external_secret(self): + task = REPOSITORY_ROOT / "benchmarks/twin2silicon/tasks/esp32s3-gpio-hil-001" + with tempfile.TemporaryDirectory() as directory: + root = Path(directory); secret = root / "secret"; secret.write_text("EXTERNAL_SECRET_SENTINEL") + agent_dir = root / "agent"; agent_dir.mkdir() + agent = executable_fixture(agent_dir, textwrap.dedent(f""" + import os, pathlib, sys + if '--version' in sys.argv: print('v'); raise SystemExit(0) + pathlib.Path('escape').symlink_to({str(secret)!r}) + os.mkfifo('special.fifo') + """)) + tool_dir = root / "tool"; tool_dir.mkdir() + tool = executable_fixture(tool_dir, "print('v')\n") + run_dir = root / "run" + result = self._run_cli(task, "--run-dir", run_dir, "--agent-bin", agent, "--model", "fixture", + "--jtag-serial", "J", "--uart-device", "/dev/null", "--openocd", tool, "--platformio", tool) + self.assertEqual(result.returncode, 2) + manifest_text = (run_dir / "run.json").read_text() + self.assertNotIn("EXTERNAL_SECRET_SENTINEL", manifest_text) + self.assertNotIn("escape", json.loads(manifest_text).get("artifacts", [])) + + def test_agent_credential_written_to_workspace_is_rejected_before_hash_or_build(self): + task = REPOSITORY_ROOT / "benchmarks/twin2silicon/tasks/esp32s3-gpio-hil-001" + with tempfile.TemporaryDirectory() as directory: + root = Path(directory); agent_dir = root / "agent"; agent_dir.mkdir() + agent = executable_fixture(agent_dir, """import os,pathlib,sys +if '--version' in sys.argv: print('v'); raise SystemExit(0) +pathlib.Path('credential.txt').write_text(os.environ['LABWIRED_ACCESS_TOKEN']) +""") + tool_dir = root / "tool"; tool_dir.mkdir(); marker = root / "tool-ran" + tool = executable_fixture(tool_dir, f"import sys\nif '--version' in sys.argv: print('v'); raise SystemExit(0)\nfrom pathlib import Path; Path({str(marker)!r}).write_text('ran')\n") + run_dir = root / "run"; env = {**os.environ, "LABWIRED_ACCESS_TOKEN": "WORKSPACE_TOKEN_SENTINEL"} + result = self._run_cli(task, "--run-dir", run_dir, "--agent-bin", agent, "--model", "fixture", + "--jtag-serial", "J", "--uart-device", "/dev/null", "--openocd", tool, "--platformio", tool, env=env) + self.assertEqual(result.returncode, 2) + self.assertFalse(marker.exists()) + manifest_text = (run_dir / "run.json").read_text() + self.assertNotIn("WORKSPACE_TOKEN_SENTINEL", manifest_text) + self.assertNotIn("credential.txt", json.loads(manifest_text).get("artifacts", [])) + def test_cli_physical_pass_uses_pty_and_records_ordered_evidence(self): task = self._short_task() with tempfile.TemporaryDirectory() as directory: @@ -1100,6 +1183,28 @@ def test_clean_nonzero_is_infrastructure_and_never_touches_identity(self): self.assertEqual((manifest["compile_status"], manifest["infrastructure_status"]), ("not_run", "error")) self.assertEqual([phase["name"] for phase in manifest["phases"]], ["prepared", "clean"]) + def test_overall_wall_deadline_bounds_cumulative_phases(self): + task = REPOSITORY_ROOT / "benchmarks/twin2silicon/tasks/esp32s3-gpio-hil-001" + with tempfile.TemporaryDirectory() as directory: + root = Path(directory); marker = root / "identity" + pio_dir = root / "pio"; pio_dir.mkdir() + pio = executable_fixture(pio_dir, """import pathlib,sys,time +if '--version' in sys.argv: print('v'); raise SystemExit(0) +time.sleep(.18) +if '--target' not in sys.argv: + p=pathlib.Path(sys.argv[sys.argv.index('--project-dir')+1])/'.pio/build/esp32s3/firmware.bin'; p.parent.mkdir(parents=True); p.write_bytes(b'fw') +""") + identity_dir = root / "id"; identity_dir.mkdir() + identity = executable_fixture(identity_dir, f"from pathlib import Path; Path({str(marker)!r}).write_text('ran')\n") + run_dir = root / "run" + result = self._run_cli(task, "--run-dir", run_dir, "--evaluate-only", "--candidate", task / "public", + "--jtag-serial", "J", "--uart-device", "/dev/null", "--openocd", pio, "--platformio", pio, + "--identity-command-json", json.dumps([str(identity)]), "--fixture-wall-time-seconds", ".45") + self.assertEqual(result.returncode, 2) + self.assertFalse(marker.exists()) + manifest = json.loads((run_dir / "run.json").read_text()) + self.assertFalse(manifest["budget_validity"]["wall_time_seconds"]["within_budget"]) + def test_uart_startup_hang_is_killed_before_bounded_finalization(self): task = REPOSITORY_ROOT / "benchmarks/twin2silicon/tasks/esp32s3-gpio-hil-001" with tempfile.TemporaryDirectory() as directory: From d63344e32869188c42696f14d3b395fda67dd707 Mon Sep 17 00:00:00 2001 From: Andrii Shylenko <14119286+w1ne@users.noreply.github.com> Date: Sat, 15 Aug 2026 14:03:00 +0200 Subject: [PATCH 22/48] fix(bench): freeze and quarantine agent workspaces --- benchmarks/twin2silicon/run_hil.py | 136 ++++++++++++++++++++++++----- tests/twin2silicon-hil.py | 68 +++++++++++++++ 2 files changed, 184 insertions(+), 20 deletions(-) diff --git a/benchmarks/twin2silicon/run_hil.py b/benchmarks/twin2silicon/run_hil.py index 38b8bfd..d452308 100644 --- a/benchmarks/twin2silicon/run_hil.py +++ b/benchmarks/twin2silicon/run_hil.py @@ -16,6 +16,7 @@ import os from pathlib import Path import stat +import tempfile import secrets import sys import time @@ -39,12 +40,18 @@ ROOT = Path(__file__).resolve().parent TASKS = ROOT / "tasks" HARNESS_REVISION = "twin2silicon-hil-1" +MAX_REGULAR_FILE_BYTES = 32 * 1024 * 1024 +MAX_TREE_BYTES = 128 * 1024 * 1024 class UartWorkerFatal(BaseException): pass +class UnsafeWorkspaceError(ValueError): + pass + + class _FileEvent: def __init__(self, path: Path) -> None: self.path = path @@ -149,26 +156,72 @@ def _safe_tree(source: Path, field: str) -> Path: return source -def _safe_files(root: Path) -> list[Path]: +def _safe_files(root: Path, deadline: float | None = None) -> list[Path]: root = root.absolute() files: list[Path] = [] + total = 0 def visit(directory: Path) -> None: + nonlocal total + if deadline is not None and time.monotonic() >= deadline: + raise UnsafeWorkspaceError("validation deadline exhausted") with os.scandir(directory) as entries: for entry in entries: info = entry.stat(follow_symlinks=False) path = directory / entry.name if stat.S_ISLNK(info.st_mode): - raise ValueError(f"unsafe symlink: {path.relative_to(root)}") + raise UnsafeWorkspaceError(f"unsafe symlink: {path.relative_to(root)}") if stat.S_ISDIR(info.st_mode): visit(path) elif stat.S_ISREG(info.st_mode): + if info.st_size > MAX_REGULAR_FILE_BYTES: + raise UnsafeWorkspaceError(f"oversize file: {path.relative_to(root)}") + total += info.st_size + if total > MAX_TREE_BYTES: + raise UnsafeWorkspaceError("tree size limit exceeded") files.append(path) else: - raise ValueError(f"unsafe non-regular file: {path.relative_to(root)}") + raise UnsafeWorkspaceError(f"unsafe non-regular file: {path.relative_to(root)}") visit(root) return sorted(files) +def _safe_remove_tree(path: Path) -> None: + try: + info = os.lstat(path) + except FileNotFoundError: + return + if not stat.S_ISDIR(info.st_mode) or stat.S_ISLNK(info.st_mode): + path.unlink(missing_ok=True) + return + with os.scandir(path) as entries: + children = [path / entry.name for entry in entries] + for child in children: + _safe_remove_tree(child) + try: path.rmdir() + except FileNotFoundError: pass + + +def _quarantine_workspace(workspace: Path) -> None: + if not workspace.exists() and not workspace.is_symlink(): + return + root = Path(tempfile.mkdtemp(prefix="hil-quarantine-", dir=workspace.parent.parent)) + quarantined = root / "workspace" + os.rename(workspace, quarantined) + _safe_remove_tree(quarantined) + _safe_remove_tree(root) + + +def _freeze_workspace(workspace: Path, deadline: float | None) -> None: + root = Path(tempfile.mkdtemp(prefix="hil-snapshot-", dir=workspace.parent.parent)) + frozen = root / "workspace" + try: + _copy_public(workspace, frozen, deadline) + _quarantine_workspace(workspace) + os.rename(frozen, workspace) + finally: + _safe_remove_tree(root) + + def _safe_hash(path: Path) -> str: flags = os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0) fd = os.open(path, flags) @@ -184,6 +237,19 @@ def _safe_hash(path: Path) -> str: os.close(fd) +def _safe_copy_file(source: Path, destination: Path) -> None: + source_fd = os.open(source, os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0)) + try: + info = os.fstat(source_fd) + if not stat.S_ISREG(info.st_mode) or info.st_size > MAX_REGULAR_FILE_BYTES: + raise UnsafeWorkspaceError("firmware artifact is unsafe or oversized") + destination_fd = os.open(destination, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600) + try: + while chunk := os.read(source_fd, 1024 * 1024): os.write(destination_fd, chunk) + finally: os.close(destination_fd) + finally: os.close(source_fd) + + def _resolve_task(value: str) -> Path: supplied = Path(value) candidate = supplied if supplied.is_absolute() or len(supplied.parts) > 1 else TASKS / supplied @@ -195,10 +261,10 @@ def _resolve_task(value: str) -> Path: return resolved -def _copy_public(source: Path, destination: Path) -> None: +def _copy_public(source: Path, destination: Path, deadline: float | None = None) -> None: source = _safe_tree(source, "candidate/public workspace") destination.mkdir() - for path in _safe_files(source): + for path in _safe_files(source, deadline): target = destination / path.relative_to(source) target.parent.mkdir(parents=True, exist_ok=True) source_fd = os.open(path, os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0)) @@ -206,6 +272,8 @@ def _copy_public(source: Path, destination: Path) -> None: target_fd = os.open(target, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600) try: while chunk := os.read(source_fd, 1024 * 1024): + if deadline is not None and time.monotonic() >= deadline: + raise UnsafeWorkspaceError("copy deadline exhausted") os.write(target_fd, chunk) finally: os.close(target_fd) @@ -217,7 +285,7 @@ def _command_record(result: Any, run_dir: Path, redactions: tuple[str, ...] = () def sanitize(value: str) -> str: for secret in redactions: if secret: - value = value.replace(secret, "") + value = value.replace(secret, "[REDACTED]") return value def relative(value: str) -> str: try: @@ -266,11 +334,13 @@ def _reject_secret_files(root: Path, secret_values: tuple[str, ...]) -> None: flags = os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0) fd = os.open(path, flags) try: - contents = bytearray() + overlap = b"" + width = max((len(needle) for needle in needles), default=1) - 1 while chunk := os.read(fd, 1024 * 1024): - contents.extend(chunk) - if any(needle in contents for needle in needles): - raise ValueError(f"credential value found in workspace artifact: {path.relative_to(root)}") + window = overlap + chunk + if any(needle in window for needle in needles): + raise UnsafeWorkspaceError(f"credential value found in workspace artifact: {path.relative_to(root)}") + overlap = window[-width:] if width else b"" finally: os.close(fd) @@ -323,8 +393,7 @@ def main(argv: list[str] | None = None) -> int: "register_assertions": [], "termination": "running", "failure_category": None, "hashes": {}, "artifacts": [], "environment": {}, "phases": [], } - def persist() -> None: - write_json_atomic(run_dir / "run.json", manifest) + workspace: Path | None = None overall_deadline: float | None = None def bounded_timeout(cap: float) -> float: if overall_deadline is None: @@ -339,6 +408,17 @@ def bounded_timeout(cap: float) -> float: "LABWIRED_ACCESS_TOKEN", "LABWIRED_PROJECT", "LABWIRED_MODEL_URL", "LABWIRED_MODEL_KEY")))) tool_env = _tool_environment() tool_runner = partial(run_command, env=tool_env, redact_values=credential_values) + def sanitize(value: Any) -> Any: + if isinstance(value, str): + for secret in (*redactions, *credential_values): + value = value.replace(secret, "[REDACTED]") + return value + if isinstance(value, list): return [sanitize(item) for item in value] + if isinstance(value, tuple): return [sanitize(item) for item in value] + if isinstance(value, dict): return {key: sanitize(item) for key, item in value.items()} + return value + def persist() -> None: + write_json_atomic(run_dir / "run.json", sanitize(manifest)) def record_phase(name: str, result: Any = None, **details: Any) -> None: phase = {"name": name, **details} if result is not None: @@ -376,7 +456,7 @@ def record_phase(name: str, result: Any = None, **details: Any) -> None: config = replace(config, identity_timeout_seconds=fixture_timeout) source = _safe_tree(args.candidate, "candidate") if args.evaluate_only else _safe_tree(public, "public") workspace = run_dir / "workspace" - _copy_public(source, workspace) + _copy_public(source, workspace, overall_deadline) nonce = secrets.token_hex(16) nonce_header = workspace / "firmware/include/run_nonce.h" nonce_header.parent.mkdir(parents=True, exist_ok=True) @@ -392,7 +472,7 @@ def record_phase(name: str, result: Any = None, **details: Any) -> None: configured = manifest["budget_validity"][name]["configured"] manifest["budget_validity"][name].update(observed=0, within_budget=0 <= configured) manifest["environment"] = {"jtag_serial_sha256": hashlib.sha256(args.jtag_serial.encode()).hexdigest(), - "uart_device": Path(args.uart_device).name} + "uart_device_sha256": hashlib.sha256(args.uart_device.encode()).hexdigest()} manifest["hashes"]["oracle_descriptor"] = sha256_file(oracle_path) manifest["hashes"]["source_initial"] = _tree_hash(workspace) record_phase("prepared") @@ -439,6 +519,7 @@ def record_phase(name: str, result: Any = None, **details: Any) -> None: return _finalize(run_dir, manifest, run_started_monotonic) record_phase("agent", agent) _reject_secret_files(workspace, credential_values) + _freeze_workspace(workspace, overall_deadline) else: manifest["model_status"] = "not_run" manifest["hashes"]["source_final"] = _tree_hash(workspace) @@ -478,7 +559,8 @@ def record_phase(name: str, result: Any = None, **details: Any) -> None: artifact = firmware / config.flash_artifact if not artifact.is_file(): raise RuntimeError("successful build produced no firmware artifact") - manifest["hashes"]["firmware"] = sha256_file(artifact) + manifest["hashes"]["firmware"] = _safe_hash(artifact) + _safe_copy_file(artifact, run_dir / "firmware.bin") persist() if args.identity_command_json: @@ -577,38 +659,52 @@ def record_phase(name: str, result: Any = None, **details: Any) -> None: record_phase("register", registers.command_result, status=registers.status, assertions=manifest["register_assertions"]) return _finalize(run_dir, manifest, run_started_monotonic) + except UnsafeWorkspaceError as error: + if workspace is not None: + _quarantine_workspace(workspace) + manifest["infrastructure_status"] = "error" + manifest["termination"] = "invalid" + manifest["failure_category"] = "unsafe_workspace" + manifest["detail"] = sanitize(str(error)) + persist() + return 2 except UartWorkerFatal as error: manifest["infrastructure_status"] = "error" manifest["termination"] = "invalid" manifest["failure_category"] = "uart_cleanup" - manifest["detail"] = str(error) + manifest["detail"] = sanitize(str(error)) persist() return 2 except (KeyboardInterrupt, SystemExit) as error: + if args.agent_bin and workspace is not None: + _quarantine_workspace(workspace) manifest["infrastructure_status"] = "error" manifest["termination"] = "interrupted" manifest["failure_category"] = "interrupt" - manifest["detail"] = type(error).__name__ + manifest["detail"] = sanitize(type(error).__name__) try: _finalize(run_dir, manifest, run_started_monotonic) except (OSError, ValueError): persist() return 2 except (OSError, ValueError, TypeError, KeyError, json.JSONDecodeError, RuntimeError, BoardLockTimeout) as error: + if args.agent_bin and workspace is not None: + _quarantine_workspace(workspace) manifest["infrastructure_status"] = "error" manifest["termination"] = "invalid" manifest["failure_category"] = "infrastructure" - manifest["detail"] = str(error) + manifest["detail"] = sanitize(str(error)) try: _finalize(run_dir, manifest, run_started_monotonic) except (OSError, ValueError) as final_error: - manifest["detail"] = f"{error}; evidence rejected: {final_error}" + manifest["detail"] = sanitize(f"{error}; evidence rejected: {final_error}") persist() - print(str(error), file=sys.stderr) + print(sanitize(str(error)), file=sys.stderr) return 2 def _finalize(run_dir: Path, manifest: dict[str, Any], started_monotonic: float) -> int: + _safe_remove_tree(run_dir / "workspace/firmware/.pio") credential_values = tuple(filter(None, (os.environ.get(name, "") for name in ( "LABWIRED_ACCESS_TOKEN", "LABWIRED_PROJECT", "LABWIRED_MODEL_URL", "LABWIRED_MODEL_KEY")))) _reject_secret_files(run_dir, credential_values) diff --git a/tests/twin2silicon-hil.py b/tests/twin2silicon-hil.py index 263d2d9..6311c4a 100644 --- a/tests/twin2silicon-hil.py +++ b/tests/twin2silicon-hil.py @@ -7,6 +7,7 @@ import pty import signal import shutil +import stat import subprocess import sys import tempfile @@ -1079,6 +1080,10 @@ def test_agent_symlink_and_fifo_are_rejected_without_reading_external_secret(sel manifest_text = (run_dir / "run.json").read_text() self.assertNotIn("EXTERNAL_SECRET_SENTINEL", manifest_text) self.assertNotIn("escape", json.loads(manifest_text).get("artifacts", [])) + for path in run_dir.rglob("*"): + info = path.lstat() + self.assertFalse(stat.S_ISLNK(info.st_mode) or stat.S_ISFIFO(info.st_mode)) + if stat.S_ISREG(info.st_mode): self.assertNotIn(b"EXTERNAL_SECRET_SENTINEL", path.read_bytes()) def test_agent_credential_written_to_workspace_is_rejected_before_hash_or_build(self): task = REPOSITORY_ROOT / "benchmarks/twin2silicon/tasks/esp32s3-gpio-hil-001" @@ -1098,6 +1103,51 @@ def test_agent_credential_written_to_workspace_is_rejected_before_hash_or_build( manifest_text = (run_dir / "run.json").read_text() self.assertNotIn("WORKSPACE_TOKEN_SENTINEL", manifest_text) self.assertNotIn("credential.txt", json.loads(manifest_text).get("artifacts", [])) + self.assertFalse((run_dir / "workspace").exists()) + + def test_agent_oversize_sparse_file_is_bounded_and_workspace_quarantined(self): + task = REPOSITORY_ROOT / "benchmarks/twin2silicon/tasks/esp32s3-gpio-hil-001" + with tempfile.TemporaryDirectory() as directory: + root = Path(directory); agent_dir = root / "agent"; agent_dir.mkdir() + agent = executable_fixture(agent_dir, """import pathlib,sys +if '--version' in sys.argv: print('v'); raise SystemExit(0) +with pathlib.Path('huge.bin').open('wb') as out: out.truncate(40 * 1024 * 1024) +""") + tool_dir=root/'tool'; tool_dir.mkdir(); tool=executable_fixture(tool_dir,"print('v')\n") + run_dir=root/'run'; started=time.monotonic() + result=self._run_cli(task,'--run-dir',run_dir,'--agent-bin',agent,'--model','m','--jtag-serial','J', + '--uart-device','/dev/null','--openocd',tool,'--platformio',tool) + self.assertEqual(result.returncode,2); self.assertLess(time.monotonic()-started,3) + self.assertFalse((run_dir/'workspace').exists()) + self.assertEqual(json.loads((run_dir/'run.json').read_text())["failure_category"],"unsafe_workspace") + + def test_detached_agent_mutator_cannot_change_frozen_workspace(self): + task = REPOSITORY_ROOT / "benchmarks/twin2silicon/tasks/esp32s3-gpio-hil-001" + with tempfile.TemporaryDirectory() as directory: + root=Path(directory); pid_path=root/'child.pid'; agent_dir=root/'agent'; agent_dir.mkdir() + child = "import os,pathlib,time; f=open('firmware/src/main.c','ab',buffering=0); pathlib.Path(%r).write_text(str(os.getpid()));\nwhile True: f.write(b'X'); time.sleep(.01)" % str(pid_path) + agent=executable_fixture(agent_dir, textwrap.dedent(f""" + import subprocess,sys,time + if '--version' in sys.argv: print('v'); raise SystemExit(0) + subprocess.Popen([sys.executable,'-c',{child!r}], start_new_session=True) + import pathlib + while not pathlib.Path({str(pid_path)!r}).exists(): time.sleep(.01) + """)) + tool_dir=root/'tool'; tool_dir.mkdir() + tool=executable_fixture(tool_dir,"import sys\nif '--version' in sys.argv: print('v'); raise SystemExit(0)\nraise SystemExit(0 if 'clean' in sys.argv else 1)\n") + run_dir=root/'run' + try: + result=self._run_cli(task,'--run-dir',run_dir,'--agent-bin',agent,'--model','m','--jtag-serial','J', + '--uart-device','/dev/null','--openocd',tool,'--platformio',tool) + self.assertEqual(result.returncode,0,result.stderr) + sys.path.insert(0,str(REPOSITORY_ROOT/'benchmarks/twin2silicon')); import run_hil + before=run_hil._tree_hash(run_dir/'workspace'); time.sleep(.2) + self.assertEqual(before,run_hil._tree_hash(run_dir/'workspace')) + self.assertEqual(before,json.loads((run_dir/'run.json').read_text())["hashes"]["source_final"]) + finally: + if pid_path.exists(): + try: os.kill(int(pid_path.read_text()),signal.SIGKILL) + except ProcessLookupError: pass def test_cli_physical_pass_uses_pty_and_records_ordered_evidence(self): task = self._short_task() @@ -1183,6 +1233,24 @@ def test_clean_nonzero_is_infrastructure_and_never_touches_identity(self): self.assertEqual((manifest["compile_status"], manifest["infrastructure_status"]), ("not_run", "error")) self.assertEqual([phase["name"] for phase in manifest["phases"]], ["prepared", "clean"]) + def test_identity_launch_detail_redacts_serial_device_and_credentials(self): + task=REPOSITORY_ROOT/'benchmarks/twin2silicon/tasks/esp32s3-gpio-hil-001' + with tempfile.TemporaryDirectory() as directory: + root=Path(directory); tool_dir=root/'tool'; tool_dir.mkdir() + tool=executable_fixture(tool_dir,"""import pathlib,sys +if '--version' in sys.argv: print('v'); raise SystemExit(0) +if '--target' not in sys.argv: + p=pathlib.Path(sys.argv[sys.argv.index('--project-dir')+1])/'.pio/build/esp32s3/firmware.bin'; p.parent.mkdir(parents=True); p.write_bytes(b'fw') +""") + serial='RAW_SERIAL_SENTINEL'; device='/dev/RAW_DEVICE_SENTINEL'; credential='RAW_CREDENTIAL_SENTINEL' + env={**os.environ,'LABWIRED_ACCESS_TOKEN':credential}; run_dir=root/'run' + result=self._run_cli(task,'--run-dir',run_dir,'--evaluate-only','--candidate',task/'public', + '--jtag-serial',serial,'--uart-device',device,'--openocd',tool,'--platformio',tool, + '--identity-command-json',json.dumps([f'/missing/{serial}/{credential}']),env=env) + self.assertEqual(result.returncode,2) + combined=(run_dir/'run.json').read_text()+result.stderr + for value in (serial,device,credential): self.assertNotIn(value,combined) + def test_overall_wall_deadline_bounds_cumulative_phases(self): task = REPOSITORY_ROOT / "benchmarks/twin2silicon/tasks/esp32s3-gpio-hil-001" with tempfile.TemporaryDirectory() as directory: From b1df5e6d07a661de2c18941864b6a933663e0334 Mon Sep 17 00:00:00 2001 From: Andrii Shylenko <14119286+w1ne@users.noreply.github.com> Date: Sat, 15 Aug 2026 14:14:45 +0200 Subject: [PATCH 23/48] fix(bench): harden quarantine and evidence freezing --- benchmarks/twin2silicon/hil/process.py | 70 +++++++++++++--- benchmarks/twin2silicon/run_hil.py | 110 +++++++++++++++++++------ tests/twin2silicon-hil.py | 64 ++++++++++++-- 3 files changed, 200 insertions(+), 44 deletions(-) diff --git a/benchmarks/twin2silicon/hil/process.py b/benchmarks/twin2silicon/hil/process.py index c842224..610e682 100644 --- a/benchmarks/twin2silicon/hil/process.py +++ b/benchmarks/twin2silicon/hil/process.py @@ -6,6 +6,7 @@ import signal import subprocess import stat +import tempfile import time from typing import Mapping, Optional, Sequence, Union @@ -129,7 +130,9 @@ def run_command( if not _wait_for_process_group_exit(process.pid, 1.0): cleanup_error = "process_group_did_not_exit" - _redact_logs((normalized_stdout, normalized_stderr), redact_values) + redaction_error = _redact_logs((normalized_stdout, normalized_stderr), redact_values) + if redaction_error is not None and cleanup_error is None: + cleanup_error = redaction_error ended_at = _utc_now() return CommandResult( command=normalized_command, @@ -145,28 +148,69 @@ def run_command( ) -def _redact_logs(paths: Sequence[str], values: Sequence[Union[str, bytes]]) -> None: +def _redact_logs(paths: Sequence[str], values: Sequence[Union[str, bytes]]) -> Optional[str]: needles = tuple(value.encode() if isinstance(value, str) else value for value in values if value) if not needles: - return + for path in paths: + try: + if os.lstat(path).st_size > 16 * 1024 * 1024: + temporary_fd, temporary = tempfile.mkstemp(prefix=".redacted-", dir=str(Path(path).parent)) + try: os.write(temporary_fd, b"[EVIDENCE LOG TOO LARGE]\n"); os.fsync(temporary_fd) + finally: os.close(temporary_fd) + os.replace(temporary, path) + return "evidence_log_too_large" + except FileNotFoundError: pass + return None + error = None for path in paths: + temporary = None try: - fd = os.open(path, os.O_RDWR | getattr(os, "O_NOFOLLOW", 0)) + if os.lstat(path).st_size > 16 * 1024 * 1024: + temporary_fd, temporary = tempfile.mkstemp(prefix=".redacted-", dir=str(Path(path).parent)) + try: + os.write(temporary_fd, b"[EVIDENCE LOG TOO LARGE]\n"); os.fsync(temporary_fd) + finally: os.close(temporary_fd) + os.replace(temporary, path); temporary = None + error = "evidence_log_too_large" + continue + fd = os.open(path, os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0)) + temporary_fd, temporary = tempfile.mkstemp(prefix=".redacted-", dir=str(Path(path).parent)) try: info = os.fstat(fd) if not stat.S_ISREG(info.st_mode): raise OSError("evidence log is not a regular file") - contents = bytearray() + overlap = b""; maximum = max(map(len, needles)) while chunk := os.read(fd, 1024 * 1024): - contents.extend(chunk) - redacted = contents - for needle in needles: - redacted = redacted.replace(needle, b"[REDACTED]") - if redacted != contents: - os.lseek(fd, 0, os.SEEK_SET) - os.ftruncate(fd, 0) - os.write(fd, redacted) + data = overlap + chunk + limit = max(0, len(data) - maximum + 1) + overlap = _write_redacted_prefix(temporary_fd, data, limit, needles) + _write_redacted_prefix(temporary_fd, overlap, len(overlap), needles) + os.fsync(temporary_fd) finally: os.close(fd) + os.close(temporary_fd) + os.replace(temporary, path); temporary = None except FileNotFoundError: pass + finally: + if temporary is not None: + try: os.unlink(temporary) + except FileNotFoundError: pass + return error + + +def _write_redacted_prefix(output_fd: int, data: bytes, limit: int, + needles: Sequence[bytes]) -> bytes: + cursor = 0 + while cursor < limit: + matches = [(position, needle) for needle in needles + if (position := data.find(needle, cursor)) != -1 and position < limit] + if not matches: + os.write(output_fd, data[cursor:limit]) + cursor = limit + break + position, needle = min(matches, key=lambda item: item[0]) + os.write(output_fd, data[cursor:position]) + os.write(output_fd, b"[REDACTED]") + cursor = position + len(needle) + return data[max(cursor, limit):] diff --git a/benchmarks/twin2silicon/run_hil.py b/benchmarks/twin2silicon/run_hil.py index d452308..8e29942 100644 --- a/benchmarks/twin2silicon/run_hil.py +++ b/benchmarks/twin2silicon/run_hil.py @@ -187,18 +187,47 @@ def visit(directory: Path) -> None: def _safe_remove_tree(path: Path) -> None: try: - info = os.lstat(path) + parent_fd = os.open(path.parent, os.O_RDONLY | os.O_DIRECTORY | getattr(os, "O_NOFOLLOW", 0)) except FileNotFoundError: return - if not stat.S_ISDIR(info.st_mode) or stat.S_ISLNK(info.st_mode): - path.unlink(missing_ok=True) - return - with os.scandir(path) as entries: - children = [path / entry.name for entry in entries] - for child in children: - _safe_remove_tree(child) - try: path.rmdir() - except FileNotFoundError: pass + try: + _remove_entry_at(parent_fd, path.name) + finally: + os.close(parent_fd) + + +def _remove_entry_at(parent_fd: int, name: str) -> None: + for _ in range(3): + try: + info = os.stat(name, dir_fd=parent_fd, follow_symlinks=False) + except FileNotFoundError: + return + if not stat.S_ISDIR(info.st_mode) or stat.S_ISLNK(info.st_mode): + try: + os.unlink(name, dir_fd=parent_fd) + return + except IsADirectoryError: + continue + try: + child_fd = os.open(name, os.O_RDONLY | os.O_DIRECTORY | getattr(os, "O_NOFOLLOW", 0), dir_fd=parent_fd) + except (FileNotFoundError, NotADirectoryError, OSError): + continue + try: + opened = os.fstat(child_fd) + if (opened.st_dev, opened.st_ino) != (info.st_dev, info.st_ino): + continue + for child_name in os.listdir(child_fd): + _remove_entry_at(child_fd, child_name) + finally: + os.close(child_fd) + try: + os.rmdir(name, dir_fd=parent_fd) + return + except (FileNotFoundError, NotADirectoryError): + return + except OSError: + continue + raise UnsafeWorkspaceError("quarantine deletion race detected") def _quarantine_workspace(workspace: Path) -> None: @@ -207,19 +236,23 @@ def _quarantine_workspace(workspace: Path) -> None: root = Path(tempfile.mkdtemp(prefix="hil-quarantine-", dir=workspace.parent.parent)) quarantined = root / "workspace" os.rename(workspace, quarantined) - _safe_remove_tree(quarantined) - _safe_remove_tree(root) + try: + _safe_remove_tree(quarantined) + _safe_remove_tree(root) + except UnsafeWorkspaceError: + pass -def _freeze_workspace(workspace: Path, deadline: float | None) -> None: +def _freeze_workspace(workspace: Path, deadline: float | None) -> Path: root = Path(tempfile.mkdtemp(prefix="hil-snapshot-", dir=workspace.parent.parent)) - frozen = root / "workspace" + frozen = root / "snapshot" try: _copy_public(workspace, frozen, deadline) _quarantine_workspace(workspace) - os.rename(frozen, workspace) - finally: + return frozen + except BaseException: _safe_remove_tree(root) + raise def _safe_hash(path: Path) -> str: @@ -394,6 +427,7 @@ def main(argv: list[str] | None = None) -> int: "hashes": {}, "artifacts": [], "environment": {}, "phases": [], } workspace: Path | None = None + trusted_snapshot: Path | None = None overall_deadline: float | None = None def bounded_timeout(cap: float) -> float: if overall_deadline is None: @@ -455,7 +489,7 @@ def record_phase(name: str, result: Any = None, **details: Any) -> None: raise ValueError("fixture identity timeout must be finite, nonnegative, and no larger than oracle timeout") config = replace(config, identity_timeout_seconds=fixture_timeout) source = _safe_tree(args.candidate, "candidate") if args.evaluate_only else _safe_tree(public, "public") - workspace = run_dir / "workspace" + workspace = run_dir / (f"agent-workspace-{secrets.token_hex(8)}" if args.agent_bin else "workspace") _copy_public(source, workspace, overall_deadline) nonce = secrets.token_hex(16) nonce_header = workspace / "firmware/include/run_nonce.h" @@ -516,10 +550,16 @@ def record_phase(name: str, result: Any = None, **details: Any) -> None: manifest["termination"], manifest["failure_category"] = "completed", "model" record_phase("agent", agent) manifest["hashes"]["source_final"] = _tree_hash(workspace) - return _finalize(run_dir, manifest, run_started_monotonic) + _quarantine_workspace(workspace) + workspace = None + return _finalize(run_dir, manifest, run_started_monotonic, trusted_snapshot) record_phase("agent", agent) _reject_secret_files(workspace, credential_values) - _freeze_workspace(workspace, overall_deadline) + frozen = _freeze_workspace(workspace, overall_deadline) + build_workspace = run_dir / f"build-workspace-{secrets.token_hex(8)}" + _copy_public(frozen, build_workspace, overall_deadline) + trusted_snapshot = frozen + workspace = build_workspace else: manifest["model_status"] = "not_run" manifest["hashes"]["source_final"] = _tree_hash(workspace) @@ -553,7 +593,7 @@ def record_phase(name: str, result: Any = None, **details: Any) -> None: manifest["hardware_status"] = "not_run" manifest["termination"], manifest["failure_category"] = "completed", "compile" record_phase("build", build) - return _finalize(run_dir, manifest, run_started_monotonic) + return _finalize(run_dir, manifest, run_started_monotonic, trusted_snapshot) manifest["compile_status"] = "pass" record_phase("build", build) artifact = firmware / config.flash_artifact @@ -641,7 +681,7 @@ def record_phase(name: str, result: Any = None, **details: Any) -> None: if flashed.status == "hardware_fail" or not uart_payload["matched"]: manifest["hardware_status"] = "fail" manifest["termination"], manifest["failure_category"] = "completed", flashed.category or "uart_nonce" - return _finalize(run_dir, manifest, run_started_monotonic) + return _finalize(run_dir, manifest, run_started_monotonic, trusted_snapshot) registers = read_registers(args.openocd, config.openocd_board_config, args.jtag_serial, config.assertions, cwd=workspace, evidence_dir=run_dir, timeout_seconds=bounded_timeout(config.openocd_command_timeout_seconds), runner=tool_runner) @@ -658,7 +698,7 @@ def record_phase(name: str, result: Any = None, **details: Any) -> None: manifest["failure_category"] = None if registers.status == "pass" else "register_mismatch" record_phase("register", registers.command_result, status=registers.status, assertions=manifest["register_assertions"]) - return _finalize(run_dir, manifest, run_started_monotonic) + return _finalize(run_dir, manifest, run_started_monotonic, trusted_snapshot) except UnsafeWorkspaceError as error: if workspace is not None: _quarantine_workspace(workspace) @@ -683,7 +723,7 @@ def record_phase(name: str, result: Any = None, **details: Any) -> None: manifest["failure_category"] = "interrupt" manifest["detail"] = sanitize(type(error).__name__) try: - _finalize(run_dir, manifest, run_started_monotonic) + _finalize(run_dir, manifest, run_started_monotonic, trusted_snapshot) except (OSError, ValueError): persist() return 2 @@ -695,7 +735,7 @@ def record_phase(name: str, result: Any = None, **details: Any) -> None: manifest["failure_category"] = "infrastructure" manifest["detail"] = sanitize(str(error)) try: - _finalize(run_dir, manifest, run_started_monotonic) + _finalize(run_dir, manifest, run_started_monotonic, trusted_snapshot) except (OSError, ValueError) as final_error: manifest["detail"] = sanitize(f"{error}; evidence rejected: {final_error}") persist() @@ -703,8 +743,26 @@ def record_phase(name: str, result: Any = None, **details: Any) -> None: return 2 -def _finalize(run_dir: Path, manifest: dict[str, Any], started_monotonic: float) -> int: - _safe_remove_tree(run_dir / "workspace/firmware/.pio") +def _finalize(run_dir: Path, manifest: dict[str, Any], started_monotonic: float, + trusted_snapshot: Path | None = None) -> int: + if trusted_snapshot is not None: + build_names = [name for name in os.listdir(run_dir) if name.startswith("build-workspace-")] + second_snapshot = None + for name in build_names: + build_path = run_dir / name + _safe_remove_tree(build_path / "firmware/.pio") + if build_path.exists(): + second_snapshot = _freeze_workspace(build_path, None) + chosen = second_snapshot if second_snapshot is not None and _tree_hash(second_snapshot) == _tree_hash(trusted_snapshot) else trusted_snapshot + delivered = run_dir / "source" + if delivered.exists(): _safe_remove_tree(delivered) + _copy_public(chosen, delivered) + if second_snapshot is not None: _safe_remove_tree(second_snapshot.parent) + _safe_remove_tree(trusted_snapshot.parent) + for name in os.listdir(run_dir): + if name.startswith("build-workspace-"): _quarantine_workspace(run_dir / name) + else: + _safe_remove_tree(run_dir / "workspace/firmware/.pio") credential_values = tuple(filter(None, (os.environ.get(name, "") for name in ( "LABWIRED_ACCESS_TOKEN", "LABWIRED_PROJECT", "LABWIRED_MODEL_URL", "LABWIRED_MODEL_KEY")))) _reject_secret_files(run_dir, credential_values) diff --git a/tests/twin2silicon-hil.py b/tests/twin2silicon-hil.py index 6311c4a..aab8e59 100644 --- a/tests/twin2silicon-hil.py +++ b/tests/twin2silicon-hil.py @@ -180,6 +180,17 @@ def test_write_json_atomic_replaces_destination_with_json(self): class ProcessContractTests(unittest.TestCase): + def test_streaming_redaction_handles_cross_chunk_secret_and_oversize_log(self): + with tempfile.TemporaryDirectory() as directory: + root=Path(directory); crossed=root/'crossed'; secret=b'CROSS_BOUNDARY_SECRET' + crossed.write_bytes(b'A'*(1024*1024-len(secret)//2)+secret+b'Z') + self.assertIsNone(process_module._redact_logs((str(crossed),),(secret,))) + self.assertNotIn(secret,crossed.read_bytes()); self.assertIn(b'[REDACTED]',crossed.read_bytes()) + huge=root/'huge'; + with huge.open('wb') as output: output.truncate(20*1024*1024) + self.assertEqual(process_module._redact_logs((str(huge),),(b'x',)),"evidence_log_too_large") + self.assertEqual(huge.read_bytes(),b'[EVIDENCE LOG TOO LARGE]\n') + def test_successful_leader_cannot_leave_mutating_descendant(self): with tempfile.TemporaryDirectory() as directory: evidence = Path(directory); child_path = evidence / "child"; mutation = evidence / "mutation" @@ -235,6 +246,17 @@ def test_run_command_redacts_exact_bytes_before_interrupt_reraises(self): finally: timer.cancel() self.assertEqual(output.read_text(), "[REDACTED]\n") + def test_interrupted_sparse_log_is_replaced_boundedly(self): + with tempfile.TemporaryDirectory() as directory: + evidence=Path(directory); output=evidence/'o' + timer=threading.Timer(.15,lambda: os.kill(os.getpid(),signal.SIGINT)); timer.start() + try: + with self.assertRaises(KeyboardInterrupt): + run_command([sys.executable,'-c',"import os,time; os.lseek(1,20*1024*1024,0); os.write(1,b'x'); time.sleep(30)"], + cwd=evidence,stdout_path=output,stderr_path=evidence/'e',timeout_seconds=30,redact_values=(b'secret',)) + finally: timer.cancel() + self.assertEqual(output.read_bytes(),b'[EVIDENCE LOG TOO LARGE]\n') + def test_run_command_interrupt_kills_sigterm_ignoring_descendant_after_leader_exits(self): with tempfile.TemporaryDirectory() as directory: evidence = Path(directory); child_path = evidence / "child" @@ -1047,7 +1069,7 @@ def test_agent_mode_repairs_workspace_with_allowlisted_home_and_no_secret_eviden self.assertEqual(result.returncode, 0, result.stderr) call = json.loads(invocation.read_text()) self.assertEqual(call["argv"][:4], ["agent", "run", "--model", "fixture/model"]) - self.assertEqual(call["cwd"], "workspace") + self.assertTrue(call["cwd"].startswith("agent-workspace-")) manifest_text = (run_dir / "run.json").read_text() self.assertNotIn("SECRET_TOKEN_SENTINEL", manifest_text) for path in run_dir.rglob("*"): @@ -1121,6 +1143,27 @@ def test_agent_oversize_sparse_file_is_bounded_and_workspace_quarantined(self): self.assertFalse((run_dir/'workspace').exists()) self.assertEqual(json.loads((run_dir/'run.json').read_text())["failure_category"],"unsafe_workspace") + def test_fd_relative_quarantine_delete_never_follows_swapped_symlink(self): + sys.path.insert(0,str(REPOSITORY_ROOT/'benchmarks/twin2silicon')); import run_hil + with tempfile.TemporaryDirectory() as directory: + root=Path(directory); victim=root/'victim'; child=victim/'child'; child.mkdir(parents=True) + for index in range(500): (child/f'f{index}').write_text('x') + external=root/'external'; external.mkdir(); sentinel=external/'sentinel'; sentinel.write_text('safe') + moved=victim/'moved'; stop=threading.Event() + def swap(): + while not stop.is_set(): + try: + child.rename(moved); child.symlink_to(external, target_is_directory=True) + child.unlink(); moved.rename(child) + except (FileNotFoundError,OSError): pass + thread=threading.Thread(target=swap); thread.start() + try: + try: run_hil._safe_remove_tree(victim) + except run_hil.UnsafeWorkspaceError: pass + finally: + stop.set(); thread.join(1) + self.assertEqual(sentinel.read_text(),'safe') + def test_detached_agent_mutator_cannot_change_frozen_workspace(self): task = REPOSITORY_ROOT / "benchmarks/twin2silicon/tasks/esp32s3-gpio-hil-001" with tempfile.TemporaryDirectory() as directory: @@ -1133,21 +1176,32 @@ def test_detached_agent_mutator_cannot_change_frozen_workspace(self): import pathlib while not pathlib.Path({str(pid_path)!r}).exists(): time.sleep(.01) """)) - tool_dir=root/'tool'; tool_dir.mkdir() - tool=executable_fixture(tool_dir,"import sys\nif '--version' in sys.argv: print('v'); raise SystemExit(0)\nraise SystemExit(0 if 'clean' in sys.argv else 1)\n") + build_pid=root/'build-child.pid'; tool_dir=root/'tool'; tool_dir.mkdir() + tool=executable_fixture(tool_dir,textwrap.dedent(f""" + import pathlib,subprocess,sys + if '--version' in sys.argv: print('v'); raise SystemExit(0) + if 'clean' in sys.argv: raise SystemExit(0) + project=pathlib.Path(sys.argv[sys.argv.index('--project-dir')+1]) + child="import os,pathlib,time; p=pathlib.Path(%r); pathlib.Path(%r).write_text(str(os.getpid()));\\nwhile True:\\n p.parent.mkdir(parents=True,exist_ok=True); open(p,'ab').write(b'Y'); time.sleep(.01)" % (str(project/'src/main.c'), {str(build_pid)!r}) + subprocess.Popen([sys.executable,'-c',child],start_new_session=True) + raise SystemExit(1) + """)) run_dir=root/'run' try: result=self._run_cli(task,'--run-dir',run_dir,'--agent-bin',agent,'--model','m','--jtag-serial','J', '--uart-device','/dev/null','--openocd',tool,'--platformio',tool) self.assertEqual(result.returncode,0,result.stderr) sys.path.insert(0,str(REPOSITORY_ROOT/'benchmarks/twin2silicon')); import run_hil - before=run_hil._tree_hash(run_dir/'workspace'); time.sleep(.2) - self.assertEqual(before,run_hil._tree_hash(run_dir/'workspace')) + before=run_hil._tree_hash(run_dir/'source'); time.sleep(.2) + self.assertEqual(before,run_hil._tree_hash(run_dir/'source')) self.assertEqual(before,json.loads((run_dir/'run.json').read_text())["hashes"]["source_final"]) finally: if pid_path.exists(): try: os.kill(int(pid_path.read_text()),signal.SIGKILL) except ProcessLookupError: pass + if build_pid.exists(): + try: os.kill(int(build_pid.read_text()),signal.SIGKILL) + except ProcessLookupError: pass def test_cli_physical_pass_uses_pty_and_records_ordered_evidence(self): task = self._short_task() From da631ba205950da0aeedb90f9b73b5829a5cd3d6 Mon Sep 17 00:00:00 2001 From: Andrii Shylenko <14119286+w1ne@users.noreply.github.com> Date: Sat, 15 Aug 2026 14:21:52 +0200 Subject: [PATCH 24/48] revert(bench): remove overengineered HIL orchestration --- benchmarks/twin2silicon/hil/esp32s3.py | 19 +- benchmarks/twin2silicon/hil/process.py | 113 +--- benchmarks/twin2silicon/run_hil.py | 792 ------------------------- tests/twin2silicon-hil.py | 685 --------------------- 4 files changed, 5 insertions(+), 1604 deletions(-) delete mode 100644 benchmarks/twin2silicon/run_hil.py diff --git a/benchmarks/twin2silicon/hil/esp32s3.py b/benchmarks/twin2silicon/hil/esp32s3.py index e8784b2..944b464 100644 --- a/benchmarks/twin2silicon/hil/esp32s3.py +++ b/benchmarks/twin2silicon/hil/esp32s3.py @@ -13,7 +13,6 @@ import time import tty from typing import Callable, Literal, Mapping, Optional, Sequence -import threading from .process import run_command from .results import CommandResult, PathLike @@ -273,13 +272,11 @@ class UartResult: matched: bool bytes_captured: int timed_out: bool - termination_reason: Literal["matched", "timeout", "max_bytes", "cancelled"] + termination_reason: Literal["matched", "timeout", "max_bytes"] def capture_uart_nonce(device: PathLike, baud: int, nonce: str, timeout_seconds: float, - log: PathLike, *, max_bytes: int = 65536, - cancel_event: Optional[threading.Event] = None, - started_event: Optional[threading.Event] = None) -> UartResult: + log: PathLike, *, max_bytes: int = 65536) -> UartResult: fd = os.open(device, os.O_RDWR | os.O_NOCTTY | os.O_NONBLOCK) try: speeds = {9600: termios.B9600, 115200: termios.B115200} @@ -293,23 +290,17 @@ def capture_uart_nonce(device: PathLike, baud: int, nonce: str, timeout_seconds: attrs[4] = attrs[5] = speeds[baud] attrs[2] = (attrs[2] & ~(termios.CSIZE | termios.PARENB | termios.CSTOPB)) | termios.CS8 | termios.CLOCAL | termios.CREAD termios.tcsetattr(fd, termios.TCSANOW, attrs) - if started_event is not None: - started_event.set() deadline = time.monotonic() + timeout_seconds captured = bytearray() pending = bytearray() matched = False expected = f"LABWIRED_READY:{nonce}".encode() while not matched and len(captured) < max_bytes: - if cancel_event is not None and cancel_event.is_set(): - break remaining = deadline - time.monotonic() if remaining <= 0: break - readable, _, _ = select.select([fd], [], [], min(remaining, 0.05) if cancel_event else remaining) + readable, _, _ = select.select([fd], [], [], remaining) if not readable: - if cancel_event is not None: - continue break try: chunk = os.read(fd, min(4096, max_bytes - len(captured))) @@ -329,11 +320,9 @@ def capture_uart_nonce(device: PathLike, baud: int, nonce: str, timeout_seconds: break Path(log).parent.mkdir(parents=True, exist_ok=True) Path(log).write_bytes(captured) - reason: Literal["matched", "timeout", "max_bytes", "cancelled"] + reason: Literal["matched", "timeout", "max_bytes"] if matched: reason = "matched" - elif cancel_event is not None and cancel_event.is_set(): - reason = "cancelled" elif len(captured) >= max_bytes: reason = "max_bytes" else: diff --git a/benchmarks/twin2silicon/hil/process.py b/benchmarks/twin2silicon/hil/process.py index 610e682..63a1c2a 100644 --- a/benchmarks/twin2silicon/hil/process.py +++ b/benchmarks/twin2silicon/hil/process.py @@ -5,10 +5,8 @@ from pathlib import Path import signal import subprocess -import stat -import tempfile import time -from typing import Mapping, Optional, Sequence, Union +from typing import Sequence, Union from .results import CommandResult, PathLike @@ -44,8 +42,6 @@ def run_command( stdout_path: PathLike, stderr_path: PathLike, timeout_seconds: float, - env: Optional[Mapping[str, str]] = None, - redact_values: Sequence[Union[str, bytes]] = (), ) -> CommandResult: normalized_command = tuple(os.fspath(part) for part in command) normalized_cwd = str(Path(cwd).resolve()) @@ -65,31 +61,9 @@ def run_command( stdout=stdout, stderr=stderr, start_new_session=True, - env=env, ) try: process.communicate(timeout=timeout_seconds) - except (KeyboardInterrupt, SystemExit) as interruption: - try: - os.killpg(process.pid, signal.SIGTERM) - except (PermissionError, ProcessLookupError): - pass - try: - process.wait(timeout=0.5) - except subprocess.TimeoutExpired: - pass - if _process_group_exists(process.pid): - try: - os.killpg(process.pid, signal.SIGKILL) - except (PermissionError, ProcessLookupError): - pass - try: - process.wait(timeout=0.5) - except subprocess.TimeoutExpired: - pass - _wait_for_process_group_exit(process.pid, 1.0) - _redact_logs((normalized_stdout, normalized_stderr), redact_values) - raise interruption except subprocess.TimeoutExpired: timed_out = True try: @@ -115,24 +89,7 @@ def run_command( group_exited = _wait_for_process_group_exit(process.pid, 0.5) if not group_exited and cleanup_error is None: cleanup_error = "process_group_did_not_exit" - else: - if _process_group_exists(process.pid): - cleanup_error = "unexpected_descendant_processes" - try: - os.killpg(process.pid, signal.SIGTERM) - except (PermissionError, ProcessLookupError): - pass - if not _wait_for_process_group_exit(process.pid, 0.5): - try: - os.killpg(process.pid, signal.SIGKILL) - except (PermissionError, ProcessLookupError): - pass - if not _wait_for_process_group_exit(process.pid, 1.0): - cleanup_error = "process_group_did_not_exit" - redaction_error = _redact_logs((normalized_stdout, normalized_stderr), redact_values) - if redaction_error is not None and cleanup_error is None: - cleanup_error = redaction_error ended_at = _utc_now() return CommandResult( command=normalized_command, @@ -146,71 +103,3 @@ def run_command( stderr_path=normalized_stderr, cleanup_error=cleanup_error, ) - - -def _redact_logs(paths: Sequence[str], values: Sequence[Union[str, bytes]]) -> Optional[str]: - needles = tuple(value.encode() if isinstance(value, str) else value for value in values if value) - if not needles: - for path in paths: - try: - if os.lstat(path).st_size > 16 * 1024 * 1024: - temporary_fd, temporary = tempfile.mkstemp(prefix=".redacted-", dir=str(Path(path).parent)) - try: os.write(temporary_fd, b"[EVIDENCE LOG TOO LARGE]\n"); os.fsync(temporary_fd) - finally: os.close(temporary_fd) - os.replace(temporary, path) - return "evidence_log_too_large" - except FileNotFoundError: pass - return None - error = None - for path in paths: - temporary = None - try: - if os.lstat(path).st_size > 16 * 1024 * 1024: - temporary_fd, temporary = tempfile.mkstemp(prefix=".redacted-", dir=str(Path(path).parent)) - try: - os.write(temporary_fd, b"[EVIDENCE LOG TOO LARGE]\n"); os.fsync(temporary_fd) - finally: os.close(temporary_fd) - os.replace(temporary, path); temporary = None - error = "evidence_log_too_large" - continue - fd = os.open(path, os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0)) - temporary_fd, temporary = tempfile.mkstemp(prefix=".redacted-", dir=str(Path(path).parent)) - try: - info = os.fstat(fd) - if not stat.S_ISREG(info.st_mode): - raise OSError("evidence log is not a regular file") - overlap = b""; maximum = max(map(len, needles)) - while chunk := os.read(fd, 1024 * 1024): - data = overlap + chunk - limit = max(0, len(data) - maximum + 1) - overlap = _write_redacted_prefix(temporary_fd, data, limit, needles) - _write_redacted_prefix(temporary_fd, overlap, len(overlap), needles) - os.fsync(temporary_fd) - finally: - os.close(fd) - os.close(temporary_fd) - os.replace(temporary, path); temporary = None - except FileNotFoundError: - pass - finally: - if temporary is not None: - try: os.unlink(temporary) - except FileNotFoundError: pass - return error - - -def _write_redacted_prefix(output_fd: int, data: bytes, limit: int, - needles: Sequence[bytes]) -> bytes: - cursor = 0 - while cursor < limit: - matches = [(position, needle) for needle in needles - if (position := data.find(needle, cursor)) != -1 and position < limit] - if not matches: - os.write(output_fd, data[cursor:limit]) - cursor = limit - break - position, needle = min(matches, key=lambda item: item[0]) - os.write(output_fd, data[cursor:position]) - os.write(output_fd, b"[REDACTED]") - cursor = position + len(needle) - return data[max(cursor, limit):] diff --git a/benchmarks/twin2silicon/run_hil.py b/benchmarks/twin2silicon/run_hil.py deleted file mode 100644 index 8e29942..0000000 --- a/benchmarks/twin2silicon/run_hil.py +++ /dev/null @@ -1,792 +0,0 @@ -#!/usr/bin/env python3 -"""Run an isolated, reproducible ESP32-S3 hardware-in-loop evaluation. - -Fixture commands are argv arrays supplied with ``--identity-command-json``; -shell command strings are deliberately unsupported. -""" - -from __future__ import annotations - -import argparse -from dataclasses import replace -from datetime import datetime, timezone -import hashlib -import json -import math -import os -from pathlib import Path -import stat -import tempfile -import secrets -import sys -import time -import signal -import subprocess -import threading -from functools import partial -from typing import Any, Mapping - -if __package__ in (None, ""): - sys.path.insert(0, str(Path(__file__).resolve().parents[2])) - -from benchmarks.twin2silicon.hil.esp32s3 import ( - BoardLock, BoardLockTimeout, Esp32S3Config, capture_uart_nonce, - flash_firmware, read_registers, validate_identity, -) -from benchmarks.twin2silicon.hil.process import run_command -from benchmarks.twin2silicon.hil.results import sha256_file, write_json_atomic - - -ROOT = Path(__file__).resolve().parent -TASKS = ROOT / "tasks" -HARNESS_REVISION = "twin2silicon-hil-1" -MAX_REGULAR_FILE_BYTES = 32 * 1024 * 1024 -MAX_TREE_BYTES = 128 * 1024 * 1024 - - -class UartWorkerFatal(BaseException): - pass - - -class UnsafeWorkspaceError(ValueError): - pass - - -class _FileEvent: - def __init__(self, path: Path) -> None: - self.path = path - - def set(self) -> None: - self.path.write_text("ready", encoding="utf-8") - - -def _uart_subprocess_worker(spec_path: Path) -> int: - spec = json.loads(spec_path.read_text(encoding="utf-8")) - if spec["mode"] == "startup-hang": - signal.signal(signal.SIGTERM, signal.SIG_IGN) - while True: - time.sleep(1) - cancel_event = threading.Event() - signal.signal(signal.SIGTERM, lambda *_: cancel_event.set()) - try: - result = capture_uart_nonce(spec["device"], spec["baud"], spec["nonce"], spec["timeout_seconds"], - spec["log_path"], cancel_event=cancel_event, - started_event=_FileEvent(Path(spec["ready_path"]))) - write_json_atomic(spec["result_path"], {"kind": "result", "matched": result.matched, - "bytes_captured": result.bytes_captured, "timed_out": result.timed_out, - "termination_reason": result.termination_reason}) - except BaseException as error: - write_json_atomic(spec["result_path"], {"kind": "error", "detail": f"{type(error).__name__}: {error}"}) - Path(spec["ready_path"]).write_text("error", encoding="utf-8") - return 0 - - -def _stop_uart_process(process: subprocess.Popen[Any]) -> bool: - if process.poll() is None: - try: os.killpg(process.pid, signal.SIGTERM) - except ProcessLookupError: pass - try: process.wait(timeout=.5) - except subprocess.TimeoutExpired: pass - if process.poll() is None: - try: os.killpg(process.pid, signal.SIGKILL) - except ProcessLookupError: pass - try: process.wait(timeout=.5) - except subprocess.TimeoutExpired: pass - return process.poll() is not None - - -def _number(value: object, field: str, *, integer: bool = False) -> int | float: - valid = isinstance(value, int if integer else (int, float)) and not isinstance(value, bool) - finite = integer or (valid and math.isfinite(value)) - if not valid or value < 0 or not finite: - raise ValueError(f"{field} must be a nonnegative finite {'integer' if integer else 'number'}") - return int(value) if integer else float(value) - - -def parse_usage(path: Path) -> dict[str, Any]: - raw = json.loads(path.read_text(encoding="utf-8")) - root_keys = {"schema_version", "requests", "tokens", "final_context_tokens", "latency_seconds", - "provider", "model", "rates_usd_per_million", "price_source", "price_effective_date"} - if not isinstance(raw, Mapping) or set(raw) != root_keys or raw.get("schema_version") != "1.0": - raise ValueError("usage schema must be exactly version 1.0") - if not isinstance(raw, Mapping) or not isinstance(raw.get("tokens"), Mapping): - raise ValueError("usage must contain an object and tokens object") - if not isinstance(raw.get("rates_usd_per_million"), Mapping): - raise ValueError("usage must contain rates_usd_per_million") - if set(raw["tokens"]) != {"fresh_input", "cached_input", "output", "reasoning"}: - raise ValueError("usage token keys are not exact") - if set(raw["rates_usd_per_million"]) != {"fresh_input", "cached_input", "output"}: - raise ValueError("usage rate keys are not exact") - tokens = {name: _number(raw["tokens"].get(name), f"tokens.{name}", integer=True) - for name in ("fresh_input", "cached_input", "output", "reasoning")} - rates = {name: _number(raw["rates_usd_per_million"].get(name), f"rates.{name}") - for name in ("fresh_input", "cached_input", "output")} - for name in ("provider", "model", "price_source", "price_effective_date"): - if not isinstance(raw.get(name), str) or not raw[name]: - raise ValueError(f"{name} must be a nonempty string") - result = { - "schema_version": "1.0", "requests": _number(raw.get("requests"), "requests", integer=True), - "tokens": tokens, "final_context_tokens": _number(raw.get("final_context_tokens"), "final_context_tokens", integer=True), - "latency_seconds": _number(raw.get("latency_seconds"), "latency_seconds"), - "provider": raw["provider"], "model": raw["model"], "rates_usd_per_million": rates, - "price_source": raw["price_source"], "price_effective_date": raw["price_effective_date"], - } - result["cost_usd"] = (tokens["fresh_input"] * rates["fresh_input"] - + tokens["cached_input"] * rates["cached_input"] - + tokens["output"] * rates["output"]) / 1_000_000 - return result - - -def _tree_hash(root: Path) -> str: - digest = hashlib.sha256() - for path in _safe_files(root): - digest.update(path.relative_to(root).as_posix().encode()) - digest.update(b"\0") - digest.update(bytes.fromhex(_safe_hash(path))) - return digest.hexdigest() - - -def _safe_tree(source: Path, field: str) -> Path: - if stat.S_ISLNK(os.lstat(source).st_mode): - raise ValueError(f"{field} root is a symlink") - source = source.resolve(strict=True) - if not source.is_dir(): - raise ValueError(f"{field} is not a directory") - _safe_files(source) - return source - - -def _safe_files(root: Path, deadline: float | None = None) -> list[Path]: - root = root.absolute() - files: list[Path] = [] - total = 0 - def visit(directory: Path) -> None: - nonlocal total - if deadline is not None and time.monotonic() >= deadline: - raise UnsafeWorkspaceError("validation deadline exhausted") - with os.scandir(directory) as entries: - for entry in entries: - info = entry.stat(follow_symlinks=False) - path = directory / entry.name - if stat.S_ISLNK(info.st_mode): - raise UnsafeWorkspaceError(f"unsafe symlink: {path.relative_to(root)}") - if stat.S_ISDIR(info.st_mode): - visit(path) - elif stat.S_ISREG(info.st_mode): - if info.st_size > MAX_REGULAR_FILE_BYTES: - raise UnsafeWorkspaceError(f"oversize file: {path.relative_to(root)}") - total += info.st_size - if total > MAX_TREE_BYTES: - raise UnsafeWorkspaceError("tree size limit exceeded") - files.append(path) - else: - raise UnsafeWorkspaceError(f"unsafe non-regular file: {path.relative_to(root)}") - visit(root) - return sorted(files) - - -def _safe_remove_tree(path: Path) -> None: - try: - parent_fd = os.open(path.parent, os.O_RDONLY | os.O_DIRECTORY | getattr(os, "O_NOFOLLOW", 0)) - except FileNotFoundError: - return - try: - _remove_entry_at(parent_fd, path.name) - finally: - os.close(parent_fd) - - -def _remove_entry_at(parent_fd: int, name: str) -> None: - for _ in range(3): - try: - info = os.stat(name, dir_fd=parent_fd, follow_symlinks=False) - except FileNotFoundError: - return - if not stat.S_ISDIR(info.st_mode) or stat.S_ISLNK(info.st_mode): - try: - os.unlink(name, dir_fd=parent_fd) - return - except IsADirectoryError: - continue - try: - child_fd = os.open(name, os.O_RDONLY | os.O_DIRECTORY | getattr(os, "O_NOFOLLOW", 0), dir_fd=parent_fd) - except (FileNotFoundError, NotADirectoryError, OSError): - continue - try: - opened = os.fstat(child_fd) - if (opened.st_dev, opened.st_ino) != (info.st_dev, info.st_ino): - continue - for child_name in os.listdir(child_fd): - _remove_entry_at(child_fd, child_name) - finally: - os.close(child_fd) - try: - os.rmdir(name, dir_fd=parent_fd) - return - except (FileNotFoundError, NotADirectoryError): - return - except OSError: - continue - raise UnsafeWorkspaceError("quarantine deletion race detected") - - -def _quarantine_workspace(workspace: Path) -> None: - if not workspace.exists() and not workspace.is_symlink(): - return - root = Path(tempfile.mkdtemp(prefix="hil-quarantine-", dir=workspace.parent.parent)) - quarantined = root / "workspace" - os.rename(workspace, quarantined) - try: - _safe_remove_tree(quarantined) - _safe_remove_tree(root) - except UnsafeWorkspaceError: - pass - - -def _freeze_workspace(workspace: Path, deadline: float | None) -> Path: - root = Path(tempfile.mkdtemp(prefix="hil-snapshot-", dir=workspace.parent.parent)) - frozen = root / "snapshot" - try: - _copy_public(workspace, frozen, deadline) - _quarantine_workspace(workspace) - return frozen - except BaseException: - _safe_remove_tree(root) - raise - - -def _safe_hash(path: Path) -> str: - flags = os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0) - fd = os.open(path, flags) - try: - before = os.lstat(path); opened = os.fstat(fd) - if not stat.S_ISREG(opened.st_mode) or (before.st_dev, before.st_ino) != (opened.st_dev, opened.st_ino): - raise ValueError(f"unsafe changed artifact: {path.name}") - digest = hashlib.sha256() - while chunk := os.read(fd, 1024 * 1024): - digest.update(chunk) - return digest.hexdigest() - finally: - os.close(fd) - - -def _safe_copy_file(source: Path, destination: Path) -> None: - source_fd = os.open(source, os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0)) - try: - info = os.fstat(source_fd) - if not stat.S_ISREG(info.st_mode) or info.st_size > MAX_REGULAR_FILE_BYTES: - raise UnsafeWorkspaceError("firmware artifact is unsafe or oversized") - destination_fd = os.open(destination, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600) - try: - while chunk := os.read(source_fd, 1024 * 1024): os.write(destination_fd, chunk) - finally: os.close(destination_fd) - finally: os.close(source_fd) - - -def _resolve_task(value: str) -> Path: - supplied = Path(value) - candidate = supplied if supplied.is_absolute() or len(supplied.parts) > 1 else TASKS / supplied - resolved = candidate.resolve(strict=True) - if resolved != TASKS.resolve() and TASKS.resolve() not in resolved.parents: - raise ValueError("task must be beneath the benchmark tasks directory") - if not resolved.is_dir() or (resolved / "task.json").is_symlink(): - raise ValueError("invalid task directory") - return resolved - - -def _copy_public(source: Path, destination: Path, deadline: float | None = None) -> None: - source = _safe_tree(source, "candidate/public workspace") - destination.mkdir() - for path in _safe_files(source, deadline): - target = destination / path.relative_to(source) - target.parent.mkdir(parents=True, exist_ok=True) - source_fd = os.open(path, os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0)) - try: - target_fd = os.open(target, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600) - try: - while chunk := os.read(source_fd, 1024 * 1024): - if deadline is not None and time.monotonic() >= deadline: - raise UnsafeWorkspaceError("copy deadline exhausted") - os.write(target_fd, chunk) - finally: - os.close(target_fd) - finally: - os.close(source_fd) - - -def _command_record(result: Any, run_dir: Path, redactions: tuple[str, ...] = ()) -> dict[str, Any]: - def sanitize(value: str) -> str: - for secret in redactions: - if secret: - value = value.replace(secret, "[REDACTED]") - return value - def relative(value: str) -> str: - try: - return Path(value).resolve().relative_to(run_dir).as_posix() - except ValueError: - return "workspace" if Path(value).name == "workspace" else Path(value).name - return {"argv": [sanitize(Path(item).name if Path(item).is_absolute() else item) for item in result.command], - "cwd": relative(result.cwd), "stdout": relative(result.stdout_path), - "stderr": relative(result.stderr_path), "started_at_utc": result.started_at_utc, - "ended_at_utc": result.ended_at_utc, "returncode": result.returncode, - "timed_out": result.timed_out, "duration_seconds": result.duration_seconds, - "cleanup_error": result.cleanup_error} - - -def _tool_version(name: str, executable: os.PathLike[str] | str, workspace: Path, - run_dir: Path, env: Mapping[str, str] | None = None, - redactions: tuple[str, ...] = (), timeout_seconds: float = 2) -> dict[str, str]: - record = {"executable": Path(executable).name, "version": "not_available"} - try: - result = run_command([executable, "--version"], cwd=workspace, - stdout_path=run_dir / f"version.{name}.stdout.log", - stderr_path=run_dir / f"version.{name}.stderr.log", timeout_seconds=timeout_seconds, env=env, - redact_values=redactions) - if result.returncode == 0 and not result.timed_out and not result.cleanup_error: - text = Path(result.stdout_path).read_text(encoding="utf-8", errors="replace").strip() - record["version"] = " ".join(text.split())[:200] or "unknown" - except OSError: - pass - return record - - -def _agent_environment() -> dict[str, str]: - allowed = ("PATH", "HOME", "LABWIRED_HOME", "XDG_CONFIG_HOME", "TMPDIR", "LANG", "LC_ALL", - "LABWIRED_ACCESS_TOKEN", "LABWIRED_PROJECT", "LABWIRED_MODEL_URL", "LABWIRED_MODEL_KEY") - return {name: os.environ[name] for name in allowed if name in os.environ} - - -def _tool_environment() -> dict[str, str]: - allowed = ("PATH", "HOME", "PLATFORMIO_CORE_DIR", "TMPDIR", "LANG", "LC_ALL") - return {name: os.environ[name] for name in allowed if name in os.environ} - - -def _reject_secret_files(root: Path, secret_values: tuple[str, ...]) -> None: - needles = tuple(value.encode() for value in secret_values if value) - for path in _safe_files(root): - flags = os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0) - fd = os.open(path, flags) - try: - overlap = b"" - width = max((len(needle) for needle in needles), default=1) - 1 - while chunk := os.read(fd, 1024 * 1024): - window = overlap + chunk - if any(needle in window for needle in needles): - raise UnsafeWorkspaceError(f"credential value found in workspace artifact: {path.relative_to(root)}") - overlap = window[-width:] if width else b"" - finally: - os.close(fd) - - -def parser() -> argparse.ArgumentParser: - value = argparse.ArgumentParser(description=__doc__) - value.add_argument("task", help="task id, or path beneath benchmarks/twin2silicon/tasks") - value.add_argument("--run-dir", required=True, type=Path) - mode = value.add_mutually_exclusive_group(required=True) - mode.add_argument("--evaluate-only", action="store_true") - mode.add_argument("--agent-bin", type=Path) - value.add_argument("--candidate", type=Path) - value.add_argument("--model") - value.add_argument("--jtag-serial", required=True) - value.add_argument("--uart-device", required=True) - value.add_argument("--openocd", required=True) - value.add_argument("--platformio", default="pio") - value.add_argument("--identity-command-json", help="fixture identity argv as a JSON array (never a shell string)") - value.add_argument("--usage-json", type=Path) - value.add_argument("--fixture-uart-timeout-seconds", type=float, - help="offline fixture only: shorten (never extend) the oracle UART timeout") - value.add_argument("--fixture-identity-timeout-seconds", type=float, - help="offline fixture only: shorten (never extend) identity/lock timeout") - value.add_argument("--fixture-uart-worker-mode", choices=("normal", "startup-hang"), default="normal", - help="offline fixture only: exercise UART worker startup cleanup") - value.add_argument("--fixture-wall-time-seconds", type=float, - help="offline fixture only: shorten (never extend) overall wall budget") - return value - - -def main(argv: list[str] | None = None) -> int: - run_started_monotonic = time.monotonic() - args = parser().parse_args(argv) - if args.evaluate_only != bool(args.candidate) or (args.agent_bin and not args.model): - parser().error("evaluate-only requires --candidate; agent mode requires --model") - run_dir = args.run_dir.absolute() - if run_dir.exists(): - print("run directory already exists", file=sys.stderr) - return 2 - # Reserve the path before resolving tools or executing any command. - run_dir.mkdir(parents=True) - manifest: dict[str, Any] = { - "schema_version": "1.0", "run": {"id": run_dir.name, "started_at_utc": datetime.now(timezone.utc).isoformat().replace("+00:00", "Z")}, - "task": {}, "harness_revision": HARNESS_REVISION, "model": None, "provider": None, - "requests": None, "tokens": {"fresh_input": None, "cached_input": None, "output": None, "reasoning": None}, - "final_context_tokens": None, "latency_seconds": None, "cost_usd": None, - "configured_budgets": {}, "budget_validity": {}, "tool_versions": {}, - "model_status": "not_run", "compile_status": "not_run", "simulator_status": "not_supported", - "hardware_status": "not_run", "infrastructure_status": "ok", "uart": None, - "register_assertions": [], "termination": "running", "failure_category": None, - "hashes": {}, "artifacts": [], "environment": {}, "phases": [], - } - workspace: Path | None = None - trusted_snapshot: Path | None = None - overall_deadline: float | None = None - def bounded_timeout(cap: float) -> float: - if overall_deadline is None: - return cap - remaining = overall_deadline - time.monotonic() - if remaining <= 0: - raise RuntimeError("overall wall-time budget exhausted") - return min(cap, remaining) - redactions = tuple(filter(None, (args.jtag_serial, args.uart_device, - *(os.environ.get(name, "") for name in ("LABWIRED_ACCESS_TOKEN", "LABWIRED_MODEL_KEY"))))) - credential_values = tuple(filter(None, (os.environ.get(name, "") for name in ( - "LABWIRED_ACCESS_TOKEN", "LABWIRED_PROJECT", "LABWIRED_MODEL_URL", "LABWIRED_MODEL_KEY")))) - tool_env = _tool_environment() - tool_runner = partial(run_command, env=tool_env, redact_values=credential_values) - def sanitize(value: Any) -> Any: - if isinstance(value, str): - for secret in (*redactions, *credential_values): - value = value.replace(secret, "[REDACTED]") - return value - if isinstance(value, list): return [sanitize(item) for item in value] - if isinstance(value, tuple): return [sanitize(item) for item in value] - if isinstance(value, dict): return {key: sanitize(item) for key, item in value.items()} - return value - def persist() -> None: - write_json_atomic(run_dir / "run.json", sanitize(manifest)) - def record_phase(name: str, result: Any = None, **details: Any) -> None: - phase = {"name": name, **details} - if result is not None: - phase["command"] = _command_record(result, run_dir, redactions) - manifest["phases"].append(phase) - persist() - persist() - try: - task_root = _resolve_task(args.task) - task = json.loads((task_root / "task.json").read_text(encoding="utf-8")) - wall_cap = float(task["budgets"]["wall_time_seconds"]) - if args.fixture_wall_time_seconds is not None: - if (not math.isfinite(args.fixture_wall_time_seconds) or args.fixture_wall_time_seconds < 0 - or args.fixture_wall_time_seconds > wall_cap): - raise ValueError("fixture wall timeout must be finite, nonnegative, and no larger than configured") - wall_cap = args.fixture_wall_time_seconds - task["budgets"]["wall_time_seconds"] = wall_cap - overall_deadline = time.monotonic() + wall_cap - if task.get("schema_version") != "1.0": - raise ValueError("unsupported task schema") - public = (task_root / task["public_dir"]).resolve(strict=True) - oracle_path = (task_root / task["hidden_oracle"]).resolve(strict=True) - if task_root not in public.parents or task_root not in oracle_path.parents: - raise ValueError("task path escapes task directory") - config = Esp32S3Config.from_oracle(json.loads(oracle_path.read_text(encoding="utf-8"))) - if args.fixture_uart_timeout_seconds is not None: - fixture_timeout = args.fixture_uart_timeout_seconds - if not math.isfinite(fixture_timeout) or fixture_timeout < 0 or fixture_timeout > config.uart_timeout_seconds: - raise ValueError("fixture UART timeout must be finite, nonnegative, and no larger than the oracle timeout") - config = replace(config, uart_timeout_seconds=fixture_timeout) - if args.fixture_identity_timeout_seconds is not None: - fixture_timeout = args.fixture_identity_timeout_seconds - if not math.isfinite(fixture_timeout) or fixture_timeout < 0 or fixture_timeout > config.identity_timeout_seconds: - raise ValueError("fixture identity timeout must be finite, nonnegative, and no larger than oracle timeout") - config = replace(config, identity_timeout_seconds=fixture_timeout) - source = _safe_tree(args.candidate, "candidate") if args.evaluate_only else _safe_tree(public, "public") - workspace = run_dir / (f"agent-workspace-{secrets.token_hex(8)}" if args.agent_bin else "workspace") - _copy_public(source, workspace, overall_deadline) - nonce = secrets.token_hex(16) - nonce_header = workspace / "firmware/include/run_nonce.h" - nonce_header.parent.mkdir(parents=True, exist_ok=True) - nonce_header.write_text(f'#pragma once\n#define LABWIRED_RUN_NONCE "{nonce}"\n', encoding="utf-8") - manifest["task"] = {"id": task["id"], "schema_version": task["schema_version"]} - manifest["configured_budgets"] = task.get("budgets", {}) - manifest["budget_validity"] = { - name: {"configured": configured, "observed": None, "within_budget": None} - for name, configured in task.get("budgets", {}).items() - } - for name in ("simulator_runs", "diagnostic_hil_runs"): - if name in manifest["budget_validity"]: - configured = manifest["budget_validity"][name]["configured"] - manifest["budget_validity"][name].update(observed=0, within_budget=0 <= configured) - manifest["environment"] = {"jtag_serial_sha256": hashlib.sha256(args.jtag_serial.encode()).hexdigest(), - "uart_device_sha256": hashlib.sha256(args.uart_device.encode()).hexdigest()} - manifest["hashes"]["oracle_descriptor"] = sha256_file(oracle_path) - manifest["hashes"]["source_initial"] = _tree_hash(workspace) - record_phase("prepared") - manifest["tool_versions"] = { - "platformio": _tool_version("platformio", args.platformio, workspace, run_dir, tool_env, credential_values, - bounded_timeout(2)), - "openocd": _tool_version("openocd", args.openocd, workspace, run_dir, tool_env, credential_values, - bounded_timeout(2)), - } - if args.agent_bin: - manifest["tool_versions"]["agent"] = _tool_version( - "agent", args.agent_bin, workspace, run_dir, _agent_environment(), credential_values, - bounded_timeout(2)) - persist() - - if args.usage_json: - usage = parse_usage(args.usage_json) - write_json_atomic(run_dir / "cost.json", usage) - for name in ("requests", "tokens", "final_context_tokens", "latency_seconds", "provider", "model", "cost_usd"): - manifest[name] = usage[name] - observed_tokens = usage["final_context_tokens"] - manifest["budget_validity"]["model_tokens"].update( - observed=observed_tokens, - within_budget=observed_tokens <= task.get("budgets", {}).get("model_tokens", math.inf)) - elif args.agent_bin: - manifest["model"] = args.model - - if args.agent_bin: - prompt = (workspace / "README.md").read_text(encoding="utf-8") - clean_env = _agent_environment() - agent = run_command([args.agent_bin, "agent", "run", "--model", args.model, prompt], cwd=workspace, - stdout_path=run_dir / "agent.stdout.log", stderr_path=run_dir / "agent.stderr.log", - timeout_seconds=bounded_timeout(float(task["budgets"]["wall_time_seconds"])), env=clean_env, - redact_values=credential_values) - if agent.timed_out or agent.cleanup_error: - manifest["infrastructure_status"] = "error" - record_phase("agent", agent) - raise RuntimeError("agent process did not terminate cleanly") - manifest["model_status"] = "pass" if agent.returncode == 0 else "fail" - if agent.returncode != 0: - manifest["termination"], manifest["failure_category"] = "completed", "model" - record_phase("agent", agent) - manifest["hashes"]["source_final"] = _tree_hash(workspace) - _quarantine_workspace(workspace) - workspace = None - return _finalize(run_dir, manifest, run_started_monotonic, trusted_snapshot) - record_phase("agent", agent) - _reject_secret_files(workspace, credential_values) - frozen = _freeze_workspace(workspace, overall_deadline) - build_workspace = run_dir / f"build-workspace-{secrets.token_hex(8)}" - _copy_public(frozen, build_workspace, overall_deadline) - trusted_snapshot = frozen - workspace = build_workspace - else: - manifest["model_status"] = "not_run" - manifest["hashes"]["source_final"] = _tree_hash(workspace) - persist() - - firmware = workspace / "firmware" - build_command = [args.platformio, "run", "--project-dir", str(firmware), "--environment", config.platformio_environment or "esp32s3"] - _safe_files(workspace) - clean = run_command(build_command + ["--target", "clean"], cwd=workspace, - stdout_path=run_dir / "clean.stdout.log", stderr_path=run_dir / "clean.stderr.log", - timeout_seconds=bounded_timeout(float(task["budgets"]["wall_time_seconds"])), env=tool_env, - redact_values=credential_values) - if clean.timed_out or clean.cleanup_error: - manifest["infrastructure_status"] = "error" - record_phase("clean", clean) - raise RuntimeError("clean process did not terminate cleanly") - if clean.returncode: - manifest["infrastructure_status"] = "error" - record_phase("clean", clean) - raise RuntimeError("clean command failed") - record_phase("clean", clean) - build = run_command(build_command, cwd=workspace, stdout_path=run_dir / "build.stdout.log", - stderr_path=run_dir / "build.stderr.log", timeout_seconds=bounded_timeout(float(task["budgets"]["wall_time_seconds"])), - env=tool_env, redact_values=credential_values) - if build.timed_out or build.cleanup_error: - manifest["infrastructure_status"] = "error" - record_phase("build", build) - raise RuntimeError("build process did not terminate cleanly") - if build.returncode: - manifest["compile_status"] = "fail" - manifest["hardware_status"] = "not_run" - manifest["termination"], manifest["failure_category"] = "completed", "compile" - record_phase("build", build) - return _finalize(run_dir, manifest, run_started_monotonic, trusted_snapshot) - manifest["compile_status"] = "pass" - record_phase("build", build) - artifact = firmware / config.flash_artifact - if not artifact.is_file(): - raise RuntimeError("successful build produced no firmware artifact") - manifest["hashes"]["firmware"] = _safe_hash(artifact) - _safe_copy_file(artifact, run_dir / "firmware.bin") - persist() - - if args.identity_command_json: - identity_command = json.loads(args.identity_command_json) - if not isinstance(identity_command, list) or not identity_command or not all(isinstance(x, str) for x in identity_command): - raise ValueError("identity command JSON must be a nonempty string array") - else: - identity_command = list(config.identity_command) - with BoardLock(run_dir.parent / ".board-locks", args.jtag_serial, - timeout_seconds=bounded_timeout(config.identity_timeout_seconds)): - identity = validate_identity(identity_command, args.jtag_serial, cwd=workspace, evidence_dir=run_dir, - timeout_seconds=bounded_timeout(config.identity_timeout_seconds), runner=tool_runner) - if identity.status != "pass": - manifest["infrastructure_status"] = "error" - record_phase("identity", identity.command_result, status=identity.status) - raise RuntimeError(identity.detail or "board identity failed") - record_phase("identity", identity.command_result, status=identity.status) - persist() - worker_spec = run_dir / ".uart-worker.json" - worker_ready = run_dir / ".uart-worker.ready" - worker_result = run_dir / ".uart-worker.result.json" - uart_timeout = bounded_timeout(config.uart_timeout_seconds) - write_json_atomic(worker_spec, {"device": args.uart_device, "baud": config.uart_baud, - "nonce": nonce, "timeout_seconds": uart_timeout, - "log_path": str(run_dir / "uart.raw.log"), "ready_path": str(worker_ready), - "result_path": str(worker_result), "mode": args.fixture_uart_worker_mode}) - worker_stdout = open(run_dir / "uart-worker.stdout.log", "wb") - worker_stderr = open(run_dir / "uart-worker.stderr.log", "wb") - uart_process: subprocess.Popen[Any] | None = None - uart_process_started = False - try: - uart_process = subprocess.Popen([sys.executable, str(Path(__file__).resolve()), - "--_uart-worker", str(worker_spec)], - cwd=workspace, stdout=worker_stdout, stderr=worker_stderr, - env=tool_env, start_new_session=True) - uart_process_started = True - ready_deadline = time.monotonic() + bounded_timeout(2) - while not worker_ready.exists() and uart_process.poll() is None and time.monotonic() < ready_deadline: - time.sleep(.01) - if not worker_ready.exists(): - raise RuntimeError("UART capture failed to start: startup timeout") - flash_command = [args.platformio, "run", "--project-dir", str(firmware), "--environment", - config.platformio_environment or "esp32s3", "--target", config.flash_target] - flashed = flash_firmware(flash_command, cwd=workspace, evidence_dir=run_dir, - timeout_seconds=bounded_timeout(config.flash_timeout_seconds), identity_validated=True, - runner=tool_runner) - if flashed.status == "infrastructure_error": - manifest["infrastructure_status"] = "error" - elif flashed.status == "hardware_fail": - manifest["hardware_status"] = "fail" - record_phase("flash", flashed.command_result, status=flashed.status, category=flashed.category) - if flashed.status == "pass": - try: uart_process.wait(timeout=bounded_timeout(uart_timeout + 0.5)) - except subprocess.TimeoutExpired: pass - finally: - uart_stopped = not uart_process_started or _stop_uart_process(uart_process) - worker_stdout.close() - worker_stderr.close() - if not uart_stopped: - raise UartWorkerFatal("UART worker could not be stopped") - uart_payload = (json.loads(worker_result.read_text(encoding="utf-8")) - if worker_result.exists() else None) - for control_path in (worker_spec, worker_ready, worker_result): - try: control_path.unlink() - except FileNotFoundError: pass - if uart_payload is None: - raise RuntimeError("UART capture produced no result") - if uart_payload["kind"] == "error": - raise RuntimeError(f"UART capture failed: {uart_payload['detail']}") - manifest["uart"] = {"matched": uart_payload["matched"], "termination": uart_payload["termination_reason"], - "bytes_captured": uart_payload["bytes_captured"]} - if not uart_payload["matched"] and flashed.status != "infrastructure_error": - manifest["hardware_status"] = "fail" - record_phase("uart", matched=uart_payload["matched"], termination=uart_payload["termination_reason"], - bytes_captured=uart_payload["bytes_captured"]) - if flashed.status == "infrastructure_error": - raise RuntimeError(flashed.detail or "flash infrastructure failure") - if flashed.status == "hardware_fail" or not uart_payload["matched"]: - manifest["hardware_status"] = "fail" - manifest["termination"], manifest["failure_category"] = "completed", flashed.category or "uart_nonce" - return _finalize(run_dir, manifest, run_started_monotonic, trusted_snapshot) - registers = read_registers(args.openocd, config.openocd_board_config, args.jtag_serial, - config.assertions, cwd=workspace, evidence_dir=run_dir, - timeout_seconds=bounded_timeout(config.openocd_command_timeout_seconds), runner=tool_runner) - if registers.status == "infrastructure_error": - manifest["infrastructure_status"] = "error" - record_phase("register", registers.command_result, status=registers.status) - raise RuntimeError(registers.detail or "OpenOCD infrastructure failure") - manifest["register_assertions"] = [ - {"name": item.name, "passed": item.passed, "observed_masked": f"0x{item.value & item.mask:08x}"} - for item in registers.evaluation.observations - ] - manifest["hardware_status"] = "pass" if registers.status == "pass" else "fail" - manifest["termination"] = "completed" - manifest["failure_category"] = None if registers.status == "pass" else "register_mismatch" - record_phase("register", registers.command_result, status=registers.status, - assertions=manifest["register_assertions"]) - return _finalize(run_dir, manifest, run_started_monotonic, trusted_snapshot) - except UnsafeWorkspaceError as error: - if workspace is not None: - _quarantine_workspace(workspace) - manifest["infrastructure_status"] = "error" - manifest["termination"] = "invalid" - manifest["failure_category"] = "unsafe_workspace" - manifest["detail"] = sanitize(str(error)) - persist() - return 2 - except UartWorkerFatal as error: - manifest["infrastructure_status"] = "error" - manifest["termination"] = "invalid" - manifest["failure_category"] = "uart_cleanup" - manifest["detail"] = sanitize(str(error)) - persist() - return 2 - except (KeyboardInterrupt, SystemExit) as error: - if args.agent_bin and workspace is not None: - _quarantine_workspace(workspace) - manifest["infrastructure_status"] = "error" - manifest["termination"] = "interrupted" - manifest["failure_category"] = "interrupt" - manifest["detail"] = sanitize(type(error).__name__) - try: - _finalize(run_dir, manifest, run_started_monotonic, trusted_snapshot) - except (OSError, ValueError): - persist() - return 2 - except (OSError, ValueError, TypeError, KeyError, json.JSONDecodeError, RuntimeError, BoardLockTimeout) as error: - if args.agent_bin and workspace is not None: - _quarantine_workspace(workspace) - manifest["infrastructure_status"] = "error" - manifest["termination"] = "invalid" - manifest["failure_category"] = "infrastructure" - manifest["detail"] = sanitize(str(error)) - try: - _finalize(run_dir, manifest, run_started_monotonic, trusted_snapshot) - except (OSError, ValueError) as final_error: - manifest["detail"] = sanitize(f"{error}; evidence rejected: {final_error}") - persist() - print(sanitize(str(error)), file=sys.stderr) - return 2 - - -def _finalize(run_dir: Path, manifest: dict[str, Any], started_monotonic: float, - trusted_snapshot: Path | None = None) -> int: - if trusted_snapshot is not None: - build_names = [name for name in os.listdir(run_dir) if name.startswith("build-workspace-")] - second_snapshot = None - for name in build_names: - build_path = run_dir / name - _safe_remove_tree(build_path / "firmware/.pio") - if build_path.exists(): - second_snapshot = _freeze_workspace(build_path, None) - chosen = second_snapshot if second_snapshot is not None and _tree_hash(second_snapshot) == _tree_hash(trusted_snapshot) else trusted_snapshot - delivered = run_dir / "source" - if delivered.exists(): _safe_remove_tree(delivered) - _copy_public(chosen, delivered) - if second_snapshot is not None: _safe_remove_tree(second_snapshot.parent) - _safe_remove_tree(trusted_snapshot.parent) - for name in os.listdir(run_dir): - if name.startswith("build-workspace-"): _quarantine_workspace(run_dir / name) - else: - _safe_remove_tree(run_dir / "workspace/firmware/.pio") - credential_values = tuple(filter(None, (os.environ.get(name, "") for name in ( - "LABWIRED_ACCESS_TOKEN", "LABWIRED_PROJECT", "LABWIRED_MODEL_URL", "LABWIRED_MODEL_KEY")))) - _reject_secret_files(run_dir, credential_values) - result = {name: manifest.get(name) for name in ( - "model_status", "compile_status", "simulator_status", "hardware_status", - "infrastructure_status", "termination", "failure_category", "uart", "register_assertions")} - write_json_atomic(run_dir / "result.json", result) - artifacts: list[str] = [] - hashes = manifest.setdefault("hashes", {}) - for path in (item for item in _safe_files(run_dir) if item.name != "run.json"): - relative = path.relative_to(run_dir).as_posix() - artifacts.append(relative) - hashes[f"artifact:{relative}"] = _safe_hash(path) - manifest["artifacts"] = artifacts - manifest["run"]["ended_at_utc"] = datetime.now(timezone.utc).isoformat().replace("+00:00", "Z") - elapsed = time.monotonic() - started_monotonic - wall = manifest.get("budget_validity", {}).get("wall_time_seconds") - if wall is not None: - wall.update(observed=elapsed, within_budget=elapsed <= wall["configured"]) - write_json_atomic(run_dir / "run.json", manifest) - return 2 if manifest["infrastructure_status"] == "error" else 0 - - -if __name__ == "__main__": - if len(sys.argv) == 3 and sys.argv[1] == "--_uart-worker": - raise SystemExit(_uart_subprocess_worker(Path(sys.argv[2]))) - raise SystemExit(main()) diff --git a/tests/twin2silicon-hil.py b/tests/twin2silicon-hil.py index aab8e59..f9b8ab4 100644 --- a/tests/twin2silicon-hil.py +++ b/tests/twin2silicon-hil.py @@ -1,13 +1,10 @@ #!/usr/bin/env python3 import hashlib -import contextlib import json import os from pathlib import Path import pty import signal -import shutil -import stat import subprocess import sys import tempfile @@ -180,112 +177,6 @@ def test_write_json_atomic_replaces_destination_with_json(self): class ProcessContractTests(unittest.TestCase): - def test_streaming_redaction_handles_cross_chunk_secret_and_oversize_log(self): - with tempfile.TemporaryDirectory() as directory: - root=Path(directory); crossed=root/'crossed'; secret=b'CROSS_BOUNDARY_SECRET' - crossed.write_bytes(b'A'*(1024*1024-len(secret)//2)+secret+b'Z') - self.assertIsNone(process_module._redact_logs((str(crossed),),(secret,))) - self.assertNotIn(secret,crossed.read_bytes()); self.assertIn(b'[REDACTED]',crossed.read_bytes()) - huge=root/'huge'; - with huge.open('wb') as output: output.truncate(20*1024*1024) - self.assertEqual(process_module._redact_logs((str(huge),),(b'x',)),"evidence_log_too_large") - self.assertEqual(huge.read_bytes(),b'[EVIDENCE LOG TOO LARGE]\n') - - def test_successful_leader_cannot_leave_mutating_descendant(self): - with tempfile.TemporaryDirectory() as directory: - evidence = Path(directory); child_path = evidence / "child"; mutation = evidence / "mutation" - script = textwrap.dedent(f""" - import pathlib, subprocess, sys - child = ''' - import os, pathlib, time - pathlib.Path({str(child_path)!r}).write_text(str(os.getpid())) - time.sleep(.4) - pathlib.Path({str(mutation)!r}).write_text('late') - time.sleep(30) - ''' - subprocess.Popen([sys.executable, '-c', child]) - while not pathlib.Path({str(child_path)!r}).exists(): pass - print('leader done') - """) - result = run_command([sys.executable, "-c", script], cwd=evidence, - stdout_path=evidence / "o", stderr_path=evidence / "e", timeout_seconds=2) - self.assertEqual(result.returncode, 0) - self.assertEqual(result.cleanup_error, "unexpected_descendant_processes") - with self.assertRaises(ProcessLookupError): os.kill(int(child_path.read_text()), 0) - time.sleep(.5) - self.assertFalse(mutation.exists()) - - def test_run_command_interrupt_terminates_process_group_before_reraising(self): - with tempfile.TemporaryDirectory() as directory: - evidence = Path(directory) - pid_path = evidence / "pid" - timer = threading.Timer(.15, lambda: os.kill(os.getpid(), signal.SIGINT)) - timer.start() - try: - with self.assertRaises(KeyboardInterrupt): - run_command([sys.executable, "-c", f"import os,pathlib,time; pathlib.Path({str(pid_path)!r}).write_text(str(os.getpid())); time.sleep(30)"], - cwd=evidence, stdout_path=evidence / "o", stderr_path=evidence / "e", timeout_seconds=30) - child = int(pid_path.read_text()) - with self.assertRaises(ProcessLookupError): - os.kill(child, 0) - finally: - timer.cancel() - if pid_path.exists(): - try: os.kill(int(pid_path.read_text()), signal.SIGKILL) - except ProcessLookupError: pass - - def test_run_command_redacts_exact_bytes_before_interrupt_reraises(self): - with tempfile.TemporaryDirectory() as directory: - evidence = Path(directory); output = evidence / "o" - timer = threading.Timer(.15, lambda: os.kill(os.getpid(), signal.SIGINT)); timer.start() - try: - with self.assertRaises(KeyboardInterrupt): - run_command([sys.executable, "-c", "import time; print('TOKEN_SENTINEL', flush=True); time.sleep(30)"], - cwd=evidence, stdout_path=output, stderr_path=evidence / "e", timeout_seconds=30, - redact_values=(b"TOKEN_SENTINEL",)) - finally: timer.cancel() - self.assertEqual(output.read_text(), "[REDACTED]\n") - - def test_interrupted_sparse_log_is_replaced_boundedly(self): - with tempfile.TemporaryDirectory() as directory: - evidence=Path(directory); output=evidence/'o' - timer=threading.Timer(.15,lambda: os.kill(os.getpid(),signal.SIGINT)); timer.start() - try: - with self.assertRaises(KeyboardInterrupt): - run_command([sys.executable,'-c',"import os,time; os.lseek(1,20*1024*1024,0); os.write(1,b'x'); time.sleep(30)"], - cwd=evidence,stdout_path=output,stderr_path=evidence/'e',timeout_seconds=30,redact_values=(b'secret',)) - finally: timer.cancel() - self.assertEqual(output.read_bytes(),b'[EVIDENCE LOG TOO LARGE]\n') - - def test_run_command_interrupt_kills_sigterm_ignoring_descendant_after_leader_exits(self): - with tempfile.TemporaryDirectory() as directory: - evidence = Path(directory); child_path = evidence / "child" - script = textwrap.dedent(f""" - import pathlib, signal, subprocess, sys, time - child = ''' - import os, pathlib, signal, time - signal.signal(signal.SIGTERM, signal.SIG_IGN) - pathlib.Path({str(child_path)!r}).write_text(str(os.getpid())) - while True: time.sleep(1) - ''' - subprocess.Popen([sys.executable, '-c', child]) - while not pathlib.Path({str(child_path)!r}).exists(): pass - signal.signal(signal.SIGTERM, lambda *_: (_ for _ in ()).throw(SystemExit(0))) - while True: time.sleep(1) - """) - timer = threading.Timer(.15, lambda: os.kill(os.getpid(), signal.SIGINT)); timer.start() - try: - with self.assertRaises(KeyboardInterrupt): - run_command([sys.executable, "-c", script], cwd=evidence, - stdout_path=evidence / "o", stderr_path=evidence / "e", timeout_seconds=30) - with self.assertRaises(ProcessLookupError): - os.kill(int(child_path.read_text()), 0) - finally: - timer.cancel() - if child_path.exists(): - try: os.kill(int(child_path.read_text()), signal.SIGKILL) - except ProcessLookupError: pass - def test_run_command_timeout_captures_evidence_and_is_bounded(self): with tempfile.TemporaryDirectory() as directory: evidence = Path(directory) @@ -723,30 +614,6 @@ def test_rejects_absent_wrong_stale_or_incomplete_nonce_with_bounded_evidence(se self.assertLess(time.monotonic() - started, .5) self.assertLessEqual(len(raw), 64) - def test_cancellation_stops_capture_and_closes_stable_log(self): - master, slave = pty.openpty() - device = os.ttyname(slave) - tty.setraw(slave) - with tempfile.TemporaryDirectory() as directory: - log = Path(directory) / "uart.log" - cancel = threading.Event() - started = threading.Event() - result = [] - thread = threading.Thread(target=lambda: result.append( - capture_uart_nonce(device, 115200, "nonce", 30, log, cancel_event=cancel, - started_event=started))) - thread.start() - self.assertTrue(started.wait(.5)) - cancel.set() - thread.join(.5) - self.assertFalse(thread.is_alive()) - self.assertEqual(result[0].termination_reason, "cancelled") - before = log.read_bytes() - time.sleep(.05) - self.assertEqual(log.read_bytes(), before) - os.close(master) - os.close(slave) - def test_rejects_unsupported_baud_and_closes_opened_fd(self): master, slave = pty.openpty() device = os.ttyname(slave) @@ -906,558 +773,6 @@ def stop(signum, frame): self.assertEqual(terminated.read_text(), "terminated") -class HilOrchestrationTests(unittest.TestCase): - def _run_cli(self, *arguments, env=None): - return subprocess.run( - [sys.executable, str(REPOSITORY_ROOT / "benchmarks/twin2silicon/run_hil.py"), *map(str, arguments)], - cwd=REPOSITORY_ROOT, text=True, capture_output=True, env=env, - ) - - def _short_task(self): - source = REPOSITORY_ROOT / "benchmarks/twin2silicon/tasks/esp32s3-gpio-hil-001" - return source - - def test_evaluate_only_prepares_isolated_workspace_and_records_compile_failure(self): - task = REPOSITORY_ROOT / "benchmarks/twin2silicon/tasks/esp32s3-gpio-hil-001" - with tempfile.TemporaryDirectory() as directory: - root = Path(directory) - candidate = root / "candidate" - shutil.copytree(task / "public", candidate) - invocations = root / "pio-invocations" - tool_dir = root / "pio" - tool_dir.mkdir() - platformio = executable_fixture(tool_dir, textwrap.dedent(f""" - import sys - from pathlib import Path - if '--version' in sys.argv: print('pio fixture 1'); raise SystemExit(0) - with Path({str(invocations)!r}).open('a') as output: - output.write(json.dumps(sys.argv[1:]) + '\\n') - raise SystemExit(0 if 'clean' in sys.argv else 1) - """).replace("import sys\n", "import json, sys\n")) - identity = executable_fixture(root, "raise AssertionError('identity must not run')\n") - run_dir = root / "run" - result = self._run_cli( - task, "--run-dir", run_dir, "--evaluate-only", "--candidate", candidate, - "--jtag-serial", "JTAG-1", "--uart-device", "/dev/null", - "--openocd", identity, "--platformio", platformio, - "--identity-command-json", json.dumps([str(identity)]), - ) - self.assertEqual(result.returncode, 0, result.stderr) - manifest = json.loads((run_dir / "run.json").read_text()) - self.assertEqual((manifest["compile_status"], manifest["hardware_status"]), ("fail", "not_run")) - pio_commands = [json.loads(line) for line in invocations.read_text().splitlines()] - self.assertEqual(len(pio_commands), 2) - self.assertEqual(pio_commands[0][-2:], ["--target", "clean"]) - nonce_header = (run_dir / "workspace/firmware/include/run_nonce.h").read_text() - nonce = nonce_header.split('"')[1] - self.assertRegex(nonce, r"^[0-9a-f]{32}$") - self.assertNotIn("hidden", {path.name for path in (run_dir / "workspace").rglob("*")}) - self.assertNotEqual(manifest["hashes"]["source_initial"], "") - self.assertEqual(manifest["hashes"]["source_initial"], manifest["hashes"]["source_final"]) - self.assertTrue(all(not Path(path).is_absolute() for path in manifest["artifacts"])) - self.assertEqual(self._run_cli( - task, "--run-dir", run_dir, "--evaluate-only", "--candidate", candidate, - "--jtag-serial", "JTAG-1", "--uart-device", "/dev/null", "--openocd", identity, - ).returncode, 2) - - def test_usage_cost_is_exact_and_rejects_boolean_numbers(self): - with tempfile.TemporaryDirectory() as directory: - root = Path(directory) - usage = root / "usage.json" - usage.write_text(json.dumps({ - "requests": 2, "tokens": {"fresh_input": 100, "cached_input": 200, - "output": 300, "reasoning": 40}, "final_context_tokens": 55, - "latency_seconds": 1.25, "provider": "fixture", "model": "m", - "rates_usd_per_million": {"fresh_input": 10, "cached_input": 1, "output": 20}, - "schema_version": "1.0", "price_source": "fixture", "price_effective_date": "2026-01-01" - })) - sys.path.insert(0, str(REPOSITORY_ROOT / "benchmarks/twin2silicon")) - import run_hil - cost = run_hil.parse_usage(usage) - self.assertEqual(cost["cost_usd"], 0.0072) - usage.write_text(usage.read_text().replace('"requests": 2', '"requests": true')) - with self.assertRaises(ValueError): - run_hil.parse_usage(usage) - - def test_usage_schema_rejects_missing_extra_and_legacy_price_date(self): - sys.path.insert(0, str(REPOSITORY_ROOT / "benchmarks/twin2silicon")) - import run_hil - base = {"schema_version": "1.0", "requests": 1, - "tokens": {"fresh_input": 1, "cached_input": 2, "output": 3, "reasoning": 4}, - "final_context_tokens": 1, "latency_seconds": 1, "provider": "p", "model": "m", - "rates_usd_per_million": {"fresh_input": 1, "cached_input": 1, "output": 1}, - "price_source": "s", "price_effective_date": "2026-01-01"} - with tempfile.TemporaryDirectory() as directory: - path = Path(directory) / "usage.json" - for invalid in ({**base, "extra": 1}, {key: value for key, value in base.items() if key != "requests"}, - {**base, "price_date": "legacy"}): - path.write_text(json.dumps(invalid)) - with self.assertRaises(ValueError): - run_hil.parse_usage(path) - - def test_finalizer_uses_total_monotonic_wall_time_not_provider_latency(self): - sys.path.insert(0, str(REPOSITORY_ROOT / "benchmarks/twin2silicon")) - import run_hil - with tempfile.TemporaryDirectory() as directory: - artifact = Path(directory) / "slow.log"; artifact.write_text("evidence") - manifest = {"model_status": "not_run", "compile_status": "not_run", - "simulator_status": "not_supported", "hardware_status": "not_run", - "infrastructure_status": "ok", "termination": "completed", "failure_category": None, - "uart": None, "register_assertions": [], "hashes": {}, "run": {}, "latency_seconds": .001, - "budget_validity": {"wall_time_seconds": {"configured": .01, "observed": None, "within_budget": None}}} - real_hash = run_hil._safe_hash - def slow_hash(path): - time.sleep(.05) - return real_hash(path) - with mock.patch.object(run_hil, "_safe_hash", side_effect=slow_hash): - run_hil._finalize(Path(directory), manifest, time.monotonic()) - wall = manifest["budget_validity"]["wall_time_seconds"] - self.assertGreaterEqual(wall["observed"], .05) - self.assertFalse(wall["within_budget"]) - - def test_candidate_symlink_escape_is_rejected_before_commands(self): - task = REPOSITORY_ROOT / "benchmarks/twin2silicon/tasks/esp32s3-gpio-hil-001" - with tempfile.TemporaryDirectory() as directory: - root = Path(directory) - candidate = root / "candidate" - candidate.mkdir() - (candidate / "escape").symlink_to(task / "hidden") - marker = root / "ran" - tool = executable_fixture(root, f"Path({str(marker)!r}).write_text('ran')\n") - result = self._run_cli( - task, "--run-dir", root / "run", "--evaluate-only", "--candidate", candidate, - "--jtag-serial", "J", "--uart-device", "/dev/null", "--openocd", tool, - "--platformio", tool, - ) - self.assertEqual(result.returncode, 2) - self.assertFalse(marker.exists()) - - def test_agent_mode_repairs_workspace_with_allowlisted_home_and_no_secret_evidence(self): - task = REPOSITORY_ROOT / "benchmarks/twin2silicon/tasks/esp32s3-gpio-hil-001" - with tempfile.TemporaryDirectory() as directory: - root = Path(directory) - invocation = root / "agent-invocation.json" - agent_dir = root / "agent" - agent_dir.mkdir() - agent = executable_fixture(agent_dir, textwrap.dedent(f""" - import json, os, pathlib, sys - if '--version' in sys.argv: print('agent fixture 1'); raise SystemExit(0) - assert os.environ['HOME'] == 'HOME_SENTINEL' - assert os.environ['LABWIRED_HOME'] == 'LABWIRED_HOME_SENTINEL' - print(os.environ['LABWIRED_ACCESS_TOKEN'], os.environ['LABWIRED_MODEL_KEY']) - assert 'hidden' not in ' '.join(sys.argv).lower() - source = pathlib.Path('firmware/src/main.c') - source.write_text(source.read_text().replace('GPIO_MODE_INPUT', 'GPIO_MODE_OUTPUT')) - pathlib.Path({str(invocation)!r}).write_text(json.dumps({{'argv': sys.argv[1:], 'cwd': pathlib.Path.cwd().name}})) - """)) - pio_dir = root / "pio" - pio_dir.mkdir() - pio = executable_fixture(pio_dir, "import os,sys\nprint(os.environ.get('LABWIRED_ACCESS_TOKEN','ABSENT'), os.environ.get('HOST_SECRET','ABSENT'))\nif '--version' in sys.argv: print('pio fixture 1'); raise SystemExit(0)\nraise SystemExit(0 if 'clean' in sys.argv else 1)\n") - env = {**os.environ, "HOME": "HOME_SENTINEL", "LABWIRED_HOME": "LABWIRED_HOME_SENTINEL", - "LABWIRED_ACCESS_TOKEN": "SECRET_TOKEN_SENTINEL", "LABWIRED_MODEL_KEY": "MODEL_KEY_SENTINEL", - "HOST_SECRET": "HOST_SECRET_SENTINEL"} - usage = root / "usage.json" - usage.write_text(json.dumps({"schema_version": "1.0", "requests": 1, - "tokens": {"fresh_input": 1000, "cached_input": 2000, "output": 3000, "reasoning": 4}, - "final_context_tokens": 55, "latency_seconds": .01, "provider": "fixture", "model": "fixture/model", - "rates_usd_per_million": {"fresh_input": 1, "cached_input": 1, "output": 1}, - "price_source": "fixture", "price_effective_date": "2026-01-01"})) - run_dir = root / "run" - result = self._run_cli(task, "--run-dir", run_dir, "--agent-bin", agent, "--model", "fixture/model", - "--jtag-serial", "J", "--uart-device", "/dev/null", "--openocd", pio, - "--platformio", pio, "--usage-json", usage, env=env) - self.assertEqual(result.returncode, 0, result.stderr) - call = json.loads(invocation.read_text()) - self.assertEqual(call["argv"][:4], ["agent", "run", "--model", "fixture/model"]) - self.assertTrue(call["cwd"].startswith("agent-workspace-")) - manifest_text = (run_dir / "run.json").read_text() - self.assertNotIn("SECRET_TOKEN_SENTINEL", manifest_text) - for path in run_dir.rglob("*"): - if path.is_file(): - contents = path.read_bytes() - for sentinel in (b"SECRET_TOKEN_SENTINEL", b"MODEL_KEY_SENTINEL", b"HOST_SECRET_SENTINEL"): - self.assertNotIn(sentinel, contents) - manifest = json.loads(manifest_text) - self.assertEqual(manifest["model_status"], "pass") - self.assertEqual(manifest["budget_validity"]["model_tokens"]["observed"], 55) - self.assertNotEqual(manifest["hashes"]["source_initial"], manifest["hashes"]["source_final"]) - - def test_agent_symlink_and_fifo_are_rejected_without_reading_external_secret(self): - task = REPOSITORY_ROOT / "benchmarks/twin2silicon/tasks/esp32s3-gpio-hil-001" - with tempfile.TemporaryDirectory() as directory: - root = Path(directory); secret = root / "secret"; secret.write_text("EXTERNAL_SECRET_SENTINEL") - agent_dir = root / "agent"; agent_dir.mkdir() - agent = executable_fixture(agent_dir, textwrap.dedent(f""" - import os, pathlib, sys - if '--version' in sys.argv: print('v'); raise SystemExit(0) - pathlib.Path('escape').symlink_to({str(secret)!r}) - os.mkfifo('special.fifo') - """)) - tool_dir = root / "tool"; tool_dir.mkdir() - tool = executable_fixture(tool_dir, "print('v')\n") - run_dir = root / "run" - result = self._run_cli(task, "--run-dir", run_dir, "--agent-bin", agent, "--model", "fixture", - "--jtag-serial", "J", "--uart-device", "/dev/null", "--openocd", tool, "--platformio", tool) - self.assertEqual(result.returncode, 2) - manifest_text = (run_dir / "run.json").read_text() - self.assertNotIn("EXTERNAL_SECRET_SENTINEL", manifest_text) - self.assertNotIn("escape", json.loads(manifest_text).get("artifacts", [])) - for path in run_dir.rglob("*"): - info = path.lstat() - self.assertFalse(stat.S_ISLNK(info.st_mode) or stat.S_ISFIFO(info.st_mode)) - if stat.S_ISREG(info.st_mode): self.assertNotIn(b"EXTERNAL_SECRET_SENTINEL", path.read_bytes()) - - def test_agent_credential_written_to_workspace_is_rejected_before_hash_or_build(self): - task = REPOSITORY_ROOT / "benchmarks/twin2silicon/tasks/esp32s3-gpio-hil-001" - with tempfile.TemporaryDirectory() as directory: - root = Path(directory); agent_dir = root / "agent"; agent_dir.mkdir() - agent = executable_fixture(agent_dir, """import os,pathlib,sys -if '--version' in sys.argv: print('v'); raise SystemExit(0) -pathlib.Path('credential.txt').write_text(os.environ['LABWIRED_ACCESS_TOKEN']) -""") - tool_dir = root / "tool"; tool_dir.mkdir(); marker = root / "tool-ran" - tool = executable_fixture(tool_dir, f"import sys\nif '--version' in sys.argv: print('v'); raise SystemExit(0)\nfrom pathlib import Path; Path({str(marker)!r}).write_text('ran')\n") - run_dir = root / "run"; env = {**os.environ, "LABWIRED_ACCESS_TOKEN": "WORKSPACE_TOKEN_SENTINEL"} - result = self._run_cli(task, "--run-dir", run_dir, "--agent-bin", agent, "--model", "fixture", - "--jtag-serial", "J", "--uart-device", "/dev/null", "--openocd", tool, "--platformio", tool, env=env) - self.assertEqual(result.returncode, 2) - self.assertFalse(marker.exists()) - manifest_text = (run_dir / "run.json").read_text() - self.assertNotIn("WORKSPACE_TOKEN_SENTINEL", manifest_text) - self.assertNotIn("credential.txt", json.loads(manifest_text).get("artifacts", [])) - self.assertFalse((run_dir / "workspace").exists()) - - def test_agent_oversize_sparse_file_is_bounded_and_workspace_quarantined(self): - task = REPOSITORY_ROOT / "benchmarks/twin2silicon/tasks/esp32s3-gpio-hil-001" - with tempfile.TemporaryDirectory() as directory: - root = Path(directory); agent_dir = root / "agent"; agent_dir.mkdir() - agent = executable_fixture(agent_dir, """import pathlib,sys -if '--version' in sys.argv: print('v'); raise SystemExit(0) -with pathlib.Path('huge.bin').open('wb') as out: out.truncate(40 * 1024 * 1024) -""") - tool_dir=root/'tool'; tool_dir.mkdir(); tool=executable_fixture(tool_dir,"print('v')\n") - run_dir=root/'run'; started=time.monotonic() - result=self._run_cli(task,'--run-dir',run_dir,'--agent-bin',agent,'--model','m','--jtag-serial','J', - '--uart-device','/dev/null','--openocd',tool,'--platformio',tool) - self.assertEqual(result.returncode,2); self.assertLess(time.monotonic()-started,3) - self.assertFalse((run_dir/'workspace').exists()) - self.assertEqual(json.loads((run_dir/'run.json').read_text())["failure_category"],"unsafe_workspace") - - def test_fd_relative_quarantine_delete_never_follows_swapped_symlink(self): - sys.path.insert(0,str(REPOSITORY_ROOT/'benchmarks/twin2silicon')); import run_hil - with tempfile.TemporaryDirectory() as directory: - root=Path(directory); victim=root/'victim'; child=victim/'child'; child.mkdir(parents=True) - for index in range(500): (child/f'f{index}').write_text('x') - external=root/'external'; external.mkdir(); sentinel=external/'sentinel'; sentinel.write_text('safe') - moved=victim/'moved'; stop=threading.Event() - def swap(): - while not stop.is_set(): - try: - child.rename(moved); child.symlink_to(external, target_is_directory=True) - child.unlink(); moved.rename(child) - except (FileNotFoundError,OSError): pass - thread=threading.Thread(target=swap); thread.start() - try: - try: run_hil._safe_remove_tree(victim) - except run_hil.UnsafeWorkspaceError: pass - finally: - stop.set(); thread.join(1) - self.assertEqual(sentinel.read_text(),'safe') - - def test_detached_agent_mutator_cannot_change_frozen_workspace(self): - task = REPOSITORY_ROOT / "benchmarks/twin2silicon/tasks/esp32s3-gpio-hil-001" - with tempfile.TemporaryDirectory() as directory: - root=Path(directory); pid_path=root/'child.pid'; agent_dir=root/'agent'; agent_dir.mkdir() - child = "import os,pathlib,time; f=open('firmware/src/main.c','ab',buffering=0); pathlib.Path(%r).write_text(str(os.getpid()));\nwhile True: f.write(b'X'); time.sleep(.01)" % str(pid_path) - agent=executable_fixture(agent_dir, textwrap.dedent(f""" - import subprocess,sys,time - if '--version' in sys.argv: print('v'); raise SystemExit(0) - subprocess.Popen([sys.executable,'-c',{child!r}], start_new_session=True) - import pathlib - while not pathlib.Path({str(pid_path)!r}).exists(): time.sleep(.01) - """)) - build_pid=root/'build-child.pid'; tool_dir=root/'tool'; tool_dir.mkdir() - tool=executable_fixture(tool_dir,textwrap.dedent(f""" - import pathlib,subprocess,sys - if '--version' in sys.argv: print('v'); raise SystemExit(0) - if 'clean' in sys.argv: raise SystemExit(0) - project=pathlib.Path(sys.argv[sys.argv.index('--project-dir')+1]) - child="import os,pathlib,time; p=pathlib.Path(%r); pathlib.Path(%r).write_text(str(os.getpid()));\\nwhile True:\\n p.parent.mkdir(parents=True,exist_ok=True); open(p,'ab').write(b'Y'); time.sleep(.01)" % (str(project/'src/main.c'), {str(build_pid)!r}) - subprocess.Popen([sys.executable,'-c',child],start_new_session=True) - raise SystemExit(1) - """)) - run_dir=root/'run' - try: - result=self._run_cli(task,'--run-dir',run_dir,'--agent-bin',agent,'--model','m','--jtag-serial','J', - '--uart-device','/dev/null','--openocd',tool,'--platformio',tool) - self.assertEqual(result.returncode,0,result.stderr) - sys.path.insert(0,str(REPOSITORY_ROOT/'benchmarks/twin2silicon')); import run_hil - before=run_hil._tree_hash(run_dir/'source'); time.sleep(.2) - self.assertEqual(before,run_hil._tree_hash(run_dir/'source')) - self.assertEqual(before,json.loads((run_dir/'run.json').read_text())["hashes"]["source_final"]) - finally: - if pid_path.exists(): - try: os.kill(int(pid_path.read_text()),signal.SIGKILL) - except ProcessLookupError: pass - if build_pid.exists(): - try: os.kill(int(build_pid.read_text()),signal.SIGKILL) - except ProcessLookupError: pass - - def test_cli_physical_pass_uses_pty_and_records_ordered_evidence(self): - task = self._short_task() - with tempfile.TemporaryDirectory() as directory: - root = Path(directory) - order = root / "order.log" - marker = root / "flash" - pio_dir = root / "pio"; pio_dir.mkdir() - pio = executable_fixture(pio_dir, textwrap.dedent(f""" - import pathlib, sys, time - if '--version' in sys.argv: print('pio fixture 1'); raise SystemExit(0) - order = pathlib.Path({str(order)!r}) - target = sys.argv[sys.argv.index('--target') + 1] if '--target' in sys.argv else 'build' - with order.open('a') as out: out.write(target + '\\n') - project = pathlib.Path(sys.argv[sys.argv.index('--project-dir') + 1]) - if target == 'build': - artifact = project / '.pio/build/esp32s3/firmware.bin'; artifact.parent.mkdir(parents=True); artifact.write_bytes(b'fw') - if target == 'upload': pathlib.Path({str(marker)!r}).write_text('flash'); time.sleep(.1) - """)) - identity_dir = root / "identity"; identity_dir.mkdir() - identity = executable_fixture(identity_dir, f"from pathlib import Path\nwith Path({str(order)!r}).open('a') as out: out.write('identity\\n')\nprint('JTAG-1')\n") - openocd_dir = root / "openocd"; openocd_dir.mkdir() - openocd = executable_fixture(openocd_dir, textwrap.dedent(f""" - import pathlib, sys - if '--version' in sys.argv: print('openocd fixture 1'); raise SystemExit(0) - with pathlib.Path({str(order)!r}).open('a') as out: out.write('openocd\\n') - print('@@REG gpio2_output_enabled 0x60004020', file=sys.stderr) - print('0x60004020: 00000004', file=sys.stderr) - print('@@REG gpio2_output_high 0x60004004', file=sys.stderr) - print('0x60004004: 00000004', file=sys.stderr) - """)) - master, slave = pty.openpty(); tty.setraw(slave) - run_dir = root / "run" - def writer(): - deadline = time.monotonic() + 2 - while not marker.exists() and time.monotonic() < deadline: time.sleep(.005) - nonce = (run_dir / "workspace/firmware/include/run_nonce.h").read_text().split('"')[1] - os.write(master, f"LABWIRED_READY:{nonce}\n".encode()) - thread = threading.Thread(target=writer); thread.start() - try: - result = self._run_cli(task, "--run-dir", run_dir, "--evaluate-only", "--candidate", task / "public", - "--jtag-serial", "JTAG-1", "--uart-device", os.ttyname(slave), - "--openocd", openocd, "--platformio", pio, - "--identity-command-json", json.dumps([str(identity)])) - finally: - thread.join(2); os.close(master); os.close(slave) - self.assertEqual(result.returncode, 0, result.stderr) - manifest = json.loads((run_dir / "run.json").read_text()) - self.assertEqual(manifest["hardware_status"], "pass") - self.assertEqual(order.read_text().splitlines(), ["clean", "build", "identity", "upload", "openocd"]) - self.assertNotIn("JTAG-1", json.dumps(manifest)) - self.assertEqual(set(manifest["budget_validity"]), - {"wall_time_seconds", "model_tokens", "repair_iterations", "simulator_runs", "diagnostic_hil_runs"}) - self.assertEqual(manifest["budget_validity"]["simulator_runs"]["observed"], 0) - self.assertEqual(manifest["budget_validity"]["diagnostic_hil_runs"]["observed"], 0) - self.assertEqual([phase["name"] for phase in manifest["phases"]], - ["prepared", "clean", "build", "identity", "flash", "uart", "register"]) - self.assertEqual(set(manifest["tool_versions"]), {"platformio", "openocd"}) - self.assertTrue(all("executable" in item and "version" in item - for item in manifest["tool_versions"].values())) - for artifact in manifest["artifacts"]: - path = run_dir / artifact - self.assertTrue(path.is_file()) - self.assertEqual(manifest["hashes"][f"artifact:{artifact}"], sha256_file(path)) - - def test_clean_nonzero_is_infrastructure_and_never_touches_identity(self): - task = REPOSITORY_ROOT / "benchmarks/twin2silicon/tasks/esp32s3-gpio-hil-001" - with tempfile.TemporaryDirectory() as directory: - root = Path(directory); marker = root / "identity-ran" - pio_dir = root / "pio"; pio_dir.mkdir() - pio = executable_fixture(pio_dir, "import sys\nif '--version' in sys.argv: print('v'); raise SystemExit(0)\nraise SystemExit(7)\n") - identity_dir = root / "identity"; identity_dir.mkdir() - identity = executable_fixture(identity_dir, f"from pathlib import Path; Path({str(marker)!r}).write_text('ran')\n") - openocd_dir = root / "openocd"; openocd_dir.mkdir() - openocd = executable_fixture(openocd_dir, "print('v')\n") - run_dir = root / "run" - result = self._run_cli(task, "--run-dir", run_dir, "--evaluate-only", "--candidate", task / "public", - "--jtag-serial", "J", "--uart-device", "/dev/null", "--openocd", openocd, - "--platformio", pio, "--identity-command-json", json.dumps([str(identity)])) - self.assertEqual(result.returncode, 2) - self.assertFalse(marker.exists()) - manifest = json.loads((run_dir / "run.json").read_text()) - self.assertEqual((manifest["compile_status"], manifest["infrastructure_status"]), ("not_run", "error")) - self.assertEqual([phase["name"] for phase in manifest["phases"]], ["prepared", "clean"]) - - def test_identity_launch_detail_redacts_serial_device_and_credentials(self): - task=REPOSITORY_ROOT/'benchmarks/twin2silicon/tasks/esp32s3-gpio-hil-001' - with tempfile.TemporaryDirectory() as directory: - root=Path(directory); tool_dir=root/'tool'; tool_dir.mkdir() - tool=executable_fixture(tool_dir,"""import pathlib,sys -if '--version' in sys.argv: print('v'); raise SystemExit(0) -if '--target' not in sys.argv: - p=pathlib.Path(sys.argv[sys.argv.index('--project-dir')+1])/'.pio/build/esp32s3/firmware.bin'; p.parent.mkdir(parents=True); p.write_bytes(b'fw') -""") - serial='RAW_SERIAL_SENTINEL'; device='/dev/RAW_DEVICE_SENTINEL'; credential='RAW_CREDENTIAL_SENTINEL' - env={**os.environ,'LABWIRED_ACCESS_TOKEN':credential}; run_dir=root/'run' - result=self._run_cli(task,'--run-dir',run_dir,'--evaluate-only','--candidate',task/'public', - '--jtag-serial',serial,'--uart-device',device,'--openocd',tool,'--platformio',tool, - '--identity-command-json',json.dumps([f'/missing/{serial}/{credential}']),env=env) - self.assertEqual(result.returncode,2) - combined=(run_dir/'run.json').read_text()+result.stderr - for value in (serial,device,credential): self.assertNotIn(value,combined) - - def test_overall_wall_deadline_bounds_cumulative_phases(self): - task = REPOSITORY_ROOT / "benchmarks/twin2silicon/tasks/esp32s3-gpio-hil-001" - with tempfile.TemporaryDirectory() as directory: - root = Path(directory); marker = root / "identity" - pio_dir = root / "pio"; pio_dir.mkdir() - pio = executable_fixture(pio_dir, """import pathlib,sys,time -if '--version' in sys.argv: print('v'); raise SystemExit(0) -time.sleep(.18) -if '--target' not in sys.argv: - p=pathlib.Path(sys.argv[sys.argv.index('--project-dir')+1])/'.pio/build/esp32s3/firmware.bin'; p.parent.mkdir(parents=True); p.write_bytes(b'fw') -""") - identity_dir = root / "id"; identity_dir.mkdir() - identity = executable_fixture(identity_dir, f"from pathlib import Path; Path({str(marker)!r}).write_text('ran')\n") - run_dir = root / "run" - result = self._run_cli(task, "--run-dir", run_dir, "--evaluate-only", "--candidate", task / "public", - "--jtag-serial", "J", "--uart-device", "/dev/null", "--openocd", pio, "--platformio", pio, - "--identity-command-json", json.dumps([str(identity)]), "--fixture-wall-time-seconds", ".45") - self.assertEqual(result.returncode, 2) - self.assertFalse(marker.exists()) - manifest = json.loads((run_dir / "run.json").read_text()) - self.assertFalse(manifest["budget_validity"]["wall_time_seconds"]["within_budget"]) - - def test_uart_startup_hang_is_killed_before_bounded_finalization(self): - task = REPOSITORY_ROOT / "benchmarks/twin2silicon/tasks/esp32s3-gpio-hil-001" - with tempfile.TemporaryDirectory() as directory: - root = Path(directory); pio_dir = root / "pio"; pio_dir.mkdir() - pio = executable_fixture(pio_dir, """import pathlib,sys -if '--version' in sys.argv: print('v'); raise SystemExit(0) -if '--target' not in sys.argv: - p=pathlib.Path(sys.argv[sys.argv.index('--project-dir')+1])/'.pio/build/esp32s3/firmware.bin'; p.parent.mkdir(parents=True); p.write_bytes(b'fw') -""") - identity_dir = root / "id"; identity_dir.mkdir() - identity = executable_fixture(identity_dir, "print('JTAG-HANG')\n") - run_dir = root / "run"; started = time.monotonic() - result = self._run_cli(task, "--run-dir", run_dir, "--evaluate-only", "--candidate", task / "public", - "--jtag-serial", "JTAG-HANG", "--uart-device", "/dev/null", "--openocd", pio, - "--platformio", pio, "--identity-command-json", json.dumps([str(identity)]), - "--fixture-uart-worker-mode", "startup-hang", "--fixture-identity-timeout-seconds", ".3") - self.assertEqual(result.returncode, 2) - self.assertLess(time.monotonic() - started, 4) - manifest_path = run_dir / "run.json"; manifest = json.loads(manifest_path.read_text()) - self.assertEqual(manifest["infrastructure_status"], "error") - before = {path: path.stat().st_mtime_ns for path in run_dir.rglob("*") if path.is_file()} - time.sleep(.2) - self.assertEqual(before, {path: path.stat().st_mtime_ns for path in run_dir.rglob("*") if path.is_file()}) - with BoardLock(run_dir.parent / ".board-locks", "JTAG-HANG", timeout_seconds=.1): pass - - def test_cli_sigint_cleans_uart_flash_descendants_and_finalizes_stable_evidence(self): - task = REPOSITORY_ROOT / "benchmarks/twin2silicon/tasks/esp32s3-gpio-hil-001" - with tempfile.TemporaryDirectory() as directory: - root = Path(directory); marker = root / "flash-pids.json" - pio_dir = root / "pio"; pio_dir.mkdir() - pio = executable_fixture(pio_dir, textwrap.dedent(f""" - import json, os, pathlib, signal, subprocess, sys, time - if '--version' in sys.argv: print('v'); raise SystemExit(0) - project = pathlib.Path(sys.argv[sys.argv.index('--project-dir') + 1]) - target = sys.argv[sys.argv.index('--target') + 1] if '--target' in sys.argv else 'build' - if target == 'build': - p=project/'.pio/build/esp32s3/firmware.bin'; p.parent.mkdir(parents=True); p.write_bytes(b'fw') - if target == 'upload': - child=subprocess.Popen([sys.executable, '-c', 'import signal,time; signal.signal(signal.SIGTERM, signal.SIG_IGN); time.sleep(30)']) - pathlib.Path({str(marker)!r}).write_text(json.dumps({{'leader': os.getpid(), 'child': child.pid}})) - time.sleep(30) - """)) - identity_dir = root / "id"; identity_dir.mkdir() - identity = executable_fixture(identity_dir, "print('JTAG-INT')\n") - master, slave = pty.openpty(); tty.setraw(slave); run_dir = root / "run" - command = [sys.executable, str(REPOSITORY_ROOT / "benchmarks/twin2silicon/run_hil.py"), str(task), - "--run-dir", str(run_dir), "--evaluate-only", "--candidate", str(task / "public"), - "--jtag-serial", "JTAG-INT", "--uart-device", os.ttyname(slave), "--openocd", str(pio), - "--platformio", str(pio), "--identity-command-json", json.dumps([str(identity)])] - process = subprocess.Popen(command, cwd=REPOSITORY_ROOT, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - deadline = time.monotonic() + 8 - while not marker.exists() and time.monotonic() < deadline: time.sleep(.01) - self.assertTrue(marker.exists(), "flash did not become active") - process.send_signal(signal.SIGINT) - stdout, stderr = process.communicate(timeout=6) - os.close(master); os.close(slave) - self.assertEqual(process.returncode, 2, stderr.decode()) - manifest = json.loads((run_dir / "run.json").read_text()) - self.assertEqual(manifest["termination"], "interrupted") - for pid in json.loads(marker.read_text()).values(): - with self.assertRaises(ProcessLookupError): os.kill(pid, 0) - with BoardLock(run_dir.parent / ".board-locks", "JTAG-INT", timeout_seconds=.1): pass - before = {p: (p.stat().st_mtime_ns, sha256_file(p)) for p in run_dir.rglob('*') if p.is_file()} - time.sleep(.2) - self.assertEqual(before, {p: (p.stat().st_mtime_ns, sha256_file(p)) for p in run_dir.rglob('*') if p.is_file()}) - for artifact in manifest["artifacts"]: - self.assertEqual(manifest["hashes"][f"artifact:{artifact}"], sha256_file(run_dir / artifact)) - - def test_cli_classifies_candidate_and_infrastructure_physical_failures(self): - task = self._short_task() - cases = ( - ("flash", 9, True, 4, 0, "fail", "ok"), - ("nonce", 0, False, 4, 0, "fail", "ok"), - ("register", 0, True, 0, 0, "fail", "ok"), - ("identity", 0, True, 4, 0, "not_run", "error"), - ("lock", 0, True, 4, 0, "not_run", "error"), - ("openocd", 0, True, 4, 7, "not_run", "error"), - ) - for name, flash_code, send_nonce, register_value, openocd_code, hardware, infrastructure in cases: - with self.subTest(name=name), tempfile.TemporaryDirectory() as directory: - root = Path(directory); marker = root / "flash" - pio_dir = root / "pio"; pio_dir.mkdir() - pio = executable_fixture(pio_dir, textwrap.dedent(f""" - import pathlib, sys, time - if '--version' in sys.argv: print('v'); raise SystemExit(0) - target = sys.argv[sys.argv.index('--target') + 1] if '--target' in sys.argv else 'build' - project = pathlib.Path(sys.argv[sys.argv.index('--project-dir') + 1]) - if target == 'build': - artifact = project / '.pio/build/esp32s3/firmware.bin'; artifact.parent.mkdir(parents=True); artifact.write_bytes(b'fw') - if target == 'upload': pathlib.Path({str(marker)!r}).write_text('x'); time.sleep(.05); raise SystemExit({flash_code}) - """)) - identity_dir = root / "identity"; identity_dir.mkdir() - identity = executable_fixture(identity_dir, f"print({('OTHER' if name == 'identity' else 'JTAG-1')!r})\n") - openocd_dir = root / "openocd"; openocd_dir.mkdir() - openocd = executable_fixture(openocd_dir, textwrap.dedent(f""" - import sys - if '--version' in sys.argv: print('v'); raise SystemExit(0) - if {openocd_code}: raise SystemExit({openocd_code}) - print('@@REG gpio2_output_enabled 0x60004020', file=sys.stderr); print('0x60004020: {register_value:08x}', file=sys.stderr) - print('@@REG gpio2_output_high 0x60004004', file=sys.stderr); print('0x60004004: {register_value:08x}', file=sys.stderr) - """)) - master, slave = pty.openpty(); tty.setraw(slave); run_dir = root / "run" - def writer(): - deadline = time.monotonic() + 2 - while not marker.exists() and time.monotonic() < deadline: time.sleep(.005) - if send_nonce and marker.exists(): - nonce = (run_dir / "workspace/firmware/include/run_nonce.h").read_text().split('"')[1] - os.write(master, f"LABWIRED_READY:{nonce}\n".encode()) - thread = threading.Thread(target=writer); thread.start() - try: - lock = (BoardLock(run_dir.parent / ".board-locks", "JTAG-1", timeout_seconds=.1) - if name == "lock" else contextlib.nullcontext()) - with lock: - cli = [task, "--run-dir", run_dir, "--evaluate-only", "--candidate", task / "public", - "--jtag-serial", "JTAG-1", "--uart-device", os.ttyname(slave), - "--openocd", openocd, "--platformio", pio, - "--fixture-uart-timeout-seconds", ".3", - "--identity-command-json", json.dumps([str(identity)])] - if name == "lock": - cli += ["--fixture-identity-timeout-seconds", ".3"] - result = self._run_cli(*cli) - finally: - thread.join(2); os.close(master); os.close(slave) - manifest = json.loads((run_dir / "run.json").read_text()) - self.assertEqual(result.returncode, 2 if infrastructure == "error" else 0, result.stderr) - self.assertEqual((manifest["hardware_status"], manifest["infrastructure_status"]), - (hardware, infrastructure)) - with BoardLock(run_dir.parent / ".board-locks", "JTAG-1", timeout_seconds=.1): - pass - - if __name__ == "__main__": if "-k" in sys.argv: pattern_index = sys.argv.index("-k") + 1 From 47ae57c4f6e03651d174e12cf82586de013e14d0 Mon Sep 17 00:00:00 2001 From: Andrii Shylenko <14119286+w1ne@users.noreply.github.com> Date: Sat, 15 Aug 2026 14:24:34 +0200 Subject: [PATCH 25/48] feat(bench): add simple ESP32-S3 HIL runner --- benchmarks/twin2silicon/run_hil.py | 189 +++++++++++++++++++++++++++++ tests/twin2silicon-hil.py | 112 +++++++++++++++++ 2 files changed, 301 insertions(+) create mode 100644 benchmarks/twin2silicon/run_hil.py diff --git a/benchmarks/twin2silicon/run_hil.py b/benchmarks/twin2silicon/run_hil.py new file mode 100644 index 0000000..d0a25e9 --- /dev/null +++ b/benchmarks/twin2silicon/run_hil.py @@ -0,0 +1,189 @@ +#!/usr/bin/env python3 +"""Run one simple ESP32-S3 build, flash, UART, and JTAG evaluation.""" + +from __future__ import annotations + +import argparse +import json +import math +import os +from pathlib import Path +import secrets +import shutil +import sys +import threading + +if __package__ in (None, ""): + sys.path.insert(0, str(Path(__file__).resolve().parents[2])) + +from benchmarks.twin2silicon.hil.esp32s3 import ( + BoardLock, + Esp32S3Config, + capture_uart_nonce, + flash_firmware, + read_registers, + validate_identity, +) +from benchmarks.twin2silicon.hil.process import run_command +from benchmarks.twin2silicon.hil.results import sha256_file, write_json_atomic + + +TASKS = Path(__file__).resolve().parent / "tasks" + + +def _usage(path: Path) -> dict: + data = json.loads(path.read_text()) + tokens = data["tokens"] + rates = data["rates_usd_per_million"] + for value in (*tokens.values(), *rates.values(), data["requests"]): + if isinstance(value, bool) or not isinstance(value, (int, float)) or not math.isfinite(value) or value < 0: + raise ValueError("usage values must be finite and nonnegative") + data["estimated_cost_usd"] = ( + tokens["fresh_input"] * rates["fresh_input"] + + tokens["cached_input"] * rates["cached_input"] + + tokens["output"] * rates["output"] + ) / 1_000_000 + return data + + +def _parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("task") + parser.add_argument("--run-dir", required=True, type=Path) + parser.add_argument("--candidate", required=True, type=Path) + parser.add_argument("--jtag-serial", required=True) + parser.add_argument("--uart-device", required=True) + parser.add_argument("--platformio", default="pio") + parser.add_argument("--openocd", required=True) + parser.add_argument("--identity-command-json") + parser.add_argument("--usage-json", type=Path) + return parser + + +def main(argv: list[str] | None = None) -> int: + args = _parser().parse_args(argv) + run_dir = args.run_dir.resolve() + if run_dir.exists(): + print("run directory already exists", file=sys.stderr) + return 2 + run_dir.mkdir(parents=True) + result = { + "schema_version": "1.0", + "status": "running", + "compile_status": "not_run", + "hardware_status": "not_run", + "infrastructure_status": "ok", + "uart": None, + "registers": [], + "cost": None, + "hashes": {}, + } + + def save() -> None: + write_json_atomic(run_dir / "run.json", result) + + save() + try: + task_root = (TASKS / args.task).resolve() if len(Path(args.task).parts) == 1 else Path(args.task).resolve() + task = json.loads((task_root / "task.json").read_text()) + oracle_path = task_root / task["hidden_oracle"] + config = Esp32S3Config.from_oracle(json.loads(oracle_path.read_text())) + workspace = run_dir / "workspace" + shutil.copytree(args.candidate.resolve(strict=True), workspace) + nonce = secrets.token_hex(16) + header = workspace / "firmware/include/run_nonce.h" + header.parent.mkdir(parents=True, exist_ok=True) + header.write_text(f'#pragma once\n#define LABWIRED_RUN_NONCE "{nonce}"\n') + result["task_id"] = task["id"] + result["hashes"]["oracle"] = sha256_file(oracle_path) + if args.usage_json: + cost = _usage(args.usage_json) + result["cost"] = cost + write_json_atomic(run_dir / "cost.json", cost) + save() + + firmware = workspace / "firmware" + build = [args.platformio, "run", "--project-dir", str(firmware), + "--environment", config.platformio_environment or "esp32s3"] + clean = run_command(build + ["--target", "clean"], cwd=workspace, + stdout_path=run_dir / "clean.stdout.log", + stderr_path=run_dir / "clean.stderr.log", + timeout_seconds=task["budgets"]["wall_time_seconds"]) + if clean.timed_out or clean.cleanup_error or clean.returncode: + raise RuntimeError("clean failed") + compiled = run_command(build, cwd=workspace, stdout_path=run_dir / "build.stdout.log", + stderr_path=run_dir / "build.stderr.log", + timeout_seconds=task["budgets"]["wall_time_seconds"]) + if compiled.timed_out or compiled.cleanup_error: + raise RuntimeError("build infrastructure failed") + if compiled.returncode: + result.update(status="fail", compile_status="fail") + save() + return 0 + result["compile_status"] = "pass" + artifact = firmware / config.flash_artifact + result["hashes"]["firmware"] = sha256_file(artifact) + save() + + identity_command = (json.loads(args.identity_command_json) + if args.identity_command_json else list(config.identity_command)) + with BoardLock(run_dir.parent / ".board-locks", args.jtag_serial, + timeout_seconds=config.identity_timeout_seconds): + identity = validate_identity(identity_command, args.jtag_serial, cwd=workspace, + evidence_dir=run_dir, + timeout_seconds=config.identity_timeout_seconds) + if identity.status != "pass": + raise RuntimeError(identity.detail or "board identity failed") + + uart_box = {} + uart_errors = [] + def capture() -> None: + try: + uart_box["result"] = capture_uart_nonce( + args.uart_device, config.uart_baud, nonce, + config.uart_timeout_seconds, run_dir / "uart.log") + except Exception as error: + uart_errors.append(error) + uart_thread = threading.Thread(target=capture) + uart_thread.start() + flash_command = build + ["--target", config.flash_target] + flashed = flash_firmware(flash_command, cwd=workspace, evidence_dir=run_dir, + timeout_seconds=config.flash_timeout_seconds, + identity_validated=True) + uart_thread.join(config.uart_timeout_seconds + 1) + if uart_thread.is_alive() or uart_errors: + raise RuntimeError("UART capture failed") + uart = uart_box["result"] + result["uart"] = {"matched": uart.matched, "termination": uart.termination_reason, + "bytes": uart.bytes_captured} + if flashed.status == "infrastructure_error": + raise RuntimeError(flashed.detail or "flash infrastructure failed") + if flashed.status == "hardware_fail" or not uart.matched: + result.update(status="fail", hardware_status="fail") + save() + return 0 + + registers = read_registers(args.openocd, config.openocd_board_config, + args.jtag_serial, config.assertions, cwd=workspace, + evidence_dir=run_dir, + timeout_seconds=config.openocd_command_timeout_seconds) + if registers.status == "infrastructure_error": + raise RuntimeError(registers.detail or "OpenOCD failed") + result["registers"] = [ + {"name": item.name, "passed": item.passed, + "observed_masked": f"0x{item.value & item.mask:08x}"} + for item in registers.evaluation.observations + ] + result["hardware_status"] = "pass" if registers.status == "pass" else "fail" + result["status"] = "pass" if registers.status == "pass" else "fail" + save() + return 0 + except Exception as error: + result.update(status="invalid", infrastructure_status="error", error=str(error)) + save() + print(error, file=sys.stderr) + return 2 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/twin2silicon-hil.py b/tests/twin2silicon-hil.py index f9b8ab4..c7c4767 100644 --- a/tests/twin2silicon-hil.py +++ b/tests/twin2silicon-hil.py @@ -773,6 +773,118 @@ def stop(signum, frame): self.assertEqual(terminated.read_text(), "terminated") +class SimpleHilRunnerTests(unittest.TestCase): + def _run_cli(self, *arguments): + return subprocess.run( + [sys.executable, str(REPOSITORY_ROOT / "benchmarks/twin2silicon/run_hil.py"), + *map(str, arguments)], + cwd=REPOSITORY_ROOT, + text=True, + capture_output=True, + timeout=15, + ) + + def test_complete_fake_hil_pass(self): + task = REPOSITORY_ROOT / "benchmarks/twin2silicon/tasks/esp32s3-gpio-hil-001" + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + candidate = root / "candidate" + import shutil + shutil.copytree(task / "public", candidate) + (candidate / "firmware/src/main.c").write_text( + (candidate / "firmware/src/main.c").read_text().replace( + "GPIO_MODE_INPUT", "GPIO_MODE_OUTPUT" + ) + ) + flash_marker = root / "flashed" + pio_dir = root / "pio" + pio_dir.mkdir() + pio = executable_fixture(pio_dir, textwrap.dedent(f""" + import pathlib, sys + args = sys.argv[1:] + project = pathlib.Path(args[args.index('--project-dir') + 1]) + if 'clean' in args: + raise SystemExit(0) + if 'upload' in args: + pathlib.Path({str(flash_marker)!r}).write_text('flashed') + raise SystemExit(0) + artifact = project / '.pio/build/esp32s3/firmware.bin' + artifact.parent.mkdir(parents=True, exist_ok=True) + artifact.write_bytes(b'firmware') + """)) + identity_dir = root / "identity" + identity_dir.mkdir() + identity = executable_fixture(identity_dir, "print('JTAG-1')\n") + openocd_dir = root / "openocd" + openocd_dir.mkdir() + openocd = executable_fixture(openocd_dir, textwrap.dedent(""" + import sys + print('@@REG gpio2_output_enabled 0x60004020', file=sys.stderr) + print('0x60004020: 00000004', file=sys.stderr) + print('@@REG gpio2_output_high 0x60004004', file=sys.stderr) + print('0x60004004: 00000004', file=sys.stderr) + """)) + master, slave = pty.openpty() + uart = os.ttyname(slave) + run_dir = root / "run" + def write_uart(): + deadline = time.monotonic() + 10 + header = run_dir / "workspace/firmware/include/run_nonce.h" + while time.monotonic() < deadline and not (flash_marker.exists() and header.exists()): + time.sleep(.01) + if flash_marker.exists() and header.exists(): + nonce = header.read_text().split('"')[1] + os.write(master, f"LABWIRED_READY:{nonce}\n".encode()) + writer = threading.Thread(target=write_uart) + writer.start() + result = self._run_cli( + task, "--run-dir", run_dir, "--candidate", candidate, + "--jtag-serial", "JTAG-1", "--uart-device", uart, + "--platformio", pio, "--openocd", openocd, + "--identity-command-json", json.dumps([str(identity)]), + ) + writer.join(10) + os.close(master) + os.close(slave) + self.assertEqual(result.returncode, 0, result.stderr) + manifest = json.loads((run_dir / "run.json").read_text()) + self.assertEqual(manifest["status"], "pass") + self.assertEqual(manifest["compile_status"], "pass") + self.assertEqual(manifest["hardware_status"], "pass") + self.assertTrue(manifest["uart"]["matched"]) + self.assertTrue(all(item["passed"] for item in manifest["registers"])) + + def test_compile_failure_never_touches_hardware(self): + task = REPOSITORY_ROOT / "benchmarks/twin2silicon/tasks/esp32s3-gpio-hil-001" + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + candidate = root / "candidate" + import shutil + shutil.copytree(task / "public", candidate) + marker = root / "hardware-ran" + pio_dir = root / "pio" + pio_dir.mkdir() + pio = executable_fixture(pio_dir, "import sys; raise SystemExit(0 if 'clean' in sys.argv else 1)\n") + hardware_dir = root / "hardware" + hardware_dir.mkdir() + hardware = executable_fixture( + hardware_dir, + f"from pathlib import Path\nPath({str(marker)!r}).write_text('ran')\n", + ) + result = self._run_cli( + task, "--run-dir", root / "run", "--candidate", candidate, + "--jtag-serial", "JTAG-1", "--uart-device", "/dev/null", + "--platformio", pio, "--openocd", hardware, + "--identity-command-json", json.dumps([str(hardware)]), + ) + self.assertEqual(result.returncode, 0, result.stderr) + manifest = json.loads((root / "run/run.json").read_text()) + self.assertEqual(manifest["status"], "fail") + self.assertEqual(manifest["compile_status"], "fail") + self.assertEqual(manifest["hardware_status"], "not_run") + self.assertFalse(marker.exists()) + + if __name__ == "__main__": if "-k" in sys.argv: pattern_index = sys.argv.index("-k") + 1 From 6cb6e332f68318f41946cefff06f8b5b8b9a15b6 Mon Sep 17 00:00:00 2001 From: Andrii Shylenko <14119286+w1ne@users.noreply.github.com> Date: Sat, 15 Aug 2026 15:46:27 +0200 Subject: [PATCH 26/48] fix(bench): run ESP32-S3 baseline on hardware --- benchmarks/twin2silicon/hil/esp32s3.py | 2 +- benchmarks/twin2silicon/run_hil.py | 30 +++++++------------ .../hidden/hil-oracle.json | 2 +- .../public/firmware/platformio.ini | 1 + .../public/firmware/src/main.c | 5 ++-- tests/twin2silicon-hil.py | 25 ++++++++++++---- 6 files changed, 36 insertions(+), 29 deletions(-) diff --git a/benchmarks/twin2silicon/hil/esp32s3.py b/benchmarks/twin2silicon/hil/esp32s3.py index 944b464..7ec2378 100644 --- a/benchmarks/twin2silicon/hil/esp32s3.py +++ b/benchmarks/twin2silicon/hil/esp32s3.py @@ -344,7 +344,7 @@ def build_openocd_command(executable: str, config: str, adapter_serial: str, "sleep 750", "halt"] for assertion in assertions: commands.extend((f'echo "@@REG {assertion.name} 0x{assertion.address:08x}"', - f"mdw 0x{assertion.address:08x} 1")) + f'echo [capture "mdw 0x{assertion.address:08x} 1"]')) commands.append("exit") return [executable, "-f", config, "-c", "; ".join(commands)] diff --git a/benchmarks/twin2silicon/run_hil.py b/benchmarks/twin2silicon/run_hil.py index d0a25e9..1bd3192 100644 --- a/benchmarks/twin2silicon/run_hil.py +++ b/benchmarks/twin2silicon/run_hil.py @@ -11,7 +11,6 @@ import secrets import shutil import sys -import threading if __package__ in (None, ""): sys.path.insert(0, str(Path(__file__).resolve().parents[2])) @@ -135,30 +134,23 @@ def save() -> None: if identity.status != "pass": raise RuntimeError(identity.detail or "board identity failed") - uart_box = {} - uart_errors = [] - def capture() -> None: - try: - uart_box["result"] = capture_uart_nonce( - args.uart_device, config.uart_baud, nonce, - config.uart_timeout_seconds, run_dir / "uart.log") - except Exception as error: - uart_errors.append(error) - uart_thread = threading.Thread(target=capture) - uart_thread.start() flash_command = build + ["--target", config.flash_target] flashed = flash_firmware(flash_command, cwd=workspace, evidence_dir=run_dir, timeout_seconds=config.flash_timeout_seconds, identity_validated=True) - uart_thread.join(config.uart_timeout_seconds + 1) - if uart_thread.is_alive() or uart_errors: - raise RuntimeError("UART capture failed") - uart = uart_box["result"] - result["uart"] = {"matched": uart.matched, "termination": uart.termination_reason, - "bytes": uart.bytes_captured} if flashed.status == "infrastructure_error": raise RuntimeError(flashed.detail or "flash infrastructure failed") - if flashed.status == "hardware_fail" or not uart.matched: + if flashed.status == "hardware_fail": + result.update(status="fail", hardware_status="fail") + save() + return 0 + + uart = capture_uart_nonce( + args.uart_device, config.uart_baud, nonce, + config.uart_timeout_seconds, run_dir / "uart.log") + result["uart"] = {"matched": uart.matched, "termination": uart.termination_reason, + "bytes": uart.bytes_captured} + if not uart.matched: result.update(status="fail", hardware_status="fail") save() return 0 diff --git a/benchmarks/twin2silicon/tasks/esp32s3-gpio-hil-001/hidden/hil-oracle.json b/benchmarks/twin2silicon/tasks/esp32s3-gpio-hil-001/hidden/hil-oracle.json index d1f98ab..6de002a 100644 --- a/benchmarks/twin2silicon/tasks/esp32s3-gpio-hil-001/hidden/hil-oracle.json +++ b/benchmarks/twin2silicon/tasks/esp32s3-gpio-hil-001/hidden/hil-oracle.json @@ -20,7 +20,7 @@ "timeout_seconds": 30 }, "openocd": { - "board_config": "esp32s3-builtin.cfg", + "board_config": "board/esp32s3-builtin.cfg", "startup_timeout_seconds": 20, "command_timeout_seconds": 10 }, diff --git a/benchmarks/twin2silicon/tasks/esp32s3-gpio-hil-001/public/firmware/platformio.ini b/benchmarks/twin2silicon/tasks/esp32s3-gpio-hil-001/public/firmware/platformio.ini index 2697a6e..473b385 100644 --- a/benchmarks/twin2silicon/tasks/esp32s3-gpio-hil-001/public/firmware/platformio.ini +++ b/benchmarks/twin2silicon/tasks/esp32s3-gpio-hil-001/public/firmware/platformio.ini @@ -4,3 +4,4 @@ board = esp32-s3-devkitc-1 framework = espidf monitor_speed = 115200 board_build.flash_size = 4MB +board_upload.flash_size = 4MB diff --git a/benchmarks/twin2silicon/tasks/esp32s3-gpio-hil-001/public/firmware/src/main.c b/benchmarks/twin2silicon/tasks/esp32s3-gpio-hil-001/public/firmware/src/main.c index f0524ee..6218356 100644 --- a/benchmarks/twin2silicon/tasks/esp32s3-gpio-hil-001/public/firmware/src/main.c +++ b/benchmarks/twin2silicon/tasks/esp32s3-gpio-hil-001/public/firmware/src/main.c @@ -13,10 +13,9 @@ void app_main(void) ESP_ERROR_CHECK(gpio_set_direction(TEST_GPIO, GPIO_MODE_INPUT)); ESP_ERROR_CHECK(gpio_set_level(TEST_GPIO, 1)); - printf("LABWIRED_READY:%s\n", LABWIRED_RUN_NONCE); - fflush(stdout); - for (;;) { + printf("LABWIRED_READY:%s\n", LABWIRED_RUN_NONCE); + fflush(stdout); vTaskDelay(pdMS_TO_TICKS(1000)); } } diff --git a/tests/twin2silicon-hil.py b/tests/twin2silicon-hil.py index c7c4767..f2bdefb 100644 --- a/tests/twin2silicon-hil.py +++ b/tests/twin2silicon-hil.py @@ -120,11 +120,19 @@ def test_esp32s3_gpio_hil_fixture_contract(self): encoding="utf-8" ) self.assertIn("gpio_set_direction(TEST_GPIO, GPIO_MODE_INPUT)", main_source) + self.assertLess( + main_source.index("for (;;)"), + main_source.index('printf("LABWIRED_READY:'), + ) sdkconfig_defaults = ( task_root / "public" / "firmware" / "sdkconfig.defaults" ).read_text(encoding="utf-8") self.assertIn("# CONFIG_ESP_CONSOLE_NONE is not set", sdkconfig_defaults) self.assertNotIn("CONFIG_ESP_CONSOLE_UART_NONE", sdkconfig_defaults) + platformio_ini = ( + task_root / "public" / "firmware" / "platformio.ini" + ).read_text(encoding="utf-8") + self.assertIn("board_upload.flash_size = 4MB", platformio_ini) class ResultContractTests(unittest.TestCase): @@ -379,7 +387,7 @@ def test_parses_shipped_oracle_exactly(self): self.assertEqual((config.flash_target, config.flash_artifact, config.flash_timeout_seconds), ("upload", ".pio/build/esp32s3/firmware.bin", 120)) self.assertEqual((config.openocd_board_config, config.openocd_startup_timeout_seconds, - config.openocd_command_timeout_seconds), ("esp32s3-builtin.cfg", 20, 10)) + config.openocd_command_timeout_seconds), ("board/esp32s3-builtin.cfg", 20, 10)) self.assertEqual((config.platformio_project_dir, config.platformio_environment), ("public/firmware", "esp32s3")) self.assertEqual(len(config.assertions), 2) @@ -665,8 +673,8 @@ def test_command_is_argv_and_requests_marked_records_at_fixed_speed(self): command = build_openocd_command("openocd", "board.cfg", "JTAG-1", self.assertions) self.assertEqual(command, ["openocd", "-f", "board.cfg", "-c", 'adapter serial JTAG-1; adapter speed 4000; init; reset run; sleep 750; halt; ' - 'echo "@@REG enable 0x60004020"; mdw 0x60004020 1; ' - 'echo "@@REG high 0x60004004"; mdw 0x60004004 1; exit']) + 'echo "@@REG enable 0x60004020"; echo [capture "mdw 0x60004020 1"]; ' + 'echo "@@REG high 0x60004004"; echo [capture "mdw 0x60004004 1"]; exit']) def test_empty_assertions_are_rejected_by_all_register_paths(self): with self.assertRaises(ValueError): @@ -797,15 +805,23 @@ def test_complete_fake_hil_pass(self): ) ) flash_marker = root / "flashed" + run_dir = root / "run" pio_dir = root / "pio" pio_dir.mkdir() pio = executable_fixture(pio_dir, textwrap.dedent(f""" - import pathlib, sys + import pathlib, sys, time args = sys.argv[1:] project = pathlib.Path(args[args.index('--project-dir') + 1]) if 'clean' in args: raise SystemExit(0) if 'upload' in args: + uart_log = pathlib.Path({str(root / "run/uart.log")!r}) + deadline = time.monotonic() + 1 + while time.monotonic() < deadline and not uart_log.exists(): + time.sleep(.01) + if uart_log.exists(): + print('UART capture started before flash', file=sys.stderr) + raise SystemExit(2) pathlib.Path({str(flash_marker)!r}).write_text('flashed') raise SystemExit(0) artifact = project / '.pio/build/esp32s3/firmware.bin' @@ -826,7 +842,6 @@ def test_complete_fake_hil_pass(self): """)) master, slave = pty.openpty() uart = os.ttyname(slave) - run_dir = root / "run" def write_uart(): deadline = time.monotonic() + 10 header = run_dir / "workspace/firmware/include/run_nonce.h" From aafa074448c5c350d742ca649bae5ebb5d619718 Mon Sep 17 00:00:00 2001 From: Andrii Shylenko <14119286+w1ne@users.noreply.github.com> Date: Sat, 15 Aug 2026 16:51:21 +0200 Subject: [PATCH 27/48] fix(agent): use DeepInfra OpenCode base URL --- config/opencode.deepinfra.json | 2 +- tests/skills-verify-all.sh | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/config/opencode.deepinfra.json b/config/opencode.deepinfra.json index 9828f5e..ec3efa4 100644 --- a/config/opencode.deepinfra.json +++ b/config/opencode.deepinfra.json @@ -48,7 +48,7 @@ "npm": "@ai-sdk/openai-compatible", "name": "DeepInfra", "options": { - "baseURL": "https://api.deepinfra.com/v1/openai", + "baseURL": "https://api.deepinfra.com/v1", "apiKey": "{env:DEEPINFRA_API_KEY}" }, "models": { diff --git a/tests/skills-verify-all.sh b/tests/skills-verify-all.sh index ccde157..876b46a 100755 --- a/tests/skills-verify-all.sh +++ b/tests/skills-verify-all.sh @@ -70,6 +70,10 @@ for cfg in "$ROOT/config/opencode.json" "$ROOT/config/opencode.hosted.json" \ done done +deepinfra_base="$(python3 -c 'import json,sys; print(json.load(open(sys.argv[1]))["provider"]["deepinfra"]["options"]["baseURL"])' "$ROOT/config/opencode.deepinfra.json")" +[[ "$deepinfra_base" == "https://api.deepinfra.com/v1" ]] \ + && pass "DeepInfra OpenCode base URL" || bad "DeepInfra OpenCode base URL: $deepinfra_base" + n=$(find "$ROOT/skills" -mindepth 1 -maxdepth 1 -type d | wc -l | tr -d ' ') # 7 domain packs + customize-labwired-agent + 14 superpowers = 22 if [[ "$n" -eq 22 ]]; then From 899858fbaf7150845b6fe69b5893895527125fb9 Mon Sep 17 00:00:00 2001 From: Andrii Shylenko <14119286+w1ne@users.noreply.github.com> Date: Sat, 15 Aug 2026 19:11:59 +0200 Subject: [PATCH 28/48] docs(bench): design cross-runtime HIL smoke test --- ...26-08-15-cross-runtime-hil-smoke-design.md | 159 ++++++++++++++++++ 1 file changed, 159 insertions(+) create mode 100644 docs/superpowers/specs/2026-08-15-cross-runtime-hil-smoke-design.md diff --git a/docs/superpowers/specs/2026-08-15-cross-runtime-hil-smoke-design.md b/docs/superpowers/specs/2026-08-15-cross-runtime-hil-smoke-design.md new file mode 100644 index 0000000..eb483fe --- /dev/null +++ b/docs/superpowers/specs/2026-08-15-cross-runtime-hil-smoke-design.md @@ -0,0 +1,159 @@ +# Cross-Runtime HIL Smoke Test Design + +## Goal + +Measure how effectively OpenCode, Codex CLI, and Claude Code use the same +LabWired hardware context to repair one ESP32-S3 firmware defect. Each runtime +uses its native/default model. The existing physical HIL oracle, not the agent, +decides success. + +## Scope + +The first smoke test runs one fresh trial for each runtime on +`esp32s3-gpio-hil-001`. It proves adapter portability and produces comparable +evidence before the benchmark grows to more tasks or repeated trials. + +This work does not add another orchestration framework, sandbox, scheduler, +leaderboard service, or generalized experiment engine. It does not claim model +or runtime superiority from three single trials. + +## Experimental Controls + +All three trials receive the same: + +- public task directory and seeded defect; +- model-neutral repair prompt; +- LabWired firmware skill instructions; +- LabWired MCP server where the runtime supports MCP; +- wall-time and repair-attempt budgets; +- physical ESP32-S3, UART port, JTAG identity, nonce policy, and hidden oracle; +- build, flash, UART, and register scoring implementation. + +The runtime and its native/default model are the independent variable. Built-in +runtime tools, context management, system prompts, and token accounting are +part of the runtime being measured and must be disclosed rather than hidden. + +## Architecture + +The implementation adds three thin candidate-generation adapters behind one +small command-line interface: + +```text +fresh public task + | + +-- OpenCode adapter ----+ + +-- Codex adapter -------+--> candidate workspace --> existing run_hil.py + +-- Claude adapter ------+ | + +--> run.json + +--> cost.json +``` + +An adapter may prepare runtime-native skill or MCP configuration, launch the +runtime, and normalize its usage output. It must not compile, flash, inspect +hidden files, score hardware, or reinterpret the oracle result in controller +code. Runtime-native compilation during repair is allowed, but final scoring +always uses `benchmarks/twin2silicon/run_hil.py`. + +## Adapter Contract + +Each adapter accepts: + +- runtime name; +- public task directory; +- fresh output directory; +- shared prompt file; +- wall-time limit. + +Each adapter produces: + +- `candidate/`: the runtime's final public workspace; +- `agent.stdout.log` and `agent.stderr.log`; +- `agent-result.json`: normalized runtime status and timing; +- `usage.json`: normalized token rates and estimated cost when the runtime + exposes enough information; otherwise explicit `null` fields and a reason. + +`agent-result.json` contains the runtime, reported native model when available, +exit status, timeout flag, elapsed seconds, repair status, and paths to raw +logs. Raw runtime output remains available for later auditing. + +## Skills and MCP + +The canonical LabWired firmware instructions remain in the repository. Each +adapter maps them into the runtime's supported instruction mechanism without +rewriting their substance: + +- OpenCode uses the existing LabWired skills and MCP configuration. +- Codex receives repository instructions plus the LabWired MCP registration. +- Claude Code receives repository instructions plus the LabWired MCP + registration. + +Tool names may differ because runtimes namespace MCP tools differently. The +adapter may map names, but the underlying LabWired MCP server and tool behavior +must be identical. + +## Evaluation Flow + +For each runtime, the controller: + +1. copies only the public task into a fresh workspace; +2. installs the runtime-specific instruction and MCP adapter; +3. invokes the native/default model with the shared prompt and time limit; +4. records raw output, normalized timing, usage, and cost; +5. passes the final candidate to the existing HIL runner; +6. records the authoritative compile, flash, UART, and register result; +7. emits one comparison row without changing or repairing the candidate. + +An adapter failure is recorded as an infrastructure failure. A clean agent exit +with an incorrect candidate is a benchmark failure. Missing token accounting is +reported as unknown rather than estimated from unrelated data. + +## Comparison Output + +The smoke test emits a JSON summary and a concise table with: + +- runtime and reported native model; +- agent exit/timeout status; +- compile and physical HIL status; +- final success; +- elapsed agent and HIL time; +- repair/tool-call counts when observable; +- fresh, cached, reasoning, and output tokens when observable; +- API or subscription cost when observable; +- invalid tool calls and infrastructure error category. + +Values unavailable from a native runtime are `null` with a reason. Subscription +access is not converted into a fictitious per-run API price. + +## Error Handling + +Every trial uses a new output directory and bounded subprocess execution. A +failed adapter cannot prevent the remaining runtime trials from running. The +controller never retries a model silently; every extra model attempt would be a +new recorded trial. It refuses to overwrite an existing trial directory and +never modifies the public fixture or hidden oracle. + +## Testing + +Offline tests use fake runtime executables to verify: + +- identical fresh candidate inputs; +- exact native/default-model behavior (no model override); +- prompt and instruction delivery; +- bounded timeout and nonzero-exit classification; +- normalized output with explicit unknown usage; +- continued execution after one adapter fails; +- invocation of the existing HIL boundary without exposing hidden files to an + adapter. + +One opt-in connected-board smoke test then runs the three installed runtimes on +the ESP32-S3 fixture. Its results are evidence, not a unit-test prerequisite. + +## Acceptance Criteria + +- OpenCode, Codex CLI, and Claude Code each run through a thin native adapter. +- No adapter specifies a non-native model override. +- All candidates are created from identical public bytes. +- All final candidates are scored by the unchanged HIL entry point. +- Results use one normalized schema while retaining raw logs. +- Unknown tokens or costs are explicit and never fabricated. +- Existing HIL tests remain green and `out/` remains untouched. From 968ac04033a7dd93850c24bb08f5d7dfbc542c08 Mon Sep 17 00:00:00 2001 From: Andrii Shylenko <14119286+w1ne@users.noreply.github.com> Date: Sat, 15 Aug 2026 19:16:02 +0200 Subject: [PATCH 29/48] docs(bench): plan cross-runtime HIL smoke test --- .../2026-08-15-cross-runtime-hil-smoke.md | 437 ++++++++++++++++++ 1 file changed, 437 insertions(+) create mode 100644 docs/superpowers/plans/2026-08-15-cross-runtime-hil-smoke.md diff --git a/docs/superpowers/plans/2026-08-15-cross-runtime-hil-smoke.md b/docs/superpowers/plans/2026-08-15-cross-runtime-hil-smoke.md new file mode 100644 index 0000000..45b1a92 --- /dev/null +++ b/docs/superpowers/plans/2026-08-15-cross-runtime-hil-smoke.md @@ -0,0 +1,437 @@ +# Cross-Runtime HIL Smoke Test Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Run OpenCode, Codex CLI, and Claude Code with their native/default models against identical LabWired firmware tasks, then score every candidate with the existing physical HIL oracle and emit one normalized comparison. + +**Architecture:** A stdlib-only adapter module builds native commands and normalizes runtime output. A single-trial controller copies public inputs, writes runtime-native instruction/MCP files, runs one bounded agent, and emits `agent-result.json` plus `usage.json`. A matrix controller runs independent trials and invokes the existing `run_hil.py`; it never interprets or replaces the hidden oracle. + +**Tech Stack:** Python 3 standard library, existing LabWired skills/MCP package, OpenCode 1.18.7, Codex CLI, Claude Code, PlatformIO, ESP-IDF, OpenOCD, UART/JTAG HIL. + +--- + +## File Structure + +- Create `benchmarks/twin2silicon/runtime_adapters.py`: runtime command construction and output normalization only. +- Create `benchmarks/twin2silicon/run_agent.py`: one fresh candidate-generation trial and its evidence. +- Create `benchmarks/twin2silicon/run_matrix.py`: sequential cross-runtime execution, HIL invocation, and summary output. +- Create `benchmarks/twin2silicon/shared-agent-instructions.md`: model-neutral firmware repair instructions mapped into each runtime. +- Create `benchmarks/twin2silicon/runtime-config/opencode.json`: existing local LabWired MCP profile with no model override. +- Create `benchmarks/twin2silicon/runtime-config/claude-mcp.json`: local LabWired MCP registration for Claude Code. +- Modify `tests/twin2silicon-hil.py`: offline adapter, trial, normalization, and matrix tests using fake executables. +- Create `tests/twin2silicon-runtime-smoke.sh`: opt-in connected-board entry point. +- Modify `package.json`: expose offline and connected smoke commands. +- Modify `benchmarks/twin2silicon/README.md` or create it if absent: document the matrix contract and usage. + +### Task 1: Define the normalized runtime contract + +**Files:** +- Create: `benchmarks/twin2silicon/runtime_adapters.py` +- Modify: `tests/twin2silicon-hil.py` + +- [ ] **Step 1: Write failing command-construction tests** + +Add `RuntimeAdapterTests` that creates an `AdapterContext` and asserts: + +```python +self.assertEqual( + build_runtime_command("codex", context)[:2], + ["codex", "exec"], +) +self.assertNotIn("--model", build_runtime_command("codex", context)) +self.assertEqual(build_runtime_command("claude", context)[:2], ["claude", "--print"]) +self.assertNotIn("--model", build_runtime_command("claude", context)) +self.assertEqual(build_runtime_command("opencode", context)[:2], ["opencode", "run"]) +self.assertNotIn("--model", build_runtime_command("opencode", context)) +``` + +Also assert each command selects structured output and the supplied workspace without embedding a hidden-oracle path. + +- [ ] **Step 2: Run the focused test and verify RED** + +Run: + +```bash +python3 tests/twin2silicon-hil.py -k RuntimeAdapterTests +``` + +Expected: import failure for `benchmarks.twin2silicon.runtime_adapters`. + +- [ ] **Step 3: Implement the minimal adapter types and commands** + +Define frozen dataclasses: + +```python +@dataclass(frozen=True) +class AdapterContext: + runtime: Literal["opencode", "codex", "claude"] + executable: str + workspace: Path + prompt: str + config_dir: Path + stdout_path: Path + stderr_path: Path + +@dataclass(frozen=True) +class NormalizedUsage: + requests: int | None + fresh_input: int | None + cached_input: int | None + reasoning: int | None + output: int | None + estimated_cost_usd: float | None + unavailable_reason: str | None +``` + +Construct native/default commands without model flags: + +```python +codex exec --json --ephemeral --skip-git-repo-check -s workspace-write -C WORKSPACE PROMPT +claude --print --output-format stream-json --no-session-persistence --permission-mode acceptEdits --mcp-config CONFIG PROMPT +opencode run --format json --dir WORKSPACE PROMPT +``` + +Use runtime-specific environment variables only for config discovery; do not copy credentials into evidence. + +- [ ] **Step 4: Write failing usage-normalization tests** + +Use compact fixtures for: + +- OpenCode `step_finish.part.tokens` and `part.cost` events; +- Codex JSONL token-usage events; +- Claude stream-json `result` usage and `total_cost_usd`; +- malformed output and successful output with no accounting. + +Expected normalized behavior: + +```python +self.assertEqual(usage.output, 1076) +self.assertEqual(usage.estimated_cost_usd, 0.007476282) +self.assertIsNone(missing.estimated_cost_usd) +self.assertEqual(missing.unavailable_reason, "runtime did not expose usage") +``` + +- [ ] **Step 5: Implement streaming parsers and GREEN the tests** + +Parse line-by-line with bounded integer/float validation. Sum OpenCode step costs, use Codex totals from the final usage event, and use Claude's final result object. Never infer subscription cost from token counts. + +Run: + +```bash +python3 tests/twin2silicon-hil.py -k RuntimeAdapterTests +``` + +Expected: all adapter tests pass. + +- [ ] **Step 6: Commit Task 1** + +```bash +git add benchmarks/twin2silicon/runtime_adapters.py tests/twin2silicon-hil.py +git commit -m "feat(bench): define native runtime adapters" +``` + +### Task 2: Add shared instructions and runtime-native MCP configuration + +**Files:** +- Create: `benchmarks/twin2silicon/shared-agent-instructions.md` +- Create: `benchmarks/twin2silicon/runtime-config/opencode.json` +- Create: `benchmarks/twin2silicon/runtime-config/claude-mcp.json` +- Modify: `tests/twin2silicon-hil.py` + +- [ ] **Step 1: Write failing configuration-contract tests** + +Assert that shared instructions: + +- require the smallest firmware repair; +- permit only public workspace access; +- prohibit hidden-oracle access and self-grading; +- require compile evidence; +- cap repair attempts at the task budget; +- name LabWired MCP tools as optional context/compile aids, not as the final oracle. + +Parse both JSON configs and assert they launch exactly: + +```json +{"command": "npx", "args": ["-y", "@labwired/mcp"]} +``` + +Assert the OpenCode config declares no `model` key. Codex MCP is supplied through isolated TOML generated by `run_agent.py`, so the contract test checks the generated TOML rather than a third static config. + +- [ ] **Step 2: Run focused tests and verify RED** + +```bash +python3 tests/twin2silicon-hil.py -k RuntimeConfigurationTests +``` + +Expected: missing instruction/config files. + +- [ ] **Step 3: Add the minimal shared instruction file** + +Keep it below 700 words and reuse the behavioral substance of `skills/develop/SKILL.md`: inspect, ground, edit, compile, report evidence, do not claim hardware success. + +- [ ] **Step 4: Add native MCP configs** + +OpenCode config uses a local `labwired` MCP entry and existing permission allowlist, but omits provider/model declarations so the installed native default remains authoritative. Claude config uses `mcpServers.labwired.command = "npx"` and `args = ["-y", "@labwired/mcp"]`. + +Generate Codex TOML in the trial config directory: + +```toml +[mcp_servers.labwired] +command = "npx" +args = ["-y", "@labwired/mcp"] +``` + +- [ ] **Step 5: Run tests and commit Task 2** + +```bash +python3 tests/twin2silicon-hil.py -k RuntimeConfigurationTests +git add benchmarks/twin2silicon/shared-agent-instructions.md benchmarks/twin2silicon/runtime-config tests/twin2silicon-hil.py +git commit -m "feat(bench): share LabWired instructions across runtimes" +``` + +### Task 3: Implement one bounded agent trial + +**Files:** +- Create: `benchmarks/twin2silicon/run_agent.py` +- Modify: `tests/twin2silicon-hil.py` + +- [ ] **Step 1: Write failing end-to-end fake-runtime tests** + +Create one fake executable per runtime. Each fake: + +- validates its expected native argv; +- confirms the candidate starts with `GPIO_MODE_INPUT`; +- changes only that token to `GPIO_MODE_OUTPUT`; +- emits representative native JSON usage; +- exits zero. + +Invoke `run_agent.py` and assert: + +```python +self.assertEqual(result["status"], "completed") +self.assertEqual(result["runtime"], runtime) +self.assertIsNone(result["model_override"]) +self.assertIn("GPIO_MODE_OUTPUT", candidate_source) +self.assertTrue((trial / "agent.stdout.log").is_file()) +self.assertTrue((trial / "agent.stderr.log").is_file()) +self.assertTrue((trial / "usage.json").is_file()) +``` + +Add cases for nonzero exit, timeout, missing executable, malformed JSON, and an existing output directory. Assert none exposes or copies the hidden oracle. + +- [ ] **Step 2: Run focused tests and verify RED** + +```bash +python3 tests/twin2silicon-hil.py -k RunAgentTests +``` + +Expected: `run_agent.py` missing. + +- [ ] **Step 3: Implement the single-trial CLI** + +CLI: + +```text +run_agent.py RUNTIME --task TASK --output TRIAL_DIR + [--executable PATH] [--timeout-seconds N] +``` + +Behavior: + +1. reject an existing output path; +2. read `task.json` only to locate `public_dir` and budgets; +3. copy only `public_dir` to `TRIAL_DIR/candidate`; +4. combine the model-neutral task prompt with shared instructions; +5. write runtime-native config under `TRIAL_DIR/runtime-config`, copy the same + shared instruction text to `candidate/AGENTS.md` for Codex/OpenCode and + `candidate/CLAUDE.md` for Claude Code; +6. invoke `run_command` with the adapter's argv and bounded timeout; +7. parse usage without failing the trial when accounting is unavailable; +8. atomically write `agent-result.json` and `usage.json`. + +Set `agent-result.status` to `completed`, `failed`, `timeout`, or `infrastructure_error`. Record monotonic elapsed time, exit code, and executable version. Do not run HIL here. + +- [ ] **Step 4: Run focused and full offline tests** + +```bash +python3 tests/twin2silicon-hil.py -k RunAgentTests +python3 tests/twin2silicon-hil.py +``` + +Expected: all tests pass. + +- [ ] **Step 5: Commit Task 3** + +```bash +git add benchmarks/twin2silicon/run_agent.py tests/twin2silicon-hil.py +git commit -m "feat(bench): run one native agent trial" +``` + +### Task 4: Add the sequential runtime matrix and normalized summary + +**Files:** +- Create: `benchmarks/twin2silicon/run_matrix.py` +- Modify: `tests/twin2silicon-hil.py` + +- [ ] **Step 1: Write failing matrix tests** + +Use fake adapters and a fake HIL executable to assert: + +- runtime order is `opencode`, `codex`, `claude`; +- each receives byte-identical public input hashes; +- one adapter failure does not prevent later trials; +- HIL runs only for completed candidates; +- the hidden-oracle path appears only in HIL argv, never adapter argv/logs; +- summary rows retain unknown usage as `null` plus a reason. + +Expected summary skeleton: + +```json +{ + "schema_version": "1.0", + "task_id": "esp32s3-gpio-hil-001", + "trials": [ + {"runtime": "opencode", "agent_status": "completed", "hil_status": "pass"}, + {"runtime": "codex", "agent_status": "failed", "hil_status": "not_run"}, + {"runtime": "claude", "agent_status": "completed", "hil_status": "fail"} + ] +} +``` + +- [ ] **Step 2: Run focused tests and verify RED** + +```bash +python3 tests/twin2silicon-hil.py -k RuntimeMatrixTests +``` + +Expected: `run_matrix.py` missing. + +- [ ] **Step 3: Implement the matrix CLI** + +CLI: + +```text +run_matrix.py --task TASK --output MATRIX_DIR + --jtag-serial SERIAL --uart-device DEVICE --openocd PATH + [--runtime opencode --runtime codex --runtime claude] + [--agent-only] +``` + +Run trials sequentially because one physical board is shared. Invoke the existing `run_hil.py` as a child process with each completed candidate. Pass `--usage-json` only when the adapter produced all numeric fields required by the existing HIL cost schema; otherwise let HIL record `cost: null` and retain the explicit unavailable reason in the matrix row. Read, do not reinterpret, each HIL `run.json`. Atomically write `matrix.json` after every trial and print a fixed-width summary at completion. + +- [ ] **Step 4: Verify matrix failure isolation and full suite** + +```bash +python3 tests/twin2silicon-hil.py -k RuntimeMatrixTests +python3 tests/twin2silicon-hil.py +``` + +Expected: all tests pass; fake matrix contains all three rows after an injected middle failure. + +- [ ] **Step 5: Commit Task 4** + +```bash +git add benchmarks/twin2silicon/run_matrix.py tests/twin2silicon-hil.py +git commit -m "feat(bench): compare native runtimes with one HIL oracle" +``` + +### Task 5: Add operator entry points and documentation + +**Files:** +- Create: `tests/twin2silicon-runtime-smoke.sh` +- Create: `benchmarks/twin2silicon/README.md` +- Modify: `package.json` +- Modify: `tests/twin2silicon-hil.py` + +- [ ] **Step 1: Write failing packaging/entry-point assertions** + +Extend the existing contract tests to require: + +```json +{ + "test:runtime-smoke:offline": "python3 tests/twin2silicon-hil.py", + "test:runtime-smoke:hardware": "bash tests/twin2silicon-runtime-smoke.sh" +} +``` + +Assert the shell entry point refuses to run unless `LABWIRED_HIL=1`, requires explicit UART/JTAG/OpenOCD values, places temporary data on `/Volumes/LabWired` when available, and never contains credentials. + +- [ ] **Step 2: Run tests and verify RED** + +```bash +python3 tests/twin2silicon-hil.py -k RuntimePackagingTests +``` + +Expected: scripts and package commands missing. + +- [ ] **Step 3: Implement the opt-in shell entry point** + +The script validates installed `opencode`, `codex`, `claude`, `pio`, and OpenOCD; prints their versions; then invokes `run_matrix.py` with explicit hardware arguments. It never reads API secrets itself—each native runtime uses its existing authenticated session. + +- [ ] **Step 4: Document experiment interpretation** + +Document: + +- this is a runtime smoke comparison, not a leaderboard claim; +- native models are intentionally not overridden; +- raw model comparisons require the separate OpenCode model matrix; +- missing subscription cost remains unknown; +- connected hardware is modified by flash operations; +- every publishable result needs multiple tasks and repeated fresh trials. + +- [ ] **Step 5: Run offline verification and commit Task 5** + +```bash +npm run test:runtime-smoke:offline +bash tests/twin2silicon-runtime-smoke.sh +``` + +Expected: offline suite passes; hardware script exits with a clear `LABWIRED_HIL=1 required` message when not opted in. + +```bash +git add benchmarks/twin2silicon/README.md tests/twin2silicon-runtime-smoke.sh package.json tests/twin2silicon-hil.py +git commit -m "docs(bench): add cross-runtime smoke entry points" +``` + +### Task 6: Run the connected-board smoke matrix + +**Files:** +- Runtime evidence only under an explicit matrix output directory outside tracked source. + +- [ ] **Step 1: Verify authentication without exposing credentials** + +Run native read-only status commands for OpenCode/LabWired, Codex, and Claude Code. Record only runtime versions and authenticated/not-authenticated status. + +- [ ] **Step 2: Confirm the target S3 identity** + +Use the existing serial/JTAG identification command to confirm the selected device is ESP32-S3 and record its serial. Do not select by port order alone. + +- [ ] **Step 3: Run one fresh native trial per runtime** + +Example: + +```bash +LABWIRED_HIL=1 \ +LABWIRED_UART_DEVICE=/dev/cu.usbmodem11101 \ +LABWIRED_JTAG_SERIAL=3C:0F:02:DF:EC:F8 \ +LABWIRED_OPENOCD="$HOME/.platformio/packages/tool-openocd-esp32/bin/openocd" \ +LABWIRED_MATRIX_OUTPUT=/Volumes/LabWired/hil-runs/runtime-smoke-$(date +%Y%m%d-%H%M%S) \ +bash tests/twin2silicon-runtime-smoke.sh +``` + +Expected: three trial directories and a parseable `matrix.json`; individual failures remain rows rather than aborting the matrix. + +- [ ] **Step 4: Verify evidence and summarize honestly** + +Check every completed candidate's `run.json`, artifact hashes, UART nonce, and register observations. Report success, time, tokens, cost availability, and infrastructure issues without claiming a general winner. + +- [ ] **Step 5: Run final repository verification** + +```bash +npm run test:runtime-smoke:offline +python3 tests/twin2silicon-hil.py +git diff --check +git status --short +``` + +Expected: all offline tests pass, diff check is clean, and only pre-existing `out/` remains untracked. From e40948aeab615a87a9194dd43a687a0976e617ed Mon Sep 17 00:00:00 2001 From: Andrii Shylenko <14119286+w1ne@users.noreply.github.com> Date: Sat, 15 Aug 2026 21:18:15 +0200 Subject: [PATCH 30/48] feat(bench): define native runtime adapters --- benchmarks/twin2silicon/runtime_adapters.py | 265 ++++++++++++++++++++ tests/twin2silicon-hil.py | 253 +++++++++++++++++++ 2 files changed, 518 insertions(+) create mode 100644 benchmarks/twin2silicon/runtime_adapters.py diff --git a/benchmarks/twin2silicon/runtime_adapters.py b/benchmarks/twin2silicon/runtime_adapters.py new file mode 100644 index 0000000..017773c --- /dev/null +++ b/benchmarks/twin2silicon/runtime_adapters.py @@ -0,0 +1,265 @@ +"""Native CLI contracts and streaming usage normalization for benchmark runtimes. + +The caller runs each command with ``AdapterContext.workspace`` as its working +directory. Claude does not expose a workspace command-line option, so its MCP +configuration is deliberately placed beneath ``config_dir``; the command uses +``config_dir / 'claude-mcp.json'`` without creating that file. +""" + +from __future__ import annotations + +from collections.abc import Iterable, Iterator +from dataclasses import dataclass +import json +import math +from pathlib import Path +from typing import Literal + + +RuntimeName = Literal["opencode", "codex", "claude"] +_MAX_USAGE_COUNT = 1_000_000_000_000 +_MAX_COST_USD = 1_000_000_000.0 +_CLAUDE_MCP_CONFIG = "claude-mcp.json" + + +@dataclass(frozen=True) +class AdapterContext: + """Execution inputs shared by the native runtime adapters.""" + + runtime: RuntimeName + executable: str + workspace: Path + prompt: str + config_dir: Path + stdout_path: Path + stderr_path: Path + + +@dataclass(frozen=True) +class NormalizedUsage: + """Usage available directly from a runtime's structured output.""" + + requests: int | None + fresh_input: int | None + cached_input: int | None + reasoning: int | None + output: int | None + estimated_cost_usd: float | None + unavailable_reason: str | None + + +def build_runtime_command(context: AdapterContext) -> list[str]: + """Build the native, model-default command for ``context.runtime``.""" + + if context.runtime == "codex": + return [ + context.executable, + "exec", + "--json", + "--ephemeral", + "--skip-git-repo-check", + "-s", + "workspace-write", + "-C", + str(context.workspace), + context.prompt, + ] + if context.runtime == "claude": + return [ + context.executable, + "--print", + "--output-format", + "stream-json", + "--no-session-persistence", + "--permission-mode", + "acceptEdits", + "--mcp-config", + str(context.config_dir / _CLAUDE_MCP_CONFIG), + context.prompt, + ] + if context.runtime == "opencode": + return [ + context.executable, + "run", + "--format", + "json", + "--dir", + str(context.workspace), + context.prompt, + ] + raise ValueError(f"unsupported runtime: {context.runtime}") + + +def normalize_usage(runtime: RuntimeName, lines: Iterable[str]) -> NormalizedUsage: + """Normalize newline-delimited structured runtime output without pricing inference.""" + + records = _json_records(lines) + if runtime == "opencode": + return _normalize_opencode(records) + if runtime == "codex": + return _normalize_codex(records) + if runtime == "claude": + return _normalize_claude(records) + raise ValueError(f"unsupported runtime: {runtime}") + + +def _json_records(lines: Iterable[str]) -> Iterator[dict[str, object]]: + source = lines.splitlines() if isinstance(lines, str) else lines + for line in source: + try: + record = json.loads(line) + except (TypeError, ValueError, json.JSONDecodeError): + continue + if isinstance(record, dict): + yield record + + +def _normalize_opencode(records: Iterable[dict[str, object]]) -> NormalizedUsage: + totals: dict[str, int] = {} + overflowed_totals: set[str] = set() + request_count = 0 + cost_total = 0.0 + has_cost = False + cost_overflowed = False + + for record in records: + if record.get("type") != "step_finish": + continue + part = _mapping(record.get("part")) + tokens = _mapping(part.get("tokens")) if part else None + values = { + "fresh_input": _bounded_int(tokens.get("input")) if tokens else None, + "cached_input": _bounded_int(_mapping(tokens.get("cache")).get("read")) + if tokens and _mapping(tokens.get("cache")) + else None, + "reasoning": _bounded_int(tokens.get("reasoning")) if tokens else None, + "output": _bounded_int(tokens.get("output")) if tokens else None, + } + cost = _bounded_float(part.get("cost")) if part else None + if all(value is None for value in values.values()) and cost is None: + continue + if request_count >= _MAX_USAGE_COUNT: + return _usage_result(0, {}, None) + request_count += 1 + for name, value in values.items(): + if value is None or name in overflowed_totals: + continue + total = totals.get(name, 0) + value + if total > _MAX_USAGE_COUNT: + totals.pop(name, None) + overflowed_totals.add(name) + else: + totals[name] = total + if cost is not None and not cost_overflowed: + total_cost = cost_total + cost + if total_cost > _MAX_COST_USD: + cost_overflowed = True + has_cost = False + else: + cost_total = total_cost + has_cost = True + + return _usage_result(request_count, totals, cost_total if has_cost else None) + + +def _normalize_codex(records: Iterable[dict[str, object]]) -> NormalizedUsage: + final_usage: dict[str, int | None] | None = None + for record in records: + if record.get("type") != "turn.completed": + continue + usage = _mapping(record.get("usage")) + final_usage = _token_values(usage) + + if final_usage is None or not any( + value is not None for value in final_usage.values() + ): + return _usage_result(0, {}, None) + return _usage_result( + 1, {name: value for name, value in final_usage.items() if value is not None}, None + ) + + +def _normalize_claude(records: Iterable[dict[str, object]]) -> NormalizedUsage: + final_usage: dict[str, int | None] | None = None + final_cost: float | None = None + for record in records: + if record.get("type") != "result": + continue + usage = _mapping(record.get("usage")) + values = { + "fresh_input": _bounded_int(usage.get("input_tokens")) if usage else None, + "cached_input": _bounded_int(usage.get("cache_read_input_tokens")) if usage else None, + "reasoning": _bounded_int(usage.get("reasoning_tokens")) if usage else None, + "output": _bounded_int(usage.get("output_tokens")) if usage else None, + } + cost = _bounded_float(record.get("total_cost_usd")) + final_usage = values + final_cost = cost + + if final_usage is None or ( + not any(value is not None for value in final_usage.values()) + and final_cost is None + ): + return _usage_result(0, {}, None) + return _usage_result( + 1, + {name: value for name, value in final_usage.items() if value is not None}, + final_cost, + ) + + +def _token_values(usage: dict[str, object] | None) -> dict[str, int | None]: + if usage is None: + return { + "fresh_input": None, + "cached_input": None, + "reasoning": None, + "output": None, + } + return { + "fresh_input": _bounded_int(usage.get("input_tokens")), + "cached_input": _bounded_int(usage.get("cached_input_tokens")), + "reasoning": _bounded_int(usage.get("reasoning_tokens")), + "output": _bounded_int(usage.get("output_tokens")), + } + + +def _usage_result( + requests: int, values: dict[str, int], cost: float | None +) -> NormalizedUsage: + if requests == 0: + return NormalizedUsage( + None, None, None, None, None, None, "runtime did not expose usage" + ) + if not values and cost is None: + return NormalizedUsage( + requests, None, None, None, None, None, "runtime did not expose usage" + ) + return NormalizedUsage( + requests, + values.get("fresh_input"), + values.get("cached_input"), + values.get("reasoning"), + values.get("output"), + cost, + None, + ) + + +def _mapping(value: object) -> dict[str, object] | None: + return value if isinstance(value, dict) else None + + +def _bounded_int(value: object) -> int | None: + if isinstance(value, bool) or not isinstance(value, int): + return None + return value if 0 <= value <= _MAX_USAGE_COUNT else None + + +def _bounded_float(value: object) -> float | None: + if isinstance(value, bool) or not isinstance(value, (int, float)): + return None + number = float(value) + if not math.isfinite(number) or not 0 <= number <= _MAX_COST_USD: + return None + return number diff --git a/tests/twin2silicon-hil.py b/tests/twin2silicon-hil.py index f2bdefb..2147dd9 100644 --- a/tests/twin2silicon-hil.py +++ b/tests/twin2silicon-hil.py @@ -41,6 +41,12 @@ read_registers, validate_identity, ) +from benchmarks.twin2silicon.runtime_adapters import ( + AdapterContext, + NormalizedUsage, + build_runtime_command, + normalize_usage, +) def executable_fixture(directory, body): @@ -50,6 +56,253 @@ def executable_fixture(directory, body): return path +class RuntimeAdapterTests(unittest.TestCase): + def adapter_context(self, runtime, executable=None): + directory = Path("/tmp/runtime-adapter-test") + return AdapterContext( + runtime=runtime, + executable=executable or runtime, + workspace=directory / "workspace", + prompt="Complete the public firmware task.", + config_dir=directory / "config", + stdout_path=directory / "stdout.log", + stderr_path=directory / "stderr.log", + ) + + def test_build_runtime_commands_use_native_structured_modes(self): + contexts = { + "codex": self.adapter_context("codex"), + "claude": self.adapter_context("claude"), + "opencode": self.adapter_context("opencode"), + } + commands = { + runtime: build_runtime_command(context) + for runtime, context in contexts.items() + } + + self.assertEqual(commands["codex"][:2], ["codex", "exec"]) + self.assertEqual(commands["claude"][:2], ["claude", "--print"]) + self.assertEqual(commands["opencode"][:2], ["opencode", "run"]) + self.assertEqual( + commands["codex"], + [ + "codex", "exec", "--json", "--ephemeral", "--skip-git-repo-check", + "-s", "workspace-write", "-C", str(contexts["codex"].workspace), + contexts["codex"].prompt, + ], + ) + self.assertEqual( + commands["claude"], + [ + "claude", "--print", "--output-format", "stream-json", + "--no-session-persistence", "--permission-mode", "acceptEdits", + "--mcp-config", str(contexts["claude"].config_dir / "claude-mcp.json"), + contexts["claude"].prompt, + ], + ) + self.assertEqual( + commands["opencode"], + [ + "opencode", "run", "--format", "json", "--dir", + str(contexts["opencode"].workspace), contexts["opencode"].prompt, + ], + ) + self.assertEqual( + Path(commands["claude"][commands["claude"].index("--mcp-config") + 1]).parent, + contexts["claude"].config_dir, + ) + hidden_oracle = "benchmarks/twin2silicon/tasks/esp32s3-gpio-hil-001/hidden/hil-oracle.json" + for command in commands.values(): + with self.subTest(command=command[:2]): + self.assertNotIn("--model", command) + self.assertNotIn(hidden_oracle, " ".join(command)) + + def test_adapter_dataclasses_are_frozen(self): + context = self.adapter_context("codex") + usage = NormalizedUsage(None, None, None, None, None, None, None) + + with self.assertRaises((AttributeError, TypeError)): + context.prompt = "another prompt" + with self.assertRaises((AttributeError, TypeError)): + usage.output = 1 + + def test_normalize_opencode_step_finish_events(self): + lines = [ + json.dumps({ + "type": "step_finish", + "part": { + "tokens": {"input": 800, "cache": {"read": 200}, "reasoning": 20, "output": 500}, + "cost": 0.002476282, + }, + }), + json.dumps({ + "type": "step_finish", + "part": { + "tokens": {"input": 600, "cache": {"read": 300}, "reasoning": 28, "output": 576}, + "cost": 0.005, + }, + }), + ] + + usage = normalize_usage("opencode", lines) + + self.assertEqual(usage.requests, 2) + self.assertEqual(usage.fresh_input, 1400) + self.assertEqual(usage.cached_input, 500) + self.assertEqual(usage.reasoning, 48) + self.assertEqual(usage.output, 1076) + self.assertAlmostEqual(usage.estimated_cost_usd, 0.007476282) + self.assertIsNone(usage.unavailable_reason) + + def test_normalize_codex_uses_final_cumulative_usage(self): + lines = [ + json.dumps({"type": "turn.completed", "usage": { + "input_tokens": 1000, "cached_input_tokens": 200, + "reasoning_tokens": 10, "output_tokens": 500, + }}), + json.dumps({"type": "turn.completed", "usage": { + "input_tokens": 1400, "cached_input_tokens": 300, + "reasoning_tokens": 48, "output_tokens": 1076, + }}), + ] + + usage = normalize_usage("codex", lines) + + self.assertEqual(usage.requests, 1) + self.assertEqual(usage.fresh_input, 1400) + self.assertEqual(usage.cached_input, 300) + self.assertEqual(usage.reasoning, 48) + self.assertEqual(usage.output, 1076) + self.assertIsNone(usage.estimated_cost_usd) + self.assertIsNone(usage.unavailable_reason) + + def test_normalize_codex_uses_the_final_terminal_event(self): + lines = [ + json.dumps({"type": "turn.completed", "usage": { + "input_tokens": 1400, "output_tokens": 1076, + }}), + json.dumps({"type": "turn.completed"}), + ] + + usage = normalize_usage("codex", lines) + + self.assertEqual( + usage, + NormalizedUsage(None, None, None, None, None, None, "runtime did not expose usage"), + ) + + def test_normalize_codex_accepts_zero_token_usage(self): + usage = normalize_usage( + "codex", + [json.dumps({ + "type": "turn.completed", + "usage": {"input_tokens": 0, "output_tokens": 0}, + })], + ) + + self.assertEqual(usage.requests, 1) + self.assertEqual(usage.fresh_input, 0) + self.assertEqual(usage.output, 0) + self.assertIsNone(usage.unavailable_reason) + + def test_normalize_claude_final_result_usage_and_cost(self): + lines = [ + json.dumps({"type": "assistant", "message": {"content": []}}), + json.dumps({ + "type": "result", + "subtype": "success", + "usage": { + "input_tokens": 1400, + "cache_read_input_tokens": 300, + "output_tokens": 1076, + }, + "total_cost_usd": 0.007476282, + }), + ] + + usage = normalize_usage("claude", lines) + + self.assertEqual(usage.requests, 1) + self.assertEqual(usage.fresh_input, 1400) + self.assertEqual(usage.cached_input, 300) + self.assertIsNone(usage.reasoning) + self.assertEqual(usage.output, 1076) + self.assertAlmostEqual(usage.estimated_cost_usd, 0.007476282) + self.assertIsNone(usage.unavailable_reason) + + def test_normalize_claude_uses_the_final_result_event(self): + lines = [ + json.dumps({ + "type": "result", + "usage": {"input_tokens": 1400, "output_tokens": 1076}, + "total_cost_usd": 0.007476282, + }), + json.dumps({"type": "result", "subtype": "success", "result": "complete"}), + ] + + usage = normalize_usage("claude", lines) + + self.assertEqual( + usage, + NormalizedUsage(None, None, None, None, None, None, "runtime did not expose usage"), + ) + + def test_normalize_claude_accepts_zero_token_usage(self): + usage = normalize_usage( + "claude", + [json.dumps({ + "type": "result", + "usage": {"input_tokens": 0, "output_tokens": 0}, + "total_cost_usd": 0.0, + })], + ) + + self.assertEqual(usage.requests, 1) + self.assertEqual(usage.fresh_input, 0) + self.assertEqual(usage.output, 0) + self.assertEqual(usage.estimated_cost_usd, 0.0) + self.assertIsNone(usage.unavailable_reason) + + def test_normalize_usage_ignores_malformed_lines(self): + usage = normalize_usage("opencode", ["not json", "{", "[]"]) + + self.assertEqual( + usage, + NormalizedUsage(None, None, None, None, None, None, "runtime did not expose usage"), + ) + + def test_normalize_opencode_rejects_aggregate_overflow(self): + lines = [ + json.dumps({ + "type": "step_finish", + "part": { + "tokens": {"output": 1_000_000_000_000}, + "cost": 1_000_000_000.0, + }, + }), + json.dumps({ + "type": "step_finish", + "part": {"tokens": {"output": 1}, "cost": 0.01}, + }), + ] + + usage = normalize_usage("opencode", lines) + + self.assertEqual(usage.requests, 2) + self.assertIsNone(usage.output) + self.assertIsNone(usage.estimated_cost_usd) + self.assertEqual(usage.unavailable_reason, "runtime did not expose usage") + + def test_normalize_usage_marks_success_without_accounting_unavailable(self): + usage = normalize_usage( + "claude", + [json.dumps({"type": "result", "subtype": "success", "result": "complete"})], + ) + + self.assertEqual(usage.estimated_cost_usd, None) + self.assertEqual(usage.unavailable_reason, "runtime did not expose usage") + + class FixtureContractTests(unittest.TestCase): def test_esp32s3_gpio_hil_fixture_contract(self): task_root = ( From 2a14165dab182f106895c6addfd4a2d1d6c510f1 Mon Sep 17 00:00:00 2001 From: Andrii Shylenko <14119286+w1ne@users.noreply.github.com> Date: Sat, 15 Aug 2026 21:20:14 +0200 Subject: [PATCH 31/48] fix(bench): make runtime command dispatch explicit --- benchmarks/twin2silicon/runtime_adapters.py | 13 ++++++------- tests/twin2silicon-hil.py | 7 +++---- 2 files changed, 9 insertions(+), 11 deletions(-) diff --git a/benchmarks/twin2silicon/runtime_adapters.py b/benchmarks/twin2silicon/runtime_adapters.py index 017773c..78d09de 100644 --- a/benchmarks/twin2silicon/runtime_adapters.py +++ b/benchmarks/twin2silicon/runtime_adapters.py @@ -26,7 +26,6 @@ class AdapterContext: """Execution inputs shared by the native runtime adapters.""" - runtime: RuntimeName executable: str workspace: Path prompt: str @@ -48,10 +47,10 @@ class NormalizedUsage: unavailable_reason: str | None -def build_runtime_command(context: AdapterContext) -> list[str]: - """Build the native, model-default command for ``context.runtime``.""" +def build_runtime_command(runtime: RuntimeName, context: AdapterContext) -> list[str]: + """Build the native, model-default command for ``runtime``.""" - if context.runtime == "codex": + if runtime == "codex": return [ context.executable, "exec", @@ -64,7 +63,7 @@ def build_runtime_command(context: AdapterContext) -> list[str]: str(context.workspace), context.prompt, ] - if context.runtime == "claude": + if runtime == "claude": return [ context.executable, "--print", @@ -77,7 +76,7 @@ def build_runtime_command(context: AdapterContext) -> list[str]: str(context.config_dir / _CLAUDE_MCP_CONFIG), context.prompt, ] - if context.runtime == "opencode": + if runtime == "opencode": return [ context.executable, "run", @@ -87,7 +86,7 @@ def build_runtime_command(context: AdapterContext) -> list[str]: str(context.workspace), context.prompt, ] - raise ValueError(f"unsupported runtime: {context.runtime}") + raise ValueError(f"unsupported runtime: {runtime}") def normalize_usage(runtime: RuntimeName, lines: Iterable[str]) -> NormalizedUsage: diff --git a/tests/twin2silicon-hil.py b/tests/twin2silicon-hil.py index 2147dd9..1d1e56f 100644 --- a/tests/twin2silicon-hil.py +++ b/tests/twin2silicon-hil.py @@ -57,11 +57,10 @@ def executable_fixture(directory, body): class RuntimeAdapterTests(unittest.TestCase): - def adapter_context(self, runtime, executable=None): + def adapter_context(self, executable="runtime"): directory = Path("/tmp/runtime-adapter-test") return AdapterContext( - runtime=runtime, - executable=executable or runtime, + executable=executable, workspace=directory / "workspace", prompt="Complete the public firmware task.", config_dir=directory / "config", @@ -76,7 +75,7 @@ def test_build_runtime_commands_use_native_structured_modes(self): "opencode": self.adapter_context("opencode"), } commands = { - runtime: build_runtime_command(context) + runtime: build_runtime_command(runtime, context) for runtime, context in contexts.items() } From e100e4251840b57eb2fd28de568b1741d91293bb Mon Sep 17 00:00:00 2001 From: Andrii Shylenko <14119286+w1ne@users.noreply.github.com> Date: Sat, 15 Aug 2026 21:21:34 +0200 Subject: [PATCH 32/48] fix(bench): validate runtime adapter dispatch --- benchmarks/twin2silicon/runtime_adapters.py | 3 +++ tests/twin2silicon-hil.py | 11 +++++++++-- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/benchmarks/twin2silicon/runtime_adapters.py b/benchmarks/twin2silicon/runtime_adapters.py index 78d09de..872870b 100644 --- a/benchmarks/twin2silicon/runtime_adapters.py +++ b/benchmarks/twin2silicon/runtime_adapters.py @@ -26,6 +26,7 @@ class AdapterContext: """Execution inputs shared by the native runtime adapters.""" + runtime: RuntimeName executable: str workspace: Path prompt: str @@ -50,6 +51,8 @@ class NormalizedUsage: def build_runtime_command(runtime: RuntimeName, context: AdapterContext) -> list[str]: """Build the native, model-default command for ``runtime``.""" + if runtime != context.runtime: + raise ValueError("runtime does not match context") if runtime == "codex": return [ context.executable, diff --git a/tests/twin2silicon-hil.py b/tests/twin2silicon-hil.py index 1d1e56f..073f7b5 100644 --- a/tests/twin2silicon-hil.py +++ b/tests/twin2silicon-hil.py @@ -57,10 +57,11 @@ def executable_fixture(directory, body): class RuntimeAdapterTests(unittest.TestCase): - def adapter_context(self, executable="runtime"): + def adapter_context(self, runtime, executable=None): directory = Path("/tmp/runtime-adapter-test") return AdapterContext( - executable=executable, + runtime=runtime, + executable=executable or runtime, workspace=directory / "workspace", prompt="Complete the public firmware task.", config_dir=directory / "config", @@ -116,6 +117,12 @@ def test_build_runtime_commands_use_native_structured_modes(self): self.assertNotIn("--model", command) self.assertNotIn(hidden_oracle, " ".join(command)) + def test_build_runtime_command_rejects_context_runtime_mismatch(self): + context = self.adapter_context("codex") + + with self.assertRaisesRegex(ValueError, "runtime does not match context"): + build_runtime_command("claude", context) + def test_adapter_dataclasses_are_frozen(self): context = self.adapter_context("codex") usage = NormalizedUsage(None, None, None, None, None, None, None) From 6af68bc1ef49b6bee9c331bad1464dff2953ed3f Mon Sep 17 00:00:00 2001 From: Andrii Shylenko <14119286+w1ne@users.noreply.github.com> Date: Sat, 15 Aug 2026 21:25:58 +0200 Subject: [PATCH 33/48] feat(bench): share LabWired instructions across runtimes --- .../runtime-config/claude-mcp.json | 11 ++++ .../twin2silicon/runtime-config/opencode.json | 46 +++++++++++++++ benchmarks/twin2silicon/runtime_adapters.py | 6 ++ .../twin2silicon/shared-agent-instructions.md | 28 +++++++++ tests/twin2silicon-hil.py | 57 +++++++++++++++++++ 5 files changed, 148 insertions(+) create mode 100644 benchmarks/twin2silicon/runtime-config/claude-mcp.json create mode 100644 benchmarks/twin2silicon/runtime-config/opencode.json create mode 100644 benchmarks/twin2silicon/shared-agent-instructions.md diff --git a/benchmarks/twin2silicon/runtime-config/claude-mcp.json b/benchmarks/twin2silicon/runtime-config/claude-mcp.json new file mode 100644 index 0000000..26a35c5 --- /dev/null +++ b/benchmarks/twin2silicon/runtime-config/claude-mcp.json @@ -0,0 +1,11 @@ +{ + "mcpServers": { + "labwired": { + "command": "npx", + "args": [ + "-y", + "@labwired/mcp" + ] + } + } +} diff --git a/benchmarks/twin2silicon/runtime-config/opencode.json b/benchmarks/twin2silicon/runtime-config/opencode.json new file mode 100644 index 0000000..65ada90 --- /dev/null +++ b/benchmarks/twin2silicon/runtime-config/opencode.json @@ -0,0 +1,46 @@ +{ + "$schema": "https://opencode.ai/config.json", + "mcp": { + "labwired": { + "type": "local", + "command": [ + "npx", + "-y", + "@labwired/mcp" + ], + "enabled": true, + "timeout": 120000, + "environment": { + "LABWIRED_CLI": "{env:LABWIRED_CLI}", + "LABWIRED_BUILDER_URL": "{env:LABWIRED_BUILDER_URL}" + } + } + }, + "permission": { + "skill": { + "brainstorming": "allow", + "develop": "allow", + "bringup": "allow", + "desk-hw": "allow", + "dispatching-parallel-agents": "allow", + "executing-plans": "allow", + "finishing-a-development-branch": "allow", + "golden-path": "allow", + "observe": "allow", + "prove": "allow", + "receiving-code-review": "allow", + "requesting-code-review": "allow", + "subagent-driven-development": "allow", + "systematic-debugging": "allow", + "test-driven-development": "allow", + "using-git-worktrees": "allow", + "using-superpowers": "allow", + "verification-before-completion": "allow", + "writing-plans": "allow", + "writing-skills": "allow", + "customize-labwired-agent": "allow", + "customize-opencode": "deny", + "import-circuit": "allow" + } + } +} diff --git a/benchmarks/twin2silicon/runtime_adapters.py b/benchmarks/twin2silicon/runtime_adapters.py index 872870b..61bb15d 100644 --- a/benchmarks/twin2silicon/runtime_adapters.py +++ b/benchmarks/twin2silicon/runtime_adapters.py @@ -22,6 +22,12 @@ _CLAUDE_MCP_CONFIG = "claude-mcp.json" +def codex_mcp_toml() -> str: + """Return the isolated local LabWired MCP registration for Codex trials.""" + + return '[mcp_servers.labwired]\ncommand = "npx"\nargs = ["-y", "@labwired/mcp"]\n' + + @dataclass(frozen=True) class AdapterContext: """Execution inputs shared by the native runtime adapters.""" diff --git a/benchmarks/twin2silicon/shared-agent-instructions.md b/benchmarks/twin2silicon/shared-agent-instructions.md new file mode 100644 index 0000000..0e4cc08 --- /dev/null +++ b/benchmarks/twin2silicon/shared-agent-instructions.md @@ -0,0 +1,28 @@ +# Firmware repair trial + +Work only inside the public workspace provided for this trial. Do not read, +search for, copy, modify, or infer from hidden task files, hidden oracle files, +or HIL result directories. Do not self-grade the repair or claim that it passes +hardware or a hidden oracle. Independent evaluation is performed outside this +workspace. + +Inspect the public task and make the smallest firmware repair that addresses +the stated problem. Preserve the project structure. Do not make unrelated +refactors, change test or evaluation files, add a workaround that bypasses the +requested behavior, or alter the task budget. + +The supplied task budget is binding. Count each edit-and-test cycle, including +the first attempt, and stop when `budgets.repair_iterations` is reached. Use +failures to make only focused repairs. + +Compile the public firmware with its existing build command. Your final report +must include compile evidence: the command, target, exit status, and relevant +diagnostics. If compilation cannot run, say why and report the evidence that is +available. Do not substitute source inspection for compile evidence. + +LabWired MCP tools are optional context and compile aids, not the final oracle. +Use them only when they help ground public hardware facts or compile the public +firmware. Their output does not prove hidden-oracle or hardware success. + +Report changed files, the repair rationale, compile evidence, and remaining +limits plainly. Do not claim more than the public evidence supports. diff --git a/tests/twin2silicon-hil.py b/tests/twin2silicon-hil.py index 073f7b5..e6872b9 100644 --- a/tests/twin2silicon-hil.py +++ b/tests/twin2silicon-hil.py @@ -45,6 +45,7 @@ AdapterContext, NormalizedUsage, build_runtime_command, + codex_mcp_toml, normalize_usage, ) @@ -309,6 +310,62 @@ def test_normalize_usage_marks_success_without_accounting_unavailable(self): self.assertEqual(usage.unavailable_reason, "runtime did not expose usage") +class RuntimeConfigurationTests(unittest.TestCase): + def test_shared_instructions_bound_repairs_and_evidence_to_public_workspace(self): + instructions_path = ( + REPOSITORY_ROOT + / "benchmarks" + / "twin2silicon" + / "shared-agent-instructions.md" + ) + instructions = instructions_path.read_text(encoding="utf-8") + + self.assertLess(len(instructions.split()), 700) + for required_text in ( + "smallest firmware repair", + "public workspace", + "hidden oracle", + "self-grade", + "compile evidence", + "repair_iterations", + "optional context and compile aids", + "not the final oracle", + ): + with self.subTest(required_text=required_text): + self.assertIn(required_text, instructions) + + def test_runtime_mcp_configs_use_only_the_local_labwired_server(self): + config_root = REPOSITORY_ROOT / "benchmarks" / "twin2silicon" / "runtime-config" + opencode = json.loads((config_root / "opencode.json").read_text(encoding="utf-8")) + claude = json.loads((config_root / "claude-mcp.json").read_text(encoding="utf-8")) + + self.assertNotIn("model", opencode) + self.assertNotIn("provider", opencode) + self.assertEqual(opencode["mcp"]["labwired"]["type"], "local") + self.assertEqual( + opencode["mcp"]["labwired"]["command"], + ["npx", "-y", "@labwired/mcp"], + ) + self.assertTrue(opencode["mcp"]["labwired"]["enabled"]) + installed_profile = json.loads( + (REPOSITORY_ROOT / "config" / "opencode.json").read_text(encoding="utf-8") + ) + self.assertEqual( + opencode["permission"]["skill"], + installed_profile["permission"]["skill"], + ) + self.assertEqual(claude["mcpServers"]["labwired"], { + "command": "npx", + "args": ["-y", "@labwired/mcp"], + }) + + def test_codex_mcp_toml_uses_the_local_labwired_server(self): + self.assertEqual( + codex_mcp_toml(), + '[mcp_servers.labwired]\ncommand = "npx"\nargs = ["-y", "@labwired/mcp"]\n', + ) + + class FixtureContractTests(unittest.TestCase): def test_esp32s3_gpio_hil_fixture_contract(self): task_root = ( From 058c74a9c04a49b959a7d51cd8bad9d59b82f3db Mon Sep 17 00:00:00 2001 From: Andrii Shylenko <14119286+w1ne@users.noreply.github.com> Date: Sat, 15 Aug 2026 21:27:51 +0200 Subject: [PATCH 34/48] fix(bench): write isolated Codex MCP config --- benchmarks/twin2silicon/runtime_adapters.py | 9 +++++++++ tests/twin2silicon-hil.py | 10 ++++++++++ 2 files changed, 19 insertions(+) diff --git a/benchmarks/twin2silicon/runtime_adapters.py b/benchmarks/twin2silicon/runtime_adapters.py index 61bb15d..46dbac9 100644 --- a/benchmarks/twin2silicon/runtime_adapters.py +++ b/benchmarks/twin2silicon/runtime_adapters.py @@ -28,6 +28,15 @@ def codex_mcp_toml() -> str: return '[mcp_servers.labwired]\ncommand = "npx"\nargs = ["-y", "@labwired/mcp"]\n' +def write_codex_mcp_config(config_dir: Path) -> Path: + """Write the isolated Codex MCP configuration and return its path.""" + + config_dir.mkdir(parents=True, exist_ok=True) + config_path = config_dir / "config.toml" + config_path.write_text(codex_mcp_toml(), encoding="utf-8") + return config_path + + @dataclass(frozen=True) class AdapterContext: """Execution inputs shared by the native runtime adapters.""" diff --git a/tests/twin2silicon-hil.py b/tests/twin2silicon-hil.py index e6872b9..fabdade 100644 --- a/tests/twin2silicon-hil.py +++ b/tests/twin2silicon-hil.py @@ -47,6 +47,7 @@ build_runtime_command, codex_mcp_toml, normalize_usage, + write_codex_mcp_config, ) @@ -365,6 +366,15 @@ def test_codex_mcp_toml_uses_the_local_labwired_server(self): '[mcp_servers.labwired]\ncommand = "npx"\nargs = ["-y", "@labwired/mcp"]\n', ) + def test_write_codex_mcp_config_creates_an_isolated_config_file(self): + with tempfile.TemporaryDirectory() as directory: + config_dir = Path(directory) / "runtime-config" + + config_path = write_codex_mcp_config(config_dir) + + self.assertEqual(config_path, config_dir / "config.toml") + self.assertEqual(config_path.read_text(encoding="utf-8"), codex_mcp_toml()) + class FixtureContractTests(unittest.TestCase): def test_esp32s3_gpio_hil_fixture_contract(self): From f8abcc2c6fa4da482cbb5db683578dfc4ab6bbeb Mon Sep 17 00:00:00 2001 From: Andrii Shylenko <14119286+w1ne@users.noreply.github.com> Date: Sat, 15 Aug 2026 21:30:24 +0200 Subject: [PATCH 35/48] fix(bench): isolate runtime MCP configuration --- benchmarks/twin2silicon/runtime_adapters.py | 1 + tests/twin2silicon-hil.py | 3 +++ 2 files changed, 4 insertions(+) diff --git a/benchmarks/twin2silicon/runtime_adapters.py b/benchmarks/twin2silicon/runtime_adapters.py index 46dbac9..e4985ce 100644 --- a/benchmarks/twin2silicon/runtime_adapters.py +++ b/benchmarks/twin2silicon/runtime_adapters.py @@ -92,6 +92,7 @@ def build_runtime_command(runtime: RuntimeName, context: AdapterContext) -> list "acceptEdits", "--mcp-config", str(context.config_dir / _CLAUDE_MCP_CONFIG), + "--strict-mcp-config", context.prompt, ] if runtime == "opencode": diff --git a/tests/twin2silicon-hil.py b/tests/twin2silicon-hil.py index fabdade..0b3d5bc 100644 --- a/tests/twin2silicon-hil.py +++ b/tests/twin2silicon-hil.py @@ -99,6 +99,7 @@ def test_build_runtime_commands_use_native_structured_modes(self): "claude", "--print", "--output-format", "stream-json", "--no-session-persistence", "--permission-mode", "acceptEdits", "--mcp-config", str(contexts["claude"].config_dir / "claude-mcp.json"), + "--strict-mcp-config", contexts["claude"].prompt, ], ) @@ -342,6 +343,8 @@ def test_runtime_mcp_configs_use_only_the_local_labwired_server(self): self.assertNotIn("model", opencode) self.assertNotIn("provider", opencode) + self.assertEqual(set(opencode["mcp"]), {"labwired"}) + self.assertEqual(set(claude["mcpServers"]), {"labwired"}) self.assertEqual(opencode["mcp"]["labwired"]["type"], "local") self.assertEqual( opencode["mcp"]["labwired"]["command"], From 686d122da6f313747300e8da36b13e7cf0ad19af Mon Sep 17 00:00:00 2001 From: Andrii Shylenko <14119286+w1ne@users.noreply.github.com> Date: Sat, 15 Aug 2026 21:36:25 +0200 Subject: [PATCH 36/48] feat(bench): run one native agent trial --- benchmarks/twin2silicon/run_agent.py | 228 +++++++++++++++++++++++++++ tests/twin2silicon-hil.py | 187 ++++++++++++++++++++++ 2 files changed, 415 insertions(+) create mode 100644 benchmarks/twin2silicon/run_agent.py diff --git a/benchmarks/twin2silicon/run_agent.py b/benchmarks/twin2silicon/run_agent.py new file mode 100644 index 0000000..c0397e8 --- /dev/null +++ b/benchmarks/twin2silicon/run_agent.py @@ -0,0 +1,228 @@ +#!/usr/bin/env python3 +"""Run one bounded native runtime trial against a public firmware task.""" + +from __future__ import annotations + +import argparse +from contextlib import contextmanager +from dataclasses import asdict +import json +import os +from pathlib import Path +import shutil +import subprocess +import sys +import time +from typing import Iterator + +if __package__ in (None, ""): + sys.path.insert(0, str(Path(__file__).resolve().parents[2])) + +from benchmarks.twin2silicon.hil.process import run_command +from benchmarks.twin2silicon.hil.results import write_json_atomic +from benchmarks.twin2silicon.runtime_adapters import ( + AdapterContext, + NormalizedUsage, + build_runtime_command, + normalize_usage, + write_codex_mcp_config, +) + + +RUNTIMES = ("opencode", "codex", "claude") +ROOT = Path(__file__).resolve().parent +TASKS = ROOT / "tasks" +INSTRUCTIONS = ROOT / "shared-agent-instructions.md" +RUNTIME_CONFIG = ROOT / "runtime-config" + + +def _positive_seconds(value: str) -> float: + try: + seconds = float(value) + except ValueError as error: + raise argparse.ArgumentTypeError("timeout must be a number") from error + if seconds <= 0: + raise argparse.ArgumentTypeError("timeout must be greater than zero") + return seconds + + +def _parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("runtime", choices=RUNTIMES) + parser.add_argument("--task", required=True) + parser.add_argument("--output", required=True, type=Path) + parser.add_argument("--executable") + parser.add_argument("--timeout-seconds", type=_positive_seconds) + return parser + + +def _task_root(value: str) -> Path: + candidate = Path(value) + if len(candidate.parts) == 1: + candidate = TASKS / candidate + return candidate.resolve() + + +def _public_inputs(task_root: Path) -> tuple[Path, float]: + task = json.loads((task_root / "task.json").read_text(encoding="utf-8")) + public_dir = task["public_dir"] + budget = task["budgets"]["wall_time_seconds"] + if isinstance(public_dir, Path) or not isinstance(public_dir, str): + raise ValueError("task public_dir must be a path string") + if isinstance(budget, bool) or not isinstance(budget, (int, float)) or budget <= 0: + raise ValueError("task wall_time_seconds must be positive") + public_root = (task_root / public_dir).resolve() + if task_root not in public_root.parents or not public_root.is_dir(): + raise ValueError("task public_dir must name a directory below the task root") + return public_root, float(budget) + + +def _prepare_runtime_config(runtime: str, config_dir: Path) -> None: + config_dir.mkdir(parents=True, exist_ok=True) + if runtime == "codex": + write_codex_mcp_config(config_dir) + return + source_name = "opencode.json" if runtime == "opencode" else "claude-mcp.json" + shutil.copyfile(RUNTIME_CONFIG / source_name, config_dir / source_name) + + +@contextmanager +def _runtime_environment(runtime: str, config_dir: Path) -> Iterator[None]: + values = { + "codex": {"CODEX_HOME": str(config_dir)}, + "opencode": {"OPENCODE_CONFIG": str(config_dir / "opencode.json")}, + "claude": {}, + }[runtime] + previous = {key: os.environ.get(key) for key in values} + os.environ.update(values) + try: + yield + finally: + for key, value in previous.items(): + if value is None: + os.environ.pop(key, None) + else: + os.environ[key] = value + + +def _version(executable: str, cwd: Path, timeout_seconds: float) -> str | None: + try: + completed = subprocess.run( + [executable, "--version"], + cwd=cwd, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + timeout=min(timeout_seconds, 5), + check=False, + ) + except (OSError, subprocess.TimeoutExpired): + return None + output = (completed.stdout or completed.stderr).strip() + return output.splitlines()[0][:4096] if output else None + + +def _prompt(candidate: Path, instructions: str) -> str: + readme = candidate / "README.md" + task_prompt = readme.read_text(encoding="utf-8") if readme.is_file() else "Repair the public firmware task." + return f"{instructions}\n\n# Public task\n\n{task_prompt}" + + +def _unavailable_usage() -> NormalizedUsage: + return NormalizedUsage(None, None, None, None, None, None, "runtime did not expose usage") + + +def _write_trial_result(trial: Path, result: dict[str, object], usage: NormalizedUsage) -> None: + write_json_atomic(trial / "agent-result.json", result) + write_json_atomic(trial / "usage.json", asdict(usage)) + + +def main(argv: list[str] | None = None) -> int: + args = _parser().parse_args(argv) + trial = args.output.resolve() + if os.path.lexists(trial): + print("output path already exists", file=sys.stderr) + return 2 + + trial.mkdir(parents=True) + usage = _unavailable_usage() + result: dict[str, object] = { + "schema_version": "1.0", + "runtime": args.runtime, + "model_override": None, + "status": "infrastructure_error", + "returncode": None, + "timed_out": False, + "elapsed_seconds": 0.0, + "executable_version": None, + "stdout_path": "agent.stdout.log", + "stderr_path": "agent.stderr.log", + } + + try: + public_root, budget_seconds = _public_inputs(_task_root(args.task)) + timeout_seconds = min(args.timeout_seconds or budget_seconds, budget_seconds) + candidate = trial / "candidate" + shutil.copytree(public_root, candidate) + instructions = INSTRUCTIONS.read_text(encoding="utf-8") + (candidate / ("CLAUDE.md" if args.runtime == "claude" else "AGENTS.md")).write_text( + instructions, encoding="utf-8" + ) + config_dir = trial / "runtime-config" + _prepare_runtime_config(args.runtime, config_dir) + executable = args.executable or args.runtime + context = AdapterContext( + runtime=args.runtime, + executable=executable, + workspace=candidate, + prompt=_prompt(candidate, instructions), + config_dir=config_dir, + stdout_path=trial / "agent.stdout.log", + stderr_path=trial / "agent.stderr.log", + ) + command = build_runtime_command(args.runtime, context) + with _runtime_environment(args.runtime, config_dir): + result["executable_version"] = _version(executable, candidate, timeout_seconds) + started = time.monotonic() + try: + completed = run_command( + command, + cwd=candidate, + stdout_path=context.stdout_path, + stderr_path=context.stderr_path, + timeout_seconds=timeout_seconds, + ) + except OSError as error: + result["elapsed_seconds"] = time.monotonic() - started + result["error"] = str(error) + else: + result.update( + returncode=completed.returncode, + timed_out=completed.timed_out, + elapsed_seconds=completed.duration_seconds, + ) + if completed.timed_out: + result["status"] = "timeout" + elif completed.cleanup_error: + result["status"] = "infrastructure_error" + result["error"] = completed.cleanup_error + elif completed.returncode: + result["status"] = "failed" + else: + result["status"] = "completed" + try: + usage = normalize_usage( + args.runtime, + context.stdout_path.read_text(encoding="utf-8", errors="replace").splitlines(), + ) + except OSError: + usage = _unavailable_usage() + except Exception as error: + result["error"] = str(error) + + _write_trial_result(trial, result, usage) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/twin2silicon-hil.py b/tests/twin2silicon-hil.py index 0b3d5bc..3f6cade 100644 --- a/tests/twin2silicon-hil.py +++ b/tests/twin2silicon-hil.py @@ -52,6 +52,7 @@ def executable_fixture(directory, body): + Path(directory).mkdir(parents=True, exist_ok=True) path = Path(directory) / "fixture.py" path.write_text("#!/usr/bin/env python3\n" + body, encoding="utf-8") path.chmod(0o755) @@ -1229,6 +1230,192 @@ def test_compile_failure_never_touches_hardware(self): self.assertFalse(marker.exists()) +class RunAgentTests(unittest.TestCase): + task = REPOSITORY_ROOT / "benchmarks/twin2silicon/tasks/esp32s3-gpio-hil-001" + script = REPOSITORY_ROOT / "benchmarks/twin2silicon/run_agent.py" + + def _fake_runtime(self, directory, runtime, mode="success"): + workspace_code = { + "codex": "workspace = Path(args[args.index('-C') + 1])\nassert args[:2] == ['exec', '--json']\nassert '--ephemeral' in args and '--skip-git-repo-check' in args\nassert args[args.index('-s') + 1] == 'workspace-write'\nassert os.environ['CODEX_HOME'] == str(Path(os.environ['EXPECTED_CONFIG']))", + "claude": "workspace = Path.cwd()\nassert args[:2] == ['--print', '--output-format']\nassert args[args.index('--output-format') + 1] == 'stream-json'\nassert args[args.index('--mcp-config') + 1] == str(Path(os.environ['EXPECTED_CONFIG']) / 'claude-mcp.json')\nassert '--strict-mcp-config' in args", + "opencode": "workspace = Path(args[args.index('--dir') + 1])\nassert args[:2] == ['run', '--format']\nassert args[args.index('--format') + 1] == 'json'\nassert os.environ['OPENCODE_CONFIG'] == str(Path(os.environ['EXPECTED_CONFIG']) / 'opencode.json')", + }[runtime] + output = { + "codex": "print(json.dumps({'type': 'turn.completed', 'usage': {'input_tokens': 12, 'output_tokens': 3}}))", + "claude": "print(json.dumps({'type': 'result', 'usage': {'input_tokens': 12, 'output_tokens': 3}, 'total_cost_usd': 0.01}))", + "opencode": "print(json.dumps({'type': 'step_finish', 'part': {'tokens': {'input': 12, 'output': 3}, 'cost': 0.01}}))", + }[runtime] + body = textwrap.dedent(f""" + import json + import os + from pathlib import Path + import sys + import time + + args = sys.argv[1:] + if args == ['--version']: + print('fake-{runtime} 1.0') + raise SystemExit(0) + assert '--model' not in args + assert Path(os.environ['EXPECTED_INSTRUCTIONS']).read_text(encoding='utf-8') in args[-1] + assert 'GPIO 2 is driven high' in args[-1] + assert 'hil-oracle.json' not in args[-1] + # workspace checks + source = workspace / 'firmware/src/main.c' + contents = source.read_text(encoding='utf-8') + assert 'GPIO_MODE_INPUT' in contents + assert 'GPIO_MODE_OUTPUT' not in contents + assert not (workspace / 'hidden').exists() + instruction = workspace / {'CLAUDE.md' if runtime == 'claude' else 'AGENTS.md'!r} + assert instruction.read_text(encoding='utf-8') == Path(os.environ['EXPECTED_INSTRUCTIONS']).read_text(encoding='utf-8') + if {mode!r} == 'timeout': + time.sleep(30) + source.write_text(contents.replace('GPIO_MODE_INPUT', 'GPIO_MODE_OUTPUT'), encoding='utf-8') + if {mode!r} == 'nonzero': + raise SystemExit(9) + if {mode!r} == 'malformed': + print('not json') + elif {mode!r} == 'missing': + pass + else: + {output} + """).replace("# workspace checks", workspace_code) + return executable_fixture(directory, body) + + def _run_cli(self, runtime, executable, trial, timeout_seconds=2): + environment = os.environ.copy() + environment.update({ + "EXPECTED_CONFIG": str((trial / "runtime-config").resolve()), + "EXPECTED_INSTRUCTIONS": str( + REPOSITORY_ROOT / "benchmarks/twin2silicon/shared-agent-instructions.md" + ), + }) + return subprocess.run( + [ + sys.executable, str(self.script), runtime, + "--task", str(self.task), "--output", str(trial), + "--executable", str(executable), + "--timeout-seconds", str(timeout_seconds), + ], + cwd=REPOSITORY_ROOT, + env=environment, + text=True, + capture_output=True, + timeout=10, + ) + + def _assert_trial_does_not_expose_hidden_oracle(self, trial): + hidden_name = "hil-oracle.json" + for path in trial.rglob("*"): + with self.subTest(path=path): + self.assertNotIn(hidden_name, str(path)) + if path.is_file(): + self.assertNotIn(hidden_name, path.read_text(encoding="utf-8", errors="replace")) + self.assertFalse((trial / "candidate/hidden").exists()) + + def test_native_runtimes_create_completed_public_candidates(self): + for runtime in ("opencode", "codex", "claude"): + with self.subTest(runtime=runtime), tempfile.TemporaryDirectory() as directory: + root = Path(directory) + executable = self._fake_runtime(root / runtime, runtime) + trial = root / "trial" + + completed = self._run_cli(runtime, executable, trial) + + self.assertEqual(completed.returncode, 0, completed.stderr) + result = json.loads((trial / "agent-result.json").read_text()) + usage = json.loads((trial / "usage.json").read_text()) + candidate_source = (trial / "candidate/firmware/src/main.c").read_text() + self.assertEqual(result["status"], "completed") + self.assertEqual(result["runtime"], runtime) + self.assertIsNone(result["model_override"]) + self.assertEqual(result["returncode"], 0) + self.assertFalse(result["timed_out"]) + self.assertGreaterEqual(result["elapsed_seconds"], 0) + self.assertEqual(result["executable_version"], f"fake-{runtime} 1.0") + self.assertIn("GPIO_MODE_OUTPUT", candidate_source) + self.assertTrue((trial / "agent.stdout.log").is_file()) + self.assertTrue((trial / "agent.stderr.log").is_file()) + self.assertTrue((trial / "runtime-config").is_dir()) + self.assertEqual(usage["requests"], 1) + self.assertIsNone(usage["unavailable_reason"]) + self._assert_trial_does_not_expose_hidden_oracle(trial) + + def test_nonzero_runtime_is_failed_but_retains_evidence(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + executable = self._fake_runtime(root / "runtime", "codex", "nonzero") + trial = root / "trial" + + completed = self._run_cli("codex", executable, trial) + + self.assertEqual(completed.returncode, 0, completed.stderr) + result = json.loads((trial / "agent-result.json").read_text()) + self.assertEqual(result["status"], "failed") + self.assertEqual(result["returncode"], 9) + self.assertFalse(result["timed_out"]) + self.assertTrue((trial / "agent.stdout.log").is_file()) + self._assert_trial_does_not_expose_hidden_oracle(trial) + + def test_timeout_runtime_is_recorded(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + executable = self._fake_runtime(root / "runtime", "opencode", "timeout") + trial = root / "trial" + + completed = self._run_cli("opencode", executable, trial, timeout_seconds=0.1) + + self.assertEqual(completed.returncode, 0, completed.stderr) + result = json.loads((trial / "agent-result.json").read_text()) + self.assertEqual(result["status"], "timeout") + self.assertTrue(result["timed_out"]) + self.assertNotEqual(result["returncode"], 0) + self._assert_trial_does_not_expose_hidden_oracle(trial) + + def test_missing_executable_is_an_infrastructure_error(self): + with tempfile.TemporaryDirectory() as directory: + trial = Path(directory) / "trial" + + completed = self._run_cli("claude", Path(directory) / "missing", trial) + + self.assertEqual(completed.returncode, 0, completed.stderr) + result = json.loads((trial / "agent-result.json").read_text()) + self.assertEqual(result["status"], "infrastructure_error") + self.assertIsNone(result["returncode"]) + self._assert_trial_does_not_expose_hidden_oracle(trial) + + def test_missing_or_malformed_usage_does_not_fail_a_completed_trial(self): + for mode in ("malformed", "missing"): + with self.subTest(mode=mode), tempfile.TemporaryDirectory() as directory: + root = Path(directory) + executable = self._fake_runtime(root / "runtime", "claude", mode) + trial = root / "trial" + + completed = self._run_cli("claude", executable, trial) + + self.assertEqual(completed.returncode, 0, completed.stderr) + result = json.loads((trial / "agent-result.json").read_text()) + usage = json.loads((trial / "usage.json").read_text()) + self.assertEqual(result["status"], "completed") + self.assertEqual(usage["unavailable_reason"], "runtime did not expose usage") + self._assert_trial_does_not_expose_hidden_oracle(trial) + + def test_existing_output_is_rejected_without_overwrite(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + trial = root / "trial" + trial.mkdir() + marker = trial / "keep" + marker.write_text("existing") + + completed = self._run_cli("codex", root / "missing", trial) + + self.assertEqual(completed.returncode, 2) + self.assertIn("output path already exists", completed.stderr) + self.assertEqual(marker.read_text(), "existing") + self.assertFalse((trial / "agent-result.json").exists()) + + if __name__ == "__main__": if "-k" in sys.argv: pattern_index = sys.argv.index("-k") + 1 From b9fdb6228d79a222723683534709456af7fb3979 Mon Sep 17 00:00:00 2001 From: Andrii Shylenko <14119286+w1ne@users.noreply.github.com> Date: Sat, 15 Aug 2026 21:40:15 +0200 Subject: [PATCH 37/48] fix(bench): isolate public agent inputs --- benchmarks/twin2silicon/run_agent.py | 44 +++++++++++---- tests/twin2silicon-hil.py | 81 ++++++++++++++++++++++++++-- 2 files changed, 113 insertions(+), 12 deletions(-) diff --git a/benchmarks/twin2silicon/run_agent.py b/benchmarks/twin2silicon/run_agent.py index c0397e8..1b4925d 100644 --- a/benchmarks/twin2silicon/run_agent.py +++ b/benchmarks/twin2silicon/run_agent.py @@ -7,9 +7,11 @@ from contextlib import contextmanager from dataclasses import asdict import json +import math import os from pathlib import Path import shutil +import stat import subprocess import sys import time @@ -63,18 +65,39 @@ def _task_root(value: str) -> Path: return candidate.resolve() -def _public_inputs(task_root: Path) -> tuple[Path, float]: +def _public_inputs(task_root: Path) -> tuple[Path, float, int]: task = json.loads((task_root / "task.json").read_text(encoding="utf-8")) public_dir = task["public_dir"] budget = task["budgets"]["wall_time_seconds"] + repair_iterations = task["budgets"]["repair_iterations"] if isinstance(public_dir, Path) or not isinstance(public_dir, str): raise ValueError("task public_dir must be a path string") - if isinstance(budget, bool) or not isinstance(budget, (int, float)) or budget <= 0: + if ( + isinstance(budget, bool) + or not isinstance(budget, (int, float)) + or not math.isfinite(float(budget)) + or budget <= 0 + ): raise ValueError("task wall_time_seconds must be positive") - public_root = (task_root / public_dir).resolve() - if task_root not in public_root.parents or not public_root.is_dir(): + if ( + isinstance(repair_iterations, bool) + or not isinstance(repair_iterations, int) + or repair_iterations <= 0 + ): + raise ValueError("task repair_iterations must be a positive integer") + source_root = task_root / public_dir + public_root = source_root.resolve() + if task_root not in public_root.parents or not source_root.is_dir(): raise ValueError("task public_dir must name a directory below the task root") - return public_root, float(budget) + _reject_public_symlinks(source_root) + return public_root, float(budget), repair_iterations + + +def _reject_public_symlinks(public_root: Path) -> None: + for directory, directories, files in os.walk(public_root, followlinks=False): + for path in (Path(directory), *(Path(directory) / name for name in directories + files)): + if stat.S_ISLNK(os.lstat(path).st_mode): + raise ValueError("public inputs must not contain symlinks") def _prepare_runtime_config(runtime: str, config_dir: Path) -> None: @@ -122,10 +145,13 @@ def _version(executable: str, cwd: Path, timeout_seconds: float) -> str | None: return output.splitlines()[0][:4096] if output else None -def _prompt(candidate: Path, instructions: str) -> str: +def _prompt(candidate: Path, instructions: str, repair_iterations: int) -> str: readme = candidate / "README.md" task_prompt = readme.read_text(encoding="utf-8") if readme.is_file() else "Repair the public firmware task." - return f"{instructions}\n\n# Public task\n\n{task_prompt}" + return ( + f"{instructions}\n\n# Trial limit\n\n" + f"Maximum repair attempts: {repair_iterations}\n\n# Public task\n\n{task_prompt}" + ) def _unavailable_usage() -> NormalizedUsage: @@ -160,7 +186,7 @@ def main(argv: list[str] | None = None) -> int: } try: - public_root, budget_seconds = _public_inputs(_task_root(args.task)) + public_root, budget_seconds, repair_iterations = _public_inputs(_task_root(args.task)) timeout_seconds = min(args.timeout_seconds or budget_seconds, budget_seconds) candidate = trial / "candidate" shutil.copytree(public_root, candidate) @@ -175,7 +201,7 @@ def main(argv: list[str] | None = None) -> int: runtime=args.runtime, executable=executable, workspace=candidate, - prompt=_prompt(candidate, instructions), + prompt=_prompt(candidate, instructions, repair_iterations), config_dir=config_dir, stdout_path=trial / "agent.stdout.log", stderr_path=trial / "agent.stderr.log", diff --git a/tests/twin2silicon-hil.py b/tests/twin2silicon-hil.py index 3f6cade..eb8b7e0 100644 --- a/tests/twin2silicon-hil.py +++ b/tests/twin2silicon-hil.py @@ -1234,7 +1234,7 @@ class RunAgentTests(unittest.TestCase): task = REPOSITORY_ROOT / "benchmarks/twin2silicon/tasks/esp32s3-gpio-hil-001" script = REPOSITORY_ROOT / "benchmarks/twin2silicon/run_agent.py" - def _fake_runtime(self, directory, runtime, mode="success"): + def _fake_runtime(self, directory, runtime, mode="success", repair_iterations=6): workspace_code = { "codex": "workspace = Path(args[args.index('-C') + 1])\nassert args[:2] == ['exec', '--json']\nassert '--ephemeral' in args and '--skip-git-repo-check' in args\nassert args[args.index('-s') + 1] == 'workspace-write'\nassert os.environ['CODEX_HOME'] == str(Path(os.environ['EXPECTED_CONFIG']))", "claude": "workspace = Path.cwd()\nassert args[:2] == ['--print', '--output-format']\nassert args[args.index('--output-format') + 1] == 'stream-json'\nassert args[args.index('--mcp-config') + 1] == str(Path(os.environ['EXPECTED_CONFIG']) / 'claude-mcp.json')\nassert '--strict-mcp-config' in args", @@ -1259,6 +1259,7 @@ def _fake_runtime(self, directory, runtime, mode="success"): assert '--model' not in args assert Path(os.environ['EXPECTED_INSTRUCTIONS']).read_text(encoding='utf-8') in args[-1] assert 'GPIO 2 is driven high' in args[-1] + assert 'Maximum repair attempts: {repair_iterations}' in args[-1] assert 'hil-oracle.json' not in args[-1] # workspace checks source = workspace / 'firmware/src/main.c' @@ -1282,7 +1283,7 @@ def _fake_runtime(self, directory, runtime, mode="success"): """).replace("# workspace checks", workspace_code) return executable_fixture(directory, body) - def _run_cli(self, runtime, executable, trial, timeout_seconds=2): + def _run_cli(self, runtime, executable, trial, timeout_seconds=2, task=None): environment = os.environ.copy() environment.update({ "EXPECTED_CONFIG": str((trial / "runtime-config").resolve()), @@ -1293,7 +1294,7 @@ def _run_cli(self, runtime, executable, trial, timeout_seconds=2): return subprocess.run( [ sys.executable, str(self.script), runtime, - "--task", str(self.task), "--output", str(trial), + "--task", str(task or self.task), "--output", str(trial), "--executable", str(executable), "--timeout-seconds", str(timeout_seconds), ], @@ -1337,6 +1338,7 @@ def test_native_runtimes_create_completed_public_candidates(self): self.assertTrue((trial / "agent.stdout.log").is_file()) self.assertTrue((trial / "agent.stderr.log").is_file()) self.assertTrue((trial / "runtime-config").is_dir()) + self.assertFalse((trial / "candidate/task.json").exists()) self.assertEqual(usage["requests"], 1) self.assertIsNone(usage["unavailable_reason"]) self._assert_trial_does_not_expose_hidden_oracle(trial) @@ -1415,6 +1417,79 @@ def test_existing_output_is_rejected_without_overwrite(self): self.assertEqual(marker.read_text(), "existing") self.assertFalse((trial / "agent-result.json").exists()) + def test_public_symlink_is_rejected_before_candidate_copy(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + task = root / "task" + public = task / "public" + hidden = task / "hidden" + public.mkdir(parents=True) + hidden.mkdir() + oracle = hidden / "hil-oracle.json" + oracle.write_text("hidden oracle evidence", encoding="utf-8") + (public / "leaked-oracle").symlink_to(oracle) + (task / "task.json").write_text(json.dumps({ + "public_dir": "public", + "budgets": {"wall_time_seconds": 1, "repair_iterations": 1}, + }), encoding="utf-8") + trial = root / "trial" + + completed = self._run_cli( + "codex", root / "missing", trial, task=task, + ) + + self.assertEqual(completed.returncode, 0, completed.stderr) + result = json.loads((trial / "agent-result.json").read_text()) + self.assertEqual(result["status"], "infrastructure_error") + self.assertIn("symlink", result["error"]) + self.assertFalse((trial / "candidate").exists()) + self.assertNotIn("hidden oracle evidence", (trial / "agent-result.json").read_text()) + + def test_prompt_uses_the_task_repair_iteration_budget(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + task = root / "task" + import shutil + shutil.copytree(self.task / "public", task / "public") + (task / "task.json").write_text(json.dumps({ + "public_dir": "public", + "budgets": {"wall_time_seconds": 2, "repair_iterations": 2}, + }), encoding="utf-8") + executable = self._fake_runtime( + root / "runtime", "opencode", repair_iterations=2, + ) + trial = root / "trial" + + completed = self._run_cli("opencode", executable, trial, task=task) + + self.assertEqual(completed.returncode, 0, completed.stderr) + result = json.loads((trial / "agent-result.json").read_text()) + self.assertEqual(result["status"], "completed") + self.assertFalse((trial / "candidate/task.json").exists()) + + def test_invalid_trial_budgets_are_rejected_before_candidate_copy(self): + cases = ( + ({"wall_time_seconds": 1, "repair_iterations": 0}, "repair_iterations"), + ({"wall_time_seconds": float("nan"), "repair_iterations": 1}, "wall_time_seconds"), + ) + for budgets, expected_error in cases: + with self.subTest(budgets=budgets), tempfile.TemporaryDirectory() as directory: + root = Path(directory) + task = root / "task" + (task / "public").mkdir(parents=True) + (task / "task.json").write_text(json.dumps({ + "public_dir": "public", "budgets": budgets, + }), encoding="utf-8") + trial = root / "trial" + + completed = self._run_cli("codex", root / "missing", trial, task=task) + + self.assertEqual(completed.returncode, 0, completed.stderr) + result = json.loads((trial / "agent-result.json").read_text()) + self.assertEqual(result["status"], "infrastructure_error") + self.assertIn(expected_error, result["error"]) + self.assertFalse((trial / "candidate").exists()) + if __name__ == "__main__": if "-k" in sys.argv: From cad9f859a9d5b5bf42f9eac0b5c5f0a0b776da45 Mon Sep 17 00:00:00 2001 From: Andrii Shylenko <14119286+w1ne@users.noreply.github.com> Date: Sat, 15 Aug 2026 21:43:48 +0200 Subject: [PATCH 38/48] fix(bench): preserve native runtime auth --- benchmarks/twin2silicon/run_agent.py | 2 +- benchmarks/twin2silicon/runtime_adapters.py | 4 ++++ tests/twin2silicon-hil.py | 10 +++++++--- 3 files changed, 12 insertions(+), 4 deletions(-) diff --git a/benchmarks/twin2silicon/run_agent.py b/benchmarks/twin2silicon/run_agent.py index 1b4925d..ebf905e 100644 --- a/benchmarks/twin2silicon/run_agent.py +++ b/benchmarks/twin2silicon/run_agent.py @@ -112,7 +112,7 @@ def _prepare_runtime_config(runtime: str, config_dir: Path) -> None: @contextmanager def _runtime_environment(runtime: str, config_dir: Path) -> Iterator[None]: values = { - "codex": {"CODEX_HOME": str(config_dir)}, + "codex": {}, "opencode": {"OPENCODE_CONFIG": str(config_dir / "opencode.json")}, "claude": {}, }[runtime] diff --git a/benchmarks/twin2silicon/runtime_adapters.py b/benchmarks/twin2silicon/runtime_adapters.py index e4985ce..7afb02b 100644 --- a/benchmarks/twin2silicon/runtime_adapters.py +++ b/benchmarks/twin2silicon/runtime_adapters.py @@ -20,6 +20,7 @@ _MAX_USAGE_COUNT = 1_000_000_000_000 _MAX_COST_USD = 1_000_000_000.0 _CLAUDE_MCP_CONFIG = "claude-mcp.json" +_CODEX_MCP_OVERRIDE = 'mcp_servers={labwired={command="npx",args=["-y","@labwired/mcp"]}}' def codex_mcp_toml() -> str: @@ -75,6 +76,8 @@ def build_runtime_command(runtime: RuntimeName, context: AdapterContext) -> list "--json", "--ephemeral", "--skip-git-repo-check", + "-c", + _CODEX_MCP_OVERRIDE, "-s", "workspace-write", "-C", @@ -85,6 +88,7 @@ def build_runtime_command(runtime: RuntimeName, context: AdapterContext) -> list return [ context.executable, "--print", + "--verbose", "--output-format", "stream-json", "--no-session-persistence", diff --git a/tests/twin2silicon-hil.py b/tests/twin2silicon-hil.py index eb8b7e0..62659db 100644 --- a/tests/twin2silicon-hil.py +++ b/tests/twin2silicon-hil.py @@ -90,6 +90,7 @@ def test_build_runtime_commands_use_native_structured_modes(self): commands["codex"], [ "codex", "exec", "--json", "--ephemeral", "--skip-git-repo-check", + "-c", 'mcp_servers={labwired={command="npx",args=["-y","@labwired/mcp"]}}', "-s", "workspace-write", "-C", str(contexts["codex"].workspace), contexts["codex"].prompt, ], @@ -97,7 +98,7 @@ def test_build_runtime_commands_use_native_structured_modes(self): self.assertEqual( commands["claude"], [ - "claude", "--print", "--output-format", "stream-json", + "claude", "--print", "--verbose", "--output-format", "stream-json", "--no-session-persistence", "--permission-mode", "acceptEdits", "--mcp-config", str(contexts["claude"].config_dir / "claude-mcp.json"), "--strict-mcp-config", @@ -1235,9 +1236,10 @@ class RunAgentTests(unittest.TestCase): script = REPOSITORY_ROOT / "benchmarks/twin2silicon/run_agent.py" def _fake_runtime(self, directory, runtime, mode="success", repair_iterations=6): + codex_override = 'mcp_servers={labwired={command="npx",args=["-y","@labwired/mcp"]}}' workspace_code = { - "codex": "workspace = Path(args[args.index('-C') + 1])\nassert args[:2] == ['exec', '--json']\nassert '--ephemeral' in args and '--skip-git-repo-check' in args\nassert args[args.index('-s') + 1] == 'workspace-write'\nassert os.environ['CODEX_HOME'] == str(Path(os.environ['EXPECTED_CONFIG']))", - "claude": "workspace = Path.cwd()\nassert args[:2] == ['--print', '--output-format']\nassert args[args.index('--output-format') + 1] == 'stream-json'\nassert args[args.index('--mcp-config') + 1] == str(Path(os.environ['EXPECTED_CONFIG']) / 'claude-mcp.json')\nassert '--strict-mcp-config' in args", + "codex": f"workspace = Path(args[args.index('-C') + 1])\nassert args[:2] == ['exec', '--json']\nassert '--ephemeral' in args and '--skip-git-repo-check' in args\nassert args[args.index('-c') + 1] == {codex_override!r}\nassert args[args.index('-s') + 1] == 'workspace-write'\nassert os.environ['CODEX_HOME'] == os.environ['EXPECTED_CODEX_HOME']", + "claude": "workspace = Path.cwd()\nassert args[:3] == ['--print', '--verbose', '--output-format']\nassert args[args.index('--output-format') + 1] == 'stream-json'\nassert args[args.index('--mcp-config') + 1] == str(Path(os.environ['EXPECTED_CONFIG']) / 'claude-mcp.json')\nassert '--strict-mcp-config' in args", "opencode": "workspace = Path(args[args.index('--dir') + 1])\nassert args[:2] == ['run', '--format']\nassert args[args.index('--format') + 1] == 'json'\nassert os.environ['OPENCODE_CONFIG'] == str(Path(os.environ['EXPECTED_CONFIG']) / 'opencode.json')", }[runtime] output = { @@ -1287,6 +1289,8 @@ def _run_cli(self, runtime, executable, trial, timeout_seconds=2, task=None): environment = os.environ.copy() environment.update({ "EXPECTED_CONFIG": str((trial / "runtime-config").resolve()), + "CODEX_HOME": str((trial.parent / "native-codex-home").resolve()), + "EXPECTED_CODEX_HOME": str((trial.parent / "native-codex-home").resolve()), "EXPECTED_INSTRUCTIONS": str( REPOSITORY_ROOT / "benchmarks/twin2silicon/shared-agent-instructions.md" ), From b2deab1d42fd6a3f688202fe3f19a0a5b9464424 Mon Sep 17 00:00:00 2001 From: Andrii Shylenko <14119286+w1ne@users.noreply.github.com> Date: Sat, 15 Aug 2026 21:48:36 +0200 Subject: [PATCH 39/48] fix(bench): isolate Codex MCP without config merge --- benchmarks/twin2silicon/run_agent.py | 34 ++++++++++++- benchmarks/twin2silicon/runtime_adapters.py | 3 -- tests/twin2silicon-hil.py | 56 ++++++++++++++++++--- 3 files changed, 82 insertions(+), 11 deletions(-) diff --git a/benchmarks/twin2silicon/run_agent.py b/benchmarks/twin2silicon/run_agent.py index ebf905e..de8ac50 100644 --- a/benchmarks/twin2silicon/run_agent.py +++ b/benchmarks/twin2silicon/run_agent.py @@ -14,6 +14,7 @@ import stat import subprocess import sys +import tempfile import time from typing import Iterator @@ -111,8 +112,11 @@ def _prepare_runtime_config(runtime: str, config_dir: Path) -> None: @contextmanager def _runtime_environment(runtime: str, config_dir: Path) -> Iterator[None]: + if runtime == "codex": + with _isolated_codex_home(): + yield + return values = { - "codex": {}, "opencode": {"OPENCODE_CONFIG": str(config_dir / "opencode.json")}, "claude": {}, }[runtime] @@ -128,6 +132,34 @@ def _runtime_environment(runtime: str, config_dir: Path) -> Iterator[None]: os.environ[key] = value +@contextmanager +def _isolated_codex_home() -> Iterator[None]: + configured_home = os.environ.get("CODEX_HOME") + source_home = ( + Path(configured_home).expanduser() + if configured_home + else Path.home() / ".codex" + ) + with tempfile.TemporaryDirectory(prefix="twin2silicon-codex-") as directory: + isolated_home = Path(directory) + config_path = write_codex_mcp_config(isolated_home) + config_path.chmod(0o600) + source_auth = source_home / "auth.json" + if source_auth.is_file(): + destination_auth = isolated_home / "auth.json" + shutil.copyfile(source_auth, destination_auth) + destination_auth.chmod(0o600) + previous = os.environ.get("CODEX_HOME") + os.environ["CODEX_HOME"] = str(isolated_home) + try: + yield + finally: + if previous is None: + os.environ.pop("CODEX_HOME", None) + else: + os.environ["CODEX_HOME"] = previous + + def _version(executable: str, cwd: Path, timeout_seconds: float) -> str | None: try: completed = subprocess.run( diff --git a/benchmarks/twin2silicon/runtime_adapters.py b/benchmarks/twin2silicon/runtime_adapters.py index 7afb02b..d66c7fb 100644 --- a/benchmarks/twin2silicon/runtime_adapters.py +++ b/benchmarks/twin2silicon/runtime_adapters.py @@ -20,7 +20,6 @@ _MAX_USAGE_COUNT = 1_000_000_000_000 _MAX_COST_USD = 1_000_000_000.0 _CLAUDE_MCP_CONFIG = "claude-mcp.json" -_CODEX_MCP_OVERRIDE = 'mcp_servers={labwired={command="npx",args=["-y","@labwired/mcp"]}}' def codex_mcp_toml() -> str: @@ -76,8 +75,6 @@ def build_runtime_command(runtime: RuntimeName, context: AdapterContext) -> list "--json", "--ephemeral", "--skip-git-repo-check", - "-c", - _CODEX_MCP_OVERRIDE, "-s", "workspace-write", "-C", diff --git a/tests/twin2silicon-hil.py b/tests/twin2silicon-hil.py index 62659db..2f50d17 100644 --- a/tests/twin2silicon-hil.py +++ b/tests/twin2silicon-hil.py @@ -90,7 +90,6 @@ def test_build_runtime_commands_use_native_structured_modes(self): commands["codex"], [ "codex", "exec", "--json", "--ephemeral", "--skip-git-repo-check", - "-c", 'mcp_servers={labwired={command="npx",args=["-y","@labwired/mcp"]}}', "-s", "workspace-write", "-C", str(contexts["codex"].workspace), contexts["codex"].prompt, ], @@ -1235,10 +1234,19 @@ class RunAgentTests(unittest.TestCase): task = REPOSITORY_ROOT / "benchmarks/twin2silicon/tasks/esp32s3-gpio-hil-001" script = REPOSITORY_ROOT / "benchmarks/twin2silicon/run_agent.py" - def _fake_runtime(self, directory, runtime, mode="success", repair_iterations=6): - codex_override = 'mcp_servers={labwired={command="npx",args=["-y","@labwired/mcp"]}}' + def _fake_runtime(self, directory, runtime, mode="success", repair_iterations=6, expects_auth=True): + codex_auth_assertion = ( + "assert (codex_home / 'auth.json').read_text(encoding='utf-8') == os.environ['EXPECTED_AUTH']" + if expects_auth + else "assert not (codex_home / 'auth.json').exists()" + ) + codex_auth_mode_assertion = ( + "assert (codex_home / 'auth.json').stat().st_mode & 0o777 == 0o600" + if expects_auth + else "" + ) workspace_code = { - "codex": f"workspace = Path(args[args.index('-C') + 1])\nassert args[:2] == ['exec', '--json']\nassert '--ephemeral' in args and '--skip-git-repo-check' in args\nassert args[args.index('-c') + 1] == {codex_override!r}\nassert args[args.index('-s') + 1] == 'workspace-write'\nassert os.environ['CODEX_HOME'] == os.environ['EXPECTED_CODEX_HOME']", + "codex": f"workspace = Path(args[args.index('-C') + 1])\nassert args[:2] == ['exec', '--json']\nassert '--ephemeral' in args and '--skip-git-repo-check' in args\nassert '-c' not in args\ncodex_home = Path(os.environ['CODEX_HOME'])\nassert codex_home != Path(os.environ['SOURCE_CODEX_HOME'])\nassert codex_home != Path(os.environ['EXPECTED_TRIAL'])\nassert codex_home != Path(os.environ['EXPECTED_CONFIG'])\nassert (codex_home / 'config.toml').read_text(encoding='utf-8') == '[mcp_servers.labwired]\\ncommand = \"npx\"\\nargs = [\"-y\", \"@labwired/mcp\"]\\n'\n{codex_auth_assertion}\n{codex_auth_mode_assertion}\n(workspace / 'effective-codex-home').write_text(str(codex_home), encoding='utf-8')", "claude": "workspace = Path.cwd()\nassert args[:3] == ['--print', '--verbose', '--output-format']\nassert args[args.index('--output-format') + 1] == 'stream-json'\nassert args[args.index('--mcp-config') + 1] == str(Path(os.environ['EXPECTED_CONFIG']) / 'claude-mcp.json')\nassert '--strict-mcp-config' in args", "opencode": "workspace = Path(args[args.index('--dir') + 1])\nassert args[:2] == ['run', '--format']\nassert args[args.index('--format') + 1] == 'json'\nassert os.environ['OPENCODE_CONFIG'] == str(Path(os.environ['EXPECTED_CONFIG']) / 'opencode.json')", }[runtime] @@ -1285,12 +1293,21 @@ def _fake_runtime(self, directory, runtime, mode="success", repair_iterations=6) """).replace("# workspace checks", workspace_code) return executable_fixture(directory, body) - def _run_cli(self, runtime, executable, trial, timeout_seconds=2, task=None): + def _run_cli(self, runtime, executable, trial, timeout_seconds=2, task=None, source_auth=True): + source_codex_home = (trial.parent / "source-codex-home").resolve() + source_codex_home.mkdir(parents=True) + auth_contents = "sentinel-codex-auth-credential" + auth_path = source_codex_home / "auth.json" + if source_auth: + auth_path.write_text(auth_contents, encoding="utf-8") + auth_path.chmod(0o600) environment = os.environ.copy() environment.update({ "EXPECTED_CONFIG": str((trial / "runtime-config").resolve()), - "CODEX_HOME": str((trial.parent / "native-codex-home").resolve()), - "EXPECTED_CODEX_HOME": str((trial.parent / "native-codex-home").resolve()), + "CODEX_HOME": str(source_codex_home), + "SOURCE_CODEX_HOME": str(source_codex_home), + "EXPECTED_TRIAL": str(trial.resolve()), + "EXPECTED_AUTH": auth_contents, "EXPECTED_INSTRUCTIONS": str( REPOSITORY_ROOT / "benchmarks/twin2silicon/shared-agent-instructions.md" ), @@ -1346,6 +1363,31 @@ def test_native_runtimes_create_completed_public_candidates(self): self.assertEqual(usage["requests"], 1) self.assertIsNone(usage["unavailable_reason"]) self._assert_trial_does_not_expose_hidden_oracle(trial) + if runtime == "codex": + isolated_home = Path((trial / "candidate/effective-codex-home").read_text()) + self.assertFalse(isolated_home.exists()) + for path in trial.rglob("*"): + if path.is_file(): + self.assertNotIn( + "sentinel-codex-auth-credential", + path.read_text(encoding="utf-8", errors="replace"), + ) + + def test_codex_without_source_auth_keeps_an_isolated_config(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + executable = self._fake_runtime( + root / "runtime", "codex", expects_auth=False, + ) + trial = root / "trial" + + completed = self._run_cli("codex", executable, trial, source_auth=False) + + self.assertEqual(completed.returncode, 0, completed.stderr) + result = json.loads((trial / "agent-result.json").read_text()) + self.assertEqual(result["status"], "completed") + isolated_home = Path((trial / "candidate/effective-codex-home").read_text()) + self.assertFalse(isolated_home.exists()) def test_nonzero_runtime_is_failed_but_retains_evidence(self): with tempfile.TemporaryDirectory() as directory: From a6cb3a484dc74e3492f5ebbc443dfa080e6693c9 Mon Sep 17 00:00:00 2001 From: Andrii Shylenko <14119286+w1ne@users.noreply.github.com> Date: Sat, 15 Aug 2026 21:52:31 +0200 Subject: [PATCH 40/48] feat(bench): compare native runtimes with one HIL oracle --- benchmarks/twin2silicon/run_matrix.py | 255 ++++++++++++++++++++++++++ tests/twin2silicon-hil.py | 145 +++++++++++++++ 2 files changed, 400 insertions(+) create mode 100644 benchmarks/twin2silicon/run_matrix.py diff --git a/benchmarks/twin2silicon/run_matrix.py b/benchmarks/twin2silicon/run_matrix.py new file mode 100644 index 0000000..a2f0082 --- /dev/null +++ b/benchmarks/twin2silicon/run_matrix.py @@ -0,0 +1,255 @@ +#!/usr/bin/env python3 +"""Run fresh native-runtime trials sequentially and retain one HIL oracle result per candidate.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import math +import os +from pathlib import Path +import subprocess +import sys +from typing import Any + +if __package__ in (None, ""): + sys.path.insert(0, str(Path(__file__).resolve().parents[2])) + +from benchmarks.twin2silicon.hil.results import write_json_atomic + + +RUNTIMES = ("opencode", "codex", "claude") +ROOT = Path(__file__).resolve().parent +TASKS = ROOT / "tasks" +USAGE_FIELDS = ( + "requests", + "fresh_input", + "cached_input", + "reasoning", + "output", + "estimated_cost_usd", +) + + +def _parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--task", required=True) + parser.add_argument("--output", required=True, type=Path) + parser.add_argument("--jtag-serial", required=True) + parser.add_argument("--uart-device", required=True) + parser.add_argument("--openocd", required=True) + parser.add_argument("--runtime", choices=RUNTIMES, action="append") + parser.add_argument("--agent-only", action="store_true") + # Test-only seams. They are deliberately omitted from operator help. + parser.add_argument("--agent-script", type=Path, default=ROOT / "run_agent.py", help=argparse.SUPPRESS) + parser.add_argument("--hil-script", type=Path, default=ROOT / "run_hil.py", help=argparse.SUPPRESS) + parser.add_argument("--agent-executable", action="append", default=[], help=argparse.SUPPRESS) + return parser + + +def _task_root(value: str) -> Path: + requested = Path(value) + return (TASKS / requested if len(requested.parts) == 1 else requested).resolve() + + +def _task_details(task_root: Path) -> tuple[str, Path, float]: + task = json.loads((task_root / "task.json").read_text(encoding="utf-8")) + task_id = task["id"] + public_dir = task["public_dir"] + budget = task["budgets"]["wall_time_seconds"] + if not isinstance(task_id, str) or not isinstance(public_dir, str): + raise ValueError("task id and public_dir must be strings") + if isinstance(budget, bool) or not isinstance(budget, (int, float)) or not math.isfinite(budget) or budget <= 0: + raise ValueError("task wall_time_seconds must be positive") + public_root = (task_root / public_dir).resolve() + if task_root not in public_root.parents or not public_root.is_dir(): + raise ValueError("task public_dir must name a directory below the task root") + return task_id, public_root, float(budget) + + +def _tree_sha256(root: Path) -> str: + digest = hashlib.sha256() + for path in sorted(root.rglob("*"), key=lambda item: item.relative_to(root).as_posix()): + if path.is_symlink(): + raise ValueError("public inputs must not contain symlinks") + if not path.is_file(): + continue + digest.update(path.relative_to(root).as_posix().encode("utf-8")) + digest.update(b"\0") + with path.open("rb") as source: + for chunk in iter(lambda: source.read(1024 * 1024), b""): + digest.update(chunk) + digest.update(b"\0") + return digest.hexdigest() + + +def _read_json(path: Path) -> dict[str, Any] | None: + try: + value = json.loads(path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError): + return None + return value if isinstance(value, dict) else None + + +def _number(value: object) -> int | float | None: + if isinstance(value, bool) or not isinstance(value, (int, float)) or not math.isfinite(value) or value < 0: + return None + return value + + +def _normalized_usage(path: Path) -> dict[str, object]: + source = _read_json(path) or {} + normalized = {field: _number(source.get(field)) for field in USAGE_FIELDS} + reason = source.get("unavailable_reason") + if not isinstance(reason, str) or not reason: + reason = None + if any(value is None for value in normalized.values()): + reason = reason or "runtime did not expose usage" + normalized["unavailable_reason"] = reason + return normalized + + +def _has_hil_usage_schema(path: Path) -> bool: + """Only forward evidence that run_hil.py can validate without invented rates.""" + source = _read_json(path) + if source is None: + return False + tokens = source.get("tokens") + rates = source.get("rates_usd_per_million") + if not isinstance(tokens, dict) or not isinstance(rates, dict): + return False + values = [source.get("requests")] + values.extend(tokens.get(field) for field in ("fresh_input", "cached_input", "output")) + values.extend(rates.get(field) for field in ("fresh_input", "cached_input", "output")) + return all(_number(value) is not None for value in values) + + +def _parse_executables(values: list[str], parser: argparse.ArgumentParser) -> dict[str, str]: + executables: dict[str, str] = {} + for value in values: + runtime, separator, executable = value.partition("=") + if separator != "=" or runtime not in RUNTIMES or not executable: + parser.error("--agent-executable must be RUNTIME=PATH") + executables[runtime] = executable + return executables + + +def _run_child(command: list[str], cwd: Path, stdout_path: Path, stderr_path: Path) -> tuple[int | None, str | None]: + try: + with stdout_path.open("wb") as stdout, stderr_path.open("wb") as stderr: + completed = subprocess.run(command, cwd=cwd, stdout=stdout, stderr=stderr, check=False) + except OSError as error: + return None, str(error) + return completed.returncode, None + + +def _agent_row( + runtime: str, + task_root: Path, + trial: Path, + initial_public_sha256: str, + agent_script: Path, + executable: str | None, +) -> dict[str, object]: + command = [sys.executable, str(agent_script), runtime, "--task", str(task_root), "--output", str(trial)] + if executable is not None: + command.extend(("--executable", executable)) + returncode, child_error = _run_child( + command, ROOT, + trial.parent / f"{trial.name}.matrix-agent.stdout.log", + trial.parent / f"{trial.name}.matrix-agent.stderr.log", + ) + result = _read_json(trial / "agent-result.json") + agent_status = result.get("status") if result and isinstance(result.get("status"), str) else "infrastructure_error" + row: dict[str, object] = { + "runtime": runtime, + "initial_public_sha256": initial_public_sha256, + "agent_status": agent_status, + "agent_result": result, + "agent_child_returncode": returncode, + "usage": _normalized_usage(trial / "usage.json"), + "hil_status": "not_run", + "hil_run": None, + } + if child_error is not None: + row["agent_error"] = child_error + elif result is None: + row["agent_error"] = "agent runner did not produce agent-result.json" + return row + + +def _run_hil(row: dict[str, object], task_root: Path, trial: Path, args: argparse.Namespace) -> None: + if row["agent_status"] != "completed": + return + candidate = trial / "candidate" + if not candidate.is_dir(): + row["hil_status"] = "invalid" + row["hil_error"] = "completed agent did not produce a candidate directory" + return + run_dir = trial / "hil" + command = [ + sys.executable, str(args.hil_script), str(task_root), "--run-dir", str(run_dir), + "--candidate", str(candidate), "--jtag-serial", args.jtag_serial, + "--uart-device", args.uart_device, "--openocd", args.openocd, + ] + usage_path = trial / "usage.json" + if _has_hil_usage_schema(usage_path): + command.extend(("--usage-json", str(usage_path))) + returncode, child_error = _run_child( + command, ROOT, trial / "matrix-hil.stdout.log", trial / "matrix-hil.stderr.log", + ) + run = _read_json(run_dir / "run.json") + row["hil_run"] = run + row["hil_child_returncode"] = returncode + if child_error is not None: + row["hil_status"] = "invalid" + row["hil_error"] = child_error + elif run is None: + row["hil_status"] = "invalid" + row["hil_error"] = "HIL runner did not produce run.json" + else: + status = run.get("status") + row["hil_status"] = status if isinstance(status, str) else "invalid" + + +def _print_summary(rows: list[dict[str, object]]) -> None: + print(f"{'RUNTIME':<10} {'AGENT':<22} {'HIL':<12}") + for row in rows: + print(f"{row['runtime']:<10} {row['agent_status']:<22} {row['hil_status']:<12}") + + +def main(argv: list[str] | None = None) -> int: + parser = _parser() + args = parser.parse_args(argv) + output = args.output.resolve() + if os.path.lexists(output): + print("output path already exists", file=sys.stderr) + return 2 + try: + task_root = _task_root(args.task) + task_id, public_root, _budget_seconds = _task_details(task_root) + executables = _parse_executables(args.agent_executable, parser) + initial_public_sha256 = _tree_sha256(public_root) + except (OSError, ValueError, KeyError, TypeError, json.JSONDecodeError) as error: + print(str(error), file=sys.stderr) + return 2 + + output.mkdir(parents=True) + trials_root = output / "trials" + trials_root.mkdir() + rows: list[dict[str, object]] = [] + matrix: dict[str, object] = {"schema_version": "1.0", "task_id": task_id, "trials": rows} + for runtime in args.runtime or list(RUNTIMES): + trial = trials_root / runtime + row = _agent_row(runtime, task_root, trial, initial_public_sha256, args.agent_script, executables.get(runtime)) + if not args.agent_only: + _run_hil(row, task_root, trial, args) + rows.append(row) + write_json_atomic(output / "matrix.json", matrix) + _print_summary(rows) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/twin2silicon-hil.py b/tests/twin2silicon-hil.py index 2f50d17..50c0899 100644 --- a/tests/twin2silicon-hil.py +++ b/tests/twin2silicon-hil.py @@ -380,6 +380,151 @@ def test_write_codex_mcp_config_creates_an_isolated_config_file(self): self.assertEqual(config_path.read_text(encoding="utf-8"), codex_mcp_toml()) +class RuntimeMatrixTests(unittest.TestCase): + script = REPOSITORY_ROOT / "benchmarks" / "twin2silicon" / "run_matrix.py" + task = REPOSITORY_ROOT / "benchmarks" / "twin2silicon" / "tasks" / "esp32s3-gpio-hil-001" + + def _fake_agent(self, directory): + body = textwrap.dedent(""" + import argparse + import json + from pathlib import Path + import shutil + + parser = argparse.ArgumentParser() + parser.add_argument("runtime") + parser.add_argument("--task", required=True) + parser.add_argument("--output", required=True) + parser.add_argument("--executable") + parser.add_argument("--timeout-seconds") + args = parser.parse_args() + output = Path(args.output) + task = Path(args.task) + output.mkdir(parents=True) + shutil.copytree(task / "public", output / "candidate") + status = {"opencode": "completed", "codex": "failed", "claude": "completed"}[args.runtime] + (output / "agent-result.json").write_text(json.dumps({ + "schema_version": "1.0", "runtime": args.runtime, + "status": status, "returncode": 0 if status == "completed" else 9, + })) + (output / "usage.json").write_text(json.dumps({ + "requests": 1 if args.runtime != "codex" else None, + "fresh_input": 10 if args.runtime != "codex" else None, + "cached_input": 0 if args.runtime != "codex" else None, + "reasoning": None, + "output": 5 if args.runtime != "codex" else None, + "estimated_cost_usd": None, + "unavailable_reason": "runtime did not expose usage" if args.runtime == "codex" else None, + })) + (output / "agent-invocation.json").write_text(json.dumps(vars(args), sort_keys=True)) + """) + return executable_fixture(directory, body) + + def _fake_hil(self, directory): + body = textwrap.dedent(""" + import argparse + import json + from pathlib import Path + + parser = argparse.ArgumentParser() + parser.add_argument("task") + parser.add_argument("--run-dir", required=True) + parser.add_argument("--candidate", required=True) + parser.add_argument("--jtag-serial", required=True) + parser.add_argument("--uart-device", required=True) + parser.add_argument("--openocd", required=True) + parser.add_argument("--platformio") + parser.add_argument("--usage-json") + args = parser.parse_args() + run_dir = Path(args.run_dir) + run_dir.mkdir(parents=True) + status = "pass" if Path(args.candidate).parent.name == "opencode" else "fail" + (run_dir / "run.json").write_text(json.dumps({ + "schema_version": "1.0", "status": status, "cost": None, + })) + (run_dir / "hil-invocation.json").write_text(json.dumps(vars(args), sort_keys=True)) + """) + return executable_fixture(directory, body) + + def _run_cli(self, output, agent, hil, *extra): + return subprocess.run( + [ + sys.executable, str(self.script), "--task", str(self.task), + "--output", str(output), "--jtag-serial", "fake-jtag", + "--uart-device", "/dev/fake-uart", "--openocd", "/fake/openocd", + "--agent-script", str(agent), "--hil-script", str(hil), *extra, + ], + cwd=REPOSITORY_ROOT, + text=True, + capture_output=True, + timeout=10, + ) + + def test_matrix_preserves_runtime_order_hashes_and_failure_isolation(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + completed = self._run_cli( + root / "matrix", self._fake_agent(root / "agent"), self._fake_hil(root / "hil"), + ) + + self.assertEqual(completed.returncode, 0, completed.stderr) + matrix = json.loads((root / "matrix" / "matrix.json").read_text()) + self.assertEqual(matrix["schema_version"], "1.0") + self.assertEqual(matrix["task_id"], "esp32s3-gpio-hil-001") + rows = matrix["trials"] + self.assertEqual([row["runtime"] for row in rows], ["opencode", "codex", "claude"]) + self.assertEqual([row["agent_status"] for row in rows], ["completed", "failed", "completed"]) + self.assertEqual([row["hil_status"] for row in rows], ["pass", "not_run", "fail"]) + self.assertEqual(len({row["initial_public_sha256"] for row in rows}), 1) + self.assertIsNone(rows[1]["usage"]["requests"]) + self.assertEqual(rows[1]["usage"]["unavailable_reason"], "runtime did not expose usage") + self.assertIsNone(rows[0]["hil_run"]["cost"]) + self.assertIn("RUNTIME", completed.stdout) + + hidden_name = "hil-oracle.json" + for trial in (path for path in (root / "matrix" / "trials").iterdir() if path.is_dir()): + with self.subTest(trial=trial.name): + agent_invocation = (trial / "agent-invocation.json").read_text(encoding="utf-8") + self.assertNotIn(hidden_name, agent_invocation) + if trial.name != "codex": + hil_invocation = (trial / "hil" / "hil-invocation.json").read_text(encoding="utf-8") + self.assertNotIn("--usage-json", hil_invocation) + + def test_agent_only_skips_hil_and_repeated_runtime_selects_trials(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + completed = self._run_cli( + root / "matrix", self._fake_agent(root / "agent"), self._fake_hil(root / "hil"), + "--agent-only", "--runtime", "claude", "--runtime", "opencode", + ) + + self.assertEqual(completed.returncode, 0, completed.stderr) + rows = json.loads((root / "matrix" / "matrix.json").read_text())["trials"] + self.assertEqual([row["runtime"] for row in rows], ["claude", "opencode"]) + self.assertEqual([row["hil_status"] for row in rows], ["not_run", "not_run"]) + self.assertTrue(all(row["hil_run"] is None for row in rows)) + self.assertFalse(any((root / "matrix" / "trials").glob("*/hil"))) + + def test_matrix_rejects_existing_output_and_invalid_runtime(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + agent = self._fake_agent(root / "agent") + hil = self._fake_hil(root / "hil") + output = root / "matrix" + output.mkdir() + marker = output / "keep" + marker.write_text("existing", encoding="utf-8") + + existing = self._run_cli(output, agent, hil) + self.assertEqual(existing.returncode, 2) + self.assertIn("output path already exists", existing.stderr) + self.assertEqual(marker.read_text(encoding="utf-8"), "existing") + + invalid = self._run_cli(root / "new", agent, hil, "--runtime", "unknown") + self.assertEqual(invalid.returncode, 2) + self.assertIn("invalid choice", invalid.stderr) + + class FixtureContractTests(unittest.TestCase): def test_esp32s3_gpio_hil_fixture_contract(self): task_root = ( From 21e8b7078320fbd9ab4f57c44953129bdd379f50 Mon Sep 17 00:00:00 2001 From: Andrii Shylenko <14119286+w1ne@users.noreply.github.com> Date: Sat, 15 Aug 2026 21:55:50 +0200 Subject: [PATCH 41/48] fix(bench): validate HIL usage schema fully --- benchmarks/twin2silicon/run_matrix.py | 7 ++-- tests/twin2silicon-hil.py | 46 +++++++++++++++++++++++---- 2 files changed, 43 insertions(+), 10 deletions(-) diff --git a/benchmarks/twin2silicon/run_matrix.py b/benchmarks/twin2silicon/run_matrix.py index a2f0082..05fb89d 100644 --- a/benchmarks/twin2silicon/run_matrix.py +++ b/benchmarks/twin2silicon/run_matrix.py @@ -119,9 +119,10 @@ def _has_hil_usage_schema(path: Path) -> bool: rates = source.get("rates_usd_per_million") if not isinstance(tokens, dict) or not isinstance(rates, dict): return False - values = [source.get("requests")] - values.extend(tokens.get(field) for field in ("fresh_input", "cached_input", "output")) - values.extend(rates.get(field) for field in ("fresh_input", "cached_input", "output")) + required = ("fresh_input", "cached_input", "output") + if "requests" not in source or any(field not in tokens or field not in rates for field in required): + return False + values = [source["requests"], *tokens.values(), *rates.values()] return all(_number(value) is not None for value in values) diff --git a/tests/twin2silicon-hil.py b/tests/twin2silicon-hil.py index 50c0899..e4951ff 100644 --- a/tests/twin2silicon-hil.py +++ b/tests/twin2silicon-hil.py @@ -384,8 +384,8 @@ class RuntimeMatrixTests(unittest.TestCase): script = REPOSITORY_ROOT / "benchmarks" / "twin2silicon" / "run_matrix.py" task = REPOSITORY_ROOT / "benchmarks" / "twin2silicon" / "tasks" / "esp32s3-gpio-hil-001" - def _fake_agent(self, directory): - body = textwrap.dedent(""" + def _fake_agent(self, directory, usage_schema="normalized"): + body = textwrap.dedent(f""" import argparse import json from pathlib import Path @@ -402,12 +402,12 @@ def _fake_agent(self, directory): task = Path(args.task) output.mkdir(parents=True) shutil.copytree(task / "public", output / "candidate") - status = {"opencode": "completed", "codex": "failed", "claude": "completed"}[args.runtime] - (output / "agent-result.json").write_text(json.dumps({ + status = {{"opencode": "completed", "codex": "failed", "claude": "completed"}}[args.runtime] + (output / "agent-result.json").write_text(json.dumps({{ "schema_version": "1.0", "runtime": args.runtime, "status": status, "returncode": 0 if status == "completed" else 9, - })) - (output / "usage.json").write_text(json.dumps({ + }})) + usage = {{ "requests": 1 if args.runtime != "codex" else None, "fresh_input": 10 if args.runtime != "codex" else None, "cached_input": 0 if args.runtime != "codex" else None, @@ -415,7 +415,18 @@ def _fake_agent(self, directory): "output": 5 if args.runtime != "codex" else None, "estimated_cost_usd": None, "unavailable_reason": "runtime did not expose usage" if args.runtime == "codex" else None, - })) + }} + if {usage_schema!r} == "hil-valid": + usage.update({{ + "tokens": {{"fresh_input": 10, "cached_input": 0, "output": 5}}, + "rates_usd_per_million": {{"fresh_input": 1, "cached_input": 1, "output": 1}}, + }}) + elif {usage_schema!r} == "hil-extra-invalid": + usage.update({{ + "tokens": {{"fresh_input": 10, "cached_input": 0, "output": 5, "reasoning": None}}, + "rates_usd_per_million": {{"fresh_input": 1, "cached_input": 1, "output": 1}}, + }}) + (output / "usage.json").write_text(json.dumps(usage)) (output / "agent-invocation.json").write_text(json.dumps(vars(args), sort_keys=True)) """) return executable_fixture(directory, body) @@ -505,6 +516,27 @@ def test_agent_only_skips_hil_and_repeated_runtime_selects_trials(self): self.assertTrue(all(row["hil_run"] is None for row in rows)) self.assertFalse(any((root / "matrix" / "trials").glob("*/hil"))) + def test_matrix_forwards_only_usage_that_the_hil_cost_schema_accepts(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + hil = self._fake_hil(root / "hil") + for schema, expected_usage_json in ( + ("hil-valid", True), + ("hil-extra-invalid", False), + ): + with self.subTest(schema=schema): + output = root / schema + completed = self._run_cli( + output, self._fake_agent(root / f"{schema}-agent", schema), hil, + "--runtime", "opencode", + ) + + self.assertEqual(completed.returncode, 0, completed.stderr) + invocation = json.loads( + (output / "trials" / "opencode" / "hil" / "hil-invocation.json").read_text() + ) + self.assertEqual(invocation["usage_json"] is not None, expected_usage_json) + def test_matrix_rejects_existing_output_and_invalid_runtime(self): with tempfile.TemporaryDirectory() as directory: root = Path(directory) From a8ed94f8273f3221df91e8e7c42c861c35d2b9e4 Mon Sep 17 00:00:00 2001 From: Andrii Shylenko <14119286+w1ne@users.noreply.github.com> Date: Sat, 15 Aug 2026 22:00:14 +0200 Subject: [PATCH 42/48] fix(bench): retain partial usage and gate HIL failures --- benchmarks/twin2silicon/run_matrix.py | 12 +++++++-- tests/twin2silicon-hil.py | 38 ++++++++++++++++++++++++++- 2 files changed, 47 insertions(+), 3 deletions(-) diff --git a/benchmarks/twin2silicon/run_matrix.py b/benchmarks/twin2silicon/run_matrix.py index 05fb89d..cc68d69 100644 --- a/benchmarks/twin2silicon/run_matrix.py +++ b/benchmarks/twin2silicon/run_matrix.py @@ -104,8 +104,12 @@ def _normalized_usage(path: Path) -> dict[str, object]: reason = source.get("unavailable_reason") if not isinstance(reason, str) or not reason: reason = None - if any(value is None for value in normalized.values()): - reason = reason or "runtime did not expose usage" + if reason is None: + values = tuple(normalized.values()) + if all(value is None for value in values): + reason = "runtime did not expose usage" + elif any(value is None for value in values): + reason = "one or more usage fields unavailable" normalized["unavailable_reason"] = reason return normalized @@ -163,6 +167,8 @@ def _agent_row( ) result = _read_json(trial / "agent-result.json") agent_status = result.get("status") if result and isinstance(result.get("status"), str) else "infrastructure_error" + if child_error is not None or returncode != 0: + agent_status = "infrastructure_error" row: dict[str, object] = { "runtime": runtime, "initial_public_sha256": initial_public_sha256, @@ -175,6 +181,8 @@ def _agent_row( } if child_error is not None: row["agent_error"] = child_error + elif returncode != 0: + row["agent_error"] = f"agent runner exited with status {returncode}" elif result is None: row["agent_error"] = "agent runner did not produce agent-result.json" return row diff --git a/tests/twin2silicon-hil.py b/tests/twin2silicon-hil.py index e4951ff..eb5953d 100644 --- a/tests/twin2silicon-hil.py +++ b/tests/twin2silicon-hil.py @@ -384,7 +384,7 @@ class RuntimeMatrixTests(unittest.TestCase): script = REPOSITORY_ROOT / "benchmarks" / "twin2silicon" / "run_matrix.py" task = REPOSITORY_ROOT / "benchmarks" / "twin2silicon" / "tasks" / "esp32s3-gpio-hil-001" - def _fake_agent(self, directory, usage_schema="normalized"): + def _fake_agent(self, directory, usage_schema="normalized", child_returncode=0): body = textwrap.dedent(f""" import argparse import json @@ -426,8 +426,11 @@ def _fake_agent(self, directory, usage_schema="normalized"): "tokens": {{"fresh_input": 10, "cached_input": 0, "output": 5, "reasoning": None}}, "rates_usd_per_million": {{"fresh_input": 1, "cached_input": 1, "output": 1}}, }}) + elif {usage_schema!r} == "partial": + usage["estimated_cost_usd"] = 0.000015 (output / "usage.json").write_text(json.dumps(usage)) (output / "agent-invocation.json").write_text(json.dumps(vars(args), sort_keys=True)) + raise SystemExit({child_returncode!r}) """) return executable_fixture(directory, body) @@ -537,6 +540,39 @@ def test_matrix_forwards_only_usage_that_the_hil_cost_schema_accepts(self): ) self.assertEqual(invocation["usage_json"] is not None, expected_usage_json) + def test_matrix_preserves_known_partial_usage_with_an_accurate_reason(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + completed = self._run_cli( + root / "matrix", self._fake_agent(root / "agent", "partial"), self._fake_hil(root / "hil"), + "--agent-only", "--runtime", "opencode", + ) + + self.assertEqual(completed.returncode, 0, completed.stderr) + usage = json.loads((root / "matrix" / "matrix.json").read_text())["trials"][0]["usage"] + self.assertEqual(usage["fresh_input"], 10) + self.assertEqual(usage["output"], 5) + self.assertEqual(usage["estimated_cost_usd"], 0.000015) + self.assertIsNone(usage["reasoning"]) + self.assertEqual(usage["unavailable_reason"], "one or more usage fields unavailable") + + def test_matrix_does_not_run_hil_after_a_nonzero_agent_child_exit(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + completed = self._run_cli( + root / "matrix", self._fake_agent(root / "agent", child_returncode=7), self._fake_hil(root / "hil"), + "--runtime", "opencode", + ) + + self.assertEqual(completed.returncode, 0, completed.stderr) + row = json.loads((root / "matrix" / "matrix.json").read_text())["trials"][0] + self.assertEqual(row["agent_status"], "infrastructure_error") + self.assertEqual(row["agent_result"]["status"], "completed") + self.assertEqual(row["agent_child_returncode"], 7) + self.assertIn("exited with status 7", row["agent_error"]) + self.assertEqual(row["hil_status"], "not_run") + self.assertFalse((root / "matrix" / "trials" / "opencode" / "hil").exists()) + def test_matrix_rejects_existing_output_and_invalid_runtime(self): with tempfile.TemporaryDirectory() as directory: root = Path(directory) From 1aa0bca138755adebf11e4de66b3b55552ece0dc Mon Sep 17 00:00:00 2001 From: Andrii Shylenko <14119286+w1ne@users.noreply.github.com> Date: Sat, 15 Aug 2026 22:03:23 +0200 Subject: [PATCH 43/48] docs(bench): add cross-runtime smoke entry points --- benchmarks/twin2silicon/README.md | 51 ++++++++++++++++++++ package.json | 4 +- tests/twin2silicon-hil.py | 55 ++++++++++++++++++++++ tests/twin2silicon-runtime-smoke.sh | 73 +++++++++++++++++++++++++++++ 4 files changed, 182 insertions(+), 1 deletion(-) create mode 100644 benchmarks/twin2silicon/README.md create mode 100755 tests/twin2silicon-runtime-smoke.sh diff --git a/benchmarks/twin2silicon/README.md b/benchmarks/twin2silicon/README.md new file mode 100644 index 0000000..56e8b61 --- /dev/null +++ b/benchmarks/twin2silicon/README.md @@ -0,0 +1,51 @@ +# Cross-runtime HIL smoke comparison + +This is a runtime smoke comparison, not a leaderboard. It runs OpenCode, Codex +CLI, and Claude Code against the same public firmware task, then sends each +completed candidate to the existing physical HIL oracle. + +## Interpretation + +Native models are intentionally not overridden: each runtime uses its installed +default. A raw-model comparison belongs in the separate OpenCode model matrix, +not this cross-runtime smoke test. A runtime may report token usage but +subscription cost is unknown unless that runtime explicitly provides a cost; +the harness never estimates subscription cost. + +The hardware command flashes the connected board. Treat it as a destructive, +opt-in operation and identify the UART and JTAG device deliberately. One smoke +run is useful only as operational evidence. Any publishable result needs +multiple tasks and repeated fresh trials before drawing a conclusion. + +## Commands + +Run the offline contract suite: + +```bash +npm run test:runtime-smoke:offline +``` + +Run a connected-board comparison only with explicit hardware and output +locations. Existing authenticated runtime sessions are used as-is. + +```bash +LABWIRED_HIL=1 \ +LABWIRED_UART_DEVICE=/dev/cu.usbmodem11101 \ +LABWIRED_JTAG_SERIAL=3C:0F:02:DF:EC:F8 \ +LABWIRED_OPENOCD="$HOME/.platformio/packages/tool-openocd-esp32/bin/openocd" \ +LABWIRED_MATRIX_OUTPUT=/Volumes/LabWired/hil-runs/runtime-smoke-$(date +%Y%m%d-%H%M%S) \ +npm run test:runtime-smoke:hardware +``` + +The entry point checks OpenCode, Codex CLI, Claude Code, PlatformIO, and +OpenOCD and records their versions before running. Temporary data is placed on +`/Volumes/LabWired` when that volume is available. It does not configure +runtime accounts; authenticate each native runtime before invoking it. + +## Evidence + +`LABWIRED_MATRIX_OUTPUT` must be a new directory. The matrix writes +`matrix.json` after each runtime, plus per-runtime trial directories under +`trials/`. Completed candidates contain runtime logs, `agent-result.json`, +and `usage.json`; HIL evidence, including `run.json`, is stored in that +trial's `hil/` directory. Preserve these records when reporting a smoke run. diff --git a/package.json b/package.json index 234e913..165d1e6 100644 --- a/package.json +++ b/package.json @@ -151,7 +151,9 @@ "test:tool-names": "bash tests/public-tool-names.sh", "test:public-install-safety": "bash tests/public-install-safety.sh", "test:install": "bash tests/install-smoke.sh", - "test:llm": "bash tests/llm-deepinfra.sh" + "test:llm": "bash tests/llm-deepinfra.sh", + "test:runtime-smoke:offline": "python3 tests/twin2silicon-hil.py", + "test:runtime-smoke:hardware": "bash tests/twin2silicon-runtime-smoke.sh" }, "publishConfig": { "access": "public" diff --git a/tests/twin2silicon-hil.py b/tests/twin2silicon-hil.py index eb5953d..ae57c83 100644 --- a/tests/twin2silicon-hil.py +++ b/tests/twin2silicon-hil.py @@ -593,6 +593,61 @@ def test_matrix_rejects_existing_output_and_invalid_runtime(self): self.assertIn("invalid choice", invalid.stderr) +class RuntimePackagingTests(unittest.TestCase): + script = REPOSITORY_ROOT / "tests" / "twin2silicon-runtime-smoke.sh" + + def test_runtime_smoke_entry_points_are_opt_in_and_documented(self): + package = json.loads((REPOSITORY_ROOT / "package.json").read_text(encoding="utf-8")) + self.assertEqual( + package["scripts"]["test:runtime-smoke:offline"], + "python3 tests/twin2silicon-hil.py", + ) + self.assertEqual( + package["scripts"]["test:runtime-smoke:hardware"], + "bash tests/twin2silicon-runtime-smoke.sh", + ) + + source = self.script.read_text(encoding="utf-8") + for variable in ( + "LABWIRED_HIL", + "LABWIRED_UART_DEVICE", + "LABWIRED_JTAG_SERIAL", + "LABWIRED_OPENOCD", + "LABWIRED_MATRIX_OUTPUT", + "/Volumes/LabWired", + ): + with self.subTest(variable=variable): + self.assertIn(variable, source) + self.assertNotRegex(source.lower(), r"api[_-]?key|credential|secret|token") + + refused = subprocess.run( + ["bash", str(self.script)], + cwd=REPOSITORY_ROOT, + text=True, + capture_output=True, + timeout=5, + ) + self.assertNotEqual(refused.returncode, 0) + self.assertIn("LABWIRED_HIL=1 required", refused.stderr) + + def test_runtime_smoke_readme_states_the_comparison_limits_and_hardware_effect(self): + readme = ( + REPOSITORY_ROOT / "benchmarks" / "twin2silicon" / "README.md" + ).read_text(encoding="utf-8").lower() + for wording in ( + "smoke comparison", + "not a leaderboard", + "not overridden", + "opencode model matrix", + "unknown", + "flash", + "multiple tasks", + "repeated fresh trials", + ): + with self.subTest(wording=wording): + self.assertIn(wording, readme) + + class FixtureContractTests(unittest.TestCase): def test_esp32s3_gpio_hil_fixture_contract(self): task_root = ( diff --git a/tests/twin2silicon-runtime-smoke.sh b/tests/twin2silicon-runtime-smoke.sh new file mode 100755 index 0000000..c07a701 --- /dev/null +++ b/tests/twin2silicon-runtime-smoke.sh @@ -0,0 +1,73 @@ +#!/usr/bin/env bash +# Run the connected-board runtime smoke matrix only after an explicit opt-in. +set -euo pipefail + +if [[ "${LABWIRED_HIL:-}" != "1" ]]; then + echo "LABWIRED_HIL=1 required; this command flashes connected hardware." >&2 + exit 2 +fi + +require_variable() { + local name="$1" + if [[ -z "${!name:-}" ]]; then + echo "$name required" >&2 + exit 2 + fi +} + +require_command() { + local name="$1" + if ! command -v "$name" >/dev/null 2>&1; then + echo "$name is required but was not found on PATH" >&2 + exit 2 + fi +} + +print_version() { + local label="$1" + shift + local version + if ! version="$("$@" 2>&1)"; then + echo "unable to determine $label version" >&2 + exit 2 + fi + printf '%s version: %s\n' "$label" "${version%%$'\n'*}" +} + +require_variable LABWIRED_UART_DEVICE +require_variable LABWIRED_JTAG_SERIAL +require_variable LABWIRED_OPENOCD +require_variable LABWIRED_MATRIX_OUTPUT +require_command opencode +require_command codex +require_command claude +require_command pio + +if [[ ! -x "$LABWIRED_OPENOCD" ]]; then + echo "LABWIRED_OPENOCD must name an executable OpenOCD binary" >&2 + exit 2 +fi + +if [[ -d /Volumes/LabWired && -w /Volumes/LabWired ]]; then + temporary_root="$(mktemp -d /Volumes/LabWired/twin2silicon-runtime-smoke.XXXXXX)" +else + temporary_root="$(mktemp -d)" +fi +trap 'rm -rf "$temporary_root"' EXIT +export TMPDIR="$temporary_root" + +print_version opencode opencode --version +print_version codex codex --version +print_version claude claude --version +print_version pio pio --version +print_version openocd "$LABWIRED_OPENOCD" --version + +repository_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +task="${LABWIRED_TASK:-esp32s3-gpio-hil-001}" + +python3 "$repository_root/benchmarks/twin2silicon/run_matrix.py" \ + --task "$task" \ + --output "$LABWIRED_MATRIX_OUTPUT" \ + --jtag-serial "$LABWIRED_JTAG_SERIAL" \ + --uart-device "$LABWIRED_UART_DEVICE" \ + --openocd "$LABWIRED_OPENOCD" From 50b0d6e656b01fd42cc1e02b25abedec08d45188 Mon Sep 17 00:00:00 2001 From: Andrii Shylenko <14119286+w1ne@users.noreply.github.com> Date: Sat, 15 Aug 2026 22:07:10 +0200 Subject: [PATCH 44/48] docs(bench): clarify runtime version evidence --- benchmarks/twin2silicon/README.md | 3 ++- tests/twin2silicon-hil.py | 2 ++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/benchmarks/twin2silicon/README.md b/benchmarks/twin2silicon/README.md index 56e8b61..e9c3f55 100644 --- a/benchmarks/twin2silicon/README.md +++ b/benchmarks/twin2silicon/README.md @@ -38,7 +38,8 @@ npm run test:runtime-smoke:hardware ``` The entry point checks OpenCode, Codex CLI, Claude Code, PlatformIO, and -OpenOCD and records their versions before running. Temporary data is placed on +OpenOCD and prints their versions before running; operators may capture stdout +with the smoke evidence. Temporary data is placed on `/Volumes/LabWired` when that volume is available. It does not configure runtime accounts; authenticate each native runtime before invoking it. diff --git a/tests/twin2silicon-hil.py b/tests/twin2silicon-hil.py index ae57c83..491f674 100644 --- a/tests/twin2silicon-hil.py +++ b/tests/twin2silicon-hil.py @@ -643,9 +643,11 @@ def test_runtime_smoke_readme_states_the_comparison_limits_and_hardware_effect(s "flash", "multiple tasks", "repeated fresh trials", + "prints their versions", ): with self.subTest(wording=wording): self.assertIn(wording, readme) + self.assertNotIn("records their versions", readme) class FixtureContractTests(unittest.TestCase): From 6b40ab2fd3766f51ef876c79d04e76755436c106 Mon Sep 17 00:00:00 2001 From: Andrii Shylenko <14119286+w1ne@users.noreply.github.com> Date: Sat, 15 Aug 2026 22:17:03 +0200 Subject: [PATCH 45/48] fix(bench): validate board identity in runtime matrix --- benchmarks/twin2silicon/README.md | 4 + .../twin2silicon/identify_pio_device.py | 76 ++++++++++ benchmarks/twin2silicon/run_matrix.py | 4 + .../twin2silicon/shared-agent-instructions.md | 3 + tests/twin2silicon-hil.py | 130 +++++++++++++++++- tests/twin2silicon-runtime-smoke.sh | 11 +- 6 files changed, 223 insertions(+), 5 deletions(-) create mode 100644 benchmarks/twin2silicon/identify_pio_device.py diff --git a/benchmarks/twin2silicon/README.md b/benchmarks/twin2silicon/README.md index e9c3f55..c4479ca 100644 --- a/benchmarks/twin2silicon/README.md +++ b/benchmarks/twin2silicon/README.md @@ -27,6 +27,10 @@ npm run test:runtime-smoke:offline Run a connected-board comparison only with explicit hardware and output locations. Existing authenticated runtime sessions are used as-is. +The trial instructions are noninteractive: this command authorizes inspection, +the smallest in-scope repair, and compilation without a confirmation prompt. +Before any flash, the harness checks that the selected UART's PlatformIO device +entry reports the supplied JTAG serial. ```bash LABWIRED_HIL=1 \ diff --git a/benchmarks/twin2silicon/identify_pio_device.py b/benchmarks/twin2silicon/identify_pio_device.py new file mode 100644 index 0000000..b38120a --- /dev/null +++ b/benchmarks/twin2silicon/identify_pio_device.py @@ -0,0 +1,76 @@ +#!/usr/bin/env python3 +"""Confirm that a selected UART is reported by PlatformIO for one JTAG serial.""" + +from __future__ import annotations + +import argparse +import json +import re +import subprocess +import sys +from typing import Any + + +_DEFAULT_TIMEOUT_SECONDS = 5.0 + + +def _positive_seconds(value: str) -> float: + try: + seconds = float(value) + except ValueError as error: + raise argparse.ArgumentTypeError("timeout must be a number") from error + if seconds <= 0: + raise argparse.ArgumentTypeError("timeout must be greater than zero") + return seconds + + +def _parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--uart-device", required=True) + parser.add_argument("--jtag-serial", required=True) + parser.add_argument("--timeout-seconds", type=_positive_seconds, default=_DEFAULT_TIMEOUT_SECONDS, + help=argparse.SUPPRESS) + return parser + + +def _has_exact_serial(hwid: str, serial: str) -> bool: + return re.search(r"(?:^|\s)SER=" + re.escape(serial) + r"(?=\s|$)", hwid) is not None + + +def _devices(timeout_seconds: float) -> list[dict[str, Any]] | None: + try: + completed = subprocess.run( + ["pio", "device", "list", "--json-output"], + stdin=subprocess.DEVNULL, + stdout=subprocess.PIPE, + stderr=subprocess.DEVNULL, + text=True, + timeout=timeout_seconds, + check=False, + ) + except (OSError, subprocess.TimeoutExpired): + return None + if completed.returncode != 0: + return None + try: + value = json.loads(completed.stdout) + except json.JSONDecodeError: + return None + return value if isinstance(value, list) and all(isinstance(item, dict) for item in value) else None + + +def main(argv: list[str] | None = None) -> int: + args = _parser().parse_args(argv) + devices = _devices(args.timeout_seconds) + if devices is None: + return 2 + for device in devices: + if device.get("port") == args.uart_device and isinstance(device.get("hwid"), str): + if _has_exact_serial(device["hwid"], args.jtag_serial): + print(args.jtag_serial) + return 0 + return 2 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/benchmarks/twin2silicon/run_matrix.py b/benchmarks/twin2silicon/run_matrix.py index cc68d69..2fc0633 100644 --- a/benchmarks/twin2silicon/run_matrix.py +++ b/benchmarks/twin2silicon/run_matrix.py @@ -39,6 +39,7 @@ def _parser() -> argparse.ArgumentParser: parser.add_argument("--jtag-serial", required=True) parser.add_argument("--uart-device", required=True) parser.add_argument("--openocd", required=True) + parser.add_argument("--identity-command-json") parser.add_argument("--runtime", choices=RUNTIMES, action="append") parser.add_argument("--agent-only", action="store_true") # Test-only seams. They are deliberately omitted from operator help. @@ -201,6 +202,7 @@ def _run_hil(row: dict[str, object], task_root: Path, trial: Path, args: argpars sys.executable, str(args.hil_script), str(task_root), "--run-dir", str(run_dir), "--candidate", str(candidate), "--jtag-serial", args.jtag_serial, "--uart-device", args.uart_device, "--openocd", args.openocd, + "--identity-command-json", args.identity_command_json, ] usage_path = trial / "usage.json" if _has_hil_usage_schema(usage_path): @@ -231,6 +233,8 @@ def _print_summary(rows: list[dict[str, object]]) -> None: def main(argv: list[str] | None = None) -> int: parser = _parser() args = parser.parse_args(argv) + if not args.agent_only and not args.identity_command_json: + parser.error("--identity-command-json is required unless --agent-only") output = args.output.resolve() if os.path.lexists(output): print("output path already exists", file=sys.stderr) diff --git a/benchmarks/twin2silicon/shared-agent-instructions.md b/benchmarks/twin2silicon/shared-agent-instructions.md index 0e4cc08..14e4b1d 100644 --- a/benchmarks/twin2silicon/shared-agent-instructions.md +++ b/benchmarks/twin2silicon/shared-agent-instructions.md @@ -1,5 +1,8 @@ # Firmware repair trial +This trial is noninteractive. This prompt authorizes you to inspect, make the +smallest in-scope repair, and compile. Do not pause or ask for confirmation. + Work only inside the public workspace provided for this trial. Do not read, search for, copy, modify, or infer from hidden task files, hidden oracle files, or HIL result directories. Do not self-grade the repair or claim that it passes diff --git a/tests/twin2silicon-hil.py b/tests/twin2silicon-hil.py index 491f674..cdd5cf4 100644 --- a/tests/twin2silicon-hil.py +++ b/tests/twin2silicon-hil.py @@ -448,6 +448,7 @@ def _fake_hil(self, directory): parser.add_argument("--uart-device", required=True) parser.add_argument("--openocd", required=True) parser.add_argument("--platformio") + parser.add_argument("--identity-command-json") parser.add_argument("--usage-json") args = parser.parse_args() run_dir = Path(args.run_dir) @@ -460,14 +461,17 @@ def _fake_hil(self, directory): """) return executable_fixture(directory, body) - def _run_cli(self, output, agent, hil, *extra): - return subprocess.run( - [ + def _run_cli(self, output, agent, hil, *extra, identity_command_json='["fake-identity"]'): + command = [ sys.executable, str(self.script), "--task", str(self.task), "--output", str(output), "--jtag-serial", "fake-jtag", "--uart-device", "/dev/fake-uart", "--openocd", "/fake/openocd", "--agent-script", str(agent), "--hil-script", str(hil), *extra, - ], + ] + if identity_command_json is not None and "--identity-command-json" not in extra: + command.extend(("--identity-command-json", identity_command_json)) + return subprocess.run( + command, cwd=REPOSITORY_ROOT, text=True, capture_output=True, @@ -519,6 +523,49 @@ def test_agent_only_skips_hil_and_repeated_runtime_selects_trials(self): self.assertTrue(all(row["hil_run"] is None for row in rows)) self.assertFalse(any((root / "matrix" / "trials").glob("*/hil"))) + def test_matrix_requires_identity_unless_agent_only(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + agent = self._fake_agent(root / "agent") + hil = self._fake_hil(root / "hil") + + missing = self._run_cli(root / "missing", agent, hil, identity_command_json=None) + self.assertEqual(missing.returncode, 2) + self.assertIn("--identity-command-json is required unless --agent-only", missing.stderr) + self.assertFalse((root / "missing").exists()) + + empty = self._run_cli(root / "empty", agent, hil, identity_command_json="") + self.assertEqual(empty.returncode, 2) + self.assertIn("--identity-command-json is required unless --agent-only", empty.stderr) + self.assertFalse((root / "empty").exists()) + + agent_only = self._run_cli( + root / "agent-only", agent, hil, "--agent-only", "--runtime", "opencode", + identity_command_json=None, + ) + self.assertEqual(agent_only.returncode, 0, agent_only.stderr) + self.assertFalse((root / "agent-only" / "trials" / "opencode" / "hil").exists()) + + def test_matrix_forwards_identity_only_to_hil(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + identity_command_json = json.dumps([ + sys.executable, "identify_pio_device.py", "--uart-device", "/dev/fake-uart", + "--jtag-serial", "fake-jtag", + ]) + completed = self._run_cli( + root / "matrix", self._fake_agent(root / "agent"), self._fake_hil(root / "hil"), + "--runtime", "opencode", identity_command_json=identity_command_json, + ) + + self.assertEqual(completed.returncode, 0, completed.stderr) + trial = root / "matrix" / "trials" / "opencode" + invocation = json.loads((trial / "hil" / "hil-invocation.json").read_text()) + self.assertEqual(invocation["identity_command_json"], identity_command_json) + agent_invocation = (trial / "agent-invocation.json").read_text(encoding="utf-8") + self.assertNotIn("identity_command_json", agent_invocation) + self.assertNotIn("identify_pio_device.py", agent_invocation) + def test_matrix_forwards_only_usage_that_the_hil_cost_schema_accepts(self): with tempfile.TemporaryDirectory() as directory: root = Path(directory) @@ -615,6 +662,8 @@ def test_runtime_smoke_entry_points_are_opt_in_and_documented(self): "LABWIRED_OPENOCD", "LABWIRED_MATRIX_OUTPUT", "/Volumes/LabWired", + "identify_pio_device.py", + "--identity-command-json", ): with self.subTest(variable=variable): self.assertIn(variable, source) @@ -649,6 +698,77 @@ def test_runtime_smoke_readme_states_the_comparison_limits_and_hardware_effect(s self.assertIn(wording, readme) self.assertNotIn("records their versions", readme) + def test_runtime_smoke_constructs_a_pio_identity_command_for_the_selected_uart_and_jtag(self): + source = self.script.read_text(encoding="utf-8") + + self.assertIn("json.dumps([sys.executable, sys.argv[1], \"--uart-device\", sys.argv[2],", source) + self.assertIn('"--jtag-serial", sys.argv[3]])', source) + self.assertIn('"$LABWIRED_UART_DEVICE"', source) + self.assertIn('"$LABWIRED_JTAG_SERIAL"', source) + + +class PioIdentityDeviceTests(unittest.TestCase): + script = REPOSITORY_ROOT / "benchmarks" / "twin2silicon" / "identify_pio_device.py" + + def _fake_pio(self, directory, body): + path = Path(directory) / "pio" + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text("#!/usr/bin/env python3\n" + body, encoding="utf-8") + path.chmod(0o755) + return path + + def _run_cli(self, pio, *extra): + environment = os.environ.copy() + environment["PATH"] = str(pio.parent) + os.pathsep + environment["PATH"] + return subprocess.run( + [ + sys.executable, str(self.script), "--uart-device", "/dev/fake-uart", + "--jtag-serial", "JTAG-1", *extra, + ], + cwd=REPOSITORY_ROOT, + env=environment, + text=True, + capture_output=True, + timeout=5, + ) + + def test_exact_port_and_serial_match_prints_only_the_requested_serial(self): + with tempfile.TemporaryDirectory() as directory: + pio = self._fake_pio(Path(directory), textwrap.dedent(""" + import json, sys + assert sys.argv[1:] == ["device", "list", "--json-output"] + print(json.dumps([ + {"port": "/dev/other", "hwid": "SER=JTAG-1"}, + {"port": "/dev/fake-uart", "hwid": "USB VID:PID=10C4:EA60 SER=JTAG-1 LOCATION=1-1"}, + ])) + """)) + + completed = self._run_cli(pio) + + self.assertEqual(completed.returncode, 0, completed.stderr) + self.assertEqual(completed.stdout, "JTAG-1\n") + + def test_wrong_mapping_and_malformed_output_fail_without_stdout(self): + cases = { + "wrong": "import json; print(json.dumps([{\"port\": \"/dev/fake-uart\", \"hwid\": \"SER=OTHER\"}]))\n", + "malformed": "print('not json')\n", + } + for name, body in cases.items(): + with self.subTest(name=name), tempfile.TemporaryDirectory() as directory: + completed = self._run_cli(self._fake_pio(Path(directory), body)) + + self.assertNotEqual(completed.returncode, 0) + self.assertEqual(completed.stdout, "") + + def test_timeout_fails_without_stdout(self): + with tempfile.TemporaryDirectory() as directory: + pio = self._fake_pio(Path(directory), "import time; time.sleep(30)\n") + + completed = self._run_cli(pio, "--timeout-seconds", "0.05") + + self.assertNotEqual(completed.returncode, 0) + self.assertEqual(completed.stdout, "") + class FixtureContractTests(unittest.TestCase): def test_esp32s3_gpio_hil_fixture_contract(self): @@ -1538,6 +1658,7 @@ def _fake_runtime(self, directory, runtime, mode="success", repair_iterations=6, raise SystemExit(0) assert '--model' not in args assert Path(os.environ['EXPECTED_INSTRUCTIONS']).read_text(encoding='utf-8') in args[-1] + assert 'This trial is noninteractive.' in args[-1] assert 'GPIO 2 is driven high' in args[-1] assert 'Maximum repair attempts: {repair_iterations}' in args[-1] assert 'hil-oracle.json' not in args[-1] @@ -1549,6 +1670,7 @@ def _fake_runtime(self, directory, runtime, mode="success", repair_iterations=6, assert not (workspace / 'hidden').exists() instruction = workspace / {'CLAUDE.md' if runtime == 'claude' else 'AGENTS.md'!r} assert instruction.read_text(encoding='utf-8') == Path(os.environ['EXPECTED_INSTRUCTIONS']).read_text(encoding='utf-8') + assert 'This trial is noninteractive.' in instruction.read_text(encoding='utf-8') if {mode!r} == 'timeout': time.sleep(30) source.write_text(contents.replace('GPIO_MODE_INPUT', 'GPIO_MODE_OUTPUT'), encoding='utf-8') diff --git a/tests/twin2silicon-runtime-smoke.sh b/tests/twin2silicon-runtime-smoke.sh index c07a701..35199af 100755 --- a/tests/twin2silicon-runtime-smoke.sh +++ b/tests/twin2silicon-runtime-smoke.sh @@ -64,10 +64,19 @@ print_version openocd "$LABWIRED_OPENOCD" --version repository_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" task="${LABWIRED_TASK:-esp32s3-gpio-hil-001}" +identity_command_json="$(python3 - "$repository_root/benchmarks/twin2silicon/identify_pio_device.py" "$LABWIRED_UART_DEVICE" "$LABWIRED_JTAG_SERIAL" <<'PY' +import json +import sys + +print(json.dumps([sys.executable, sys.argv[1], "--uart-device", sys.argv[2], + "--jtag-serial", sys.argv[3]])) +PY +)" python3 "$repository_root/benchmarks/twin2silicon/run_matrix.py" \ --task "$task" \ --output "$LABWIRED_MATRIX_OUTPUT" \ --jtag-serial "$LABWIRED_JTAG_SERIAL" \ --uart-device "$LABWIRED_UART_DEVICE" \ - --openocd "$LABWIRED_OPENOCD" + --openocd "$LABWIRED_OPENOCD" \ + --identity-command-json "$identity_command_json" From a51e6179421022aa7b982532caa00f586c6d76be Mon Sep 17 00:00:00 2001 From: Andrii Shylenko <14119286+w1ne@users.noreply.github.com> Date: Sat, 15 Aug 2026 22:24:06 +0200 Subject: [PATCH 46/48] fix(bench): pin HIL flash to identified UART --- benchmarks/twin2silicon/run_hil.py | 2 +- tests/twin2silicon-hil.py | 7 +++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/benchmarks/twin2silicon/run_hil.py b/benchmarks/twin2silicon/run_hil.py index 1bd3192..f9268d6 100644 --- a/benchmarks/twin2silicon/run_hil.py +++ b/benchmarks/twin2silicon/run_hil.py @@ -134,7 +134,7 @@ def save() -> None: if identity.status != "pass": raise RuntimeError(identity.detail or "board identity failed") - flash_command = build + ["--target", config.flash_target] + flash_command = build + ["--upload-port", args.uart_device, "--target", config.flash_target] flashed = flash_firmware(flash_command, cwd=workspace, evidence_dir=run_dir, timeout_seconds=config.flash_timeout_seconds, identity_validated=True) diff --git a/tests/twin2silicon-hil.py b/tests/twin2silicon-hil.py index cdd5cf4..c1bafbe 100644 --- a/tests/twin2silicon-hil.py +++ b/tests/twin2silicon-hil.py @@ -1526,6 +1526,9 @@ def test_complete_fake_hil_pass(self): ) flash_marker = root / "flashed" run_dir = root / "run" + master, slave = pty.openpty() + uart = os.ttyname(slave) + wrong_uart = "/dev/cu.usbmodem11201" pio_dir = root / "pio" pio_dir.mkdir() pio = executable_fixture(pio_dir, textwrap.dedent(f""" @@ -1535,6 +1538,8 @@ def test_complete_fake_hil_pass(self): if 'clean' in args: raise SystemExit(0) if 'upload' in args: + assert args[args.index('--upload-port') + 1] == {uart!r} + assert {wrong_uart!r} not in args uart_log = pathlib.Path({str(root / "run/uart.log")!r}) deadline = time.monotonic() + 1 while time.monotonic() < deadline and not uart_log.exists(): @@ -1560,8 +1565,6 @@ def test_complete_fake_hil_pass(self): print('@@REG gpio2_output_high 0x60004004', file=sys.stderr) print('0x60004004: 00000004', file=sys.stderr) """)) - master, slave = pty.openpty() - uart = os.ttyname(slave) def write_uart(): deadline = time.monotonic() + 10 header = run_dir / "workspace/firmware/include/run_nonce.h" From 8b02a039f55670ed15c80a18372a62e01ab4f000 Mon Sep 17 00:00:00 2001 From: Andrii Shylenko <14119286+w1ne@users.noreply.github.com> Date: Sat, 15 Aug 2026 22:36:37 +0200 Subject: [PATCH 47/48] fix(bench): emit complete runtime comparison evidence --- benchmarks/twin2silicon/run_agent.py | 11 +- benchmarks/twin2silicon/run_matrix.py | 92 +++++++++++- benchmarks/twin2silicon/runtime_adapters.py | 79 ++++++++++- .../native-002/claude.stdout.jsonl | 2 + .../native-002/codex.stdout.jsonl | 2 + tests/twin2silicon-hil.py | 134 +++++++++++++++++- 6 files changed, 300 insertions(+), 20 deletions(-) create mode 100644 fixtures/twin2silicon/native-002/claude.stdout.jsonl create mode 100644 fixtures/twin2silicon/native-002/codex.stdout.jsonl diff --git a/benchmarks/twin2silicon/run_agent.py b/benchmarks/twin2silicon/run_agent.py index de8ac50..998f158 100644 --- a/benchmarks/twin2silicon/run_agent.py +++ b/benchmarks/twin2silicon/run_agent.py @@ -27,6 +27,7 @@ AdapterContext, NormalizedUsage, build_runtime_command, + extract_native_model, normalize_usage, write_codex_mcp_config, ) @@ -208,6 +209,7 @@ def main(argv: list[str] | None = None) -> int: "schema_version": "1.0", "runtime": args.runtime, "model_override": None, + "native_model": None, "status": "infrastructure_error", "returncode": None, "timed_out": False, @@ -269,10 +271,11 @@ def main(argv: list[str] | None = None) -> int: else: result["status"] = "completed" try: - usage = normalize_usage( - args.runtime, - context.stdout_path.read_text(encoding="utf-8", errors="replace").splitlines(), - ) + stdout_lines = context.stdout_path.read_text( + encoding="utf-8", errors="replace" + ).splitlines() + usage = normalize_usage(args.runtime, stdout_lines) + result["native_model"] = extract_native_model(args.runtime, stdout_lines) except OSError: usage = _unavailable_usage() except Exception as error: diff --git a/benchmarks/twin2silicon/run_matrix.py b/benchmarks/twin2silicon/run_matrix.py index 2fc0633..c08706c 100644 --- a/benchmarks/twin2silicon/run_matrix.py +++ b/benchmarks/twin2silicon/run_matrix.py @@ -11,6 +11,7 @@ from pathlib import Path import subprocess import sys +import time from typing import Any if __package__ in (None, ""): @@ -99,6 +100,22 @@ def _number(value: object) -> int | float | None: return value +def _safe_text(value: object) -> str | None: + if not isinstance(value, str): + return None + text = " ".join(value.split()) + return text[:4096] if text else None + + +def _result_number(result: dict[str, Any] | None, name: str) -> int | float | None: + return _number(result.get(name)) if result else None + + +def _result_bool(result: dict[str, Any] | None, name: str) -> bool | None: + value = result.get(name) if result else None + return value if isinstance(value, bool) else None + + def _normalized_usage(path: Path) -> dict[str, object]: source = _read_json(path) or {} normalized = {field: _number(source.get(field)) for field in USAGE_FIELDS} @@ -168,17 +185,43 @@ def _agent_row( ) result = _read_json(trial / "agent-result.json") agent_status = result.get("status") if result and isinstance(result.get("status"), str) else "infrastructure_error" + infrastructure_category: str | None = None + infrastructure_error: str | None = None if child_error is not None or returncode != 0: agent_status = "infrastructure_error" + infrastructure_category = "agent_runner" + infrastructure_error = _safe_text( + child_error if child_error is not None else f"agent runner exited with status {returncode}" + ) + elif result is None: + infrastructure_category = "agent_runner" + infrastructure_error = "agent runner did not produce agent-result.json" + elif agent_status == "infrastructure_error": + infrastructure_category = "agent_runtime" + infrastructure_error = _safe_text(result.get("error")) or "agent runtime infrastructure error" + native_model = _safe_text(result.get("native_model")) if result else None row: dict[str, object] = { "runtime": runtime, + "native_model": native_model, "initial_public_sha256": initial_public_sha256, "agent_status": agent_status, + "agent_returncode": _result_number(result, "returncode"), + "agent_timed_out": _result_bool(result, "timed_out"), + "elapsed_agent_seconds": _result_number(result, "elapsed_seconds"), "agent_result": result, "agent_child_returncode": returncode, "usage": _normalized_usage(trial / "usage.json"), + "repair_count": None, + "tool_call_count": None, + "invalid_call_count": None, + "observability_reason": "runtime did not expose repair/tool-call counts", + "compile_status": "not_run", "hil_status": "not_run", "hil_run": None, + "elapsed_hil_seconds": None, + "final_success": False, + "infrastructure_category": infrastructure_category, + "infrastructure_error": infrastructure_error, } if child_error is not None: row["agent_error"] = child_error @@ -196,6 +239,9 @@ def _run_hil(row: dict[str, object], task_root: Path, trial: Path, args: argpars if not candidate.is_dir(): row["hil_status"] = "invalid" row["hil_error"] = "completed agent did not produce a candidate directory" + row["compile_status"] = "invalid" + row["infrastructure_category"] = "candidate" + row["infrastructure_error"] = row["hil_error"] return run_dir = trial / "hil" command = [ @@ -207,27 +253,69 @@ def _run_hil(row: dict[str, object], task_root: Path, trial: Path, args: argpars usage_path = trial / "usage.json" if _has_hil_usage_schema(usage_path): command.extend(("--usage-json", str(usage_path))) + started = time.monotonic() returncode, child_error = _run_child( command, ROOT, trial / "matrix-hil.stdout.log", trial / "matrix-hil.stderr.log", ) + row["elapsed_hil_seconds"] = time.monotonic() - started run = _read_json(run_dir / "run.json") row["hil_run"] = run row["hil_child_returncode"] = returncode if child_error is not None: row["hil_status"] = "invalid" row["hil_error"] = child_error + row["compile_status"] = "invalid" + row["infrastructure_category"] = "hil_runner" + row["infrastructure_error"] = _safe_text(child_error) elif run is None: row["hil_status"] = "invalid" row["hil_error"] = "HIL runner did not produce run.json" + row["compile_status"] = "invalid" + row["infrastructure_category"] = "hil_runner" + row["infrastructure_error"] = row["hil_error"] else: status = run.get("status") row["hil_status"] = status if isinstance(status, str) else "invalid" + compile_status = run.get("compile_status") + row["compile_status"] = compile_status if isinstance(compile_status, str) else None + row["final_success"] = row["hil_status"] == "pass" + + +def _display(value: object, width: int) -> str: + text = "-" if value is None else str(value) + return text[:width] + + +def _seconds(value: object) -> str: + return f"{value:.3f}" if isinstance(value, (int, float)) else "-" + + +def _compact_usage(usage: object) -> str: + if not isinstance(usage, dict): + return "-" + fresh = usage.get("fresh_input") + cached = usage.get("cached_input") + output = usage.get("output") + reasoning = usage.get("reasoning") + cost = usage.get("estimated_cost_usd") + tokens = f"i={fresh if fresh is not None else '-'}+{cached if cached is not None else '-'}" + tokens += f" o={output if output is not None else '-'} r={reasoning if reasoning is not None else '-'}" + return f"{tokens} ${cost:.6g}" if isinstance(cost, (int, float)) else tokens def _print_summary(rows: list[dict[str, object]]) -> None: - print(f"{'RUNTIME':<10} {'AGENT':<22} {'HIL':<12}") + print( + f"{'RUNTIME':<10} {'MODEL':<20} {'AGENT':<18} {'COMPILE':<12} {'HIL':<12} " + f"{'SUCCESS':<7} {'A_SEC':>8} {'H_SEC':>8} TOKENS/COST" + ) for row in rows: - print(f"{row['runtime']:<10} {row['agent_status']:<22} {row['hil_status']:<12}") + print( + f"{_display(row['runtime'], 10):<10} {_display(row['native_model'], 20):<20} " + f"{_display(row['agent_status'], 18):<18} {_display(row['compile_status'], 12):<12} " + f"{_display(row['hil_status'], 12):<12} {_display(row['final_success'], 7):<7} " + f"{_seconds(row['elapsed_agent_seconds']):>8} {_seconds(row['elapsed_hil_seconds']):>8} " + f"{_compact_usage(row['usage'])}" + ) def main(argv: list[str] | None = None) -> int: diff --git a/benchmarks/twin2silicon/runtime_adapters.py b/benchmarks/twin2silicon/runtime_adapters.py index d66c7fb..1fb1adf 100644 --- a/benchmarks/twin2silicon/runtime_adapters.py +++ b/benchmarks/twin2silicon/runtime_adapters.py @@ -4,6 +4,12 @@ directory. Claude does not expose a workspace command-line option, so its MCP configuration is deliberately placed beneath ``config_dir``; the command uses ``config_dir / 'claude-mcp.json'`` without creating that file. + +Codex reports ``output_tokens`` as its total output token count, so its +``reasoning_output_tokens`` is retained as a subset rather than subtracted. +Claude's ``output_tokens_details.thinking_tokens`` is also a subset of the +reported output total; for a non-overlapping comparison output field we retain +thinking as ``reasoning`` and subtract it from ``output``. """ from __future__ import annotations @@ -122,6 +128,36 @@ def normalize_usage(runtime: RuntimeName, lines: Iterable[str]) -> NormalizedUsa raise ValueError(f"unsupported runtime: {runtime}") +def extract_native_model(runtime: RuntimeName, lines: Iterable[str]) -> str | None: + """Return a model only when a runtime's structured event explicitly names it. + + Runtime versions and prompts are not model evidence. The narrow event + locations below are deliberately conservative so an arbitrary JSON field + cannot be reported as a native model selection. + """ + + for record in _json_records(lines): + if runtime == "claude": + if record.get("type") == "system" and record.get("subtype") == "init": + model = _model_text(record.get("model")) + if model is not None: + return model + elif runtime == "codex": + if record.get("type") == "turn.started": + model = _model_text(record.get("model")) + if model is not None: + return model + elif runtime == "opencode": + if record.get("type") == "step_start": + part = _mapping(record.get("part")) + model = _model_text(part.get("model")) if part else None + if model is not None: + return model + else: + raise ValueError(f"unsupported runtime: {runtime}") + return None + + def _json_records(lines: Iterable[str]) -> Iterator[dict[str, object]]: source = lines.splitlines() if isinstance(lines, str) else lines for line in source: @@ -187,7 +223,7 @@ def _normalize_codex(records: Iterable[dict[str, object]]) -> NormalizedUsage: if record.get("type") != "turn.completed": continue usage = _mapping(record.get("usage")) - final_usage = _token_values(usage) + final_usage = _codex_token_values(usage) if final_usage is None or not any( value is not None for value in final_usage.values() @@ -205,11 +241,23 @@ def _normalize_claude(records: Iterable[dict[str, object]]) -> NormalizedUsage: if record.get("type") != "result": continue usage = _mapping(record.get("usage")) + details = _mapping(usage.get("output_tokens_details")) if usage else None + total_output = _bounded_int(usage.get("output_tokens")) if usage else None + thinking = _bounded_int(details.get("thinking_tokens")) if details else None + if ( + total_output is not None + and thinking is not None + and thinking > total_output + ): + thinking = None + output = None + else: + output = total_output - thinking if total_output is not None and thinking is not None else total_output values = { "fresh_input": _bounded_int(usage.get("input_tokens")) if usage else None, "cached_input": _bounded_int(usage.get("cache_read_input_tokens")) if usage else None, - "reasoning": _bounded_int(usage.get("reasoning_tokens")) if usage else None, - "output": _bounded_int(usage.get("output_tokens")) if usage else None, + "reasoning": thinking, + "output": output, } cost = _bounded_float(record.get("total_cost_usd")) final_usage = values @@ -227,7 +275,7 @@ def _normalize_claude(records: Iterable[dict[str, object]]) -> NormalizedUsage: ) -def _token_values(usage: dict[str, object] | None) -> dict[str, int | None]: +def _codex_token_values(usage: dict[str, object] | None) -> dict[str, int | None]: if usage is None: return { "fresh_input": None, @@ -235,10 +283,20 @@ def _token_values(usage: dict[str, object] | None) -> dict[str, int | None]: "reasoning": None, "output": None, } + total_input = _bounded_int(usage.get("input_tokens")) + cached_input = _bounded_int(usage.get("cached_input_tokens")) + if total_input is not None and cached_input is None: + cached_input = 0 + if total_input is None or cached_input is None or cached_input > total_input: + fresh_input = None + if total_input is not None and cached_input is not None and cached_input > total_input: + cached_input = None + else: + fresh_input = total_input - cached_input return { - "fresh_input": _bounded_int(usage.get("input_tokens")), - "cached_input": _bounded_int(usage.get("cached_input_tokens")), - "reasoning": _bounded_int(usage.get("reasoning_tokens")), + "fresh_input": fresh_input, + "cached_input": cached_input, + "reasoning": _bounded_int(usage.get("reasoning_output_tokens")), "output": _bounded_int(usage.get("output_tokens")), } @@ -282,3 +340,10 @@ def _bounded_float(value: object) -> float | None: if not math.isfinite(number) or not 0 <= number <= _MAX_COST_USD: return None return number + + +def _model_text(value: object) -> str | None: + if not isinstance(value, str): + return None + text = value.strip() + return text[:4096] if text else None diff --git a/fixtures/twin2silicon/native-002/claude.stdout.jsonl b/fixtures/twin2silicon/native-002/claude.stdout.jsonl new file mode 100644 index 0000000..0afd5db --- /dev/null +++ b/fixtures/twin2silicon/native-002/claude.stdout.jsonl @@ -0,0 +1,2 @@ +{"type":"system","subtype":"init","model":"claude-sonnet-4-20250514"} +{"type":"result","subtype":"success","usage":{"input_tokens":5732,"cache_read_input_tokens":2048,"output_tokens":961,"output_tokens_details":{"thinking_tokens":417}},"total_cost_usd":0.042} diff --git a/fixtures/twin2silicon/native-002/codex.stdout.jsonl b/fixtures/twin2silicon/native-002/codex.stdout.jsonl new file mode 100644 index 0000000..2961acd --- /dev/null +++ b/fixtures/twin2silicon/native-002/codex.stdout.jsonl @@ -0,0 +1,2 @@ +{"type":"thread.started","thread_id":"thread-native-002"} +{"type":"turn.completed","usage":{"input_tokens":16431,"cached_input_tokens":12288,"reasoning_output_tokens":621,"output_tokens":1472}} diff --git a/tests/twin2silicon-hil.py b/tests/twin2silicon-hil.py index c1bafbe..34faf88 100644 --- a/tests/twin2silicon-hil.py +++ b/tests/twin2silicon-hil.py @@ -46,6 +46,7 @@ NormalizedUsage, build_runtime_command, codex_mcp_toml, + extract_native_model, normalize_usage, write_codex_mcp_config, ) @@ -168,24 +169,52 @@ def test_normalize_codex_uses_final_cumulative_usage(self): lines = [ json.dumps({"type": "turn.completed", "usage": { "input_tokens": 1000, "cached_input_tokens": 200, - "reasoning_tokens": 10, "output_tokens": 500, + "reasoning_output_tokens": 10, "output_tokens": 500, }}), json.dumps({"type": "turn.completed", "usage": { "input_tokens": 1400, "cached_input_tokens": 300, - "reasoning_tokens": 48, "output_tokens": 1076, + "reasoning_output_tokens": 48, "output_tokens": 1076, }}), ] usage = normalize_usage("codex", lines) self.assertEqual(usage.requests, 1) - self.assertEqual(usage.fresh_input, 1400) + self.assertEqual(usage.fresh_input, 1100) self.assertEqual(usage.cached_input, 300) self.assertEqual(usage.reasoning, 48) self.assertEqual(usage.output, 1076) self.assertIsNone(usage.estimated_cost_usd) self.assertIsNone(usage.unavailable_reason) + def test_normalize_codex_native_002_usage_keeps_total_output_with_reasoning_subset(self): + """Codex native-002 output_tokens is total output, including reasoning.""" + lines = (REPOSITORY_ROOT / "fixtures/twin2silicon/native-002/codex.stdout.jsonl").read_text().splitlines() + + usage = normalize_usage("codex", lines) + + self.assertEqual( + usage, + NormalizedUsage(1, 4143, 12288, 621, 1472, None, None), + ) + + def test_normalize_codex_rejects_cached_input_larger_than_total(self): + usage = normalize_usage("codex", [json.dumps({ + "type": "turn.completed", + "usage": { + "input_tokens": 10, + "cached_input_tokens": 11, + "reasoning_output_tokens": 2, + "output_tokens": 5, + }, + })]) + + self.assertEqual(usage.requests, 1) + self.assertIsNone(usage.fresh_input) + self.assertIsNone(usage.cached_input) + self.assertEqual(usage.reasoning, 2) + self.assertEqual(usage.output, 5) + def test_normalize_codex_uses_the_final_terminal_event(self): lines = [ json.dumps({"type": "turn.completed", "usage": { @@ -240,6 +269,54 @@ def test_normalize_claude_final_result_usage_and_cost(self): self.assertAlmostEqual(usage.estimated_cost_usd, 0.007476282) self.assertIsNone(usage.unavailable_reason) + def test_normalize_claude_native_002_usage_excludes_thinking_from_output(self): + lines = (REPOSITORY_ROOT / "fixtures/twin2silicon/native-002/claude.stdout.jsonl").read_text().splitlines() + + usage = normalize_usage("claude", lines) + + self.assertEqual( + usage, + NormalizedUsage(1, 5732, 2048, 417, 544, 0.042, None), + ) + + def test_normalize_claude_rejects_thinking_larger_than_total_output(self): + usage = normalize_usage("claude", [json.dumps({ + "type": "result", + "usage": { + "input_tokens": 10, + "output_tokens": 3, + "output_tokens_details": {"thinking_tokens": 4}, + }, + })]) + + self.assertEqual(usage.requests, 1) + self.assertEqual(usage.fresh_input, 10) + self.assertIsNone(usage.reasoning) + self.assertIsNone(usage.output) + + def test_extract_native_model_accepts_only_runtime_event_model_fields(self): + self.assertEqual( + extract_native_model("claude", [json.dumps({ + "type": "system", "subtype": "init", "model": "claude-sonnet-4-20250514", + })]), + "claude-sonnet-4-20250514", + ) + self.assertEqual( + extract_native_model("codex", [json.dumps({ + "type": "turn.started", "model": "gpt-5-codex", + })]), + "gpt-5-codex", + ) + self.assertEqual( + extract_native_model("opencode", [json.dumps({ + "type": "step_start", "part": {"model": "deepseek-r1"}, + })]), + "deepseek-r1", + ) + self.assertIsNone(extract_native_model("codex", [json.dumps({ + "type": "turn.completed", "model": "untrusted", "usage": {}, + })])) + def test_normalize_claude_uses_the_final_result_event(self): lines = [ json.dumps({ @@ -405,7 +482,9 @@ def _fake_agent(self, directory, usage_schema="normalized", child_returncode=0): status = {{"opencode": "completed", "codex": "failed", "claude": "completed"}}[args.runtime] (output / "agent-result.json").write_text(json.dumps({{ "schema_version": "1.0", "runtime": args.runtime, + "native_model": f"fake-{{args.runtime}}-model", "status": status, "returncode": 0 if status == "completed" else 9, + "timed_out": False, "elapsed_seconds": 0.25, }})) usage = {{ "requests": 1 if args.runtime != "codex" else None, @@ -455,7 +534,8 @@ def _fake_hil(self, directory): run_dir.mkdir(parents=True) status = "pass" if Path(args.candidate).parent.name == "opencode" else "fail" (run_dir / "run.json").write_text(json.dumps({ - "schema_version": "1.0", "status": status, "cost": None, + "schema_version": "1.0", "status": status, + "compile_status": status, "cost": None, })) (run_dir / "hil-invocation.json").write_text(json.dumps(vars(args), sort_keys=True)) """) @@ -508,6 +588,43 @@ def test_matrix_preserves_runtime_order_hashes_and_failure_isolation(self): hil_invocation = (trial / "hil" / "hil-invocation.json").read_text(encoding="utf-8") self.assertNotIn("--usage-json", hil_invocation) + def test_matrix_emits_complete_comparison_evidence(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + completed = self._run_cli( + root / "matrix", self._fake_agent(root / "agent"), self._fake_hil(root / "hil"), + ) + + self.assertEqual(completed.returncode, 0, completed.stderr) + rows = json.loads((root / "matrix" / "matrix.json").read_text())["trials"] + passed, failed, agent_only = rows + + self.assertEqual(passed["native_model"], "fake-opencode-model") + self.assertEqual(passed["agent_returncode"], 0) + self.assertFalse(passed["agent_timed_out"]) + self.assertEqual(passed["compile_status"], "pass") + self.assertEqual(passed["hil_status"], "pass") + self.assertTrue(passed["final_success"]) + self.assertEqual(passed["elapsed_agent_seconds"], 0.25) + self.assertGreaterEqual(passed["elapsed_hil_seconds"], 0) + self.assertIsNone(passed["repair_count"]) + self.assertIsNone(passed["tool_call_count"]) + self.assertIsNone(passed["invalid_call_count"]) + self.assertEqual(passed["observability_reason"], "runtime did not expose repair/tool-call counts") + self.assertIsNone(passed["infrastructure_category"]) + self.assertIsNone(passed["infrastructure_error"]) + + self.assertEqual(failed["agent_status"], "failed") + self.assertEqual(failed["compile_status"], "not_run") + self.assertFalse(failed["final_success"]) + self.assertEqual(agent_only["compile_status"], "fail") + self.assertFalse(agent_only["final_success"]) + self.assertIn("MODEL", completed.stdout) + self.assertIn("COMPILE", completed.stdout) + self.assertIn("A_SEC", completed.stdout) + self.assertIn("H_SEC", completed.stdout) + self.assertIn("TOKENS/COST", completed.stdout) + def test_agent_only_skips_hil_and_repeated_runtime_selects_trials(self): with tempfile.TemporaryDirectory() as directory: root = Path(directory) @@ -522,6 +639,8 @@ def test_agent_only_skips_hil_and_repeated_runtime_selects_trials(self): self.assertEqual([row["hil_status"] for row in rows], ["not_run", "not_run"]) self.assertTrue(all(row["hil_run"] is None for row in rows)) self.assertFalse(any((root / "matrix" / "trials").glob("*/hil"))) + self.assertTrue(all(row["compile_status"] == "not_run" for row in rows)) + self.assertTrue(all(not row["final_success"] for row in rows)) def test_matrix_requires_identity_unless_agent_only(self): with tempfile.TemporaryDirectory() as directory: @@ -1644,9 +1763,9 @@ def _fake_runtime(self, directory, runtime, mode="success", repair_iterations=6, "opencode": "workspace = Path(args[args.index('--dir') + 1])\nassert args[:2] == ['run', '--format']\nassert args[args.index('--format') + 1] == 'json'\nassert os.environ['OPENCODE_CONFIG'] == str(Path(os.environ['EXPECTED_CONFIG']) / 'opencode.json')", }[runtime] output = { - "codex": "print(json.dumps({'type': 'turn.completed', 'usage': {'input_tokens': 12, 'output_tokens': 3}}))", - "claude": "print(json.dumps({'type': 'result', 'usage': {'input_tokens': 12, 'output_tokens': 3}, 'total_cost_usd': 0.01}))", - "opencode": "print(json.dumps({'type': 'step_finish', 'part': {'tokens': {'input': 12, 'output': 3}, 'cost': 0.01}}))", + "codex": "print(json.dumps({'type': 'turn.started', 'model': 'fake-codex-model'})); print(json.dumps({'type': 'turn.completed', 'usage': {'input_tokens': 12, 'cached_input_tokens': 0, 'output_tokens': 3}}))", + "claude": "print(json.dumps({'type': 'system', 'subtype': 'init', 'model': 'fake-claude-model'})); print(json.dumps({'type': 'result', 'usage': {'input_tokens': 12, 'output_tokens': 3}, 'total_cost_usd': 0.01}))", + "opencode": "print(json.dumps({'type': 'step_start', 'part': {'model': 'fake-opencode-model'}})); print(json.dumps({'type': 'step_finish', 'part': {'tokens': {'input': 12, 'output': 3}, 'cost': 0.01}}))", }[runtime] body = textwrap.dedent(f""" import json @@ -1746,6 +1865,7 @@ def test_native_runtimes_create_completed_public_candidates(self): self.assertEqual(result["status"], "completed") self.assertEqual(result["runtime"], runtime) self.assertIsNone(result["model_override"]) + self.assertEqual(result["native_model"], f"fake-{runtime}-model") self.assertEqual(result["returncode"], 0) self.assertFalse(result["timed_out"]) self.assertGreaterEqual(result["elapsed_seconds"], 0) From d13400e372d4c0a8138ce2d7369d1161fff62a82 Mon Sep 17 00:00:00 2001 From: Andrii Shylenko <14119286+w1ne@users.noreply.github.com> Date: Sat, 15 Aug 2026 22:39:54 +0200 Subject: [PATCH 48/48] fix(bench): complete runtime evidence details --- benchmarks/twin2silicon/run_agent.py | 4 ++ benchmarks/twin2silicon/run_matrix.py | 20 ++++-- benchmarks/twin2silicon/runtime_adapters.py | 17 +++-- .../native-002/codex.stdout.jsonl | 2 +- tests/twin2silicon-hil.py | 67 +++++++++++++++++-- 5 files changed, 97 insertions(+), 13 deletions(-) diff --git a/benchmarks/twin2silicon/run_agent.py b/benchmarks/twin2silicon/run_agent.py index 998f158..8fadc84 100644 --- a/benchmarks/twin2silicon/run_agent.py +++ b/benchmarks/twin2silicon/run_agent.py @@ -210,6 +210,7 @@ def main(argv: list[str] | None = None) -> int: "runtime": args.runtime, "model_override": None, "native_model": None, + "native_model_unavailable_reason": "runtime did not expose model", "status": "infrastructure_error", "returncode": None, "timed_out": False, @@ -276,6 +277,9 @@ def main(argv: list[str] | None = None) -> int: ).splitlines() usage = normalize_usage(args.runtime, stdout_lines) result["native_model"] = extract_native_model(args.runtime, stdout_lines) + result["native_model_unavailable_reason"] = ( + None if result["native_model"] is not None else "runtime did not expose model" + ) except OSError: usage = _unavailable_usage() except Exception as error: diff --git a/benchmarks/twin2silicon/run_matrix.py b/benchmarks/twin2silicon/run_matrix.py index c08706c..6b73258 100644 --- a/benchmarks/twin2silicon/run_matrix.py +++ b/benchmarks/twin2silicon/run_matrix.py @@ -200,9 +200,17 @@ def _agent_row( infrastructure_category = "agent_runtime" infrastructure_error = _safe_text(result.get("error")) or "agent runtime infrastructure error" native_model = _safe_text(result.get("native_model")) if result else None + native_model_unavailable_reason = ( + None + if native_model is not None + else _safe_text(result.get("native_model_unavailable_reason")) if result else None + ) + if native_model is None and native_model_unavailable_reason is None: + native_model_unavailable_reason = "runtime did not expose model" row: dict[str, object] = { "runtime": runtime, "native_model": native_model, + "native_model_unavailable_reason": native_model_unavailable_reason, "initial_public_sha256": initial_public_sha256, "agent_status": agent_status, "agent_returncode": _result_number(result, "returncode"), @@ -305,14 +313,18 @@ def _compact_usage(usage: object) -> str: def _print_summary(rows: list[dict[str, object]]) -> None: print( - f"{'RUNTIME':<10} {'MODEL':<20} {'AGENT':<18} {'COMPILE':<12} {'HIL':<12} " - f"{'SUCCESS':<7} {'A_SEC':>8} {'H_SEC':>8} TOKENS/COST" + f"{'RUNTIME':<10} {'MODEL':<20} {'AGENT':<18} {'RETURN':<7} {'TIMEOUT':<7} " + f"{'REPAIR':<7} {'TOOLS':<7} {'INVALID':<7} {'COMPILE':<12} {'HIL':<12} " + f"{'SUCCESS':<7} {'INFRA':<14} {'A_SEC':>8} {'H_SEC':>8} TOKENS/COST" ) for row in rows: print( f"{_display(row['runtime'], 10):<10} {_display(row['native_model'], 20):<20} " - f"{_display(row['agent_status'], 18):<18} {_display(row['compile_status'], 12):<12} " - f"{_display(row['hil_status'], 12):<12} {_display(row['final_success'], 7):<7} " + f"{_display(row['agent_status'], 18):<18} {_display(row['agent_returncode'], 7):<7} " + f"{_display(row['agent_timed_out'], 7):<7} {_display(row['repair_count'], 7):<7} " + f"{_display(row['tool_call_count'], 7):<7} {_display(row['invalid_call_count'], 7):<7} " + f"{_display(row['compile_status'], 12):<12} {_display(row['hil_status'], 12):<12} " + f"{_display(row['final_success'], 7):<7} {_display(row['infrastructure_category'], 14):<14} " f"{_seconds(row['elapsed_agent_seconds']):>8} {_seconds(row['elapsed_hil_seconds']):>8} " f"{_compact_usage(row['usage'])}" ) diff --git a/benchmarks/twin2silicon/runtime_adapters.py b/benchmarks/twin2silicon/runtime_adapters.py index 1fb1adf..79466e9 100644 --- a/benchmarks/twin2silicon/runtime_adapters.py +++ b/benchmarks/twin2silicon/runtime_adapters.py @@ -5,8 +5,9 @@ configuration is deliberately placed beneath ``config_dir``; the command uses ``config_dir / 'claude-mcp.json'`` without creating that file. -Codex reports ``output_tokens`` as its total output token count, so its -``reasoning_output_tokens`` is retained as a subset rather than subtracted. +``NormalizedUsage.output`` consistently excludes separately reported reasoning +tokens. Codex reports ``output_tokens`` as its total output token count, so +its ``reasoning_output_tokens`` is subtracted when both values are valid. Claude's ``output_tokens_details.thinking_tokens`` is also a subset of the reported output total; for a non-overlapping comparison output field we retain thinking as ``reasoning`` and subtract it from ``output``. @@ -293,11 +294,19 @@ def _codex_token_values(usage: dict[str, object] | None) -> dict[str, int | None cached_input = None else: fresh_input = total_input - cached_input + reasoning = _bounded_int(usage.get("reasoning_output_tokens")) + output = _bounded_int(usage.get("output_tokens")) + if reasoning is not None and output is not None: + if reasoning > output: + reasoning = None + output = None + else: + output -= reasoning return { "fresh_input": fresh_input, "cached_input": cached_input, - "reasoning": _bounded_int(usage.get("reasoning_output_tokens")), - "output": _bounded_int(usage.get("output_tokens")), + "reasoning": reasoning, + "output": output, } diff --git a/fixtures/twin2silicon/native-002/codex.stdout.jsonl b/fixtures/twin2silicon/native-002/codex.stdout.jsonl index 2961acd..46c7f98 100644 --- a/fixtures/twin2silicon/native-002/codex.stdout.jsonl +++ b/fixtures/twin2silicon/native-002/codex.stdout.jsonl @@ -1,2 +1,2 @@ {"type":"thread.started","thread_id":"thread-native-002"} -{"type":"turn.completed","usage":{"input_tokens":16431,"cached_input_tokens":12288,"reasoning_output_tokens":621,"output_tokens":1472}} +{"type":"turn.completed","usage":{"input_tokens":16431,"cached_input_tokens":12288,"reasoning_output_tokens":621,"output_tokens":2892}} diff --git a/tests/twin2silicon-hil.py b/tests/twin2silicon-hil.py index 34faf88..dd9db45 100644 --- a/tests/twin2silicon-hil.py +++ b/tests/twin2silicon-hil.py @@ -183,19 +183,18 @@ def test_normalize_codex_uses_final_cumulative_usage(self): self.assertEqual(usage.fresh_input, 1100) self.assertEqual(usage.cached_input, 300) self.assertEqual(usage.reasoning, 48) - self.assertEqual(usage.output, 1076) + self.assertEqual(usage.output, 1028) self.assertIsNone(usage.estimated_cost_usd) self.assertIsNone(usage.unavailable_reason) - def test_normalize_codex_native_002_usage_keeps_total_output_with_reasoning_subset(self): - """Codex native-002 output_tokens is total output, including reasoning.""" + def test_normalize_codex_native_002_usage_excludes_reasoning_from_output(self): lines = (REPOSITORY_ROOT / "fixtures/twin2silicon/native-002/codex.stdout.jsonl").read_text().splitlines() usage = normalize_usage("codex", lines) self.assertEqual( usage, - NormalizedUsage(1, 4143, 12288, 621, 1472, None, None), + NormalizedUsage(1, 4143, 12288, 621, 2271, None, None), ) def test_normalize_codex_rejects_cached_input_larger_than_total(self): @@ -213,8 +212,31 @@ def test_normalize_codex_rejects_cached_input_larger_than_total(self): self.assertIsNone(usage.fresh_input) self.assertIsNone(usage.cached_input) self.assertEqual(usage.reasoning, 2) + self.assertEqual(usage.output, 3) + + def test_normalize_codex_keeps_output_total_when_reasoning_is_absent(self): + usage = normalize_usage("codex", [json.dumps({ + "type": "turn.completed", + "usage": {"input_tokens": 10, "cached_input_tokens": 0, "output_tokens": 5}, + })]) + + self.assertIsNone(usage.reasoning) self.assertEqual(usage.output, 5) + def test_normalize_codex_rejects_reasoning_larger_than_total_output(self): + usage = normalize_usage("codex", [json.dumps({ + "type": "turn.completed", + "usage": { + "input_tokens": 10, + "cached_input_tokens": 0, + "reasoning_output_tokens": 6, + "output_tokens": 5, + }, + })]) + + self.assertIsNone(usage.reasoning) + self.assertIsNone(usage.output) + def test_normalize_codex_uses_the_final_terminal_event(self): lines = [ json.dumps({"type": "turn.completed", "usage": { @@ -483,6 +505,7 @@ def _fake_agent(self, directory, usage_schema="normalized", child_returncode=0): (output / "agent-result.json").write_text(json.dumps({{ "schema_version": "1.0", "runtime": args.runtime, "native_model": f"fake-{{args.runtime}}-model", + "native_model_unavailable_reason": None, "status": status, "returncode": 0 if status == "completed" else 9, "timed_out": False, "elapsed_seconds": 0.25, }})) @@ -600,6 +623,7 @@ def test_matrix_emits_complete_comparison_evidence(self): passed, failed, agent_only = rows self.assertEqual(passed["native_model"], "fake-opencode-model") + self.assertIsNone(passed["native_model_unavailable_reason"]) self.assertEqual(passed["agent_returncode"], 0) self.assertFalse(passed["agent_timed_out"]) self.assertEqual(passed["compile_status"], "pass") @@ -621,9 +645,16 @@ def test_matrix_emits_complete_comparison_evidence(self): self.assertFalse(agent_only["final_success"]) self.assertIn("MODEL", completed.stdout) self.assertIn("COMPILE", completed.stdout) + self.assertIn("RETURN", completed.stdout) + self.assertIn("TIMEOUT", completed.stdout) + self.assertIn("REPAIR", completed.stdout) + self.assertIn("TOOLS", completed.stdout) + self.assertIn("INVALID", completed.stdout) + self.assertIn("INFRA", completed.stdout) self.assertIn("A_SEC", completed.stdout) self.assertIn("H_SEC", completed.stdout) self.assertIn("TOKENS/COST", completed.stdout) + self.assertIn("fake-opencode-model", completed.stdout) def test_agent_only_skips_hil_and_repeated_runtime_selects_trials(self): with tempfile.TemporaryDirectory() as directory: @@ -1866,6 +1897,7 @@ def test_native_runtimes_create_completed_public_candidates(self): self.assertEqual(result["runtime"], runtime) self.assertIsNone(result["model_override"]) self.assertEqual(result["native_model"], f"fake-{runtime}-model") + self.assertIsNone(result["native_model_unavailable_reason"]) self.assertEqual(result["returncode"], 0) self.assertFalse(result["timed_out"]) self.assertGreaterEqual(result["elapsed_seconds"], 0) @@ -1963,6 +1995,33 @@ def test_missing_or_malformed_usage_does_not_fail_a_completed_trial(self): self.assertEqual(usage["unavailable_reason"], "runtime did not expose usage") self._assert_trial_does_not_expose_hidden_oracle(trial) + def test_missing_native_model_has_an_explicit_unavailable_reason(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + executable = executable_fixture(root / "runtime", textwrap.dedent(""" + import json + import sys + + if sys.argv[1:] == ["--version"]: + print("fake-claude 1.0") + raise SystemExit(0) + print(json.dumps({ + "type": "result", + "usage": {"input_tokens": 1, "output_tokens": 1}, + })) + """)) + trial = root / "trial" + + completed = self._run_cli("claude", executable, trial) + + self.assertEqual(completed.returncode, 0, completed.stderr) + result = json.loads((trial / "agent-result.json").read_text()) + self.assertIsNone(result["native_model"]) + self.assertEqual( + result["native_model_unavailable_reason"], + "runtime did not expose model", + ) + def test_existing_output_is_rejected_without_overwrite(self): with tempfile.TemporaryDirectory() as directory: root = Path(directory)