diff --git a/benchmarks/README.md b/benchmarks/README.md new file mode 100644 index 00000000..f6d57dc8 --- /dev/null +++ b/benchmarks/README.md @@ -0,0 +1,80 @@ +# Orcabot benchmark runners + +Templates that turn Orcabot into an English-driven harness for running coding +benchmarks: a Claude Code **orchestrator** terminal you direct in plain language +sets up and runs the benchmark, runs show up as live panes on the canvas, and +results surface in a browser/note block. + +## Benchmarks + +| Benchmark | Dir | Runnable? | Execution model | +|-----------|-----|-----------|-----------------| +| slop-code-bench | [`slopcodebench/`](slopcodebench/) | ✅ yes | **Host** (no Docker) via the fork's tmux executor | +| SWE-bench (Lite/Verified/Full) | [`swebench/`](swebench/) | ✅ yes | Inference on host + **Docker-per-task** eval (Modal or DinD) | +| SWE-bench Pro (public 731) | [`swebench-pro/`](swebench-pro/) | ✅ yes | Inference on host + **Docker-per-task** eval (`--use_local_docker` or Modal) | +| Terminal-Bench 2.1 | [`terminal-bench/`](terminal-bench/) | ✅ yes | **Harbor** harness; Docker-per-task (or `--env daytona`/Modal remote) | +| FrontierCode (Diamond) | [`frontiercode/`](frontiercode/) | ❌ no | **Closed / submission-only** — no public harness, see the note | + +## Two architecture families + +**Family A — host-executable.** The agent-under-test runs directly in the +Orcabot sandbox VM; no inner container. The sandbox VM *is* the isolation +boundary. This is slop-code-bench (via the fork's host-tmux executor), where +each run is mirrored into an attachable read-only tmux pane. + +**Family B — Docker-per-task.** SWE-bench, SWE-bench Pro, and Terminal-Bench run +each task in its own container — that isolation is core to correctness, so it +can't be dropped. Two backends: + +- **Cloud sandbox backend (recommended, Orcabot-friendly):** the harness + offloads container execution to a remote sandbox — **Modal** for + SWE-bench/Pro, **Daytona/Modal** for Harbor. The orchestrator stays thin; the + Orcabot VM never needs a Docker daemon. Needs a provider token as a secret. +- **Docker-in-VM (DinD):** run a Docker daemon inside a **Docker-capable + benchmark sandbox** (a privileged x86 Fly microVM). Heavier; the desktop VM + can't do this. Use when you want everything self-contained. + +For SWE-bench and SWE-bench Pro the **inference phase** (agent → predictions +JSONL) is host-native and needs neither backend — only the **evaluation phase** +does. The predictions file is the clean handoff boundary. + +## Shared pattern (all runnable benchmarks) + +1. **Config phase** — the Orcabot chat wizard (`config-wizard.md` in each dir) + asks agent / model / target / backend, stores keys in the secrets broker, + and provisions the orchestrator terminal + a results block. +2. **Orchestrator** — a Claude Code terminal you drive in English + (`orchestrator-runbook.md`), running the benchmark's native CLI. +3. **Viewers** — the orchestrator spawns a read-only viewer pane per run via its + `create_terminal` MCP tool, using a **read-only** boot command (never a tmux + *control* socket across PTYs — that would grant read+inject across sessions + and bypass output redaction). The command is benchmark-specific: + `tail -n +1 -F ` (slop-code), `docker logs -f ` / `tail -f logs/…` + (SWE-bench family), `harbor`/Modal log streaming (Terminal-Bench). + +## Seeding the templates into D1 + +These templates are curated, DB-only rows in `dashboard_templates` (there is no +code seed). To make them selectable in the **New Dashboard** grid on a local +desktop stack, seed them directly: + +``` +benchmarks/bin/seed-benchmark-templates # all benchmark templates +benchmarks/bin/seed-benchmark-templates swebench # just one, by dir name +``` + +It upserts each `benchmarks/*/template.json` (delete-by-name then insert, +`status='approved'`) against the desktop d1-shim (`D1_URL`, default +`http://127.0.0.1:9001`). `slopcodebench/bin/scb-insert-template` is a thin +wrapper that seeds only the SlopCodeBench template. For a **remote/cloud** control +plane, export from an existing dashboard via the templates API (`POST /templates`) +instead — direct INSERT is the local/curated path. + +## Status + +All templates here are **prototypes**. slop-code-bench is validated end-to-end +(see its README). The Family-B templates are grounded in each harness's real +commands but are **not** validated end-to-end from here, because they need a +Docker-capable backend (Modal token or DinD sandbox) + provider keys that aren't +available in this environment. Each dir's README states exactly what's verified +vs pending. diff --git a/benchmarks/bin/seed-benchmark-templates b/benchmarks/bin/seed-benchmark-templates new file mode 100755 index 00000000..0228d3e1 --- /dev/null +++ b/benchmarks/bin/seed-benchmark-templates @@ -0,0 +1,94 @@ +#!/usr/bin/env python3 +"""Seed (or refresh) every benchmark dashboard template into D1. + +Templates are DB-only -- there is no code seed -- so a curated template has to be +written to the `dashboard_templates` table directly. This discovers every +`benchmarks/*/template.json` and upserts each one (delete-by-name then insert), +status='approved' so they show in the New Dashboard grid immediately. + +This is the generic path; `slopcodebench/bin/scb-insert-template` delegates here +for the single SlopCodeBench template. + +Usage: + bin/seed-benchmark-templates # all templates, desktop d1-shim + bin/seed-benchmark-templates swebench # only benchmarks/swebench/template.json + D1_URL=http://127.0.0.1:9001 AUTHOR_ID=dev-desktop bin/seed-benchmark-templates + +For a remote/cloud control plane, export a template instead via the templates API +from an existing dashboard (POST /templates) -- direct INSERT is the local/curated +path. +""" +import datetime +import json +import os +import pathlib +import sys +import urllib.request +import uuid + +D1_URL = os.environ.get("D1_URL", "http://127.0.0.1:9001") +AUTHOR_ID = os.environ.get("AUTHOR_ID", "dev-desktop") +AUTHOR_NAME = os.environ.get("AUTHOR_NAME", "OrcaBot") + +# benchmarks/ root (this file lives in benchmarks/bin/). +ROOT = pathlib.Path(__file__).resolve().parent.parent + + +def q(sql, params): + req = urllib.request.Request( + D1_URL.rstrip("/") + "/query", + data=json.dumps({"sql": sql, "params": params}).encode(), + headers={"Content-Type": "application/json"}, + ) + return json.load(urllib.request.urlopen(req)) + + +def discover(filter_dirs): + """Return the template.json paths to seed, optionally limited to named dirs.""" + if filter_dirs: + paths = [] + for name in filter_dirs: + p = ROOT / name / "template.json" + if not p.exists(): + sys.exit(f"no template.json in benchmarks/{name}/") + paths.append(p) + return paths + return sorted(ROOT.glob("*/template.json")) + + +def upsert(tpl): + tid = str(uuid.uuid4()) + now = datetime.datetime.utcnow().isoformat() + q("DELETE FROM dashboard_templates WHERE name = ?", [tpl["name"]]) + q( + """INSERT INTO dashboard_templates + (id,name,description,category,author_id,author_name,items_json,edges_json, + viewport_json,setup_guide,item_count,is_featured,use_count,status,created_at,updated_at) + VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)""", + [ + tid, tpl["name"], tpl["description"], tpl.get("category", "coding"), + AUTHOR_ID, AUTHOR_NAME, + json.dumps(tpl["items"]), json.dumps(tpl["edges"]), + json.dumps(tpl.get("viewport")), tpl.get("setupGuide"), len(tpl["items"]), + 1, 0, "approved", now, now, + ], + ) + return tid + + +def main(): + paths = discover(sys.argv[1:]) + # Ensure the setup_guide column exists (idempotent) so older DBs accept the insert. + try: + q("ALTER TABLE dashboard_templates ADD COLUMN setup_guide TEXT", []) + except Exception: + pass + for path in paths: + tpl = json.loads(path.read_text()) + tid = upsert(tpl) + print(f"seeded '{tpl['name']}' (id={tid}) from {path.relative_to(ROOT.parent)}") + print(f"done: {len(paths)} template(s) -> {D1_URL}") + + +if __name__ == "__main__": + main() diff --git a/benchmarks/frontiercode/NOT-RUNNABLE.md b/benchmarks/frontiercode/NOT-RUNNABLE.md new file mode 100644 index 00000000..044fc2b0 --- /dev/null +++ b/benchmarks/frontiercode/NOT-RUNNABLE.md @@ -0,0 +1,50 @@ +# FrontierCode (Diamond) — not runnable as an Orcabot template + +**Verdict: no runner template is possible.** FrontierCode is real and +well-identified, but it is **closed / submission-only**. There is nothing to +`pip install`, no dataset to download, and no public harness to drive — so an +Orcabot "runner" would be a fiction. This note exists so the gap is documented +rather than papered over with a fake template. + +## What it is + +- **FrontierCode** — a coding benchmark from **Cognition AI** (the Devin team), + announced June 2026: +- Premise: goes *beyond correctness*. Instead of "do the tests pass" (SWE-bench + style) it asks **"would an open-source maintainer actually merge this PR?"** — + judging regression safety, test quality, scope discipline, and repo + conventions. Built with 20+ maintainers across 36 OSS repos. +- **Diamond** is the hardest nested subset (your GPQA-Diamond analogy is right): + Diamond = 50 hardest tasks ⊂ Main = 100 ⊂ Extended = 150. Scores are brutal + (Claude Opus 4.8 ≈ 13.4% on Diamond). + +## Why it can't be run locally + +- **Tasks are deliberately held out** — Cognition states they don't plan to + release tasks publicly, to avoid contamination. +- **No public repo, no HuggingFace dataset, no harness, no download.** +- **Access is submission-only:** model creators request that Cognition runs the + eval. Grading (unit tests + reverse-classical tests + adaptive LLM grading + + rubric review + scope checks) happens entirely on Cognition's side. +- Scores are reported as **model + agent-harness** combinations (Claude Code, + Codex, Gemini CLI, mini-swe-agent, Devin) — so it's agentic, but you still + can't reproduce it without the held-out tasks and private graders. + +## The only partial path (and why it's not a real score) + +You *could* point an Orcabot agent harness (Claude Code, mini-swe-agent, …) at +public OSS repos and grade "mergeability" with your own rubric — but that +produces a *different* benchmark, not a FrontierCode score, since neither the +tasks nor the graders match. Not worth a template. + +## If/when this changes + +If Cognition later publishes tasks or a submission API, the natural Orcabot fit +would be an **inference-only** template (run the agent harness against the +provided repos, emit submissions in their format) — mirroring the SWE-bench +inference phase. Until then: **not runnable.** + +_Sources: [Cognition blog](https://cognition.ai/blog/frontier-code) · +[HN discussion](https://news.ycombinator.com/item?id=48451723). Not to be +confused with Epoch AI's **FrontierMath** (a math benchmark with a similar +hardest-tier structure)._ diff --git a/benchmarks/slopcodebench/README.md b/benchmarks/slopcodebench/README.md new file mode 100644 index 00000000..6bfe194f --- /dev/null +++ b/benchmarks/slopcodebench/README.md @@ -0,0 +1,145 @@ +# SlopCodeBench Runner for Orcabot + +Turn Orcabot into a **1-click, English-driven harness for running +[slop-code-bench](https://github.com/SprocketLab/slop-code-bench)** — a coding +agent you direct in plain language sets up and runs the benchmark, and every +agent-under-test run shows up as a live, read-only pane on the canvas. + +This directory is the **orchestration layer**. It pairs with a branch on the +slop-code-bench fork that adds the no-Docker, watchable executor: + +| Piece | Where | What | +|-------|-------|------| +| Host (no-Docker) executor + tmux mirror | `robdmac/slop-code-bench` branch `feat/host-tmux-executor` | Runs the agent-under-test directly in the VM and mirrors each run into an attachable tmux window | +| Orchestration layer (this dir) | `orcabot3` branch `feat/benchmark-tmux-viewers` | Orchestrator runbook, read-only viewer convention, results browser, config wizard, importable template | + +## Why this shape + +slop-code-bench normally spawns a Docker container per problem. Inside Orcabot +the sandbox is *already* an isolated VM, so we drop the inner container and run +on the host — which also sidesteps Docker-in-Docker and works on the desktop +VM. The cost is weaker isolation/reproducibility than the pinned image (fine for +relative comparisons; use the Docker env for leaderboard-exact numbers). See +the fork's `docs/HOST_TMUX_EXECUTOR.md`. + +Keeping a Claude Code agent as the **orchestrator** (not a deterministic script) +is deliberate: you can drive stop/start/chaining/interim-analysis/debugging in +English via Remote Control, which is the actual product value here. + +## Architecture + +``` +┌─ Terminal: ORCHESTRATOR ───────────┐ ┌─ Browser block ───────────┐ +│ Claude Code, driven in English │ │ slop-code Dash dashboard │ +│ runs: slop-code run (local-tmux) │ │ http://localhost:8050 │ +│ has Orcabot MCP tools │ └───────────────────────────┘ +│ (create_terminal, connect_nodes) │ +└───┬───────────┬───────────┬─────────┘ + │ spawns viewer panes + edges per run + ▼ ▼ ▼ +┌─ ▶ run ──┐ ┌─ ▶ run ──┐ ┌─ ▶ run ──┐ boot_command: +│ tail -F │ │ tail -F │ │ tail -F │ tail -n +1 -F +└──────────┘ └──────────┘ └──────────┘ (read-only viewers; no inject) + ▲ per-run logfiles (the executor tees each run to one) + │ + host-tmux executor in the slop-code-bench fork +``` + +Viewer panes **tail the per-run logfile**, not the executor's tmux socket — a +separate viewer PTY may run under a different uid, and bridging a tmux *control* +socket across uids would need world-accessibility (read+inject across sessions, +bypassing output redaction). Logfile tail is read-only and reaches only that +run. The executor's tmux session stays private to its own uid (its multiplexing +convenience); `scb-attach` is for same-uid interactive use only. + +## Prerequisites + +- **tmux** — already in the sandbox image (`sandbox/docker/Dockerfile`), and the + desktop VM rootfs is `docker export`ed from that same image, so it's present + on both prod and desktop. No infra change needed. +- The **fork branch** `feat/host-tmux-executor` checked out in the workspace as + `slop-code-bench`, with `uv sync` run. +- Problems repo `scb-problems` checked out. +- Keys in the Orcabot **secrets broker** for whatever provider you benchmark. + +## Quick start + +1. **Config phase** — pick the "SlopCodeBench Runner" template (or ask the + Orcabot chat to "set up a benchmark"). The chat wizard + (`config-wizard.md`) asks CLI / auth / target / problems, stores keys in the + broker, and provisions the orchestrator terminal + results browser. +2. **Drive the orchestrator** in English (`orchestrator-runbook.md`): + - "Run `file_backup` and `execution_server` with claude_code on opus-4.5." + - "Show me the runs." → it spawns a read-only viewer pane per run. + - "execution_server looks stuck — stop it." + - "Score the finished runs and open the dashboard." +3. **Watch** runs in the viewer panes; **read results** in the browser block. + +## Files + +| File | Purpose | +|------|---------| +| `orchestrator-runbook.md` | System prompt + runbook for the orchestrator agent | +| `config-wizard.md` | Orcabot-chat config phase (Part 1 of the plan) | +| `template.json` | Importable `DashboardTemplateWithData` (orchestrator + note + results browser) | +| `bin/scb-runs` | List active runs + read-only attach commands | +| `bin/scb-attach` | Attach read-only to a run | +| `bin/scb-dashboard` | Serve the Dash results dashboard on :8050 | + +## Importing the template + +`template.json` matches `DashboardTemplateWithData` +(`frontend/src/types/dashboard.ts`). Templates are **DB-only** — there is no code +seed and nothing imports this file automatically. To make it appear in the +**New Dashboard** grid, insert it into the `dashboard_templates` table: + +```bash +benchmarks/slopcodebench/bin/scb-insert-template # desktop d1-shim :9001 +``` + +(For a cloud control plane, export a template from an existing dashboard via the +`POST /templates` API instead — direct INSERT is the local/curated path.) + +The orchestrator terminal's `bootCommand` appends the operating briefing to a +`CLAUDE.md` in its working dir, so Claude Code boots **already knowing** the +validated `slop-code` commands (this is what was missing when a stock agent had +"no idea" how to run a benchmark). Viewer panes are intentionally **not** in the +template; the orchestrator can create them per run at runtime. + +> **"Set up a benchmark" in the in-app chat is not a feature.** There is no +> benchmark chat intent — that phrase just spins up a generic agent with no +> slop-code context. Use this template (above) instead. + +### One-time VM setup (manual, required before driving the orchestrator) +1. Fork at `/workspace/slop-code-bench` with venv `/workspace/scb-venv` + (`uv sync --python 3.12`, managed python on `/workspace/.uv-python`). On + desktop this is the host-shared workspace, so it persists across reboots. +2. Add your provider key in the **secrets panel**, broker-protected + (e.g. `OPENROUTER_API_KEY`). + +Then drive the orchestrator in English: *"Run `file_backup` with opencode on +kimi-k2.6."* The validated arm + the `localhost`→`127.0.0.1` broker note live in +`RESUME-kimi-opencode.md`. + +## Status — prototype + +**Validated** +- Host-tmux executor: unit + real-tmux lifecycle tests pass; end-to-end smoke + run confirms harness output is unchanged while a tmux window mirrors it + (`feat/host-tmux-executor`). +- `scb-runs` / `scb-attach` against a real tmux session; read-only resolution. +- `template.json` parses; every embedded `content` string is valid. + +**Not yet validated end-to-end (needs a live Orcabot VM + keys)** +- A full `slop-code run` of a real problem under `local-tmux-py.yaml` with a + real agent CLI. +- The orchestrator calling `create_terminal`/`connect_nodes` to spawn viewer + panes from inside an Orcabot terminal. +- The Dash dashboard rendering in a browser block at :8050. +- Skills-variant copy-in path for OMC / SuperPowers / Karpathy (only GSD has a + ready-made local env + prompt; others follow the documented pattern). + +**Deliberately deferred** +- Docker-env parity path (for leaderboard-exact runs) — orthogonal; the Docker + configs already exist upstream. +- Per-run cgroup memory caps for safe parallelism — documented, not wired. diff --git a/benchmarks/slopcodebench/RESUME-kimi-opencode.md b/benchmarks/slopcodebench/RESUME-kimi-opencode.md new file mode 100644 index 00000000..ea7a2d9e --- /dev/null +++ b/benchmarks/slopcodebench/RESUME-kimi-opencode.md @@ -0,0 +1,194 @@ +# Resume plan — run a Kimi K2.6 / opencode arm in the desktop VM + +Goal: run **one** slop-code-bench arm — model **Kimi K2.6 via OpenRouter**, agent +**opencode** — end-to-end in the Orcabot **desktop VM**, with the API key kept in +the **secrets broker** and the run watchable via the **host-tmux logfile mirror**. + +> **RESOLVED 2026-06-23.** The "hang" was an IPv4/IPv6 loopback mismatch, not a +> streaming bug. The broker listens on `127.0.0.1` only; opencode (Node/undici) +> resolves `localhost` to IPv6 `::1` first, can't reach the IPv4 broker, and +> retry-storms into an apparent hang. Fix: hand harnesses `127.0.0.1`, not +> `localhost`. With that, the arm runs end-to-end (`cost>0`, steps advance through +> checkpoints). See §2 + §4. The rest of this doc is kept as the run recipe. + +--- + +## 1. Status at a glance + +**Working / validated (don't re-litigate):** +- Broker → Kimi works: a direct broker call returned "PONG" (key never leaves the + broker; PTY only ever sees a `[BROKERED]` placeholder + `OPENROUTER_BASE_URL`). +- opencode 1.17.9 baked into the sandbox image (`/usr/local/bin/opencode`); VM + image bumped to 4 GB. +- Official `slop-code run` orchestrates: resolves the catalog problem + (`file_backup`), auto-downloads the problem catalog to + `/root/.cache/scbench/problems`, runs the checkpoint loop. +- Host-tmux executor tees each run to a logfile (`…/checkpoint_1/snapshot/.scb_tmux/`). +- **opencode auth through the broker is SOLVED** — no more 401. opencode launches + `--model=scb-openrouter/moonshotai/kimi-k2.6`, reads `baseURL`+`apiKey` from its + config, request reaches OpenRouter via the broker. + +**Root cause of the former "hang" (FIXED):** opencode's `@ai-sdk/openai-compatible` +provider could not connect to the broker. opencode debug log +(`--print-logs --log-level DEBUG`) showed `AI_APICallError: Cannot connect to API` +→ `AI_RetryError: Failed after 3 attempts`, repeating with exponential backoff +across the title + build sub-agents — i.e. a retry storm, not a stalled stream. +The broker binds `127.0.0.1:8082` (IPv4 only; `secrets_broker.go:630`), but the +baseURL was `http://localhost:8082/...`. Node/undici tries IPv6 `::1` first → +ECONNREFUSED. curl/Go/Python prefer IPv4, which is why the direct streaming curl +through the broker worked perfectly (chunks in 3 s) and masked the bug. +**Fix:** use `127.0.0.1`, not `localhost`. Two places — see §2. + +--- + +## 2. What was fixed, and where (commits) + +| Repo / branch | Commit | Change | +|---|---|---| +| fork `feat/host-tmux-executor` | `8956077` | host-tmux executor: tee streamed output to a logfile + tmux mirror (the no-Docker watchable run) | +| fork `feat/host-tmux-executor` | `bfa9aa6` | opencode local-config materialize + apiKey injection; `kimi-k2.6.yaml` `agent_specific.opencode.endpoint: openai` | +| fork `feat/host-tmux-executor` | `0adc65c` | **the auth unlock** — `LocalStreamingRuntime` now applies spawn `env_vars`; opencode uses a synthetic `scb-` `@ai-sdk/openai-compatible` provider | +| orcabot `feat/benchmark-templates-swe-tbench` | `5bd9360` | re-enable opencode in `sandbox/docker/Dockerfile` | +| orcabot `feat/benchmark-templates-swe-tbench` | `b3b7751` | default VM image size → 4 GB (5bd9360 only set it via env, not source) | +| orcabot `feat/benchmark-templates-swe-tbench` | `17e7cd4` | **the hang fix** — broker env URLs use `127.0.0.1`, not `localhost` (`sandbox/cmd/server/env.go:138,162`). Product-wide; needs a VM image rebuild to reach the deployed sandbox | + +Why `0adc65c` was the key: `LocalStreamingRuntime` was **dropping** the agent's +spawn-time `env_vars`, so opencode ran with `HOME=/root` and never read its +generated config (always fell back to the built-in openrouter provider → 401). +And a provider id that matches a built-in (`openrouter`) makes opencode ignore +`options.apiKey`/`baseURL`; a **synthetic** id backed by `@ai-sdk/openai-compatible` +honors both and sends a Bearer header the broker swaps for the real key. + +--- + +## 3. Environment / how to resume the stack + +Host paths: +- orcabot repo: `~/work/hyper/orcabot3` (branch `feat/benchmark-templates-swe-tbench`) +- slop fork: `~/work/hyper/slop-code-bench` (branch `feat/host-tmux-executor`) +- `orcabot` CLI: `~/work/hyper/orcabot3/desktop/app/src-tauri/target/release/orcabot` +- host-shared VM workspace: `~/Library/Application Support/com.orcabot.desktop/workspace` + (maps to the VM's `/workspace`) + +In the VM (`/workspace` survives reboots; rootfs does NOT): +- fork: `/workspace/slop-code-bench` +- venv: `/workspace/scb-venv` (uv `--python 3.12`; managed python on + `/workspace/.uv-python`). Run via `/workspace/scb-venv/bin/slop-code` directly — + **no uv needed**. (VM has no compiler; deps must be wheels → python 3.12, not 3.14.) +- problem catalog: `/root/.cache/scbench/problems` (auto-downloaded; on rootfs, so + re-downloads after a reboot — that's fine/automatic). + +Bring it up: +```bash +cd ~/work/hyper/orcabot3 +OB=desktop/app/src-tauri/target/release/orcabot +$OB up --timeout 240 # control plane :8787 + sandbox VM :8080 +$OB exec "command -v opencode && opencode --version" # sanity: 1.17.9 +``` +If the venv's `slop-code` errors with "required file not found" after a reboot +(rootfs wiped its uv-managed python), re-link it: +```bash +# reinstall uv + re-sync (deps cached on /workspace; python lands on /workspace) +# (launch detached from a PTY; see §6 launcher pattern, swap the run cmd for:) +# curl -LsSf https://astral.sh/uv/install.sh | sh +# UV_PYTHON_INSTALL_DIR=/workspace/.uv-python UV_CACHE_DIR=/workspace/.uv-cache \ +# UV_PROJECT_ENVIRONMENT=/workspace/scb-venv /root/.local/bin/uv sync --python 3.12 +``` + +Broker key: brokered `OPENROUTER_API_KEY` secret on dashboard +`f054e883-ef4e-4d86-ab16-dd90a74e4b9f`. If gone, re-add (user runs it, key never +in chat): +```bash +curl -s -X POST -H "X-User-ID: dev-desktop" -H 'Content-Type: application/json' \ + --data-raw "{\"name\":\"OPENROUTER_API_KEY\",\"value\":\"$OPENROUTER_KEY\",\"type\":\"secret\",\"brokerProtected\":true,\"dashboardId\":\"f054e883-ef4e-4d86-ab16-dd90a74e4b9f\"}" \ + http://127.0.0.1:8787/secrets +``` + +--- + +## 4. How it was resolved (record) + +The diagnosis was done from a PTY in the Kimi dashboard (which has +`$OPENROUTER_BASE_URL` + the `[BROKERED]` placeholder key in env): + +1. **Streaming curl through the broker** (`"stream":true`) — returned SSE chunks in + ~3 s, `http=200`, real cost. So the broker streams fine; the earlier non-stream + PONG was not the gap. → hang is opencode-side. +2. **Ran opencode standalone** with the same synthetic-provider config + a `timeout` + so it couldn't hang forever; added `--print-logs --log-level DEBUG`. The log + showed `AI_APICallError: Cannot connect to API` → `AI_RetryError: Failed after 3 + attempts`, retrying with backoff. Config loaded fine (`scb-openrouter` selected) — + it simply couldn't open the socket. +3. **Switched the baseURL `localhost` → `127.0.0.1`** and re-ran: the agent loop + completed instantly (reasoning + text + `step_finish` reason=stop). Root cause = + IPv4-only broker vs Node/undici's IPv6-first `localhost` resolution. + +**Fixes applied:** +- Product source: `sandbox/cmd/server/env.go:138,162` now emits `127.0.0.1` + (commit `17e7cd4`). Fixes every brokered provider for all Node harnesses; needs a + VM image rebuild to reach the deployed sandbox. +- Run recipe (works on the *current* image without a rebuild): the launcher rewrites + `providers.yaml`'s broker `api_base` with `localhost`→`127.0.0.1` (see §6). + +Success criterion met: the arm advanced past `checkpoint_1` with `cost=0.01027` +(`steps=1`) into `checkpoint_2`. To score it, drop `--no-evaluate`. + +--- + +## 5. Key files + +| File | Why | +|---|---| +| `slop-code-bench/src/slop_code/agent_runner/agents/opencode/agent.py` | opencode agent: synthetic provider injection (`_from_config`), config materialize (`_materialize_local_config`), `run()` stream loop (add timeout) | +| `slop-code-bench/src/slop_code/execution/local_streaming.py` | `LocalStreamingRuntime` — now applies spawn env_vars + tmux mirror | +| `slop-code-bench/configs/models/kimi-k2.6.yaml` | `agent_specific.opencode.endpoint: openai` | +| `slop-code-bench/configs/providers.yaml` | openrouter `openai` endpoint `api_base` (rewritten to the broker URL at launch — see launcher) | +| `orcabot3/sandbox/internal/broker/secrets_broker.go` | broker proxy: `ServeHTTP` (291), forward `client.Do` (480), SSE stream+flush (524-540), `WriteTimeout` (633) | +| `orcabot3/sandbox/docker/Dockerfile` | opencode install (re-enabled) | + +--- + +## 6. Exact run launcher (copy-paste) + +Create a fresh PTY in the Kimi dashboard, then send this (base64-wrapped so +`$OPENROUTER_*` expand in the PTY, not the host). `DID=f054e883-…`; get the debug +token from `/tmp/vz-console.log` (`grep 'debug-exec] auth token:'`). + +The launcher (written to a host temp file via a quoted heredoc, then +`echo | base64 -d | bash` sent via `orcabot tail --send`): +```bash +setsid bash -c ' +export HOME=/root TMPDIR=/workspace/.uvtmp +cd /workspace/slop-code-bench +cp configs/providers.yaml.orig configs/providers.yaml # restore pristine +python3 - <<'PY' +import os +p="configs/providers.yaml"; s=open(p).read() +# 127.0.0.1, NOT localhost: opencode (Node/undici) resolves localhost to IPv6 ::1 +# first and the broker is IPv4-only -> retry-storm hang. (env.go now emits +# 127.0.0.1 too; this swap keeps the recipe working on a not-yet-rebuilt image.) +url=os.environ["OPENROUTER_BASE_URL"].replace("localhost","127.0.0.1") +open(p,"w").write(s.replace("api_base: https://openrouter.ai/api/v1","api_base: "+url)) +PY +/workspace/scb-venv/bin/slop-code run --agent opencode --model openrouter/kimi-k2.6 \ + --environment configs/environments/local-tmux-py.yaml \ + --prompt configs/prompts/just-solve.jinja \ + --problem file_backup --no-evaluate thinking=low 2>&1 +echo "RUN_DONE_rc=$?" +' > /workspace/.scb-run.log 2>&1 < /dev/null & +``` +Watch: `tail -f /workspace/.scb-run.log` (via `/debug/exec`), and the run dir +`outputs/kimi-k2.6/opencode-*/file_backup/checkpoint_1/agent/{messages.jsonl,stdout.txt}`. +Keep a `.orig` copy of `providers.yaml` so each launch re-patches a clean file with +the current session's broker URL. + +--- + +## 7. Fallbacks if the streaming fix is hard +- **Different agent:** `claude_code` or `codex` are installed and could be pointed + at OpenRouter via the broker's anthropic/openai-compatible endpoints — but Kimi + isn't configured for them in the catalog (only `opencode`). +- **Docker env instead of host-tmux:** run opencode in the upstream + `docker-python3.12-uv.yaml` env (needs Docker-in-VM — the path we avoided). +- **Inference vs eval:** keep `--no-evaluate` until the agent loop works; add eval + (runs pytest in the local env via `uv init`/`uv add`) once inference is green. diff --git a/benchmarks/slopcodebench/bin/scb-attach b/benchmarks/slopcodebench/bin/scb-attach new file mode 100755 index 00000000..1d339c80 --- /dev/null +++ b/benchmarks/slopcodebench/bin/scb-attach @@ -0,0 +1,32 @@ +#!/usr/bin/env bash +# REVISION: scb-viewers-v1-attach +# Attach READ-ONLY to a slop-code-bench run's tmux window. +# +# This is what a viewer terminal block runs as its boot command. The `-r` flag +# makes the attachment read-only: the viewer can watch but cannot send input, +# resize, or kill the run. +# +# Usage: scb-attach +# scb-attach # attaches to the most recently created run +# Env: SCB_SESSION (default: scb) +set -euo pipefail + +SESSION="${SCB_SESSION:-scb}" +WINDOW="${1:-}" + +if ! tmux has-session -t "$SESSION" 2>/dev/null; then + echo "no slop-code runs yet (tmux session '$SESSION' not started)" >&2 + exit 1 +fi + +if [ -z "$WINDOW" ]; then + # Most recently created non-placeholder window. + WINDOW=$(tmux list-windows -t "$SESSION" -F '#{window_name}' \ + | grep -v '^_scb_runs$' | tail -n1 || true) + if [ -z "$WINDOW" ]; then + echo "no active runs to attach to" >&2 + exit 1 + fi +fi + +exec tmux attach-session -r -t "${SESSION}:${WINDOW}" diff --git a/benchmarks/slopcodebench/bin/scb-dashboard b/benchmarks/slopcodebench/bin/scb-dashboard new file mode 100755 index 00000000..246d65ac --- /dev/null +++ b/benchmarks/slopcodebench/bin/scb-dashboard @@ -0,0 +1,27 @@ +#!/usr/bin/env bash +# REVISION: scb-viewers-v1-dashboard +# Launch the slop-code-bench results dashboard (Dash) inside the sandbox VM. +# +# An Orcabot browser block pointed at http://localhost:8050 then renders it on +# the canvas, next to the orchestrator and the run viewers. +# +# Usage: scb-dashboard [runs_dir] +# runs_dir directory containing run outputs (default: outputs) +# +# Note: upstream slop_code.dashboard.app serves on Dash's default 127.0.0.1:8050 +# (it calls app.run(debug=True) with no port arg). Point the browser block at +# http://localhost:8050. To change the port you must patch the upstream app. +# +# Run from the slop-code-bench checkout (where `uv run` resolves the package). +set -euo pipefail + +RUNS_DIR="${1:-outputs}" + +if [ ! -d "$RUNS_DIR" ]; then + echo "runs dir '$RUNS_DIR' not found — run a benchmark first" >&2 + exit 1 +fi + +echo "[scb-dashboard] serving '$RUNS_DIR' on http://localhost:8050" +echo "[scb-dashboard] point an Orcabot browser block at http://localhost:8050" +exec uv run python -m slop_code.dashboard.app "$RUNS_DIR" diff --git a/benchmarks/slopcodebench/bin/scb-insert-template b/benchmarks/slopcodebench/bin/scb-insert-template new file mode 100755 index 00000000..cd4181a7 --- /dev/null +++ b/benchmarks/slopcodebench/bin/scb-insert-template @@ -0,0 +1,19 @@ +#!/usr/bin/env python3 +"""Insert (or refresh) the SlopCodeBench Runner dashboard template into D1. + +Thin wrapper around the generic seeder (`benchmarks/bin/seed-benchmark-templates`) +scoped to just this benchmark, kept for backwards compatibility. To seed every +benchmark template at once, run the generic seeder with no argument. + +Usage: + bin/scb-insert-template # desktop d1-shim on 127.0.0.1:9001 + D1_URL=http://127.0.0.1:9001 AUTHOR_ID=dev-desktop bin/scb-insert-template +""" +import os +import pathlib +import sys + +# benchmarks/ root: this file is benchmarks/slopcodebench/bin/scb-insert-template. +SEEDER = pathlib.Path(__file__).resolve().parents[2] / "bin" / "seed-benchmark-templates" + +os.execv(sys.executable, [sys.executable, str(SEEDER), "slopcodebench", *sys.argv[1:]]) diff --git a/benchmarks/slopcodebench/bin/scb-runs b/benchmarks/slopcodebench/bin/scb-runs new file mode 100755 index 00000000..7849a8af --- /dev/null +++ b/benchmarks/slopcodebench/bin/scb-runs @@ -0,0 +1,41 @@ +#!/usr/bin/env bash +# REVISION: scb-viewers-v1-runs +# List active slop-code-bench runs in the shared tmux session. +# +# Each agent run launched by the host-tmux executor appears as a tmux window +# in the `scb` session. This prints one line per run with a read-only attach +# command for SAME-UID INTERACTIVE use (a human in the executor's own shell). +# +# NOTE: cross-PTY Orcabot viewer *panes* must NOT attach this tmux socket — +# a separate viewer PTY may run under a different uid, and bridging the tmux +# control socket across uids needs world-accessibility (read+inject across +# sessions, bypassing output redaction). Those panes tail the per-run logfile +# instead (see orchestrator-runbook.md). `tmux attach -r` here is fine only +# because it's the same uid as the executor. +# +# Usage: scb-runs +# Env: SCB_SESSION (default: scb) +set -euo pipefail + +SESSION="${SCB_SESSION:-scb}" + +if ! command -v tmux >/dev/null 2>&1; then + echo "tmux not found on PATH" >&2 + exit 1 +fi + +if ! tmux has-session -t "$SESSION" 2>/dev/null; then + echo "no slop-code runs yet (tmux session '$SESSION' not started)" + exit 0 +fi + +printf '%-28s %-10s %s\n' "RUN" "STATUS" "ATTACH (read-only)" +tmux list-windows -t "$SESSION" \ + -F '#{window_name} #{?window_active,active,idle} #{pane_dead}' \ + | while IFS=$'\t' read -r name _active dead; do + # Skip the placeholder window that keeps the session alive. + [ "$name" = "_scb_runs" ] && continue + status="running" + [ "$dead" = "1" ] && status="finished" + printf '%-28s %-10s tmux attach -r -t %s:%s\n' "$name" "$status" "$SESSION" "$name" + done diff --git a/benchmarks/slopcodebench/config-wizard.md b/benchmarks/slopcodebench/config-wizard.md new file mode 100644 index 00000000..17d10175 --- /dev/null +++ b/benchmarks/slopcodebench/config-wizard.md @@ -0,0 +1,81 @@ +# Config-phase wizard (Orcabot chat) + +This is the **configuration phase** from the original plan: before the +benchmark template is usable, the Orcabot chat (the Gemini-powered assistant in +`controlplane/src/chat/handler.ts`) walks the user through a short back-and-forth +and then provisions the dashboard. + +Wire this as an additional system-prompt block the chat loads when the user +picks the "SlopCodeBench Runner" template, or as a standalone "Set up a +benchmark" intent. It uses **only tools the chat already has**: `secrets_create`, +`create_terminal` (with `boot_command`), `create_browser`, `connect_nodes`. + +--- + +## Conversation flow + +Ask these in order, one at a time, accepting natural-language answers: + +1. **Which CLI should be the agent under test?** + `claude_code` · `codex` · `gemini` + → sets `--agent`. (This is the harness being benchmarked, *not* the + orchestrator — the orchestrator is always Claude Code.) + +2. **How should that agent authenticate?** + - **API key** → ask the user to paste it into the secrets panel (never into + chat). Store via `secrets_create` under the broker name the provider + expects (`ANTHROPIC_API_KEY`, `OPENROUTER_API_KEY`, `OPENAI_API_KEY`, + `GEMINI_API_KEY`). It stays in the broker; the agent only ever sees a + placeholder. + - **Subscription login** (`codex login`, `gemini` OAuth) → note that this + uses a file-based credential (`~/.codex/auth.json` etc.) that the broker + does **not** manage, and that the orchestrator will run the login flow in + its terminal. + +3. **What are you benchmarking?** + - **A model** → ask for `provider/model` (e.g. `anthropic/opus-4.5`, + `openrouter/…`). → `--model`. + - **A CLI/harness** → same agent, you'll compare versions; capture + `version=`. + - **A skill/plugin** → pick `gsd` · `omc` · `superpowers` · `karpathy`. + → selects `local-tmux-py-with-.yaml` + + `just-solve-with--local-trigger.jinja`, and you'll clone the skill + repo + set `SCB_SKILL__DIR` in the orchestrator's setup. + +4. **Which problems / how many?** (default: a small smoke set like + `file_backup`) → repeated `--problem` flags. + +5. **Thinking budget?** `none|low|medium|high` → `thinking=`. + +--- + +## What the wizard then does + +1. `secrets_create` for any API key the user supplied (broker-scoped). +2. `create_terminal` for the **orchestrator** (Claude Code), `boot_command: + "claude"`, working dir `slop-code-bench`, `skipApprovals: true`. Seed its + first message with the assembled command, e.g.: + ``` + uv run slop-code run --agent codex --model codex_auth/gpt-5.5 \ + --environment configs/environments/local-tmux-py-with-gsd.yaml \ + --prompt configs/prompts/just-solve-with-gsd-local-trigger.jinja \ + --problem file_backup thinking=high version=0.136.0 + ``` +3. `create_browser` pointed at `http://localhost:8050` for results. +4. Hand off: tell the user they can now drive the orchestrator in English + ("run it", "show me the runs", "stop X", "score and open the dashboard"). + See `orchestrator-runbook.md`. + +--- + +## Guardrails + +- **Never** accept or display a raw API key in chat — always route through the + secrets panel + `secrets_create`. +- Confirm the assembled command back to the user before creating the + orchestrator terminal. +- If the user picks the skills variant, remember it needs the matching + `*-local-trigger.jinja` prompt **and** `SCB_SKILL__DIR` — don't run the + base prompt against a skills env (the skill won't be read). +- Subscription auth (codex/gemini login) bypasses the broker by design; say so + rather than implying every credential is broker-protected. diff --git a/benchmarks/slopcodebench/orchestrator-runbook.md b/benchmarks/slopcodebench/orchestrator-runbook.md new file mode 100644 index 00000000..985bf193 --- /dev/null +++ b/benchmarks/slopcodebench/orchestrator-runbook.md @@ -0,0 +1,121 @@ +# Benchmark Orchestrator — Runbook / System Prompt + +This is the operating guide for the **Benchmark Orchestrator**: a Claude Code +agent running in an Orcabot terminal that you drive in plain English (via +Remote Control) to run [slop-code-bench](https://github.com/SprocketLab/slop-code-bench). + +Paste the **System prompt** section into the orchestrator terminal's agent (or +a `CLAUDE.md` in its working dir). The rest documents the moving parts. + +--- + +## What you (the orchestrator) control + +- You run benchmark jobs with the `slop-code` CLI using the **host-tmux + executor** (no Docker), so every agent-under-test run is a live tmux window. +- You make each run **watchable** by spawning a **read-only viewer pane** for it + on the Orcabot canvas, using your `create_terminal` MCP tool. +- You score finished runs and surface results in a **browser block**. +- You start, stop, chain, and debug runs on the user's spoken instruction. + +You do **not** type the agent's code or solve problems yourself — slop-code +drives the agent-under-test. You are the operator around it. + +--- + +## System prompt + +> You are the **Benchmark Orchestrator** for slop-code-bench inside Orcabot. +> The user directs you in English; translate that into `slop-code` commands and +> Orcabot canvas actions. Be concise; report what you ran and what you observed. +> +> **Environment** +> - Work in the `slop-code-bench` checkout (your working dir). If it's missing, +> clone `https://github.com/robdmac/slop-code-bench` (branch +> `reproduce-public-skills`) and run `uv sync`. +> - Problems come from a separate repo. If `scb-problems` is absent, clone +> `https://github.com/gabeorlanski/scb-problems` and pass it to slop-code. +> - API keys are injected by the Orcabot secrets broker as env vars +> (`ANTHROPIC_API_KEY`, `OPENROUTER_API_KEY`, …). **Never print, echo, or +> `cat` a key.** If a needed key is missing, ask the user to add it via the +> secrets panel rather than requesting the value in chat. +> +> **Running a benchmark — always use the host-tmux environment** so runs are +> watchable: +> ``` +> uv run slop-code run \ +> --agent \ +> --model \ +> --environment configs/environments/local-tmux-py.yaml \ +> --prompt configs/prompts/just-solve.jinja \ +> --problem [--problem ...] \ +> thinking= version= +> ``` +> For the **skills variant**, first clone the skill and export +> `SCB_SKILL__DIR`, then use `local-tmux-py-with-.yaml` + +> `just-solve-with--local-trigger.jinja`. +> +> **After starting runs, make them visible.** Read `/.scb_tmux/runs.jsonl` +> for the active runs (each record has `window` and `logfile`). For each run +> that does not already have a viewer pane, call the **`create_terminal`** MCP +> tool with: +> - `boot_command`: `tail -n +1 -F ` ← read-only by construction +> - a name like `▶ ` +> Then call **`connect_nodes`** to draw an edge from your own terminal to the +> new viewer. Never create a second viewer for a run that already has one. +> +> **Security — do NOT make viewers attach the tmux socket.** A viewer pane is a +> separate PTY (potentially a different uid under the egress UID pool). Bridging +> it to the run's tmux *control* socket would require a world-accessible socket, +> which grants read+inject across sessions and bypasses output redaction — a +> local lateral channel the egress proxy never sees. Tailing the per-run logfile +> is read-only, reaches only that one run, and stays within the shared +> `/workspace` trust boundary. (A human attaching `tmux attach -r` from *your own* +> shell — same uid, default socket — is fine; that's what `scb-attach` is for. +> The rule is specifically about cross-PTY viewer panes.) +> +> **Scoring + results** +> - When runs finish, score them: `uv run slop-code eval outputs//`. +> - Launch the results dashboard once: `python -m slop_code.dashboard.app +> outputs` (serves http://localhost:8050). Ensure a browser block points +> there (create one with `create_browser` if absent). +> +> **Lifecycle / debugging** +> - Stop a stuck run: `tmux send-keys -t scb: C-c` then kill it, or kill +> the slop-code job. Report what you did. +> - Read interim output without attaching: `tail -n 50 ` (logfiles are +> listed in `/.scb_tmux/runs.jsonl`). +> - Chain runs (e.g. sweep models or problems) by issuing them sequentially and +> reporting progress after each. +> - Concurrency is bounded by VM RAM, not Docker — don't launch more parallel +> runs than the VM can hold. When in doubt, ask before fanning out widely. + +--- + +## Why read-only viewers + +The host-tmux executor (slop-code-bench `feat/host-tmux-executor`) mirrors each +run's output into a tmux window via a tailed logfile. Attaching with `-r` lets +a viewer watch the run **without** being able to send keystrokes, resize, or +kill it — so a viewer pane can never corrupt a benchmark in progress. The +harness's own captured output is untouched by the mirror. + +## Discovery: `runs.jsonl` + +Each run appends a record to `/.scb_tmux/runs.jsonl`: +```json +{"target":"scb:file_backup","session":"scb","window":"file_backup","logfile":"…/file_backup..log","created":} +``` +Use it to find logfiles for `tail`, or `tmux list-windows -t scb` for the live +window list. The `bin/` helpers (`scb-runs`, `scb-attach`, `scb-dashboard`) +wrap these; copy them into the workspace if you want them on `$PATH`. + +## Helpers (optional, in `bin/`) + +| Helper | Does | +|--------|------| +| `scb-runs` | List active runs + their read-only attach commands | +| `scb-attach [window]` | Attach read-only to a run (most-recent if no arg) | +| `scb-dashboard [runs_dir]` | Serve the Dash results dashboard on :8050 | + +They only use `tmux` + `uv`, both present in the sandbox image. diff --git a/benchmarks/slopcodebench/template.json b/benchmarks/slopcodebench/template.json new file mode 100644 index 00000000..c581a0db --- /dev/null +++ b/benchmarks/slopcodebench/template.json @@ -0,0 +1,59 @@ +{ + "_comment": "Importable Orcabot dashboard template (DashboardTemplateWithData). Inserted into D1 via benchmarks/slopcodebench/bin/insert-template or the templates API. The orchestrator's bootCommand appends a briefing to CLAUDE.md so Claude Code boots knowing the validated slop-code commands. Manual one-time VM setup (fork + venv) still required.", + "name": "SlopCodeBench Runner", + "description": "Run slop-code-bench arms with a Claude Code orchestrator you drive in English. One-time VM setup required (fork + venv).", + "category": "coding", + "items": [ + { + "placeholderId": "orchestrator", + "type": "terminal", + "content": "{\"name\": \"Claude Code\", \"agentic\": true, \"bootCommand\": \"B='CjwhLS0gU0NCLU9SQ0ggLS0+CiMgU2xvcENvZGVCZW5jaCBPcmNoZXN0cmF0b3IKCllvdSBvcGVyYXRlIHNsb3AtY29kZS1iZW5jaCBpbiB0aGlzIE9yY2Fib3QgVk0uIFRoZSB1c2VyIGRpcmVjdHMgeW91IGluIEVuZ2xpc2g7CnlvdSB0cmFuc2xhdGUgdGhhdCBpbnRvIGBzbG9wLWNvZGVgIGNvbW1hbmRzIGFuZCByZXBvcnQgd2hhdCB5b3UgcmFuICsgb2JzZXJ2ZWQuCllvdSBkbyBOT1Qgc29sdmUgdGhlIGJlbmNobWFyayBwcm9ibGVtcyB5b3Vyc2VsZiAtLSB0aGUgYWdlbnQtdW5kZXItdGVzdCBkb2VzLgoKIyMgRW52aXJvbm1lbnQgKG1hbnVhbCBzZXR1cCAtLSBhc3N1bWUgZG9uZSBpZiAvd29ya3NwYWNlL3NjYi12ZW52IGV4aXN0cykKLSBGb3JrIGNoZWNrb3V0ID0gdGhpcyBkaXJlY3RvcnkgKGAvd29ya3NwYWNlL3Nsb3AtY29kZS1iZW5jaGApLgotIFJ1biB0aGUgQ0xJIHZpYSB0aGUgdmVudiBkaXJlY3RseTogYC93b3Jrc3BhY2Uvc2NiLXZlbnYvYmluL3Nsb3AtY29kZWAgKG5vIGB1dmApLgotIFRoZSBwcm9ibGVtIGNhdGFsb2cgYXV0by1kb3dubG9hZHMgdG8gYH4vLmNhY2hlL3NjYmVuY2gvcHJvYmxlbXNgIG9uIGZpcnN0IHJ1bi4KCiMjIFNlY3JldHMgLS0gTkVWRVIgZWNobywgY2F0LCBvciBwcmludCBhIGtleQpQcm92aWRlciBrZXlzIGFycml2ZSBmcm9tIHRoZSBPcmNhYm90IHNlY3JldHMgYnJva2VyIGFzIGVudiB2YXJzIChlLmcuCmBPUEVOUk9VVEVSX0FQSV9LRVlgIGhvbGRzIGEgYFtCUk9LRVJFRF1gIHBsYWNlaG9sZGVyOyB0aGUgYnJva2VyIGluamVjdHMgdGhlCnJlYWwga2V5IHNlcnZlci1zaWRlKS4gSWYgYSBuZWVkZWQga2V5IGlzIG1pc3NpbmcsIGFzayB0aGUgdXNlciB0byBhZGQgaXQgaW4gdGhlCnNlY3JldHMgcGFuZWwgLS0gbmV2ZXIgcmVxdWVzdCB0aGUgdmFsdWUgaW4gY2hhdC4KCiMjIEJyb2tlciBiYXNlIFVSTCAtLSB1c2UgMTI3LjAuMC4xLCBOT1QgbG9jYWxob3N0CkJlZm9yZSBlYWNoIHJ1biwgcG9pbnQgcHJvdmlkZXJzLnlhbWwncyBicm9rZXIgYGFwaV9iYXNlYCBhdCAxMjcuMC4wLjEuIG9wZW5jb2RlCihOb2RlL3VuZGljaSkgcmVzb2x2ZXMgYGxvY2FsaG9zdGAgdG8gSVB2NiBgOjoxYCwgd2hpY2ggdGhlIElQdjQtb25seSBicm9rZXIKbmV2ZXIgYW5zd2VycyAtPiBhIG11bHRpLW1pbnV0ZSAiaGFuZyIuIFBhdGNoIGl0OgpgYGAKY3AgY29uZmlncy9wcm92aWRlcnMueWFtbC5vcmlnIGNvbmZpZ3MvcHJvdmlkZXJzLnlhbWwgMj4vZGV2L251bGwKcHl0aG9uMyAtIDw8J1BZJwppbXBvcnQgb3M7IHA9ImNvbmZpZ3MvcHJvdmlkZXJzLnlhbWwiOyBzPW9wZW4ocCkucmVhZCgpCnVybD1vcy5lbnZpcm9uWyJPUEVOUk9VVEVSX0JBU0VfVVJMIl0ucmVwbGFjZSgibG9jYWxob3N0IiwiMTI3LjAuMC4xIikKb3BlbihwLCJ3Iikud3JpdGUocy5yZXBsYWNlKCJhcGlfYmFzZTogaHR0cHM6Ly9vcGVucm91dGVyLmFpL2FwaS92MSIsImFwaV9iYXNlOiAiK3VybCkpClBZCmBgYAoKIyMgUnVuIG9uZSBhcm0gKHZhbGlkYXRlZDogb3BlbmNvZGUgKyBLaW1pIEsyLjYgdmlhIE9wZW5Sb3V0ZXIpCmBgYAovd29ya3NwYWNlL3NjYi12ZW52L2Jpbi9zbG9wLWNvZGUgcnVuIC0tYWdlbnQgb3BlbmNvZGUgLS1tb2RlbCBvcGVucm91dGVyL2tpbWktazIuNiBcCiAgLS1lbnZpcm9ubWVudCBjb25maWdzL2Vudmlyb25tZW50cy9sb2NhbC10bXV4LXB5LnlhbWwgXAogIC0tcHJvbXB0IGNvbmZpZ3MvcHJvbXB0cy9qdXN0LXNvbHZlLmppbmphIFwKICAtLXByb2JsZW0gZmlsZV9iYWNrdXAgLS1uby1ldmFsdWF0ZSB0aGlua2luZz1sb3cKYGBgCkxhdW5jaCBkZXRhY2hlZCB0byBhIGxvZ2ZpbGUgc28gdGhlIHVzZXIgY2FuIHdhdGNoLCB0aGVuIHRhaWwgaXQ6CmBgYApzZXRzaWQgYmFzaCAtYyAnPHRoZSBydW4gY29tbWFuZCBhYm92ZT4nID4gL3dvcmtzcGFjZS8uc2NiLXJ1bi5sb2cgMj4mMSA8IC9kZXYvbnVsbCAmCnRhaWwgLWYgL3dvcmtzcGFjZS8uc2NiLXJ1bi5sb2cKYGBgClN1Y2Nlc3MgPSB0aGUgcnVuIGFkdmFuY2VzIHBhc3QgYGNoZWNrcG9pbnRfMWAgd2l0aCBgY29zdD4wYCBhbmQgYHN0ZXBzPjBgLgoKIyMgU2NvcmUKRHJvcCBgLS1uby1ldmFsdWF0ZWAgdG8gc2NvcmUgaW5saW5lLCBvciBydW4KYC93b3Jrc3BhY2Uvc2NiLXZlbnYvYmluL3Nsb3AtY29kZSBldmFsIG91dHB1dHMvPHJ1bi1kaXI+L2AgYWZ0ZXIuCgojIyBXYXRjaCAocmVhZC1vbmx5IG9ubHkpClBlci1ydW4gbG9nZmlsZXMgbGl2ZSB1bmRlciB0aGUgcnVuJ3Mgb3V0cHV0IGRpciAvIGAuc2NiX3RtdXgvcnVucy5qc29ubGAuIFRhaWwKdGhlbS4gRG8gTk9UIGJyaWRnZSBhIGNyb3NzLXNlc3Npb24gdG11eCBjb250cm9sIHNvY2tldCAtLSB0aGF0IGdyYW50cwpyZWFkK2luamVjdCBhY3Jvc3Mgc2Vzc2lvbnMgYW5kIGJ5cGFzc2VzIG91dHB1dCByZWRhY3Rpb24uCg=='; grep -q SCB-ORCH CLAUDE.md 2>/dev/null || echo \\\"$B\\\" | base64 -d >> CLAUDE.md; exec claude\", \"workingDir\": \"slop-code-bench\", \"skipApprovals\": true}", + "position": { + "x": 0, + "y": 0 + }, + "size": { + "width": 640, + "height": 520 + } + }, + { + "placeholderId": "runbook", + "type": "note", + "content": "# SlopCodeBench Runner\n\nRun slop-code-bench arms with a **Claude Code orchestrator** you drive in plain English.\n\n## One-time setup (per VM)\n1. Fork at `/workspace/slop-code-bench` with venv `/workspace/scb-venv` (uv sync `--python 3.12`). On desktop this is the host-shared workspace.\n2. Add your provider key in the **secrets panel**, broker-protected (e.g. `OPENROUTER_API_KEY`).\n\n## Drive the Claude Code terminal (it already knows these from CLAUDE.md)\n- \"Run `file_backup` with opencode on kimi-k2.6\"\n- \"Show me the run log\" / \"is it past checkpoint_1 with cost>0?\"\n- \"Score the finished run\"\n\nValidated arm: opencode + Kimi K2.6 via OpenRouter. Broker URL must be `127.0.0.1`, not `localhost`. Full docs: `benchmarks/slopcodebench/RESUME-kimi-opencode.md`.", + "position": { + "x": 680, + "y": 0 + }, + "size": { + "width": 400, + "height": 320 + } + }, + { + "placeholderId": "results", + "type": "browser", + "content": "{\"name\": \"SlopCodeBench Results\", \"url\": \"http://localhost:8050\"}", + "position": { + "x": 680, + "y": 360 + }, + "size": { + "width": 720, + "height": 500 + } + } + ], + "edges": [ + { + "sourcePlaceholderId": "orchestrator", + "targetPlaceholderId": "results" + } + ], + "viewport": { + "x": 0, + "y": 0, + "zoom": 1 + }, + "setupGuide": "# SlopCodeBench setup walkthrough\n\nGoal: get this dashboard ready to run a slop-code-bench arm, then run one.\nBe brief; do each step with your tools and confirm before the next.\n\n1. **Choose the harness source.** Ask the user which to use:\n - (a) **Our fork** `https://github.com/robdmac/slop-code-bench` (branch\n `feat/host-tmux-executor`) \u2014 recommended; it has the no-Docker watchable\n executor + the broker-compatible routing this needs. Default to this.\n - (b) a **custom fork/branch URL** they paste.\n - (c) **already present** at `/workspace/slop-code-bench` \u2014 skip the clone.\n\n2. **Clone it** (unless (c)) via terminal_input in a shell terminal:\n `git clone --branch /workspace/slop-code-bench`\n (needs network; if a domain is held by the egress proxy, tell the user to approve it.)\n\n3. **Create the venv** (one-time, slow): in `/workspace/slop-code-bench` run\n `UV_PYTHON_INSTALL_DIR=/workspace/.uv-python UV_PROJECT_ENVIRONMENT=/workspace/scb-venv uv sync --python 3.12`\n (Python 3.12 \u2014 3.14 has no wheels and the VM has no compiler.)\n\n4. **Provider key.** Ensure an `OPENROUTER_API_KEY` broker secret exists for this\n dashboard. If not, ask the user to add it (secrets_create, broker-protected).\n Never display or echo the value.\n\n5. **Point the broker URL at 127.0.0.1** (not localhost \u2014 Node/opencode can't\n reach the IPv4-only broker via `localhost`). In the checkout:\n `cp configs/providers.yaml.orig configs/providers.yaml` then replace the\n openrouter `api_base` with `$OPENROUTER_BASE_URL` after swapping\n `localhost`->`127.0.0.1`.\n\n6. **Run one arm** and watch it: launch detached to a logfile, then tail it:\n `/workspace/scb-venv/bin/slop-code run --agent opencode --model openrouter/kimi-k2.6 --environment configs/environments/local-tmux-py.yaml --prompt configs/prompts/just-solve.jinja --problem file_backup --no-evaluate thinking=low`\n Success = advances past `checkpoint_1` with `cost>0` and `steps>0`. Report when confirmed.\n" +} \ No newline at end of file diff --git a/benchmarks/slopcodebench/tests/viewer-smoke.sh b/benchmarks/slopcodebench/tests/viewer-smoke.sh new file mode 100755 index 00000000..8776d559 --- /dev/null +++ b/benchmarks/slopcodebench/tests/viewer-smoke.sh @@ -0,0 +1,99 @@ +#!/usr/bin/env bash +# Light smoke test for the benchmark *viewer pattern* (branch 2) on the desktop VM. +# +# Validates the SECURE viewer mechanism: a read-only Orcabot terminal watches a +# run by tailing its per-run logfile (`tail -F`), NOT by attaching a shared tmux +# control socket. +# +# Why not a shared tmux socket: a world-accessible tmux control socket would let +# any in-VM process attach = read every pane AND inject commands into other +# sessions, bypassing output redaction (CLAUDE.md:96) and the "two terminals +# cannot see each other" / "no cross-session access" isolation invariants +# (CLAUDE.md:94, sandbox/CLAUDE.md:248), via a local channel the egress proxy +# never sees. A logfile tail has no inject capability and no reach into other +# runs — it stays within the already-shared /workspace trust boundary. +# +# This mirrors the real host-tmux executor: it tees each run's output to a +# per-run logfile (its tmux window stays PRIVATE to the executor's own uid). +# A cross-PTY viewer tails that same logfile. +# +# Flow: +# 1. (in VM, via /debug/exec) a "run" writes a marker + heartbeats to a logfile. +# 2. create an Orcabot terminal whose bootCommand is `tail -F `. +# 3. `orcabot tail` the viewer -> assert it shows the marker (watching works). +# 4. send keystrokes to the viewer -> assert they do NOT reach the run's +# logfile (read-only holds; tail cannot write back to the run). +# +# Usage: benchmarks/slopcodebench/tests/viewer-smoke.sh +# Requires: stack running (`orcabot up`), python3, curl. Exits non-zero on failure. +set -uo pipefail + +ROOT="$(cd "$(dirname "$0")/../../.." && pwd)" +ORCABOT="${ORCABOT_BIN:-$ROOT/desktop/app/src-tauri/target/release/orcabot}" +CP="http://127.0.0.1:8787" +SB="http://127.0.0.1:8080" +U="X-User-ID: dev-desktop" +VZ_LOG="/tmp/vz-console.log" + +# Per-boot surface token: the control plane only trusts dev-auth when the +# X-Orcabot-Surface header matches (devAuthSurfaceTrusted). Read it from the same +# file the desktop app / CLI use so these host-side curls authenticate like the CLI. +# Empty on pre-enforcement stacks (harmless — dev-auth trusted headerless there). +SURFACE_TOKEN_FILE="${ORCABOT_SURFACE_TOKEN_FILE:-$HOME/Library/Application Support/com.orcabot.desktop/surface-token}" +ST="$(cat "$SURFACE_TOKEN_FILE" 2>/dev/null | tr -d '[:space:]')" +AUTH=(-H "$U" -H "X-Orcabot-Surface: $ST") # dev-auth headers for every control-plane call +RUNDIR="/workspace/.scb-smoke" +LOG="$RUNDIR/run1.log" +MARKER="SMOKE_VIEWER_MARKER_OK" +INJECT="INJECT_RO_SHOULD_NOT_APPEAR" + +PASS=0; FAIL=0 +ok() { echo " PASS: $1"; PASS=$((PASS+1)); } +bad() { echo " FAIL: $1"; FAIL=$((FAIL+1)); } +jq_py(){ python3 -c "$1"; } + +TOK="$(grep -a 'debug-exec] auth token:' "$VZ_LOG" 2>/dev/null | tail -1 | sed 's/.*auth token: //' | tr -cd 'a-f0-9')" +# run a shell command in the VM as root, print stdout +X(){ local b; b="$(python3 -c "import json,sys;print(json.dumps({'cmd':sys.argv[1],'timeout_ms':10000}))" "$1")" + curl -s -m 14 -X POST -H "X-Debug-Exec-Token: $TOK" -H 'Content-Type: application/json' \ + -d "$b" "$SB/debug/exec" | python3 -c "import json,sys;print(json.load(sys.stdin).get('stdout','').strip())"; } + +# ---- preflight ---- +[ -x "$ORCABOT" ] || { echo "orcabot binary not found at $ORCABOT" >&2; exit 2; } +curl -s -m3 "$CP/health" >/dev/null || { echo "control plane down on :8787 — run 'orcabot up'." >&2; exit 2; } +[ -n "$TOK" ] || { echo "no /debug/exec token in $VZ_LOG (need VZ_CONSOLE_DIRECT=1 / 'orcabot up')." >&2; exit 2; } + +echo "[1] start a 'run' that tees output to a per-run logfile" +X "rm -rf $RUNDIR; mkdir -p $RUNDIR" >/dev/null +# marker now; then a detached heartbeat writer so tail -F has something live. +X "echo $MARKER > $LOG; nohup sh -c 'for i in \$(seq 1 120); do echo hb_\$i >> $LOG; sleep 1; done' >/dev/null 2>&1 &" >/dev/null +sleep 1 +HAVE="$(X "grep -c $MARKER $LOG 2>/dev/null")" +[ "$HAVE" = "1" ] && ok "run logfile created with marker" || bad "logfile/marker not present (got '$HAVE')" + +echo "[2] spawn a read-only viewer terminal (tail -F the logfile)" +DID="$(curl -s -X POST "${AUTH[@]}" -H 'Content-Type: application/json' --data-raw '{"name":"viewer-smoke"}' "$CP/dashboards" | jq_py "import json,sys;print(json.load(sys.stdin)['dashboard']['id'])")" +CONTENT="{\"name\":\"viewer\",\"bootCommand\":\"tail -n +1 -F $LOG\"}" +# Build the request body with single-quoted python reading $CONTENT from the env, +# so bash never brace-expands the JSON dict. +BODY="$(CONTENT="$CONTENT" python3 -c 'import json,os;print(json.dumps({"type":"terminal","content":os.environ["CONTENT"]}))')" +IT="$(curl -s -X POST "${AUTH[@]}" -H 'Content-Type: application/json' --data-raw "$BODY" \ + "$CP/dashboards/$DID/items" | jq_py "import json,sys;print(json.load(sys.stdin)['item']['id'])")" +curl -s -o /dev/null -X POST "${AUTH[@]}" -H 'Content-Type: application/json' -d '{}' "$CP/dashboards/$DID/items/$IT/session"; sleep 3 +echo " dashboard=$DID item=$IT" + +echo "[3] viewer shows the run (watching works)" +OUT="$(timeout 12 "$ORCABOT" tail "$IT" --dash "$DID" --secs 6 2>/dev/null)" +echo "$OUT" | grep -q "$MARKER" && ok "viewer shows marker (tailing the run logfile)" || bad "viewer did not show marker" + +echo "[4] viewer is read-only (input cannot reach the run)" +timeout 12 "$ORCABOT" tail "$IT" --dash "$DID" --secs 5 --send "$INJECT" >/dev/null 2>&1; sleep 1 +LOGTXT="$(X "cat $LOG")" +echo "$LOGTXT" | grep -q "$INJECT" && bad "INPUT LEAKED into run logfile (not read-only!)" || ok "read-only holds (injected text absent from run logfile)" + +# ---- teardown ---- +X "pkill -f 'seq 1 120' 2>/dev/null; rm -rf $RUNDIR" >/dev/null +curl -s -o /dev/null -X DELETE "${AUTH[@]}" "$CP/dashboards/$DID" +echo +echo "================ $PASS passed, $FAIL failed ================" +[ "$FAIL" -eq 0 ] diff --git a/benchmarks/swebench-pro/README.md b/benchmarks/swebench-pro/README.md new file mode 100644 index 00000000..0e23d23d --- /dev/null +++ b/benchmarks/swebench-pro/README.md @@ -0,0 +1,83 @@ +# SWE-bench Pro runner for Orcabot + +Run [Scale AI's SWE-bench Pro](https://labs.scale.com/leaderboard/swe_bench_pro_public) +**public** set (731 tasks) from Orcabot. Harder, contamination-resistant +successor to SWE-bench — top models score ~23% vs ~70% on SWE-bench Verified. + +## Runnable? Yes — the public set self-scores + +The eval harness and golden tests are **open source**, so you score the public +731 locally without submitting to Scale: + +- Harness: (MIT) +- Dataset: `ScaleAI/SWE-bench_Pro` on HuggingFace, `test` split, 731 rows +- Prebuilt per-task images: `jefzda/sweap-images:` + +(The private/held-out splits — 276 + 858 tasks — are Scale-internal only.) + +## Task + format + +Same shape as SWE-bench: repo + issue → patch → hidden `fail_to_pass` / +`pass_to_pass` tests. Each dataset row carries a `dockerhub_tag` naming its +image. + +## Two phases + +**1. Inference (host, model keys, no Docker).** Drive an agent (the bundled +SWE-agent submodule, or Claude Code/Codex) over each instance's +`problem_statement` to produce per-instance `.pred` files, then gather: +``` +python helper_code/gather_patches.py \ + --directory swe_bench_pro_results/sample1 \ + --prefix sample1 \ + --output sample1_patches.json +``` +Predictions format (array): `[{"instance_id":"…","patch":"diff --git …","prefix":"sample1"}]` + +**2. Evaluation (Docker-per-task, no model keys).** +``` +python swe_bench_pro_eval.py \ + --raw_sample_path=swe_bench_pro_full.csv \ + --patch_path=sample1_patches.json \ + --output_dir= \ + --scripts_dir=run_scripts \ + --num_workers= \ + --dockerhub_username=jefzda +``` + +## Backends for eval + +- **`--use_local_docker`** (beta): run `jefzda/sweap-images:*` on a local Docker + daemon — needs a **Docker-capable benchmark sandbox** (x86 DinD microVM). +- **Modal** (default in the harness): `modal setup` → `~/.modal.toml`; container + execution runs in Modal's cloud, no local Docker. The harness defaults to + `--num_workers=100`, i.e. it's built for cloud parallelism. + +## Resource reality + +Dataset is tiny (~8 MB). The cost is **one prebuilt image per instance** (731 +multi-GB images across 41 repos) — tens-to-hundreds of GB if pulled locally. +Strongly prefer **Modal** unless you've provisioned a fat DinD sandbox, and +start with a small instance subset. + +## Setup + +``` +git clone --recurse-submodules https://github.com/scaleapi/SWE-bench_Pro-os +pip install -r requirements.txt +# Docker (for --use_local_docker) OR `modal setup` for the Modal backend +``` + +## Files + +`orchestrator-runbook.md`, `config-wizard.md`, `template.json`. + +## Status — prototype, partially validated + +**Verified:** commands/flags/formats are from the harness README + dataset card; +`template.json` parses with valid embedded content. + +**Not validated from here** (needs Docker/Modal + keys): a real inference run, a +real `swe_bench_pro_eval.py`, image pulls. Note the inference step delegates +exact LLM env-var names to the SWE-agent submodule — confirm them against the +submodule config before finalizing. `--use_local_docker` is upstream-beta. diff --git a/benchmarks/swebench-pro/config-wizard.md b/benchmarks/swebench-pro/config-wizard.md new file mode 100644 index 00000000..414a8e0d --- /dev/null +++ b/benchmarks/swebench-pro/config-wizard.md @@ -0,0 +1,36 @@ +# SWE-bench Pro config-phase wizard (Orcabot chat) + +Config phase before provisioning the SWE-bench Pro dashboard. Uses existing chat +tools: `secrets_create`, `create_terminal`, `create_browser`, `connect_nodes`. + +## Conversation flow + +1. **Inference agent?** SWE-agent (bundled) · Claude Code · Codex. → Phase-1 + driver. +2. **Model + key?** Ask provider/model; user pastes the key into the secrets + panel (never chat) → `secrets_create`. Note: SWE-agent reads provider keys + per its submodule config — confirm the exact var name. +3. **Eval backend?** + - **Modal** (default): user adds Modal token (`~/.modal.toml`) as a secret. + - **Local Docker** (`--use_local_docker`): only if a Docker-capable x86 + benchmark sandbox exists; warn about heavy `jefzda/sweap-images` pulls. +4. **How many instances?** Default to a small subset for the first run, not all + 731. + +## What the wizard does + +1. `secrets_create` for the model key (and Modal token if chosen). +2. `create_terminal` for the **orchestrator** (Claude Code, `boot_command: + "claude"`, working dir `SWE-bench_Pro-os`, `skipApprovals: true`), seeded + with the two-phase plan (inference → `gather_patches.py` → `swe_bench_pro_eval.py`). +3. `create_browser` → `https://labs.scale.com/leaderboard/swe_bench_pro_public` + (leaderboard reference; actual results are local under `--output_dir`). +4. Hand off to the orchestrator runbook. + +## Guardrails + +- Never accept/echo a raw key — secrets panel only. +- Prefer Modal; only offer local Docker if a DinD sandbox is provisioned, and + warn about tens-to-hundreds of GB of image pulls. +- Public set only (731). The private/held-out splits are not accessible — don't + imply otherwise. diff --git a/benchmarks/swebench-pro/orchestrator-runbook.md b/benchmarks/swebench-pro/orchestrator-runbook.md new file mode 100644 index 00000000..4c24849d --- /dev/null +++ b/benchmarks/swebench-pro/orchestrator-runbook.md @@ -0,0 +1,65 @@ +# SWE-bench Pro Orchestrator — Runbook / System Prompt + +Operating guide for the **SWE-bench Pro Orchestrator** (Claude Code in an +Orcabot terminal, driven in English). Paste the System prompt into the agent. + +## System prompt + +> You are the **SWE-bench Pro Orchestrator** inside Orcabot. Run Scale's public +> SWE-bench Pro (731 tasks). Two phases — keep them separate. Be concise; report +> what you ran and observed. +> +> **Environment** +> - Work in the `SWE-bench_Pro-os` checkout (your working dir). If missing: +> `git clone --recurse-submodules https://github.com/scaleapi/SWE-bench_Pro-os` +> then `pip install -r requirements.txt`. +> - Dataset: `ScaleAI/SWE-bench_Pro` (HuggingFace, `test` split, 731 rows). Each +> row has `instance_id`, `problem_statement`, `dockerhub_tag`, `fail_to_pass`, +> `pass_to_pass`. +> - Model keys come from the Orcabot secrets broker (SWE-agent reads them as env +> vars). **Never print or echo a key.** +> - Default to the **Modal** eval backend and a **small instance subset**. +> +> **Phase 1 — inference → patches.** Drive an agent over each instance's +> `problem_statement` to emit per-instance `.pred` files, then gather them: +> ``` +> python helper_code/gather_patches.py --directory swe_bench_pro_results/ \ +> --prefix --output _patches.json +> ``` +> Predictions are an array of `{instance_id, patch, prefix}`. No Docker here. +> Report patch count + any errored instances. +> +> **Phase 2 — evaluation (Docker-per-task, no model keys).** +> ``` +> python swe_bench_pro_eval.py \ +> --raw_sample_path=swe_bench_pro_full.csv \ +> --patch_path=_patches.json \ +> --output_dir= --scripts_dir=run_scripts \ +> --num_workers= --dockerhub_username=jefzda +> ``` +> - **Modal backend** (default): ensure `~/.modal.toml` exists (`modal setup`). +> Keep `--num_workers` high only if Modal capacity allows. +> - **Local Docker** (`--use_local_docker`, beta): only on a Docker-capable +> benchmark sandbox; expect heavy multi-GB image pulls from +> `jefzda/sweap-images`. Use a small subset and low workers. +> - Do **not** manually invoke bash inside these images (their entrypoint runs +> bash by default). +> +> **Make runs visible.** Surface per-task progress with a viewer pane: call the +> **`create_terminal`** MCP tool with a boot command that tails the eval +> output_dir logs (or `docker logs -f ` for the local backend, or +> Modal log streaming for Modal), and **`connect_nodes`** an edge from yourself. +> No duplicate viewers. +> +> **Results.** The harness reports the **Resolve Rate** over the evaluated set. +> Report it and point the user at `--output_dir`. +> +> **Lifecycle.** Start with a handful of instances; expand once the backend is +> proven. Re-run only failed instances. Stop on Ctrl-C; report partials. + +## Notes + +- The `{instance_id, patch, prefix}` JSON is the clean handoff: Phase 1 on the + desktop VM, Phase 2 on Modal or a DinD sandbox. +- Exact LLM env-var names for inference live in the SWE-agent submodule config + (e.g. `claude.yaml`) — verify them rather than assuming. diff --git a/benchmarks/swebench-pro/template.json b/benchmarks/swebench-pro/template.json new file mode 100644 index 00000000..4e9f44f4 --- /dev/null +++ b/benchmarks/swebench-pro/template.json @@ -0,0 +1,34 @@ +{ + "_comment": "Importable Orcabot dashboard template for SWE-bench Pro (public 731). Shape matches DashboardTemplateWithData. Viewer panes are spawned by the orchestrator at runtime.", + "name": "SWE-bench Pro Runner", + "description": "Run Scale's public SWE-bench Pro (731 tasks) with a Claude Code orchestrator you drive in English. Inference produces a {instance_id,patch,prefix} JSON on host; evaluation runs Docker-per-task via Modal or local Docker.", + "category": "coding", + "items": [ + { + "placeholderId": "orchestrator", + "type": "terminal", + "content": "{\"name\":\"SWE-bench Pro Orchestrator\",\"agentic\":true,\"bootCommand\":\"claude\",\"workingDir\":\"SWE-bench_Pro-os\",\"skipApprovals\":true}", + "position": { "x": 0, "y": 0 }, + "size": { "width": 640, "height": 520 } + }, + { + "placeholderId": "runbook", + "type": "note", + "content": "# SWE-bench Pro Runner\n\nDrive the **SWE-bench Pro Orchestrator** (Claude Code) in English:\n\n- *\"Run inference on 5 instances with SWE-agent.\"*\n- *\"Gather the patches into a predictions JSON.\"*\n- *\"Evaluate on Modal.\"*\n- *\"Show me the running task logs.\"* -> spawns a viewer pane\n\nPublic 731 set, self-scoring. Two phases: inference (host, needs key) -> patches JSON -> eval (Docker-per-task, Modal or local). Keys in the secrets broker. Guide: benchmarks/swebench-pro/orchestrator-runbook.md", + "position": { "x": 680, "y": 0 }, + "size": { "width": 400, "height": 320 } + }, + { + "placeholderId": "leaderboard", + "type": "browser", + "content": "{\"name\":\"SWE-bench Pro Leaderboard\",\"url\":\"https://labs.scale.com/leaderboard/swe_bench_pro_public\"}", + "position": { "x": 680, "y": 360 }, + "size": { "width": 720, "height": 500 } + } + ], + "edges": [ + { "sourcePlaceholderId": "orchestrator", "targetPlaceholderId": "leaderboard" } + ], + "viewport": { "x": 0, "y": 0, "zoom": 1 }, + "setupGuide": "# SWE-bench Pro setup walkthrough\n\nGoal: get ready to run Scale's SWE-bench Pro (731 tasks), then run a small subset.\nBe brief; do each step with your tools and confirm before the next. Two phases —\nkeep them separate.\n\n1. **Backend + scope.** Default to the **Modal** eval backend and a **small instance\n subset** (not all 731). Confirm with the user.\n\n2. **Get the harness** (skip if `SWE-bench_Pro-os` already exists):\n `git clone --recurse-submodules https://github.com/scaleapi/SWE-bench_Pro-os` then\n `pip install -r requirements.txt` in it. The dataset `ScaleAI/SWE-bench_Pro` (HF,\n `test` split, 731 rows) is pulled automatically.\n\n3. **Provider key.** Ensure the agent's model key is a broker-protected secret for\n this dashboard (SWE-agent reads it as an env var). Ask the user to add it if\n missing. Never display or echo a key.\n\n4. **Modal backend.** Needs `~/.modal.toml`; if absent, tell the user to run\n `modal setup`. (Local Docker via `--use_local_docker` is beta and pulls multi-GB\n images — only on a Docker-capable sandbox.)\n\n5. **Phase 1 — inference (small subset).** Drive an agent over each instance's\n `problem_statement` to emit per-instance `.pred` files, then gather them:\n `python helper_code/gather_patches.py --directory swe_bench_pro_results/ --prefix --output _patches.json`.\n No Docker. Report the patch count and any errored instances.\n\n6. **Phase 2 — evaluation.**\n `python swe_bench_pro_eval.py --raw_sample_path=swe_bench_pro_full.csv --patch_path=_patches.json --output_dir= --scripts_dir=run_scripts --num_workers= --dockerhub_username=jefzda`.\n Spawn a viewer terminal for progress and connect an edge to it. Do NOT manually run\n bash inside the task images (their entrypoint runs bash). Report the **resolved\n count / rate**.\n" +} diff --git a/benchmarks/swebench/README.md b/benchmarks/swebench/README.md new file mode 100644 index 00000000..eb9908a6 --- /dev/null +++ b/benchmarks/swebench/README.md @@ -0,0 +1,83 @@ +# SWE-bench runner for Orcabot + +Run [SWE-bench](https://github.com/SWE-bench/SWE-bench) from Orcabot: an +English-driven Claude Code orchestrator generates patches, then scores them. + +## The task + variants + +Each instance = a real GitHub issue + a repo snapshot. The agent produces a +`model_patch`; the harness applies it and runs the repo's tests. **Resolved** = +designated `FAIL_TO_PASS` tests pass and `PASS_TO_PASS` tests still pass. + +| Variant | Dataset (`--dataset_name`) | Size | Use | +|---------|----------------------------|------|-----| +| Lite | `princeton-nlp/SWE-bench_Lite` | ~300 | cheap smoke test | +| Verified | `princeton-nlp/SWE-bench_Verified` | 500 | standard leaderboard target | +| Full | `princeton-nlp/SWE-bench` | ~2294 | full run | +| Multimodal | `princeton-nlp/SWE-bench_Multimodal` | — | visual/JS (test split private) | + +## Two phases — this is the key architecture + +**1. Inference (host-native, needs model keys, no Docker).** Run an agent +scaffold against each instance to emit a **predictions JSONL**, one object/line: +```json +{"instance_id":"astropy__astropy-12345","model_name_or_path":"my-agent","model_patch":"diff --git ..."} +``` +Scaffolds: SWE-agent (canonical agentic), Agentless, plain model API +(`python -m swebench.inference.run_api`), or any harness (Claude Code/Codex) +that can write the JSONL. **Runs anywhere, including the desktop VM.** + +**2. Evaluation (Docker-per-task, no model keys).** +``` +python -m swebench.harness.run_evaluation \ + --dataset_name princeton-nlp/SWE-bench_Verified \ + --predictions_path preds.jsonl \ + --run_id my-run \ + --max_workers 8 \ + --cache_level env +``` +Every instance runs in its own container. **Docker is mandatory — there is no +host/local eval mode.** No model API keys needed for this phase. + +## Backends for the eval phase + +- **Modal (recommended, no local Docker):** add `--modal true`. Container + execution runs on Modal's cloud; the Orcabot VM stays thin. Needs a Modal + token (`~/.modal.toml`) as a secret. +- **Docker-in-VM (DinD):** run a Docker daemon in a **Docker-capable x86 Fly + microVM** benchmark sandbox, then run eval locally. Heavier; not the desktop + VM. On arm64 you must pass `--namespace ''` to build images locally (prebuilt + images are x86_64-only) — slow and partly experimental. + +## Resource reality + +Minimum ~**120 GB disk, 16 GB RAM, 8 cores** for local eval. `--cache_level +env` (default) ≈ 100 GB; `instance` ≈ 2 TB (avoid). Keep `--max_workers` ≤ +`min(0.75·cores, 24)`; ~8 for Lite, ~12 for Full. Start with **Lite** + Modal. + +## Sanity check before real runs + +``` +python -m swebench.harness.run_evaluation \ + --predictions_path gold --max_workers 1 \ + --instance_ids sympy__sympy-20590 --run_id validate-gold +``` +`gold` runs the reference patches — confirms your Docker/Modal backend works +before you spend inference tokens. + +## Files + +| File | Purpose | +|------|---------| +| `orchestrator-runbook.md` | System prompt + runbook for the orchestrator agent | +| `config-wizard.md` | Orcabot-chat config phase | +| `template.json` | Importable dashboard template | + +## Status — prototype, partially validated + +**Verified:** commands/flags/formats above are from the SWE-bench repo + docs; +`template.json` parses with valid embedded content. + +**Not validated from here** (needs Docker/Modal + keys): a real inference run, a +real `run_evaluation` (local or Modal), arm64 local-build path. Treat the +runbook as grounded instructions, not a turnkey-verified pipeline. diff --git a/benchmarks/swebench/config-wizard.md b/benchmarks/swebench/config-wizard.md new file mode 100644 index 00000000..aae59211 --- /dev/null +++ b/benchmarks/swebench/config-wizard.md @@ -0,0 +1,44 @@ +# SWE-bench config-phase wizard (Orcabot chat) + +The configuration phase the Orcabot chat runs before provisioning the SWE-bench +dashboard. Uses only existing chat tools: `secrets_create`, `create_terminal`, +`create_browser`, `connect_nodes`. + +## Conversation flow (one question at a time) + +1. **Which variant?** `Lite` (default, cheap) · `Verified` (leaderboard) · + `Full` · `Multimodal` (inference only). → `--dataset_name`. +2. **Which agent/scaffold for inference?** plain model API + (`swebench.inference.run_api`) · SWE-agent · Agentless · a CLI harness + (Claude Code/Codex). → determines the Phase-1 command. +3. **Which model + key?** Ask for the provider/model and have the user paste the + key into the secrets panel (never chat) → `secrets_create` + (`ANTHROPIC_API_KEY` / `OPENAI_API_KEY`). +4. **Eval backend?** + - **Modal** (default, no local Docker) → have the user add their Modal token + (`~/.modal.toml`) as a secret; eval uses `--modal true`. + - **Docker-in-VM** → only if a Docker-capable benchmark sandbox is + provisioned; warn this needs an x86 DinD machine and ~120 GB disk. +5. **Subset size?** Default to a handful of `--instance_ids` for a first run, + not the whole variant. + +## What the wizard then does + +1. `secrets_create` for the model key (and Modal token if chosen). +2. `create_terminal` for the **orchestrator** (Claude Code, `boot_command: + "claude"`, working dir `SWE-bench`, `skipApprovals: true`). Seed its first + message with the assembled two-phase plan (inference command → `preds.jsonl` + → `run_evaluation` with the chosen backend). +3. `create_browser` pointed at `https://www.swebench.com` (leaderboard + + reference) — results themselves are local JSON, so also tell the user where + the report lands. +4. Hand off to the orchestrator runbook. + +## Guardrails + +- Never accept/echo a raw API key — route through the secrets panel. +- Always run the `--predictions_path gold` sanity check on a new eval backend + before real inference, and say why (it validates Docker/Modal for free). +- Be explicit that **Full @ `--cache_level instance` ≈ 2 TB** — steer to Lite + + `env` cache for a first run. +- Multimodal: inference only (private test split) — don't promise a score. diff --git a/benchmarks/swebench/orchestrator-runbook.md b/benchmarks/swebench/orchestrator-runbook.md new file mode 100644 index 00000000..45559580 --- /dev/null +++ b/benchmarks/swebench/orchestrator-runbook.md @@ -0,0 +1,72 @@ +# SWE-bench Orchestrator — Runbook / System Prompt + +Operating guide for the **SWE-bench Orchestrator**: a Claude Code agent in an +Orcabot terminal you drive in English to run SWE-bench. Paste the System prompt +into the orchestrator agent (or a `CLAUDE.md` in its working dir). + +## System prompt + +> You are the **SWE-bench Orchestrator** inside Orcabot. The user directs you in +> English; translate that into SWE-bench commands and Orcabot canvas actions. +> Be concise; report what you ran and what you observed. SWE-bench has two +> phases — keep them separate. +> +> **Environment** +> - Work in the `SWE-bench` checkout (your working dir). If missing, clone +> `https://github.com/SWE-bench/SWE-bench` and `pip install -e .`. +> - Datasets are pulled from HuggingFace automatically by `--dataset_name`. +> - Model keys (`ANTHROPIC_API_KEY`, `OPENAI_API_KEY`) come from the Orcabot +> secrets broker as env vars. **Never print or echo a key.** +> - Default to the **Lite** variant and the **Modal** eval backend unless told +> otherwise — it's the cheapest, lowest-infra path. +> +> **Phase 1 — inference → predictions JSONL.** Produce one JSON object per +> instance with keys `instance_id`, `model_name_or_path`, `model_patch`: +> - Simple/API: `python -m swebench.inference.run_api --dataset_name_or_path +> --model_name_or_path --output_dir ./outputs`. +> - Agentic: drive SWE-agent / Agentless, or any harness, as long as it emits +> the JSONL. Write it to `preds.jsonl`. +> - This phase needs model keys and **no Docker**. Report the count of patches +> produced and any instances that errored. +> +> **Phase 2 — evaluation (Docker-per-task, no model keys).** First sanity-check +> the backend with gold patches: +> ``` +> python -m swebench.harness.run_evaluation --predictions_path gold \ +> --max_workers 1 --instance_ids sympy__sympy-20590 --run_id validate-gold +> ``` +> Then evaluate your predictions: +> ``` +> python -m swebench.harness.run_evaluation \ +> --dataset_name --predictions_path preds.jsonl \ +> --run_id --max_workers --cache_level env [--modal true] +> ``` +> - Use `--modal true` for the Modal backend (needs `~/.modal.toml`); omit it +> only if a local Docker daemon is available (DinD sandbox). +> - On arm64 with local Docker, add `--namespace ''` to build images locally. +> - Keep `--max_workers` modest (≤ 8 for Lite). Watch disk — `--cache_level env`, +> never `instance`. +> +> **Make runs visible.** Per-instance logs stream to +> `logs/run_evaluation////`. For instances you want +> to watch, call the **`create_terminal`** MCP tool with a boot command that +> tails the log, e.g. `tail -F logs/run_evaluation////run_instance.log`, +> and **`connect_nodes`** an edge from yourself to it. For the Modal backend, +> stream Modal's logs instead. Don't create duplicate viewers. +> +> **Results.** The harness writes a summary report JSON (counts: submitted / +> completed / **resolved**, plus the **resolution rate %**) and per-instance +> logs. Report the resolution rate and point the user at the report file. There +> is no bundled local dashboard. +> +> **Lifecycle.** Subset with `--instance_ids` to debug a single failing case. +> Re-run only failures by passing their ids. Stop a run with Ctrl-C; report +> partial results. Don't fan out `--max_workers` beyond the VM/backend capacity. + +## Notes + +- The predictions JSONL is the clean boundary: you can run Phase 1 on the + desktop VM and Phase 2 on a Docker-capable backend, handing off the file. +- `--predictions_path gold` is a free correctness check of your backend before + spending inference tokens — always run it once on a new backend. +- Multimodal's test split is private; you can only do inference there. diff --git a/benchmarks/swebench/template.json b/benchmarks/swebench/template.json new file mode 100644 index 00000000..d4509fbc --- /dev/null +++ b/benchmarks/swebench/template.json @@ -0,0 +1,34 @@ +{ + "_comment": "Importable Orcabot dashboard template for running SWE-bench. Shape matches DashboardTemplateWithData in frontend/src/types/dashboard.ts. Viewer panes (per-instance log tails) are spawned by the orchestrator at runtime, not baked into the template.", + "name": "SWE-bench Runner", + "description": "Run SWE-bench (Lite/Verified/Full) with a Claude Code orchestrator you drive in English. Inference produces a predictions JSONL on host; evaluation runs Docker-per-task via Modal or a DinD sandbox.", + "category": "coding", + "items": [ + { + "placeholderId": "orchestrator", + "type": "terminal", + "content": "{\"name\":\"SWE-bench Orchestrator\",\"agentic\":true,\"bootCommand\":\"claude\",\"workingDir\":\"SWE-bench\",\"skipApprovals\":true}", + "position": { "x": 0, "y": 0 }, + "size": { "width": 640, "height": 520 } + }, + { + "placeholderId": "runbook", + "type": "note", + "content": "# SWE-bench Runner\n\nDrive the **SWE-bench Orchestrator** (Claude Code) in English:\n\n- *\"Run inference on 5 Lite instances with claude.\"* -> writes preds.jsonl\n- *\"Sanity-check the Modal backend with gold.\"*\n- *\"Evaluate preds.jsonl on Lite via Modal.\"*\n- *\"Show me instance astropy__astropy-12345.\"* -> spawns a log viewer pane\n\nTwo phases: inference (host, needs key) -> predictions JSONL -> eval (Docker-per-task, Modal or DinD). Keys live in the secrets broker. Full guide: benchmarks/swebench/orchestrator-runbook.md", + "position": { "x": 680, "y": 0 }, + "size": { "width": 400, "height": 320 } + }, + { + "placeholderId": "leaderboard", + "type": "browser", + "content": "{\"name\":\"SWE-bench Leaderboard\",\"url\":\"https://www.swebench.com\"}", + "position": { "x": 680, "y": 360 }, + "size": { "width": 720, "height": 500 } + } + ], + "edges": [ + { "sourcePlaceholderId": "orchestrator", "targetPlaceholderId": "leaderboard" } + ], + "viewport": { "x": 0, "y": 0, "zoom": 1 }, + "setupGuide": "# SWE-bench setup walkthrough\n\nGoal: get this dashboard ready to run SWE-bench, then run one instance end to end.\nBe brief; do each step with your tools and confirm before the next. Keep SWE-bench's\ntwo phases (inference, then evaluation) separate.\n\n1. **Pick variant + eval backend.** Default to **Lite** (cheapest) and the **Modal**\n eval backend (no local Docker). Confirm with the user, or use those.\n\n2. **Get the harness** (skip if the `SWE-bench` checkout already exists): in a shell\n terminal `git clone https://github.com/SWE-bench/SWE-bench` then `pip install -e .`\n in it. Needs network; if a domain is held by the egress proxy, ask the user to approve it.\n\n3. **Provider key.** Ensure the model key the run needs (`ANTHROPIC_API_KEY` or\n `OPENAI_API_KEY`) exists as a broker-protected secret for this dashboard. If not,\n ask the user to add it in the secrets panel. Never display or echo a key.\n\n4. **Modal backend (eval phase).** It needs `~/.modal.toml`; if absent, tell the user\n to run `modal setup` (opens a browser to authenticate). Skip if you're using a\n local Docker (DinD) sandbox instead.\n\n5. **Phase 1 — inference (one instance).** Produce a `preds.jsonl` with keys\n `instance_id`, `model_name_or_path`, `model_patch`. Simplest is\n `python -m swebench.inference.run_api --dataset_name_or_path --model_name_or_path --output_dir ./outputs`,\n or drive an agent that emits the JSONL. No Docker. Report the patch count.\n\n6. **Phase 2 — evaluation.** Sanity-check the backend with gold first:\n `python -m swebench.harness.run_evaluation --predictions_path gold --max_workers 1 --instance_ids sympy__sympy-20590 --run_id validate-gold`,\n then evaluate your predictions with `--dataset_name --predictions_path preds.jsonl\n --run_id --cache_level env [--modal true]`. Make it watchable: spawn a viewer\n terminal (create_terminal) tailing\n `logs/run_evaluation////run_instance.log` and connect an\n edge to it. Report the **resolution rate %** and point the user at the report JSON.\n" +} diff --git a/benchmarks/terminal-bench/README.md b/benchmarks/terminal-bench/README.md new file mode 100644 index 00000000..5fdc9505 --- /dev/null +++ b/benchmarks/terminal-bench/README.md @@ -0,0 +1,87 @@ +# Terminal-Bench 2.1 runner for Orcabot + +Run [Terminal-Bench](https://www.tbench.ai) **2.1** from Orcabot — an agent +operating autonomously in a real terminal to complete end-to-end tasks +(compile, train, debug, set up servers, security fixes…). + +## ⚠️ 2.x uses Harbor, not `tb` + +The original `tb` / `terminal-bench` pip package was the v0.x/1.x line. +**Terminal-Bench 2.0 / 2.1 run on the new Harbor framework** (`harbor` CLI, same +Laude Institute team). Target **Harbor** — do not build against `tb` for 2.1. + +## Task + Docker model + +Task unit = a Dockerized environment + a verifier. The agent runs **headless +inside the task's container**; a verifier container computes a reward. **Docker +is mandatory by default** (each task = its own container). Prereqs: `uv` + +Docker. + +## Harness — install + run + +``` +uv tool install harbor # or: pip install harbor + +# Validate the env first with the oracle agent (runs reference solutions, NO key): +uv run harbor run --dataset terminal-bench@2.0 --agent oracle --n-concurrent 4 + +# Terminal-Bench 2.1 with an agent under test: +harbor run --dataset terminal-bench/terminal-bench-2-1 \ + --agent terminus-2 \ + --model anthropic/claude-opus-4-1 \ + --n-concurrent 4 +``` +Flags: `--dataset`/`-d` (slug `org/dataset@version`), `--agent`/`-a`, +`--model`/`-m` (LiteLLM `provider/model`), `--n-concurrent`, `--env` (runtime +backend), `--task-id` (single task), `--agent-import-path` (custom agent). +Harbor auto-downloads datasets from `hub.harborframework.com`. + +## Agent under test + +- **Internal:** `terminus` / `terminus-1` / `terminus-2` (reference agent), plus + `oracle` (runs known-good solution — env check, no key) and `nop`. +- **Installed CLI agents** (run headless *inside* the task container): + `claude-code`, `codex`, `gemini-cli`, `copilot-cli`, `openhands`, `aider`, + `goose`, `opencode`, `cursor-cli`, `mini-swe-agent`, `qwen-coder`, … +- **Custom:** implement `BaseAgent` and pass + `--agent-import-path module:YourAgent`. + +Select with `--agent --model `. Keys are passed into the +agent's container via env vars. + +## Backends — Docker vs remote (the Orcabot decision) + +- **Local Docker** (default): needs a Docker daemon reachable by the harness → + a **Docker-capable benchmark sandbox** (DinD on an x86 Fly microVM). The + desktop VM deliberately avoids Docker, so this won't fit there. +- **Remote sandbox** (`--env daytona` / Modal): tasks execute in a cloud + backend; the orchestrating process needs **no local Docker daemon**. This is + the most Orcabot-friendly path (works regardless of where the orchestrator + runs) — needs the backend's token as a secret. + +## Keys / env vars + +LiteLLM-style: `--model provider/model` + the matching provider key as env var +(`ANTHROPIC_API_KEY`, `OPENAI_API_KEY`, `GEMINI_API_KEY`). `oracle`/`nop` need +none. + +## Outputs / scoring + +Harbor writes `jobs//` with `config.json`, top-level `result.json` +(aggregate accuracy), and per-trial `/result.json` + +`verifier/reward.txt`. Score = per-task reward → accuracy across the dataset. +Leaderboard: `tbench.ai/leaderboard/terminal-bench/2.1`. + +## Files + +`orchestrator-runbook.md`, `config-wizard.md`, `template.json`. + +## Status — prototype, partially validated + +**Verified:** Harbor commands/flags/dataset slug/agent list are from the +Terminal-Bench + Harbor docs; `template.json` parses with valid embedded +content. + +**Not validated from here** (needs Docker or a Daytona/Modal token + keys): a +real `harbor run`, the oracle check, remote-backend execution. Validate the env +with `--agent oracle` first (no key needed) before wiring a model. diff --git a/benchmarks/terminal-bench/config-wizard.md b/benchmarks/terminal-bench/config-wizard.md new file mode 100644 index 00000000..9daa0e1b --- /dev/null +++ b/benchmarks/terminal-bench/config-wizard.md @@ -0,0 +1,42 @@ +# Terminal-Bench 2.1 config-phase wizard (Orcabot chat) + +Config phase before provisioning the Terminal-Bench dashboard. Uses existing +chat tools: `secrets_create`, `create_terminal`, `create_browser`, +`connect_nodes`. + +## Conversation flow + +1. **Which agent under test?** `terminus-2` (reference) · `claude-code` · + `codex` · `gemini-cli` · `mini-swe-agent` · custom (`--agent-import-path`). + → `--agent`. +2. **Model + key?** Ask provider/model (LiteLLM `provider/model`); user pastes + the key into the secrets panel (never chat) → `secrets_create` + (`ANTHROPIC_API_KEY` / `OPENAI_API_KEY` / `GEMINI_API_KEY`). +3. **Backend?** + - **Remote** (`--env daytona` / Modal): default; user adds the backend token + as a secret. No local Docker needed. + - **Local Docker**: only if a Docker-capable benchmark sandbox is + provisioned. +4. **Scope?** Single `--task-id` for a smoke test, or the full 2.1 set. Set + `--n-concurrent` to backend capacity. + +## What the wizard does + +1. `secrets_create` for the model key (and backend token if remote). +2. `create_terminal` for the **orchestrator** (Claude Code, `boot_command: + "claude"`, `skipApprovals: true`). Seed its first message to (a) install + Harbor, (b) run the **oracle** validation, then (c) the real run command with + the chosen agent/model/backend. +3. `create_browser` → `https://www.tbench.ai/leaderboard/terminal-bench/2.1` + (leaderboard reference; results are local under `jobs/`). +4. Hand off to the orchestrator runbook. + +## Guardrails + +- Never accept/echo a raw key — secrets panel only. +- Always start with `--agent oracle` (no key) to prove the environment before + spending model tokens — and say why. +- Use **Harbor** + the `terminal-bench/terminal-bench-2-1` slug, not legacy + `tb`. Don't conflate the two. +- Default to the remote backend; only offer local Docker if a DinD sandbox + exists (the desktop VM can't run it). diff --git a/benchmarks/terminal-bench/orchestrator-runbook.md b/benchmarks/terminal-bench/orchestrator-runbook.md new file mode 100644 index 00000000..1530c358 --- /dev/null +++ b/benchmarks/terminal-bench/orchestrator-runbook.md @@ -0,0 +1,61 @@ +# Terminal-Bench 2.1 Orchestrator — Runbook / System Prompt + +Operating guide for the **Terminal-Bench Orchestrator** (Claude Code in an +Orcabot terminal, driven in English) running Terminal-Bench 2.1 via **Harbor**. + +## System prompt + +> You are the **Terminal-Bench Orchestrator** inside Orcabot. Run Terminal-Bench +> 2.1 with the **Harbor** CLI (`harbor`, NOT the legacy `tb`). Be concise; report +> what you ran and observed. +> +> **Environment** +> - Install Harbor if absent: `uv tool install harbor`. +> - The dataset slug for 2.1 is **`terminal-bench/terminal-bench-2-1`** (Harbor +> auto-downloads it). Do not use `--dataset-name/--dataset-version` — that's +> the legacy `tb` form. +> - Model keys (`ANTHROPIC_API_KEY`, etc.) come from the Orcabot secrets broker. +> **Never print or echo a key.** Keys pass into the agent's task container. +> - Pick a backend: **local Docker** (needs a Docker-capable sandbox) or +> **`--env daytona`/Modal** (remote, no local Docker — preferred unless a DinD +> sandbox is provisioned). Use the remote backend by default. +> +> **Always validate the environment first** (no API key, runs reference +> solutions): +> ``` +> uv run harbor run --dataset terminal-bench/terminal-bench-2-1 --agent oracle --n-concurrent 4 +> ``` +> Only proceed to a real model run once oracle passes. +> +> **Run the benchmark:** +> ``` +> harbor run --dataset terminal-bench/terminal-bench-2-1 \ +> --agent \ +> --model \ +> --n-concurrent [--env daytona] [--task-id ] +> ``` +> - For a smoke test, use `--task-id ` to run a single task. +> - Installed CLI agents (claude-code/codex/gemini-cli) run **headless inside the +> task container** — you don't drive them interactively; Harbor does. +> - Keep `--n-concurrent` within the backend's capacity. +> +> **Make runs visible.** Harbor writes a live `jobs//` tree. Spawn a +> viewer pane by calling the **`create_terminal`** MCP tool with a boot command +> that tails progress — e.g. `tail -F jobs///result.json` or +> `docker logs -f ` (local backend) / the backend's log stream +> (remote) — and **`connect_nodes`** an edge from yourself. No duplicate viewers. +> +> **Results.** Read `jobs//result.json` for the aggregate accuracy and +> per-trial `verifier/reward.txt` for individual rewards. Report the accuracy and +> point the user at the jobs dir. The public leaderboard is at +> tbench.ai/leaderboard/terminal-bench/2.1 (not auto-submitted). +> +> **Lifecycle.** Start with `--agent oracle`, then a single `--task-id`, then the +> full set. Re-run a failed task by id. Stop with Ctrl-C; report partials. + +## Notes + +- 2.1 is a hardened iteration of 2.0 (26 tasks fixed for timeouts/resources/ + reward-hacking) — use the 2-1 slug for current numbers. +- The remote backend (`--env daytona`/Modal) is the only path that works when no + Docker daemon is available to the orchestrator (e.g. desktop). diff --git a/benchmarks/terminal-bench/template.json b/benchmarks/terminal-bench/template.json new file mode 100644 index 00000000..88456772 --- /dev/null +++ b/benchmarks/terminal-bench/template.json @@ -0,0 +1,34 @@ +{ + "_comment": "Importable Orcabot dashboard template for Terminal-Bench 2.1 (Harbor). Shape matches DashboardTemplateWithData. Viewer panes are spawned by the orchestrator at runtime.", + "name": "Terminal-Bench 2.1 Runner", + "description": "Run Terminal-Bench 2.1 via the Harbor harness with a Claude Code orchestrator you drive in English. Docker-per-task; use the Daytona/Modal remote backend to avoid local Docker.", + "category": "coding", + "items": [ + { + "placeholderId": "orchestrator", + "type": "terminal", + "content": "{\"name\":\"Terminal-Bench Orchestrator\",\"agentic\":true,\"bootCommand\":\"claude\",\"workingDir\":\"terminal-bench\",\"skipApprovals\":true}", + "position": { "x": 0, "y": 0 }, + "size": { "width": 640, "height": 520 } + }, + { + "placeholderId": "runbook", + "type": "note", + "content": "# Terminal-Bench 2.1 Runner (Harbor)\n\nDrive the **Terminal-Bench Orchestrator** (Claude Code) in English:\n\n- *\"Validate the env with the oracle agent.\"* (no key)\n- *\"Run task hello-world with claude-code on opus.\"*\n- *\"Run the full 2.1 set on Daytona, 4 concurrent.\"*\n- *\"Show me the running job.\"* -> spawns a viewer pane\n\nUses **Harbor** (not legacy tb). Dataset slug: terminal-bench/terminal-bench-2-1. Docker-per-task; prefer --env daytona/Modal. Keys in the secrets broker. Guide: benchmarks/terminal-bench/orchestrator-runbook.md", + "position": { "x": 680, "y": 0 }, + "size": { "width": 400, "height": 340 } + }, + { + "placeholderId": "leaderboard", + "type": "browser", + "content": "{\"name\":\"Terminal-Bench 2.1 Leaderboard\",\"url\":\"https://www.tbench.ai/leaderboard/terminal-bench/2.1\"}", + "position": { "x": 680, "y": 380 }, + "size": { "width": 720, "height": 480 } + } + ], + "edges": [ + { "sourcePlaceholderId": "orchestrator", "targetPlaceholderId": "leaderboard" } + ], + "viewport": { "x": 0, "y": 0, "zoom": 1 }, + "setupGuide": "# Terminal-Bench setup walkthrough\n\nGoal: get ready to run Terminal-Bench 2.1 (via the **Harbor** CLI), then smoke one\ntask. Be brief; do each step with your tools and confirm before the next.\n\n1. **Install Harbor** (skip if `harbor` is on PATH): `uv tool install harbor`. Use\n Harbor, NOT the legacy `tb`.\n\n2. **Pick a backend.** Prefer the **remote** backend (`--env daytona`/Modal — no local\n Docker). Use **local Docker** only on a Docker-capable (DinD) sandbox. Confirm with\n the user.\n\n3. **Provider key.** Ensure the model key (`ANTHROPIC_API_KEY`, etc.) is a\n broker-protected secret for this dashboard — Harbor passes it into the agent's task\n container. Ask the user to add it if missing. Never display or echo a key.\n\n4. **Validate the environment FIRST** (no key; runs reference solutions):\n `uv run harbor run --dataset terminal-bench/terminal-bench-2-1 --agent oracle --n-concurrent 4`.\n Only proceed to a model run once oracle passes.\n\n5. **Smoke one task.**\n `harbor run --dataset terminal-bench/terminal-bench-2-1 --agent --model --task-id [--env daytona]`.\n The CLI agents run headless inside the task container — Harbor drives them, you don't.\n\n6. **Watch + results.** Spawn a viewer terminal (create_terminal) tailing\n `jobs///result.json` (or the backend's log stream) and connect an edge\n to it. Read `jobs//result.json` for the aggregate accuracy; report it and\n point the user at the jobs dir.\n" +} diff --git a/controlplane/src/chat/handler.ts b/controlplane/src/chat/handler.ts index 5e4a28f7..c74fe455 100644 --- a/controlplane/src/chat/handler.ts +++ b/controlplane/src/chat/handler.ts @@ -378,6 +378,20 @@ const GUIDANCE_TOOLS: GeminiTool[] = [ required: ['message'], }, }, + { + name: 'complete_setup_walkthrough', + description: 'Mark this dashboard\'s setup walkthrough as complete so it is no longer injected. Call ONLY after the user confirms setup is done (e.g. the first job ran successfully). Only relevant when an ACTIVE SETUP WALKTHROUGH is present.', + inputSchema: { + type: 'object', + properties: { + dashboard_id: { + type: 'string', + description: 'The dashboard whose setup walkthrough is complete', + }, + }, + required: ['dashboard_id'], + }, + }, { name: 'ui_highlight', description: 'Highlight a UI element to draw the user\'s attention. Use for onboarding and guidance.', @@ -601,6 +615,18 @@ async function executeTool( return { result: data as Record, isError: !response.ok }; } + if (toolName === 'complete_setup_walkthrough') { + // Clear the dashboard's setup guide so streamMessage stops injecting it. + try { + await env.DB.prepare(`UPDATE dashboards SET setup_guide = NULL WHERE id = ?`) + .bind(args.dashboard_id as string) + .run(); + return { result: { ok: true }, isError: false }; + } catch { + return { result: { ok: false }, isError: false }; + } + } + if (toolName === 'dashboard_rename') { const response = await dashboards.updateDashbоard( env, @@ -1327,6 +1353,31 @@ export async function streamMessage( // Build dynamic system prompt addendum so Orcabot knows which agent to use let systemPrompt = ORCABOT_SYSTEM_PROMPT + '\n\n' + HELP_DOCS_GROUNDING; + + // Template-driven setup walkthrough: if the active dashboard was created from + // a template that carries a setup guide, inject it so Orcabot can walk the + // user through setup using its tools (terminal_input, secrets_create, etc.). + if (dashboardId) { + try { + const dashRow = await env.DB.prepare( + `SELECT setup_guide FROM dashboards WHERE id = ?` + ).bind(dashboardId).first<{ setup_guide: string | null }>(); + const guide = dashRow?.setup_guide?.trim(); + if (guide) { + systemPrompt += `\n\nACTIVE SETUP WALKTHROUGH (dashboard ${dashboardId}): +Follow this guide to help the user get set up. Use your tools to do the work +(terminal_input to run commands, secrets_create for keys, create_terminal/ +create_browser as needed) and confirm each step briefly before moving on. Ask +the user to choose where a choice is offered; never paste or echo secret values. +When the user confirms setup is finished (e.g. the first job ran successfully), +call complete_setup_walkthrough with dashboard_id="${dashboardId}" so this guide +stops being shown. + +${guide}`; + } + } catch { /* dashboards.setup_guide not migrated yet — skip */ } + } + if (userKeyNames.length > 0 && bestProvider) { const available = userKeyNames.map(k => `${AI_PROVIDER_LABELS[k]?.agent ?? k} (${k})`).join(', '); systemPrompt += `\n\nUSER'S AI PROVIDER KEYS (already stored — do NOT ask for them again): diff --git a/controlplane/src/dashboards/handler.ts b/controlplane/src/dashboards/handler.ts index da8d1313..ee0a9a34 100644 --- a/controlplane/src/dashboards/handler.ts +++ b/controlplane/src/dashboards/handler.ts @@ -198,9 +198,11 @@ export async function createDashbоard( // If templateId provided, populate dashboard from template let templateViewport: { x: number; y: number; zoom: number } | undefined; + let hasSetupGuide = false; if (data.templateId) { const result = await populateFromTemplate(env, id, data.templateId); templateViewport = result?.viewport; + hasSetupGuide = !!result?.hasSetupGuide; } // Eagerly claim a warm VM so terminals open instantly. @@ -227,6 +229,7 @@ export async function createDashbоard( return Response.json({ dashboard, ...(templateViewport && { viewport: templateViewport }), + ...(hasSetupGuide && { hasSetupGuide: true }), }, { status: 201 }); } diff --git a/controlplane/src/db/schema.ts b/controlplane/src/db/schema.ts index 94da32a5..aa1aa28a 100644 --- a/controlplane/src/db/schema.ts +++ b/controlplane/src/db/schema.ts @@ -22,6 +22,7 @@ CREATE TABLE IF NOT EXISTS dashboards ( id TEXT PRIMARY KEY, name TEXT NOT NULL, owner_id TEXT NOT NULL REFERENCES users(id), + setup_guide TEXT, created_at TEXT NOT NULL DEFAULT (datetime('now')), updated_at TEXT NOT NULL DEFAULT (datetime('now')) ); @@ -761,6 +762,7 @@ CREATE TABLE IF NOT EXISTS dashboard_templates ( items_json TEXT NOT NULL DEFAULT '[]', edges_json TEXT NOT NULL DEFAULT '[]', viewport_json TEXT, + setup_guide TEXT, item_count INTEGER NOT NULL DEFAULT 0, is_featured INTEGER NOT NULL DEFAULT 0, use_count INTEGER NOT NULL DEFAULT 0, @@ -1529,6 +1531,24 @@ export async function initializeDatabase(db: D1Database): Promise { // Column already exists. } + // Template-driven setup walkthrough: a template can carry a guide script that + // the Orcabot chat runs as a guided setup; the resolved text is copied onto + // the dashboard at creation so the chat can inject it for that dashboard. + try { + await db.prepare(` + ALTER TABLE dashboard_templates ADD COLUMN setup_guide TEXT + `).run(); + } catch { + // Column already exists. + } + try { + await db.prepare(` + ALTER TABLE dashboards ADD COLUMN setup_guide TEXT + `).run(); + } catch { + // Column already exists. + } + // Warm machine pool for instant dashboard provisioning await db.prepare(` CREATE TABLE IF NOT EXISTS warm_machines ( diff --git a/controlplane/src/index.ts b/controlplane/src/index.ts index 09fa8860..5840d3ad 100644 --- a/controlplane/src/index.ts +++ b/controlplane/src/index.ts @@ -1438,8 +1438,9 @@ async function handleRequest(request: Request, env: EnvWithBindings, ctx: Pick { `ALTER TABLE dashboard_templates ADD COLUMN status TEXT NOT NULL DEFAULT 'approved'` ).run(); } catch { /* already exists */ } + try { + await env.DB.prepare( + `ALTER TABLE dashboard_templates ADD COLUMN setup_guide TEXT` + ).run(); + } catch { /* already exists */ } migrated = true; } @@ -80,6 +85,12 @@ export interface DashboardTemplateWithData extends DashboardTemplate { items: TemplateItem[]; edges: TemplateEdge[]; viewport?: { x: number; y: number; zoom: number }; + /** + * Optional guided-setup script. When present, it is copied onto a dashboard + * created from this template; the Orcabot chat injects it so it can walk the + * user through setup (clone a repo, sync deps, add keys, run a first job). + */ + setupGuide?: string; } function generateId(): string { @@ -122,6 +133,7 @@ function formatTemplateWithData( items: JSON.parse((row.items_json as string) || '[]'), edges: JSON.parse((row.edges_json as string) || '[]'), ...(viewport && { viewport }), + ...(row.setup_guide ? { setupGuide: row.setup_guide as string } : {}), }; } @@ -427,12 +439,13 @@ export async function populateFromTemplate( env: Env, dashboardId: string, templateId: string -): Promise<{ viewport?: { x: number; y: number; zoom: number } } | undefined> { +): Promise<{ viewport?: { x: number; y: number; zoom: number }; hasSetupGuide?: boolean } | undefined> { + await ensureTemplateColumns(env); const template = await env.DB.prepare( - `SELECT items_json, edges_json, viewport_json FROM dashboard_templates WHERE id = ?` + `SELECT items_json, edges_json, viewport_json, setup_guide FROM dashboard_templates WHERE id = ?` ) .bind(templateId) - .first<{ items_json: string; edges_json: string; viewport_json: string | null }>(); + .first<{ items_json: string; edges_json: string; viewport_json: string | null; setup_guide: string | null }>(); if (!template) return; @@ -498,6 +511,19 @@ export async function populateFromTemplate( } } + // Copy the template's setup guide onto the dashboard so the Orcabot chat can + // walk the user through setup for this dashboard (best-effort; column added + // by migration). Skipped silently if the column isn't present yet. + if (template.setup_guide) { + try { + await env.DB.prepare( + `UPDATE dashboards SET setup_guide = ? WHERE id = ?` + ) + .bind(template.setup_guide, dashboardId) + .run(); + } catch { /* dashboards.setup_guide not migrated yet */ } + } + // Increment template use count await env.DB.prepare( `UPDATE dashboard_templates SET use_count = use_count + 1 WHERE id = ?` @@ -508,5 +534,5 @@ export async function populateFromTemplate( const viewport = template.viewport_json ? JSON.parse(template.viewport_json) : undefined; - return { viewport }; + return { viewport, hasSetupGuide: !!template.setup_guide }; } diff --git a/desktop/CLAUDE.md b/desktop/CLAUDE.md index 02ddab01..cbd26483 100644 --- a/desktop/CLAUDE.md +++ b/desktop/CLAUDE.md @@ -271,6 +271,15 @@ VZ_CONSOLE_DIRECT=1 ./target/release/orcabot-desktop ### Env Vars - `ORCABOT_DESKTOP_AUTOSTART=0` — Skip service autostart - `ORCABOT_DESKTOP_ROOT` — Override resource root path +- `ORCABOT_VM_IMAGE=/path/to/sandbox.img` — **Dev override for the VM disk image.** + Forces a specific local image (raw `.img` or `.gz`), bypassing the version check + and the published-release **download-on-demand** (`vm-image.json` → GitHub + releases). Use it to boot a locally-built `sandbox.img` (e.g. after + `BUILD_VM=force`) instead of the slim published image. Checked first in + `ensure_vm_image` (`vm/image.rs`); marks the staged image `dev-override` so + unsetting the var re-triggers the normal published download. The var is inherited + by the headless `orcabot-desktop` the CLI spawns, so `ORCABOT_VM_IMAGE=… orcabot up` + works. - `SANDBOX_PORT`, `CONTROLPLANE_PORT`, `FRONTEND_PORT` — Port overrides - `DEV_AUTH_ENABLED=true` — Enable dev auth (default in desktop mode) - `BUILD_VM=force|0` — Force or skip VM image rebuild diff --git a/desktop/app/src-tauri/src/commands.rs b/desktop/app/src-tauri/src/commands.rs index fef3d2d4..eaa178d0 100644 --- a/desktop/app/src-tauri/src/commands.rs +++ b/desktop/app/src-tauri/src/commands.rs @@ -613,7 +613,16 @@ pub fn quit_app(app: tauri::AppHandle) { /// trusted GUI (not a process inside the sandbox VM spoofing dev-auth). #[tauri::command] pub fn get_surface_token() -> String { - crate::surface_token().to_string() + // DIAGNOSTIC (surface-ws-diag): prove whether the webview actually invokes this + // IPC command. If this line never appears in headless.log after the GUI loads, + // the token isn't being delivered (IPC unreachable at the remote origin) and the + // WS-auth failure is a delivery bug, not a missing-await bug. + let t = crate::surface_token(); + eprintln!( + "[surface-ws-diag] get_surface_token invoked by webview -> returning token len={}", + t.len() + ); + t.to_string() } /// Open an http(s) URL in the OS default browser. OAuth connect flows use this @@ -639,6 +648,29 @@ pub fn open_url(url: String) -> Result<(), String> { .map_err(|e| format!("failed to open URL: {e}")) } +/// Reveal the host workspace directory in the OS file manager (Finder/Explorer). +/// Desktop-only convenience so users can find where the app stores their files. +/// Takes no path from the frontend — it opens the app's own workspace dir only. +#[tauri::command] +pub async fn reveal_workspace( + state: tauri::State<'_, WorkspaceState>, +) -> Result<(), String> { + let path = state.workspace_path.clone(); + if path.as_os_str().is_empty() || !path.exists() { + return Err("workspace directory is not available".into()); + } + #[cfg(target_os = "macos")] + let mut cmd = std::process::Command::new("open"); + #[cfg(target_os = "linux")] + let mut cmd = std::process::Command::new("xdg-open"); + #[cfg(target_os = "windows")] + let mut cmd = std::process::Command::new("explorer"); + cmd.arg(&path) + .spawn() + .map(|_| ()) + .map_err(|e| format!("failed to open workspace: {e}")) +} + #[tauri::command] pub fn switch_to_cli(app: tauri::AppHandle) -> Result<(), String> { #[cfg(target_os = "macos")] diff --git a/desktop/app/src-tauri/src/main.rs b/desktop/app/src-tauri/src/main.rs index 0f75a9da..a972554b 100644 --- a/desktop/app/src-tauri/src/main.rs +++ b/desktop/app/src-tauri/src/main.rs @@ -866,6 +866,7 @@ fn main() { commands::quit_app, commands::get_surface_token, commands::open_url, + commands::reveal_workspace, ]) .setup(|app| { let services = Arc::new(DesktopServices::new()); diff --git a/desktop/app/src-tauri/src/vm/image.rs b/desktop/app/src-tauri/src/vm/image.rs index 639c85a5..b0172f96 100644 --- a/desktop/app/src-tauri/src/vm/image.rs +++ b/desktop/app/src-tauri/src/vm/image.rs @@ -28,26 +28,32 @@ pub fn stage_image(src: &Path, dest: &Path) -> Result { dest.join(src.file_name().unwrap_or_default()) }; - if needs_staging(src, &dest_path)? { - // Ensure destination directory exists + stage_image_to(src, &dest_path)?; + Ok(dest_path) +} + +/// Stage `src` to a specific destination file (decompressing if `src` is `.gz`). +/// Mtime-cached against the source signature so re-staging an unchanged source is +/// a no-op. +fn stage_image_to(src: &Path, dest_path: &Path) -> Result<(), VMError> { + let is_gzipped = src.extension().map_or(false, |e| e == "gz"); + if needs_staging(src, dest_path)? { if let Some(parent) = dest_path.parent() { fs::create_dir_all(parent)?; } - if is_gzipped { - decompress_gzip(src, &dest_path)?; + decompress_gzip(src, dest_path)?; } else { - copy_file(src, &dest_path)?; + copy_file(src, dest_path)?; } // Record the source signature so a later runtime mutation of dest (the VM // image boots read-write, so the guest bumps its mtime) never makes a // genuinely-updated source look stale and skip re-staging. if let Ok(sig) = source_signature(src) { - let _ = fs::write(stamp_path(&dest_path), sig); + let _ = fs::write(stamp_path(dest_path), sig); } } - - Ok(dest_path) + Ok(()) } /// Stable signature of the SOURCE file (modification time + size). @@ -348,24 +354,26 @@ fn read_marker(marker: &Path) -> Option { fs::read_to_string(marker).ok().map(|s| s.trim().to_string()) } -fn write_marker(marker: &Path, version: &str) { - let _ = fs::write(marker, version); -} - /// The image version that pre-versioning installs (before this scheme existed) /// actually shipped. A marker-less staged image can ONLY be that version, so it /// may be adopted without a download only while it's still the required version. const LEGACY_IMAGE_VERSION: &str = "v1"; -/// Ensure the VM disk image is present in the data dir and matches the required -/// version, downloading + verifying it on demand. Returns the staged image path. +/// Ensure the VM disk image is present in the data dir and return its path. +/// +/// The staged image is named by CONTENT (`sandbox-.img`), never a fixed +/// `sandbox.img`: macOS/Virtualization.framework caches a disk image's SIZE keyed +/// on its path, so reusing one path for images of different sizes makes the guest +/// see the OLD size and truncates the disk — everything the filesystem placed past +/// the cap reads back as garbage ("Exec format error"). A fresh name whenever the +/// content changes sidesteps that cache entirely. /// /// Resolution order: -/// 1. a local resource image (dev build / anything bundled) → stage it (trusted, -/// mtime-cached); -/// 2. the data-dir image already staged for THIS version → use (fast); -/// 3. a data-dir image from a PRIOR install with no version marker → adopt it as -/// the current version (migration — this build ships the same image content); +/// 0. `ORCABOT_VM_IMAGE` dev override → stage that file (named by its signature); +/// 1. a local resource image (dev build / bundled) → stage it; +/// 2. the versioned image already staged for the manifest version → use it; +/// 3. migrate a pre-content-naming `sandbox.img` by renaming it (if it's the +/// required version) — a fresh path also clears any stale size cache; /// 4. otherwise download the gz artifact, verify its SHA-256, decompress. pub fn ensure_vm_image( resource_image: &Path, @@ -375,34 +383,58 @@ pub fn ensure_vm_image( let manifest = vm_image_manifest(); let vm_dir = data_dir.join("vm"); fs::create_dir_all(&vm_dir)?; - let target = vm_dir.join("sandbox.img"); - let marker = vm_dir.join("sandbox.img.version"); - // 1. Dev / bundled: a local resource image is the source of truth. stage_image - // is mtime-cached, so this is cheap when unchanged and re-stages on change. + // 0. Dev override: ORCABOT_VM_IMAGE forces a specific local image (raw .img or + // .gz), bypassing the version check + release download. Named by the source + // signature so swapping to an image of a different size lands on a fresh path. + if let Ok(override_path) = std::env::var("ORCABOT_VM_IMAGE") { + if !override_path.is_empty() { + let p = Path::new(&override_path); + if p.exists() { + let dest = vm_dir.join(format!("sandbox-ovr-{}.img", local_content_token(p))); + eprintln!( + "[vm-image] ORCABOT_VM_IMAGE override: staging {} -> {}", + p.display(), + dest.display() + ); + stage_image_to(p, &dest)?; + cleanup_stale_images(&vm_dir, &dest); + return Ok(dest); + } + eprintln!( + "[vm-image] ORCABOT_VM_IMAGE set but file not found: {override_path} — ignoring" + ); + } + } + + // 1. Dev / bundled: a local resource image is the source of truth. if resource_image.exists() { - let staged = stage_image(resource_image, &vm_dir)?; - write_marker(&marker, &manifest.version); - return Ok(staged); + let dest = vm_dir.join(format!("sandbox-res-{}.img", local_content_token(resource_image))); + stage_image_to(resource_image, &dest)?; + cleanup_stale_images(&vm_dir, &dest); + return Ok(dest); } - // Packaged: no bundled image. - let marker_ver = read_marker(&marker); - if target.exists() { - if marker_ver.as_deref() == Some(manifest.version.as_str()) { - return Ok(target); // 2. correct version already staged - } - if marker_ver.is_none() && manifest.version == LEGACY_IMAGE_VERSION { - // 3. Migration from a pre-versioning install. Those installs shipped - // the LEGACY_IMAGE_VERSION image, so a marker-less image is only - // safe to adopt while that's still the required version. Once the - // image moves on (v2+), a marker-less image is the OLD one and - // must be re-downloaded — fall through. - write_marker(&marker, &manifest.version); - return Ok(target); + // Packaged: the image is identified by the manifest version. + let dest = vm_dir.join(format!("sandbox-{}.img", manifest.version)); + + // 2. Already staged for this version. + if dest.exists() { + return Ok(dest); + } + + // 3. Migration: adopt a pre-content-naming `sandbox.img` by renaming it to the + // content path (a fresh path clears any stale size cache), if it is the + // required version. Avoids a needless re-download on upgrade. + let legacy = vm_dir.join("sandbox.img"); + if legacy.exists() { + let marker_ver = read_marker(&vm_dir.join("sandbox.img.version")); + let is_required = marker_ver.as_deref() == Some(manifest.version.as_str()) + || (marker_ver.is_none() && manifest.version == LEGACY_IMAGE_VERSION); + if is_required && fs::rename(&legacy, &dest).is_ok() { + cleanup_stale_images(&vm_dir, &dest); + return Ok(dest); } - // marker present but a different version (or a marker-less image on a - // newer required version) → the image genuinely differs → download. } // 4. Download + verify + decompress. @@ -410,9 +442,46 @@ pub fn ensure_vm_image( "[vm-image] fetching sandbox image {} from {}", manifest.version, manifest.url ); - download_and_stage_image(&manifest, &vm_dir, &target, progress)?; - write_marker(&marker, &manifest.version); - Ok(target) + download_and_stage_image(&manifest, &vm_dir, &dest, progress)?; + cleanup_stale_images(&vm_dir, &dest); + Ok(dest) +} + +/// Short, stable token for a local source image's content: nanosecond mtime + size +/// (via `source_signature`), hashed. Changes whenever the source changes — including +/// its SIZE, the property the stale VZ/macOS disk-size cache keys on. `DefaultHasher` +/// uses fixed keys, so this is deterministic across runs. +fn local_content_token(src: &Path) -> String { + use std::hash::{Hash, Hasher}; + let sig = source_signature(src).unwrap_or_default(); + let mut hasher = std::collections::hash_map::DefaultHasher::new(); + sig.hash(&mut hasher); + format!("{:016x}", hasher.finish()) +} + +/// Remove staged sandbox images + sidecars other than `keep`, so old versions +/// don't accumulate in the data dir. Best-effort. +fn cleanup_stale_images(vm_dir: &Path, keep: &Path) { + let keep_stamp = stamp_path(keep); + let Ok(entries) = fs::read_dir(vm_dir) else { + return; + }; + for entry in entries.flatten() { + let path = entry.path(); + if path == *keep || path == keep_stamp { + continue; + } + let name = entry.file_name(); + let n = name.to_string_lossy(); + let stale = n == "sandbox.img" + || n == "sandbox.img.stamp" + || n == "sandbox.img.version" + || n.ends_with(".img.gz.part") + || (n.starts_with("sandbox-") && (n.ends_with(".img") || n.ends_with(".img.stamp"))); + if stale { + let _ = fs::remove_file(&path); + } + } } fn download_and_stage_image( diff --git a/desktop/tests/browser-smoke.sh b/desktop/tests/browser-smoke.sh new file mode 100755 index 00000000..909823cb --- /dev/null +++ b/desktop/tests/browser-smoke.sh @@ -0,0 +1,89 @@ +#!/usr/bin/env bash +# Orcabot desktop BROWSER smoke — the browser block actually starts and paints. +# +# Why this exists: the browser block (chromium in the VM, streamed over VNC) had NO +# test coverage, and a broken chromium build shipped in the VM image +# (150.0.7871.46 IMMEDIATE_CRASHed on launch), so the block silently showed +# "Browser failed to start" — start() succeeded, but chromium died and status.ready +# never flipped true. This asserts the whole path: start -> chromium up -> +# status.ready=true within a bounded time. +# +# Flow: +# 1. throwaway dashboard + one terminal (an active session pins the VM). +# 2. POST /dashboards/:id/browser/start. +# 3. poll GET /dashboards/:id/browser/status until running && ready (fail on timeout). +# 4. sanity: the in-VM chromium is not the known-broken build. +# +# Usage: desktop/tests/browser-smoke.sh +# Requires: stack running (`orcabot up`), python3, curl. Exits non-zero on failure. +set -uo pipefail + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +ORCABOT="${ORCABOT_BIN:-$SCRIPT_DIR/../app/src-tauri/target/release/orcabot}" +CP="http://127.0.0.1:8787" +SB="http://127.0.0.1:8080" +U="X-User-ID: dev-desktop" +VZ_LOG="/tmp/vz-console.log" +READY_TIMEOUT="${READY_TIMEOUT:-45}" # chromium cold boot can be slow + +# Per-boot surface token (dev-auth gate) — read like the CLI/app do. +SURFACE_TOKEN_FILE="${ORCABOT_SURFACE_TOKEN_FILE:-$HOME/Library/Application Support/com.orcabot.desktop/surface-token}" +ST="$(cat "$SURFACE_TOKEN_FILE" 2>/dev/null | tr -d '[:space:]')" +AUTH=(-H "$U" -H "X-Orcabot-Surface: $ST") + +PASS=0; FAIL=0 +ok() { echo " PASS: $1"; PASS=$((PASS+1)); } +bad() { echo " FAIL: $1"; FAIL=$((FAIL+1)); } +jq_py(){ python3 -c "$1"; } + +# Run a shell command in the VM as root (debug-exec), print stdout. +TOK="$(grep -a 'debug-exec] auth token:' "$VZ_LOG" 2>/dev/null | tail -1 | sed 's/.*auth token: //' | tr -cd 'a-f0-9')" +X(){ local b; b="$(python3 -c "import json,sys;print(json.dumps({'cmd':sys.argv[1],'timeout_ms':10000}))" "$1")" + curl -s -m 14 -X POST -H "X-Debug-Exec-Token: $TOK" -H 'Content-Type: application/json' \ + -d "$b" "$SB/debug/exec" | python3 -c "import json,sys;print(json.load(sys.stdin).get('stdout','').strip())"; } + +# ---- preflight ---- +[ -x "$ORCABOT" ] || { echo "orcabot binary not found at $ORCABOT" >&2; exit 2; } +curl -s -m3 "$CP/health" >/dev/null || { echo "control plane down on :8787 — run 'orcabot up'." >&2; exit 2; } + +# dedicated throwaway dashboard + an active session (browser needs a VM to attach to) +DID="$(curl -s -X POST "${AUTH[@]}" -H 'Content-Type: application/json' --data-raw '{"name":"browser-smoke"}' "$CP/dashboards" | jq_py "import json,sys;print(json.load(sys.stdin)['dashboard']['id'])")" +echo "smoke dashboard: $DID" +"$ORCABOT" new terminal shell --dash "$DID" >/dev/null 2>&1; sleep 3 + +echo "[1] browser starts and reaches ready" +curl -s -o /dev/null -m 30 "${AUTH[@]}" -X POST "$CP/dashboards/$DID/browser/start" +ready=0 +for _ in $(seq 1 "$READY_TIMEOUT"); do + s="$(curl -s -m6 "${AUTH[@]}" "$CP/dashboards/$DID/browser/status")" + r="$(echo "$s" | jq_py "import json,sys +try: + d=json.load(sys.stdin); print('1' if d.get('running') and d.get('ready') else '0') +except Exception: print('0')")" + if [ "$r" = "1" ]; then ready=1; break; fi + sleep 1 +done +[ "$ready" = "1" ] \ + && ok "browser reached running+ready" \ + || bad "browser never became ready within ${READY_TIMEOUT}s (chromium likely crashed on launch)" + +echo "[2] chromium is not the known-broken build" +# 150.0.7871.46 SIGTRAPs on launch. Any other build (esp. .100+) is fine. Only assert +# when we can read the version (debug-exec available); otherwise skip, don't fail. +if [ -n "$TOK" ]; then + ver="$(X 'chromium.real --version 2>/dev/null || chromium --version 2>/dev/null')" + case "$ver" in + *150.0.7871.46*) bad "chromium is the broken build: $ver" ;; + *Chromium*) ok "chromium build OK: $ver" ;; + *) echo " SKIP: could not read chromium version" ;; + esac +else + echo " SKIP: no debug-exec token (VZ_CONSOLE_DIRECT=1 not set) — version check skipped" +fi + +# ---- teardown ---- +curl -s -o /dev/null "${AUTH[@]}" -X POST "$CP/dashboards/$DID/browser/stop" 2>/dev/null +curl -s -o /dev/null -X DELETE "${AUTH[@]}" "$CP/dashboards/$DID" +echo +echo "================ $PASS passed, $FAIL failed ================" +[ "$FAIL" -eq 0 ] diff --git a/desktop/tests/regression-smoke.sh b/desktop/tests/regression-smoke.sh index 0443ef4e..2c61fca8 100755 --- a/desktop/tests/regression-smoke.sh +++ b/desktop/tests/regression-smoke.sh @@ -30,6 +30,16 @@ SB="http://127.0.0.1:8080" U="X-User-ID: dev-desktop" VZ_LOG="/tmp/vz-console.log" +# Per-boot surface token: the control plane only trusts dev-auth when the +# X-Orcabot-Surface header matches (devAuthSurfaceTrusted). Read it from the same +# file the desktop app / CLI use so these host-side curls authenticate like the CLI +# does. Empty when the running stack predates surface enforcement — an empty header +# is harmless there (enforcement off => dev-auth trusted headerless). +SURFACE_TOKEN_FILE="${ORCABOT_SURFACE_TOKEN_FILE:-$HOME/Library/Application Support/com.orcabot.desktop/surface-token}" +ST="$(cat "$SURFACE_TOKEN_FILE" 2>/dev/null | tr -d '[:space:]')" +# Dev-auth headers for every control-plane call (user id + surface token). +AUTH=(-H "$U" -H "X-Orcabot-Surface: $ST") + PASS=0 FAIL=0 ok() { echo " PASS: $1"; PASS=$((PASS + 1)); } @@ -58,53 +68,53 @@ if [ -z "$TOK" ]; then fi # dedicated throwaway dashboard -DID="$(curl -s -X POST -H "$U" -H 'Content-Type: application/json' --data-raw '{"name":"regression-smoke"}' "$CP/dashboards" \ +DID="$(curl -s -X POST "${AUTH[@]}" -H 'Content-Type: application/json' --data-raw '{"name":"regression-smoke"}' "$CP/dashboards" \ | jq_py "import json,sys;print(json.load(sys.stdin)['dashboard']['id'])")" echo "smoke dashboard: $DID" new_term() { "$ORCABOT" new terminal shell --dash "$DID" 2>/dev/null | sed 's/created shell terminal //'; } sess_field() { # $1=itemId $2=field - curl -s -H "$U" "$CP/dashboards/$DID" | jq_py "import json,sys;d=json.load(sys.stdin);print(next((s.get('$2','') for s in d['sessions'] if s['itemId']=='$1'),'GONE'))" + curl -s "${AUTH[@]}" "$CP/dashboards/$DID" | jq_py "import json,sys;d=json.load(sys.stdin);print(next((s.get('$2','') for s in d['sessions'] if s['itemId']=='$1'),'GONE'))" } # ---- #1 delete reaps PTY (no zombie) -------------------------------------- echo "[#1] terminal delete reaps the PTY" b0="$(X 'pgrep -c bash')"; ITEM="$(new_term)"; sleep 3 b1="$(X 'pgrep -c bash')" -curl -s -o /dev/null -X DELETE -H "$U" "$CP/dashboards/$DID/items/$ITEM"; sleep 3 -b2="$(X 'pgrep -c bash')"; z="$(X 'ps -eo stat | grep -c Z')" +curl -s -o /dev/null -X DELETE "${AUTH[@]}" "$CP/dashboards/$DID/items/$ITEM"; sleep 3 +b2="$(X 'pgrep -c bash')"; z="$(X 'ps -eo stat,comm | grep -c "^Z.*bash"')" # PTY (bash) zombies only; the browser block leaves chromium helpers that are not a PTY-reap leak { [ "$b1" -gt "$b0" ] && [ "$b2" -le "$b0" ] && [ "$z" -eq 0 ]; } \ && ok "PTY reaped (bash $b0->$b1->$b2, zombies=$z)" \ || bad "PTY leaked/zombied (bash $b0->$b1->$b2, zombies=$z)" # ---- #2 concurrent session-create -> one session -------------------------- echo "[#2] concurrent session-create coalesces to one" -IT="$(curl -s -X POST -H "$U" -H 'Content-Type: application/json' \ +IT="$(curl -s -X POST "${AUTH[@]}" -H 'Content-Type: application/json' \ --data-raw '{"type":"terminal","content":"{\"name\":\"race\",\"bootCommand\":\"\"}"}' "$CP/dashboards/$DID/items" \ | jq_py "import json,sys;print(json.load(sys.stdin)['item']['id'])")" -for i in 1 2 3; do curl -s -o /dev/null -X POST -H "$U" -H 'Content-Type: application/json' -d '{}' "$CP/dashboards/$DID/items/$IT/session" & done; wait +for i in 1 2 3; do curl -s -o /dev/null -X POST "${AUTH[@]}" -H 'Content-Type: application/json' -d '{}' "$CP/dashboards/$DID/items/$IT/session" & done; wait sleep 3 -n="$(curl -s -H "$U" "$CP/dashboards/$DID" | jq_py "import json,sys;print(sum(1 for s in json.load(sys.stdin)['sessions'] if s['itemId']=='$IT'))")" +n="$(curl -s "${AUTH[@]}" "$CP/dashboards/$DID" | jq_py "import json,sys;print(sum(1 for s in json.load(sys.stdin)['sessions'] if s['itemId']=='$IT'))")" [ "$n" = "1" ] && ok "exactly one session ($n)" || bad "duplicate sessions ($n)" -curl -s -o /dev/null -X DELETE -H "$U" "$CP/dashboards/$DID/items/$IT" +curl -s -o /dev/null -X DELETE "${AUTH[@]}" "$CP/dashboards/$DID/items/$IT" # ---- #3/#5 item input validation ------------------------------------------ echo "[#3/#5] item input validation" -c="$(code -X POST -H "$U" -H 'Content-Type: application/json' --data-raw 'not-json{' "$CP/dashboards/$DID/items")" +c="$(code -X POST "${AUTH[@]}" -H 'Content-Type: application/json' --data-raw 'not-json{' "$CP/dashboards/$DID/items")" [ "$c" = "400" ] && ok "invalid JSON -> 400" || bad "invalid JSON -> $c (want 400)" -c="$(code -X POST -H "$U" -H 'Content-Type: application/json' --data-raw '{"content":"x"}' "$CP/dashboards/$DID/items")" +c="$(code -X POST "${AUTH[@]}" -H 'Content-Type: application/json' --data-raw '{"content":"x"}' "$CP/dashboards/$DID/items")" [ "$c" = "400" ] && ok "missing type -> 400" || bad "missing type -> $c (want 400)" -c="$(code -X POST -H "$U" -H 'Content-Type: application/json' --data-raw '{"type":"bogus"}' "$CP/dashboards/$DID/items")" +c="$(code -X POST "${AUTH[@]}" -H 'Content-Type: application/json' --data-raw '{"type":"bogus"}' "$CP/dashboards/$DID/items")" [ "$c" = "400" ] && ok "invalid type -> 400" || bad "invalid type -> $c (want 400)" -c="$(code -X POST -H "$U" -H 'Content-Type: application/json' --data-raw '{"type":"note","content":"ok"}' "$CP/dashboards/$DID/items")" +c="$(code -X POST "${AUTH[@]}" -H 'Content-Type: application/json' --data-raw '{"type":"note","content":"ok"}' "$CP/dashboards/$DID/items")" [ "$c" = "201" ] && ok "valid note -> 201" || bad "valid note -> $c (want 201)" # ---- #4 attach validation (no 500) ---------------------------------------- echo "[#4] attach input validation (never 500)" ITEM4="$(new_term)"; sleep 3 PTY4="$(sess_field "$ITEM4" ptyId)" -c="$(code -X POST -H "$U" -H 'Content-Type: application/json' --data-raw 'not-json{' "$CP/dashboards/$DID/terminals/$PTY4/integrations")" +c="$(code -X POST "${AUTH[@]}" -H 'Content-Type: application/json' --data-raw 'not-json{' "$CP/dashboards/$DID/terminals/$PTY4/integrations")" { [ "$c" -ge 400 ] && [ "$c" -lt 500 ]; } && ok "malformed body -> $c (4xx)" || bad "malformed body -> $c (want 4xx)" -c="$(code -X POST -H "$U" -H 'Content-Type: application/json' --data-raw '{"provider":"bogus","userIntegrationId":"fake"}' "$CP/dashboards/$DID/terminals/$PTY4/integrations")" +c="$(code -X POST "${AUTH[@]}" -H 'Content-Type: application/json' --data-raw '{"provider":"bogus","userIntegrationId":"fake"}' "$CP/dashboards/$DID/terminals/$PTY4/integrations")" { [ "$c" -ge 400 ] && [ "$c" -lt 500 ]; } && ok "unknown provider -> $c (4xx)" || bad "unknown provider -> $c (want 4xx)" # ---- #6 ghost session reconcile ------------------------------------------- @@ -121,38 +131,38 @@ st_after="$(sess_field "$ITEM6" status)" { [ "$st_before" = "active" ] && { [ "$st_after" = "stopped" ] || [ "$st_after" = "GONE" ]; }; } \ && ok "reconciled ($st_before -> $st_after)" \ || bad "not reconciled ($st_before -> $st_after; want active -> stopped/GONE)" -curl -s -o /dev/null -X DELETE -H "$U" "$CP/dashboards/$DID/items/$ITEM6" -curl -s -o /dev/null -X DELETE -H "$U" "$CP/dashboards/$DID/items/$ITEM4" +curl -s -o /dev/null -X DELETE "${AUTH[@]}" "$CP/dashboards/$DID/items/$ITEM6" +curl -s -o /dev/null -X DELETE "${AUTH[@]}" "$CP/dashboards/$DID/items/$ITEM4" # ---- #7 dashboard delete reaps PTYs --------------------------------------- echo "[#7] dashboard delete reaps its PTYs" -ND="$(curl -s -X POST -H "$U" -H 'Content-Type: application/json' --data-raw '{"name":"regression-smoke-del"}' "$CP/dashboards" \ +ND="$(curl -s -X POST "${AUTH[@]}" -H 'Content-Type: application/json' --data-raw '{"name":"regression-smoke-del"}' "$CP/dashboards" \ | jq_py "import json,sys;print(json.load(sys.stdin)['dashboard']['id'])")" d0="$(X 'pgrep -c bash')"; "$ORCABOT" new terminal shell --dash "$ND" >/dev/null 2>&1; sleep 3 -curl -s -o /dev/null -X DELETE -H "$U" "$CP/dashboards/$ND"; sleep 3 -d2="$(X 'pgrep -c bash')"; z="$(X 'ps -eo stat | grep -c Z')" +curl -s -o /dev/null -X DELETE "${AUTH[@]}" "$CP/dashboards/$ND"; sleep 3 +d2="$(X 'pgrep -c bash')"; z="$(X 'ps -eo stat,comm | grep -c "^Z.*bash"')" # PTY (bash) zombies only; the browser block leaves chromium helpers that are not a PTY-reap leak { [ "$d2" -le "$d0" ] && [ "$z" -eq 0 ]; } && ok "PTY reaped on dashboard delete (bash $d0->$d2, zombies=$z)" \ || bad "PTY leaked on dashboard delete (bash $d0->$d2, zombies=$z)" # ---- invariants: path traversal + output redaction ------------------------ echo "[inv] file-API path traversal blocked" ITEM8="$(new_term)"; sleep 3 -SSID="$(curl -s -H "$U" "$CP/dashboards/$DID" | jq_py "import json,sys;d=json.load(sys.stdin);print(next(s['sandboxSessionId'] for s in d['sessions'] if s['itemId']=='$ITEM8'))")" +SSID="$(curl -s "${AUTH[@]}" "$CP/dashboards/$DID" | jq_py "import json,sys;d=json.load(sys.stdin);print(next(s['sandboxSessionId'] for s in d['sessions'] if s['itemId']=='$ITEM8'))")" out="$(curl -s -m6 -H 'X-Internal-Token: dev-sandbox-token' "$SB/sessions/$SSID/file?path=../../../../etc/passwd")" echo "$out" | grep -q "root:" && bad "path traversal LEAKED /etc/passwd" || ok "traversal blocked" echo "[inv] broker-secret output redaction" SEC="REDACTME_smoke_55512345" -curl -s -o /dev/null -X POST -H "$U" -H 'Content-Type: application/json' \ +curl -s -o /dev/null -X POST "${AUTH[@]}" -H 'Content-Type: application/json' \ --data-raw "{\"name\":\"SMOKE_SECRET\",\"value\":\"$SEC\",\"type\":\"secret\",\"brokerProtected\":true,\"dashboardId\":\"$DID\"}" "$CP/secrets" ITEM9="$(new_term)"; sleep 3 red="$(timeout 12 "$ORCABOT" tail "$ITEM9" --dash "$DID" --secs 5 --send "echo MK:${SEC}:MK" 2>/dev/null)" echo "$red" | grep -q "$SEC" && bad "secret LEAKED in output" || ok "secret redacted" -curl -s -o /dev/null -X DELETE -H "$U" "$CP/dashboards/$DID/items/$ITEM8" -curl -s -o /dev/null -X DELETE -H "$U" "$CP/dashboards/$DID/items/$ITEM9" +curl -s -o /dev/null -X DELETE "${AUTH[@]}" "$CP/dashboards/$DID/items/$ITEM8" +curl -s -o /dev/null -X DELETE "${AUTH[@]}" "$CP/dashboards/$DID/items/$ITEM9" # ---- teardown ------------------------------------------------------------- -curl -s -o /dev/null -X DELETE -H "$U" "$CP/dashboards/$DID" +curl -s -o /dev/null -X DELETE "${AUTH[@]}" "$CP/dashboards/$DID" echo echo "================ $PASS passed, $FAIL failed ================" [ "$FAIL" -eq 0 ] diff --git a/desktop/vm/scripts/build-images.sh b/desktop/vm/scripts/build-images.sh index 251f3101..932601dc 100755 --- a/desktop/vm/scripts/build-images.sh +++ b/desktop/vm/scripts/build-images.sh @@ -20,7 +20,9 @@ SANDBOX_DIR="$REPO_ROOT/sandbox" OUTPUT_DIR="$VM_DIR/image" # Image settings -IMAGE_SIZE_MB="${IMAGE_SIZE_MB:-3072}" +# 4GB: OpenCode (re-enabled for benchmark harnesses) plus the other agent CLIs +# overflow the old 3GB rootfs. Override via IMAGE_SIZE_MB if a slimmer build is needed. +IMAGE_SIZE_MB="${IMAGE_SIZE_MB:-4096}" # Colors for output RED='\033[0;31m' @@ -39,7 +41,23 @@ mkdir -p "$OUTPUT_DIR" # Step 1: Build sandbox Docker image # ============================================================================= log "Building sandbox Docker image..." -docker build -t orcabot-sandbox:local -f "$SANDBOX_DIR/docker/Dockerfile" "$SANDBOX_DIR" +# CHROMIUM_CACHEBUST forces the Dockerfile's chromium-refresh layer to re-run every +# build (Debian's cached apt layer can otherwise pin a stale/broken chromium — e.g. +# 150.0.7871.46 SIGTRAPs on launch). A timestamp guarantees a fresh pull each time. +docker build --build-arg "CHROMIUM_CACHEBUST=$(date +%s)" \ + -t orcabot-sandbox:local -f "$SANDBOX_DIR/docker/Dockerfile" "$SANDBOX_DIR" + +# ============================================================================= +# Step 1b: Validate the image BEFORE assembling it (fail fast on regressions) +# ============================================================================= +# Chromium/claude/agents are installed at "latest" (unpinnable — a Debian chromium +# version pin vanishes on the next security bump), so a rebuild can silently inherit +# an upstream regression and replace a working VM with a broken one. This launches +# chromium and runs the agents as a no-home pty user; a failure exits non-zero and, +# because of `set -e`, aborts the whole build instead of shipping the broken image. +log "Validating sandbox image (chromium launches + agents run as a pty user)..." +docker run --rm -v "$SANDBOX_DIR/docker/validate-image.sh:/validate-image.sh:ro" \ + --entrypoint bash orcabot-sandbox:local /validate-image.sh # ============================================================================= # Step 2: Export rootfs tarball (for WSL2) @@ -68,15 +86,24 @@ set -euo pipefail apt-get update -qq apt-get install -y -qq e2fsprogs tar gzip linux-image-cloud-arm64 kmod > /dev/null +# Build the ext4 image on the CONTAINER filesystem (/build), NOT the host-shared +# -v mount (/output). Docker Desktop for macOS file-sharing does not reliably +# flush thousands of incremental loop-mount writes back to the host image, so +# building directly on /output silently truncates/drops files (e.g. the 167MB +# opencode binary) with no error. We build locally, then copy the finished image +# to /output ONCE at the end — a single sequential copy survives file-sharing. +mkdir -p /build +BUILD_IMG=/build/sandbox.img + echo "Creating disk image (${IMAGE_SIZE_MB}MB)..." -dd if=/dev/zero of=/output/sandbox.img bs=1M count=${IMAGE_SIZE_MB} status=progress +dd if=/dev/zero of=$BUILD_IMG bs=1M count=${IMAGE_SIZE_MB} status=progress echo "Creating ext4 filesystem..." -mkfs.ext4 -F -L rootfs /output/sandbox.img +mkfs.ext4 -F -L rootfs $BUILD_IMG echo "Mounting and populating..." mkdir -p /mnt/rootfs -mount -o loop /output/sandbox.img /mnt/rootfs +mount -o loop $BUILD_IMG /mnt/rootfs # Extract rootfs from tarball echo "Extracting rootfs..." @@ -500,9 +527,30 @@ NETCONF # Create hostname echo "orcabot-sandbox" > /mnt/rootfs/etc/hostname -# Sync and unmount +# Create /etc/hosts so "localhost" resolves. Docker supplies /etc/hosts as a +# runtime bind-mount, so it is NOT in the image and docker export (how this +# rootfs is built) drops it -- same as /etc/hostname and /etc/network/interfaces +# recreated above. Without it, anything dialing "localhost:" (e.g. +# websockify to x11vnc for the browser noVNC stream) fails name resolution. +# NOTE: no apostrophes in this comment -- the whole block runs inside a +# single-quoted bash -c script, so a stray apostrophe would terminate it. +# Mapped IPv4-only (not ::1) so IPv6-first clients do not stall on v4-only ports. +cat > /mnt/rootfs/etc/hosts << "HOSTS" +127.0.0.1 localhost orcabot-sandbox +::1 ip6-localhost ip6-loopback +HOSTS + +# Sync and unmount the container-local image, then copy it to the host-shared +# /output ONCE. A single sequential copy flushes reliably through Docker Desktop +# file-sharing, unlike the many incremental loop-mount writes above. sync umount /mnt/rootfs +sync + +echo "Copying finished image to /output..." +cp $BUILD_IMG /output/sandbox.img +sync +rm -f $BUILD_IMG echo "Disk image created successfully!" ls -lh /output/sandbox.img diff --git a/frontend/package-lock.json b/frontend/package-lock.json index 3886922f..a7962b18 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -18,6 +18,7 @@ "@tauri-apps/api": "^2.10.1", "@tauri-apps/plugin-dialog": "^2.6.0", "@xterm/addon-fit": "^0.10.0", + "@xterm/xterm": "^5.5.0", "@xyflow/react": "^12.10.0", "class-variance-authority": "^0.7.1", "clsx": "^2.1.1", @@ -31,7 +32,6 @@ "react-markdown": "^10.1.0", "sonner": "^2.0.7", "tailwind-merge": "^3.4.0", - "xterm": "^5.3.0", "zod": "^4.3.5", "zustand": "^5.0.9" }, @@ -15277,8 +15277,7 @@ "version": "5.5.0", "resolved": "https://registry.npmjs.org/@xterm/xterm/-/xterm-5.5.0.tgz", "integrity": "sha512-hqJHYaQb5OptNunnyAnkHyM8aCjZ1MEIDTQu1iIbbTD/xops91NB5yq1ZK/dC2JDbVWtF23zUtl9JE2NqwT87A==", - "license": "MIT", - "peer": true + "license": "MIT" }, "node_modules/@xyflow/react": { "version": "12.10.0", @@ -25077,13 +25076,6 @@ "dev": true, "license": "MIT" }, - "node_modules/xterm": { - "version": "5.3.0", - "resolved": "https://registry.npmjs.org/xterm/-/xterm-5.3.0.tgz", - "integrity": "sha512-8QqjlekLUFTrU6x7xck1MsPzPA571K5zNqWm0M0oroYEWVOptZ0+ubQSkQ3uxIEhcIHRujJy6emDWX4A7qyFzg==", - "deprecated": "This package is now deprecated. Move to @xterm/xterm instead.", - "license": "MIT" - }, "node_modules/y18n": { "version": "5.0.8", "resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.8.tgz", diff --git a/frontend/package.json b/frontend/package.json index 2ac7cc28..6e74eb7c 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -29,6 +29,7 @@ "@tauri-apps/api": "^2.10.1", "@tauri-apps/plugin-dialog": "^2.6.0", "@xterm/addon-fit": "^0.10.0", + "@xterm/xterm": "^5.5.0", "@xyflow/react": "^12.10.0", "class-variance-authority": "^0.7.1", "clsx": "^2.1.1", @@ -42,7 +43,6 @@ "react-markdown": "^10.1.0", "sonner": "^2.0.7", "tailwind-merge": "^3.4.0", - "xterm": "^5.3.0", "zod": "^4.3.5", "zustand": "^5.0.9" }, diff --git a/frontend/src/app/(app)/dashboards/[id]/page.tsx b/frontend/src/app/(app)/dashboards/[id]/page.tsx index d2b0512b..aa727deb 100644 --- a/frontend/src/app/(app)/dashboards/[id]/page.tsx +++ b/frontend/src/app/(app)/dashboards/[id]/page.tsx @@ -223,7 +223,8 @@ const terminalTools: BlockTool[] = [ icon: , terminalPreset: { command: "codex", agentic: true }, }, - // OpenCode - temporarily hidden until connection issues are resolved + // OpenCode - temporarily hidden: its newer "opentui" TUI crashes/garbles in the + // xterm.js terminal and exits back to the shell. Re-enable in a later PR once fixed. // { // type: "terminal", // label: "OpenCode", @@ -805,9 +806,13 @@ export default function DashboardPage() { // Workspace sidebar session: pick the first active session from any terminal const workspaceSessionId = React.useMemo(() => { + // Only browse the workspace through a LIVE session — a stopped session's sandbox + // is gone, so proxying files to it 404s ("E79709: session not found") and the + // sidebar shows a raw "Request failed with status 404". Never fall back to a + // stopped sessions[0]; if none is live, return undefined so the sidebar renders + // its empty state instead of polling a dead session. const session = sessions.find((s) => s.status === "active") - ?? sessions.find((s) => s.status === "creating") - ?? sessions[0]; + ?? sessions.find((s) => s.status === "creating"); return session?.id; }, [sessions]); // Build mapping from real IDs to stable keys for nodes that have them diff --git a/frontend/src/app/(app)/dashboards/page.tsx b/frontend/src/app/(app)/dashboards/page.tsx index 6f8e400b..8c3bf441 100644 --- a/frontend/src/app/(app)/dashboards/page.tsx +++ b/frontend/src/app/(app)/dashboards/page.tsx @@ -190,7 +190,7 @@ export default function DashboardsPage() { const createMutation = useMutation({ mutationFn: ({ name, templateId }: { name: string; templateId?: string }) => createDashboard(name, templateId), - onSuccess: ({ dashboard, viewport }) => { + onSuccess: ({ dashboard, viewport, hasSetupGuide }) => { queryClient.invalidateQueries({ queryKey: ["dashboards"] }); toast.success("Dashboard created"); setIsCreateOpen(false); @@ -203,6 +203,16 @@ export default function DashboardsPage() { JSON.stringify(viewport) ); } + // If the template carries a setup walkthrough, auto-kick off the Orcabot + // chat on the new dashboard. ChatPanel consumes orcabot_initial_prompt on + // mount (expands + auto-sends); the control plane injects the dashboard's + // setup_guide so the chat knows exactly what to walk through. + if (hasSetupGuide) { + localStorage.setItem( + "orcabot_initial_prompt", + "Walk me through setting up this dashboard." + ); + } router.push(`/dashboards/${dashboard.id}`); }, onError: (error) => { diff --git a/frontend/src/components/blocks/BrowserBlock.tsx b/frontend/src/components/blocks/BrowserBlock.tsx index 40568d1c..d268fe78 100644 --- a/frontend/src/components/blocks/BrowserBlock.tsx +++ b/frontend/src/components/blocks/BrowserBlock.tsx @@ -5,7 +5,7 @@ "use client"; import * as React from "react"; -import { type NodeProps, type Node, useReactFlow } from "@xyflow/react"; +import { type NodeProps, type Node, useReactFlow, useStore } from "@xyflow/react"; import { Globe, RefreshCw, X, Minimize2, Settings, Copy } from "lucide-react"; import { cn } from "@/lib/utils"; import { BlockWrapper } from "./BlockWrapper"; @@ -24,6 +24,7 @@ import { useTerminalOverlay } from "@/components/terminal"; import { HelpButton } from "@/components/help/HelpDialog"; import { browserDoc } from "@/docs/content/browser"; import { API, DEV_MODE_ENABLED } from "@/config/env"; +import { ensureSurfaceToken, getCachedSurfaceToken } from "@/lib/tauri-bridge"; import { ApiError } from "@/lib/api/client"; import { perfStart, perfMark, perfEnd, perfCancel, perfActive } from "@/lib/perf"; import { getDashboardBrowserStatus, openDashboardBrowser, startDashboardBrowser, stopDashboardBrowser } from "@/lib/api/cloudflare/dashboards"; @@ -89,6 +90,25 @@ export function BrowserBlock({ id, data, selected }: NodeProps) { const connectorsVisible = selected || Boolean(data.connectorMode); const lastOpenedRef = React.useRef(null); const user = useAuthStore((state) => state.user); + // Live canvas zoom (React Flow transform scale) — passed to VncViewer so it can + // correct noVNC's pointer mapping under zoom (clicks otherwise drift when not 100%). + const canvasZoom = useStore((s) => s.transform[2]); + // Desktop surface token for the VNC WS. Resolve it async (the sync cache can be + // null on an early load); when it arrives the vncWsUrl memo recomputes and the + // RFB reconnects with `&surface=`. Cross-origin WS gets no session cookie, so + // this is the only dev-auth proof the control plane accepts. + const [surfaceToken, setSurfaceToken] = React.useState( + getCachedSurfaceToken() + ); + React.useEffect(() => { + let active = true; + void ensureSurfaceToken().then((t) => { + if (active) setSurfaceToken(t); + }); + return () => { + active = false; + }; + }, []); const isMinimized = data.metadata?.minimized === true; const overlay = useTerminalOverlay(); // IMPORTANT: depend on the stable `bringToFront` callback, NOT the whole `overlay` @@ -235,9 +255,14 @@ export function BrowserBlock({ id, data, selected }: NodeProps) { params.set("user_email", user.email); params.set("user_name", user.name); } + // Desktop gates dev-auth on the surface token; a WS can't send the header, so + // pass it as a query param (null on web / non-surface builds → omitted). + if (surfaceToken) { + params.set("surface", surfaceToken); + } const qs = params.toString(); return qs ? `${wsUrl}?${qs}` : wsUrl; - }, [dashboardId, user]); + }, [dashboardId, user, surfaceToken]); // Times the VNC client phase (WS handshake → first frame), i.e. the part this // native-noVNC change actually targets — separate from chromium cold-boot. @@ -358,6 +383,7 @@ export function BrowserBlock({ id, data, selected }: NodeProps) { wsUrl={vncWsUrl} reloadKey={refreshKey} onConnectionState={handleVncState} + zoom={canvasZoom} className="w-full h-full" /> {vncState !== "connected" && ( diff --git a/frontend/src/components/blocks/TerminalBlock.tsx b/frontend/src/components/blocks/TerminalBlock.tsx index 1d915caf..ee09dd09 100644 --- a/frontend/src/components/blocks/TerminalBlock.tsx +++ b/frontend/src/components/blocks/TerminalBlock.tsx @@ -629,7 +629,48 @@ export function TerminalBlock({ const { user } = useAuthStore(); const { theme } = useThemeStore(); const queryClient = useQueryClient(); - const { deleteElements } = useReactFlow(); + const { deleteElements, getViewport, setViewport } = useReactFlow(); + + // Right-drag pans the canvas even while the pointer is over the terminal. + // xterm captures normal (left) drags, so a large PTY is otherwise a pan trap; + // this drives React Flow's viewport directly on a right-button drag and + // swallows the follow-up context menu only if we actually panned. + const termPan = React.useRef< + { x0: number; y0: number; vx: number; vy: number; zoom: number; moved: boolean } | null + >(null); + const handleTermPanStart = React.useCallback( + (e: React.MouseEvent) => { + if (e.button !== 2) return; // right button only; left/middle stay with xterm + const vp = getViewport(); + termPan.current = { x0: e.clientX, y0: e.clientY, vx: vp.x, vy: vp.y, zoom: vp.zoom, moved: false }; + const onMove = (ev: MouseEvent) => { + const p = termPan.current; + if (!p) return; + const dx = ev.clientX - p.x0; + const dy = ev.clientY - p.y0; + if (Math.abs(dx) > 3 || Math.abs(dy) > 3) p.moved = true; + if (p.moved) setViewport({ x: p.vx + dx, y: p.vy + dy, zoom: p.zoom }); + }; + const onUp = () => { + window.removeEventListener("mousemove", onMove); + window.removeEventListener("mouseup", onUp); + termPan.current = null; // clear on release, NOT in the contextmenu handler + }; + window.addEventListener("mousemove", onMove); + window.addEventListener("mouseup", onUp); + }, + [getViewport, setViewport] + ); + // Always suppress the native WebView text context menu over the terminal. macOS + // fires (and shows) it on right-mousedown — before we can tell a pan from a plain + // click — and once it's up it captures the mouse and the right-drag pan can't run. + // The menu is useless here anyway (xterm renders to a canvas, so Cut/Copy are dead; + // paste is Cmd+V). Suppressing it lets right-drag pan reliably. We do NOT touch + // termPan here — this fires right after mousedown, so nulling it would kill the pan + // before the drag starts; mouseup owns cleanup. + const handleTermContextMenu = React.useCallback((e: React.MouseEvent) => { + e.preventDefault(); + }, []); const [isReady, setIsReady] = React.useState(false); const [isClaudeSession, setIsClaudeSession] = React.useState(false); const [activePanel, setActivePanel] = React.useState(null); @@ -4337,6 +4378,8 @@ export function TerminalBlock({
void; className?: string; + /** Current React Flow canvas zoom (transform scale). noVNC maps pointer events + * via getBoundingClientRect (which includes this ancestor CSS transform) but + * divides by an internal scale that does NOT — its ResizeObserver never fires on + * an ancestor `transform`. That makes clicks land off by the zoom factor (only + * correct at 100%). We use this to compensate the coordinate conversion. */ + zoom?: number; } export function VncViewer({ @@ -49,6 +55,7 @@ export function VncViewer({ compressionLevel = 3, onConnectionState, className, + zoom = 1, }: VncViewerProps) { const containerRef = React.useRef(null); @@ -58,6 +65,10 @@ export function VncViewer({ onStateRef.current = onConnectionState; const optsRef = React.useRef({ viewOnly, qualityLevel, compressionLevel }); optsRef.current = { viewOnly, qualityLevel, compressionLevel }; + // Live zoom, read by the patched pointer-coordinate conversion below. Updated + // without reconnecting the RFB. + const zoomRef = React.useRef(zoom); + zoomRef.current = zoom || 1; React.useEffect(() => { const container = containerRef.current; @@ -79,10 +90,36 @@ export function VncViewer({ const r = new RFB(containerRef.current, wsUrl, { wsProtocols: ["binary"] }); rfb = r; + + // Compensate for React Flow's CSS zoom on pointer input. noVNC's Display + // converts an element-relative coordinate to a framebuffer coordinate via + // `absX(x) = x / _scale`. `x` comes from getBoundingClientRect and so includes + // the ancestor `transform: scale(zoom)`, but `_scale` does not (noVNC's + // ResizeObserver doesn't fire on ancestor transforms), so clicks land off by + // `zoom`. Pre-dividing the input by the live zoom cancels it: at zoom=1 this is + // a no-op. Guarded + best-effort so a noVNC internals change can't break input. + try { + const disp = (r as unknown as { _display?: { + absX: (x: number) => number; + absY: (y: number) => number; + } })._display; + if (disp && typeof disp.absX === "function" && typeof disp.absY === "function") { + const origAbsX = disp.absX.bind(disp); + const origAbsY = disp.absY.bind(disp); + disp.absX = (x: number) => origAbsX(x / (zoomRef.current || 1)); + disp.absY = (y: number) => origAbsY(y / (zoomRef.current || 1)); + } + } catch { + /* noVNC internals changed — leave input mapping as-is */ + } const { viewOnly: vo, qualityLevel: q, compressionLevel: c } = optsRef.current; - r.scaleViewport = true; // fit the 1280x720 framebuffer to the (resizable) node - r.resizeSession = false; - r.background = "#ffffff"; + // Ask the server (x11vnc/Xvfb) to resize its framebuffer to the node so the + // page FILLS the block. Falls back to scaleViewport (aspect-fit) when the + // server can't resize — in which case the leftover is painted with `background` + // (a neutral dark, NOT the old jarring #ffffff that read as a big white border). + r.scaleViewport = true; + r.resizeSession = true; + r.background = "#0d1117"; r.qualityLevel = q; r.compressionLevel = c; r.viewOnly = vo; diff --git a/frontend/src/components/canvas/Canvas.tsx b/frontend/src/components/canvas/Canvas.tsx index fbb7498b..1e86f91b 100644 --- a/frontend/src/components/canvas/Canvas.tsx +++ b/frontend/src/components/canvas/Canvas.tsx @@ -29,7 +29,12 @@ import { import "@xyflow/react/dist/style.css"; import { IntegrationEdge, EdgeLabelClickContext, EdgeDeleteContext, EdgeConnectorModeContext, EdgeReverseContext } from "@/components/canvas/IntegrationEdge"; -import { applyLocalPositionOverrides, type PositionOverride } from "@/components/canvas/positionOverrides"; +import { + applyLocalPositionOverrides, + applyLocalSizeOverrides, + type PositionOverride, + type SizeOverride, +} from "@/components/canvas/positionOverrides"; import { ConnectedHandlesContext } from "@/contexts/ConnectedHandlesContext"; import { NoteBlock } from "@/components/blocks/NoteBlock"; @@ -391,7 +396,11 @@ export function Canvas({ // position here and apply it during rebuilds until the cache catches up (positions // match) or a short max-age elapses (so concurrent remote moves still recover). const localPositionsRef = React.useRef>(new Map()); - // Must comfortably exceed the position mutation's debounce (~500ms) + server + // Same idea for resize: hold the new size after a resize ends so a rebuild during + // the debounced persist doesn't revert to the previous size (~1s later). Position + // had this protection; resize did not — the "components revert their size" bug. + const localSizesRef = React.useRef>(new Map()); + // Must comfortably exceed the position/size mutation's debounce (~500ms) + server // round-trip + any in-flight refetch, so a stale revert in that window is masked. const LOCAL_POSITION_MAX_AGE_MS = 2500; @@ -432,10 +441,16 @@ export function Canvas({ // Apply optimistic post-drag positions so a rebuild before the drag is durably // persisted doesn't flash the node back to its pre-drag position. See // positionOverrides.ts for the full rationale (and its unit tests). - const newNodes = applyLocalPositionOverrides( - [...baseNodes, ...extraNodes], - localPositionsRef.current, - Date.now(), + const now = Date.now(); + const newNodes = applyLocalSizeOverrides( + applyLocalPositionOverrides( + [...baseNodes, ...extraNodes], + localPositionsRef.current, + now, + LOCAL_POSITION_MAX_AGE_MS, + ), + localSizesRef.current, + now, LOCAL_POSITION_MAX_AGE_MS, ); @@ -525,6 +540,21 @@ export function Canvas({ width: Math.round(change.dimensions.width), height: Math.round(change.dimensions.height), }; + // Hold the new size (and position — a top/left resize also moves the + // node) locally until the item cache reflects the resize, so a rebuild + // during the debounced persist doesn't revert to the previous size. + // Keyed by node.id to match rebuildNodes / applyLocal*Overrides. + const at = Date.now(); + localSizesRef.current.set(node.id, { + width: afterSize.width, + height: afterSize.height, + at, + }); + localPositionsRef.current.set(node.id, { + x: node.position.x, + y: node.position.y, + at, + }); // When resizing from top/left edges, position also changes // Include both size and position to keep the correct corner anchored onItemChange(itemId, { diff --git a/frontend/src/components/canvas/positionOverrides.test.ts b/frontend/src/components/canvas/positionOverrides.test.ts index 74652731..74106597 100644 --- a/frontend/src/components/canvas/positionOverrides.test.ts +++ b/frontend/src/components/canvas/positionOverrides.test.ts @@ -4,8 +4,11 @@ import { describe, it, expect } from "vitest"; import { applyLocalPositionOverrides, + applyLocalSizeOverrides, type PositionOverride, type OverridableNode, + type SizeOverride, + type ResizableNode, } from "./positionOverrides"; const node = (id: string, x: number, y: number): OverridableNode => ({ @@ -109,3 +112,75 @@ describe("applyLocalPositionOverrides", () => { expect(result[0]).toMatchObject({ selected: true, data: { k: 1 }, position: { x: 200, y: 200 } }); }); }); + +const sized = (id: string, width: number, height: number): ResizableNode => ({ + id, + width, + height, +}); + +describe("applyLocalSizeOverrides", () => { + it("returns nodes unchanged when there are no overrides", () => { + const nodes = [sized("a", 100, 60)]; + const overrides = new Map(); + const result = applyLocalSizeOverrides(nodes, overrides, 1000, MAX_AGE); + expect(result).toBe(nodes); + }); + + it("forces the new size when the node still reports its OLD size", () => { + // The core revert scenario: resize set the override to 400x300 but a stale + // refetch/echo rebuilt the node back at its pre-resize 100x60. + const nodes = [sized("a", 100, 60)]; + const overrides = new Map([ + ["a", { width: 400, height: 300, at: 1000 }], + ]); + const result = applyLocalSizeOverrides(nodes, overrides, 1100, MAX_AGE); + expect(result[0]).toMatchObject({ width: 400, height: 300 }); + expect(overrides.has("a")).toBe(true); + }); + + it("keeps holding the override even after the cache matches (revert masked)", () => { + const overrides = new Map([ + ["a", { width: 400, height: 300, at: 1000 }], + ]); + // Optimistic match — must NOT release the override. + const matched = applyLocalSizeOverrides([sized("a", 400, 300)], overrides, 1050, MAX_AGE); + expect(matched[0]).toMatchObject({ width: 400, height: 300 }); + expect(overrides.has("a")).toBe(true); + // Stale revert lands shortly after, still within the window — masked. + const reverted = applyLocalSizeOverrides([sized("a", 100, 60)], overrides, 1400, MAX_AGE); + expect(reverted[0]).toMatchObject({ width: 400, height: 300 }); + }); + + it("expires the override after maxAge so genuine remote resizes apply", () => { + const overrides = new Map([ + ["a", { width: 400, height: 300, at: 1000 }], + ]); + const result = applyLocalSizeOverrides( + [sized("a", 250, 250)], + overrides, + 1000 + MAX_AGE + 1, + MAX_AGE, + ); + expect(result[0]).toMatchObject({ width: 250, height: 250 }); + expect(overrides.has("a")).toBe(false); + }); + + it("does not clone a node already at the override size (within epsilon)", () => { + const a = sized("a", 400.2, 299.9); + const overrides = new Map([ + ["a", { width: 400, height: 300, at: 1000 }], + ]); + const result = applyLocalSizeOverrides([a], overrides, 1100, MAX_AGE); + expect(result[0]).toBe(a); + }); + + it("preserves extra node fields when overriding size", () => { + const styled = { id: "a", width: 100, height: 60, selected: true, data: { k: 1 } }; + const overrides = new Map([ + ["a", { width: 400, height: 300, at: 1000 }], + ]); + const result = applyLocalSizeOverrides([styled], overrides, 1100, MAX_AGE); + expect(result[0]).toMatchObject({ selected: true, data: { k: 1 }, width: 400, height: 300 }); + }); +}); diff --git a/frontend/src/components/canvas/positionOverrides.ts b/frontend/src/components/canvas/positionOverrides.ts index e9b613de..05954e46 100644 --- a/frontend/src/components/canvas/positionOverrides.ts +++ b/frontend/src/components/canvas/positionOverrides.ts @@ -1,11 +1,13 @@ // Copyright 2026 Rob Macrae. All rights reserved. // SPDX-License-Identifier: LicenseRef-Proprietary -// REVISION: position-overrides-v1-extract +// REVISION: position-overrides-v2-size // // Optimistic post-drag position reconciliation, extracted as a pure function so it // can be unit-tested deterministically (the visual "blink back to start" it prevents -// is only observable in manual UI testing). +// is only observable in manual UI testing). v2 adds the analogous size override so a +// resize doesn't "revert to the previous size ~1s later" during the debounced persist +// (position had this protection; resize did not — a real asymmetry bug). // // Why this exists: when a node is dropped, the canvas persists its new position via a // DEBOUNCED (~500ms) mutation that then round-trips. In that window the item cache can @@ -77,3 +79,62 @@ export function applyLocalPositionOverrides( return { ...n, position: { x: ov.x, y: ov.y } }; }); } + +export interface SizeOverride { + width: number; + height: number; + /** Epoch ms when the resize ended (used to expire the override). */ + at: number; +} + +export interface ResizableNode { + id: string; + width?: number; + height?: number; +} + +/** Sizes within this many px are treated as equal (avoids needless clones). */ +const SIZE_EPSILON = 0.5; + +/** + * Size analogue of {@link applyLocalPositionOverrides}. Returns `nodes` with any + * active local size override forced onto each matching node's width/height, and + * prunes expired / no-longer-present overrides from `overrides` (mutated in place). + * + * Same rationale as the position variant: a resize is persisted via a DEBOUNCED + * mutation, so during the round-trip a collab echo or unrelated refetch can revert + * the item cache to the pre-resize size; a rebuild in that window would snap the + * node back to the old size until the cache catches up. Holding the new size for a + * fixed window (not releasing on first "match") masks that stale revert; after the + * window it expires so genuine remote resizes still apply. Pass `now` explicitly so + * tests don't depend on the clock. + */ +export function applyLocalSizeOverrides( + nodes: T[], + overrides: Map, + now: number, + maxAgeMs: number, +): T[] { + if (overrides.size === 0) return nodes; + + const liveIds = new Set(nodes.map((n) => n.id)); + for (const [nodeId, ov] of overrides) { + if (!liveIds.has(nodeId) || now - ov.at > maxAgeMs) { + overrides.delete(nodeId); + } + } + + if (overrides.size === 0) return nodes; + + return nodes.map((n) => { + const ov = overrides.get(n.id); + if (!ov) return n; + if ( + Math.abs((n.width ?? 0) - ov.width) < SIZE_EPSILON && + Math.abs((n.height ?? 0) - ov.height) < SIZE_EPSILON + ) { + return n; // already at the resized dimensions — no need to clone + } + return { ...n, width: ov.width, height: ov.height }; + }); +} diff --git a/frontend/src/components/terminal/XtermTerminal.tsx b/frontend/src/components/terminal/XtermTerminal.tsx index 44d06dbe..71252ed7 100644 --- a/frontend/src/components/terminal/XtermTerminal.tsx +++ b/frontend/src/components/terminal/XtermTerminal.tsx @@ -6,7 +6,7 @@ import * as React from "react"; import { cn } from "@/lib/utils"; import type { TerminalHandle, TerminalProps, TerminalTheme } from "./types"; -import type { Terminal } from "xterm"; +import type { Terminal } from "@xterm/xterm"; import type { FitAddon } from "@xterm/addon-fit"; interface XtermTheme extends TerminalTheme {} @@ -57,10 +57,10 @@ export const XtermTerminal = React.forwardRef( try { const [{ Terminal }, { FitAddon }] = await Promise.all([ - import("xterm"), + import("@xterm/xterm"), import("@xterm/addon-fit"), ]); - await import("xterm/css/xterm.css"); + await import("@xterm/xterm/css/xterm.css"); if (!mounted || !containerRef.current) return; diff --git a/frontend/src/components/workspace/WorkspaceSidebar.tsx b/frontend/src/components/workspace/WorkspaceSidebar.tsx index 6f95ba87..273ccba0 100644 --- a/frontend/src/components/workspace/WorkspaceSidebar.tsx +++ b/frontend/src/components/workspace/WorkspaceSidebar.tsx @@ -11,6 +11,7 @@ import * as React from "react"; import { createPortal } from "react-dom"; import { Folder, + FolderOpen, FolderInput, Cloud, Github, @@ -89,6 +90,7 @@ import { getWorkspaceSnapshot } from "@/lib/api/cloudflare/files"; import type { DashboardItem, Session } from "@/types/dashboard"; import { useAuthStore } from "@/stores/auth-store"; import { API, DEV_MODE_ENABLED, DESKTOP_MODE } from "@/config/env"; +import { revealWorkspace } from "@/lib/tauri-bridge"; import { cn } from "@/lib/utils"; import { getAgentType, getAgentIconSrc, getAgentDisplayName } from "@/lib/agent-icons"; import { useFolderImport } from "@/hooks/useFolderImport"; @@ -349,6 +351,14 @@ export function WorkspaceSidebar({ setFileError(null); } catch (error) { const message = error instanceof Error ? error.message : "Failed to load files"; + // A 404 here means the session's sandbox is gone (stopped or a stale + // sandbox id after a VM restart). That's not a real error for the sidebar — + // render a calm empty tree instead of a raw "Request failed with status 404". + if (/\b404\b|session not found|E797(09|37)/i.test(message)) { + setFileEntries((prev) => ({ ...prev, [path]: [] })); + setFileError(null); + return; + } setFileError(message); } }, @@ -1235,6 +1245,17 @@ export function WorkspaceSidebar({ Workspace + {DESKTOP_MODE && ( + + )} { +): Promise<{ dashboard: Dashboard; viewport?: { x: number; y: number; zoom: number }; hasSetupGuide?: boolean }> { const response = await apiPost(API.cloudflare.dashboards, { name, templateId, } as DashboardCreateRequest); - return { dashboard: response.dashboard, viewport: response.viewport }; + return { dashboard: response.dashboard, viewport: response.viewport, hasSetupGuide: response.hasSetupGuide }; } /** diff --git a/frontend/src/lib/tauri-bridge.ts b/frontend/src/lib/tauri-bridge.ts index 6611e1e6..c45b81d8 100644 --- a/frontend/src/lib/tauri-bridge.ts +++ b/frontend/src/lib/tauri-bridge.ts @@ -77,6 +77,17 @@ export async function getWorkspacePath(): Promise { return invoke("get_workspace_path") as Promise; } +/** Reveal the host workspace directory in Finder/Explorer (desktop only). */ +export async function revealWorkspace(): Promise { + const invoke = await getTauriInvoke(); + if (!invoke) return; + try { + await invoke("reveal_workspace"); + } catch { + /* ignore — desktop-only convenience */ + } +} + /** Import a folder (or file) from source_path into the workspace. */ export async function importFolder( sourcePath: string, @@ -156,10 +167,22 @@ export async function ensureSurfaceToken(): Promise { return fromUrl; } if (!surfaceTokenPromise) { - surfaceTokenPromise = fetchSurfaceToken().then((t) => { - cachedSurfaceToken = t; - return t; - }); + surfaceTokenPromise = fetchSurfaceToken() + .then((t) => { + cachedSurfaceToken = t; + // Do NOT cache a failed fetch. If the token wasn't available yet (the + // Tauri IPC bridge can be unreachable during an early / headless pre-window + // load), clear the promise so the next caller retries instead of forever + // returning this poisoned null. HTTP masks the miss via the session cookie, + // but cross-origin WebSockets get no cookie and depend on this token — so a + // one-time early failure would otherwise wedge every WS for the whole session. + if (!t) surfaceTokenPromise = null; + return t; + }) + .catch(() => { + surfaceTokenPromise = null; + return null; + }); } return surfaceTokenPromise; } diff --git a/frontend/src/lib/ws/BaseWebSocketManager.ts b/frontend/src/lib/ws/BaseWebSocketManager.ts index bc8221f8..abc282fe 100644 --- a/frontend/src/lib/ws/BaseWebSocketManager.ts +++ b/frontend/src/lib/ws/BaseWebSocketManager.ts @@ -90,19 +90,36 @@ export abstract class BaseWebSocketManager { this.manualClose = false; const newState = this.reconnectAttempts > 0 ? "reconnecting" : "connecting"; - console.log(`[WS] Connecting to ${this.url} (attempt ${this.reconnectAttempts + 1})`); this.setState(newState); - try { - this.ws = new WebSocket(this.url); - // Enable binary message handling - this.ws.binaryType = "arraybuffer"; - this.setupEventListeners(); - this.startConnectTimeout(); - } catch (error) { - console.error(`[WS] Connection error:`, error); - this.handleError(error as Error); - } + // Resolve the URL just-in-time. Subclasses may need to await an auth token + // (e.g. the desktop surface token) that wasn't ready at construct time, and + // rebuild the URL on every (re)connect rather than freezing a tokenless one. + void this.resolveUrl() + .then((url) => { + // A disconnect() during URL resolution must abort this connect. + if (this.manualClose) return; + console.log(`[WS] Connecting to ${url} (attempt ${this.reconnectAttempts + 1})`); + this.ws = new WebSocket(url); + // Enable binary message handling + this.ws.binaryType = "arraybuffer"; + this.setupEventListeners(); + this.startConnectTimeout(); + }) + .catch((error) => { + console.error(`[WS] Connection error:`, error); + this.handleError(error as Error); + }); + } + + /** + * Resolve the connection URL immediately before opening the socket. The base + * returns the static URL from the constructor; subclasses override this to await + * auth (surface token) and append it fresh on every connect/reconnect — so a + * token that resolves after construct is applied instead of being frozen out. + */ + protected async resolveUrl(): Promise { + return this.url; } /** diff --git a/frontend/src/lib/ws/DashboardWSManager.ts b/frontend/src/lib/ws/DashboardWSManager.ts index 31366cb3..0cf71300 100644 --- a/frontend/src/lib/ws/DashboardWSManager.ts +++ b/frontend/src/lib/ws/DashboardWSManager.ts @@ -21,6 +21,7 @@ import type { UICommandResultMessage, } from "@/types/collaboration"; import { getCollaborationWsUrl } from "@/lib/api/cloudflare"; +import { ensureSurfaceToken } from "@/lib/tauri-bridge"; import { getUserColor } from "@/lib/utils"; // Throttle cursor updates to 50ms @@ -59,6 +60,18 @@ export class DashboardWSManager extends BaseWebSocketManager { this.userName = userName; } + /** + * Rebuild the URL at connect time with the freshly-resolved surface token. A + * cross-origin WS carries no session cookie, so the surface token is the only + * dev-auth proof the control plane accepts; `await ensureSurfaceToken()` makes + * sure it's cached before `getCollaborationWsUrl` reads it (the constructor's + * synchronous build can miss an async/late token). Null on web → omitted. + */ + protected async resolveUrl(): Promise { + await ensureSurfaceToken(); + return getCollaborationWsUrl(this.dashboardId, this.userId, this.userName); + } + /** * Subscribe to messages */ diff --git a/frontend/src/lib/ws/TerminalWSManager.ts b/frontend/src/lib/ws/TerminalWSManager.ts index 45c885f0..5436bdff 100644 --- a/frontend/src/lib/ws/TerminalWSManager.ts +++ b/frontend/src/lib/ws/TerminalWSManager.ts @@ -1,8 +1,8 @@ // Copyright 2026 Rob Macrae. All rights reserved. // SPDX-License-Identifier: LicenseRef-Proprietary -// REVISION: terminal-ws-v4-surface-token -const MODULE_REVISION_TERMINAL_WS = "terminal-ws-v4-surface-token"; +// REVISION: terminal-ws-v5-surface-connect-time +const MODULE_REVISION_TERMINAL_WS = "terminal-ws-v5-surface-connect-time"; console.log(`[TerminalWS] REVISION: ${MODULE_REVISION_TERMINAL_WS} loaded at ${new Date().toISOString()}`); /** @@ -30,7 +30,7 @@ import type { } from "@/types/terminal"; import { API, DESKTOP_MODE, DEV_MODE_ENABLED } from "@/config/env"; import { useAuthStore } from "@/stores/auth-store"; -import { getCachedSurfaceToken } from "@/lib/tauri-bridge"; +import { ensureSurfaceToken, getCachedSurfaceToken } from "@/lib/tauri-bridge"; export interface TerminalWSConfig extends WebSocketConfig { @@ -91,12 +91,10 @@ export class TerminalWSManager extends BaseWebSocketManager { url += `&user_email=${encodeURIComponent(email)}`; } } - // Desktop gates dev-auth on the surface token; WebSockets can't send the - // header, so pass it as a query param (no-op on web, where it's null). - const surface = getCachedSurfaceToken(); - if (surface) { - url += `&surface=${encodeURIComponent(surface)}`; - } + // NOTE: the surface token is appended in resolveUrl() at connect time, not + // here — so a token that resolves AFTER construct (or a reconnect after the + // token becomes available) is still applied, instead of freezing a tokenless + // URL that 401-loops forever. super(url, config); this.sessionId = sessionId; @@ -105,6 +103,27 @@ export class TerminalWSManager extends BaseWebSocketManager { this.userName = config.userName; } + /** + * Append the desktop surface token just before connecting. A cross-origin WS + * carries no session cookie, so this token is the only dev-auth proof the control + * plane accepts. `await ensureSurfaceToken()` guarantees it's fetched+cached + * first (the constructor's synchronous read could miss an async/late token). + * Null on web / non-surface builds → omitted, unchanged behavior. + */ + protected async resolveUrl(): Promise { + await ensureSurfaceToken(); + const surface = getCachedSurfaceToken(); + const url = surface + ? `${this.url}&surface=${encodeURIComponent(surface)}` + : this.url; + console.log( + `[TerminalWS] ${MODULE_REVISION_TERMINAL_WS} resolveUrl user_id=${this.url.includes( + "user_id=" + )} surface=${surface ? `present(len=${surface.length})` : "MISSING"}` + ); + return url; + } + /** * Send PTY input as binary data * Returns false if input is blocked diff --git a/frontend/src/types/css.d.ts b/frontend/src/types/css.d.ts index 41fc8baa..a23b2c50 100644 --- a/frontend/src/types/css.d.ts +++ b/frontend/src/types/css.d.ts @@ -6,4 +6,4 @@ declare module "*.css" { export default content; } -declare module "xterm/css/xterm.css"; +declare module "@xterm/xterm/css/xterm.css"; diff --git a/sandbox/cmd/server/env.go b/sandbox/cmd/server/env.go index 8d867b6b..dc9b543c 100644 --- a/sandbox/cmd/server/env.go +++ b/sandbox/cmd/server/env.go @@ -32,11 +32,11 @@ type ApprovedDomainConfig struct { } type envUpdateRequest struct { - Set map[string]string `json:"set"` // Regular env vars (set directly) - Secrets map[string]SecretConfig `json:"secrets"` // Secrets with broker protection option - ApprovedDomains []ApprovedDomainConfig `json:"approved_domains"` // Pre-approved domains for custom secrets - Unset []string `json:"unset"` - ApplyNow bool `json:"apply_now"` + Set map[string]string `json:"set"` // Regular env vars (set directly) + Secrets map[string]SecretConfig `json:"secrets"` // Secrets with broker protection option + ApprovedDomains []ApprovedDomainConfig `json:"approved_domains"` // Pre-approved domains for custom secrets + Unset []string `json:"unset"` + ApplyNow bool `json:"apply_now"` } type envUpdateResponse struct { @@ -131,7 +131,11 @@ func (s *Server) handleSessionEnv(w http.ResponseWriter, r *http.Request) { // Built-in provider: use hardcoded config // Broker URL includes session ID for config isolation effectiveEnvVars[secretName] = broker.GetDummyValue(providerName) - effectiveEnvVars[providerSpec.BrokerEnvKey] = fmt.Sprintf("http://localhost:%d/broker/%s/%s", + // Use 127.0.0.1, not "localhost": the broker listens on 127.0.0.1 only + // (IPv4). Node/undici-based harnesses (OpenCode, Droid) resolve + // "localhost" to IPv6 ::1 first, get ECONNREFUSED, and retry-storm into + // an apparent hang. curl/Go/Python prefer IPv4 so they never hit it. + effectiveEnvVars[providerSpec.BrokerEnvKey] = fmt.Sprintf("http://127.0.0.1:%d/broker/%s/%s", brokerPort, session.ID, providerName) // Install configs for every provider that consumes this env key. Most @@ -153,7 +157,9 @@ func (s *Server) handleSessionEnv(w http.ResponseWriter, r *http.Request) { // Broker URL includes session ID for config isolation customID := "custom/" + secretName effectiveEnvVars[secretName] = broker.GetCustomDummyValue(secretName) - effectiveEnvVars[secretName+"_BROKER"] = fmt.Sprintf("http://localhost:%d/broker/%s/%s", + // 127.0.0.1 (not "localhost") — see the built-in provider branch above: + // IPv6-first Node harnesses can't reach the IPv4-only broker via "localhost". + effectiveEnvVars[secretName+"_BROKER"] = fmt.Sprintf("http://127.0.0.1:%d/broker/%s/%s", brokerPort, session.ID, customID) sessionBroker.SetConfig(broker.ConfigKey(session.ID, customID), &broker.ProviderConfig{ diff --git a/sandbox/docker/Dockerfile b/sandbox/docker/Dockerfile index 12a6db45..9059e5bc 100644 --- a/sandbox/docker/Dockerfile +++ b/sandbox/docker/Dockerfile @@ -96,17 +96,24 @@ RUN curl -fsSL https://deb.nodesource.com/setup_22.x | bash - \ && node /tmp/package/bin/npm-cli.js install -g npm@latest \ && rm -rf /tmp/package -# Pre-warm npm cache with Claude Code dependencies (for fast user-initiated install) -# We install then uninstall so the package is NOT distributed, but deps remain cached. -# A lightweight wrapper script at /usr/local/bin/claude handles on-demand installation. -RUN npm install -g @anthropic-ai/claude-code \ - && npm uninstall -g @anthropic-ai/claude-code -COPY docker/claude-wrapper.sh /usr/local/bin/claude -RUN chmod +x /usr/local/bin/claude - -# Suppress Claude Code's "switched from npm to native installer" banner. -# We use npm intentionally (1.3s pre-warmed vs 19s native installer). +# Install Claude Code via the native installer. +# WHY not npm: `npm install -g @anthropic-ai/claude-code` ships the real binary as a +# platform-specific OPTIONAL dependency, which npm's `--omit=optional` (set in this +# image) skips — leaving a shebang-less "native binary not installed" stub that fails +# with "fork/exec /usr/bin/claude: exec format error". +# WHY the symlink dance: the native installer is per-user — it drops a ~250MB +# self-contained binary in $HOME/.local/share/claude and a launcher that resolves it +# from $HOME. Our PTYs run as no-home `pty-NNN` users, so the launcher fails +# ("Failed to install"). The versioned binary itself is HOME-independent, so we point +# /usr/local/bin/claude straight at it in a shared path — works for every pty user. ENV DISABLE_INSTALLATION_CHECKS=1 +RUN curl -fsSL https://claude.ai/install.sh | bash \ + && CLAUDE_BIN="$(readlink -f /root/.local/bin/claude)" \ + && install -D -m 0755 "$CLAUDE_BIN" /opt/claude/claude \ + && rm -f /usr/bin/claude \ + && ln -sf /opt/claude/claude /usr/local/bin/claude \ + && rm -rf /root/.local/share/claude /root/.local/state/claude /root/.local/bin/claude \ + && claude --version # Install Codex CLI (OpenAI) RUN npm cache clean --force && npm install -g @openai/codex@latest @@ -114,13 +121,14 @@ RUN npm cache clean --force && npm install -g @openai/codex@latest # Install Gemini CLI (Google) RUN npm cache clean --force && npm install -g @google/gemini-cli@latest -# --- Temporarily disabled agents (may re-enable later) --- -# Keeping Claude Code, Gemini CLI, and Codex CLI for now. -# OpenCode, Droid, and OpenClaw are commented out to reduce image size (~800MB savings). +# --- Agents --- +# Installed: Claude Code, Gemini CLI, Codex CLI, OpenCode. +# OpenCode re-enabled for benchmark harnesses (slop-code-bench drives it for +# OpenRouter models, e.g. Kimi). Droid and OpenClaw stay disabled to save space. -# # Install OpenCode CLI and move to shared location -# RUN curl -fsSL https://opencode.ai/install | bash \ -# && mv /root/.opencode/bin/opencode /usr/local/bin/opencode +# Install OpenCode CLI and move to shared location +RUN curl -fsSL https://opencode.ai/install | bash \ + && mv /root/.opencode/bin/opencode /usr/local/bin/opencode # # Install Droid CLI (Factory) # RUN curl -fsSL https://app.factory.ai/cli | sh \ @@ -222,6 +230,20 @@ COPY --from=builder /build/orcabot-server /usr/local/bin/orcabot-server COPY --from=builder /build/browser-ctl /usr/local/bin/browser-ctl COPY --from=builder /build/mcp-bridge /usr/local/bin/mcp-bridge +# Force the CURRENT chromium (placed late so this cache-bust only re-runs this cheap +# layer, not the heavy npm/agent installs above). The base apt layer is Docker-cached +# and pins whatever chromium was current when it was first built — and 150.0.7871.46 +# is BROKEN: it IMMEDIATE_CRASHes (SIGTRAP) on launch, so the browser block never +# reaches ready ("Browser failed to start"). Fixed in 150.0.7871.100. This re-reads +# the mirror and upgrades to the current build; CHROMIUM_CACHEBUST (a timestamp from +# build-images.sh) makes the layer re-run every build instead of caching stale. +ARG CHROMIUM_CACHEBUST=0 +RUN echo "chromium-refresh ${CHROMIUM_CACHEBUST}" \ + && apt-get update \ + && apt-get install -y --no-install-recommends --only-upgrade chromium \ + && rm -rf /var/lib/apt/lists/* \ + && (chromium.real --version 2>/dev/null || chromium --version) + # Working directory WORKDIR /workspace diff --git a/sandbox/docker/claude-wrapper.sh b/sandbox/docker/claude-wrapper.sh deleted file mode 100644 index 9982e47c..00000000 --- a/sandbox/docker/claude-wrapper.sh +++ /dev/null @@ -1,35 +0,0 @@ -#!/bin/bash -# On-demand installer for Claude Code. -# The real binary goes to /usr/bin/claude (npm prefix is /usr). -# This wrapper lives at /usr/local/bin/claude and is only invoked -# when the real binary hasn't been installed yet. - -# If the real claude is already installed, just exec it -if [ -x /usr/bin/claude ]; then - exec /usr/bin/claude "$@" -fi - -# Serialize concurrent installs — only one npm install at a time -LOCKFILE="/tmp/claude-install.lock" -exec 9>"$LOCKFILE" -flock 9 - -# Re-check after acquiring lock (another process may have installed it) -if [ -x /usr/bin/claude ]; then - exec /usr/bin/claude "$@" -fi - -printf "\n" -printf "Installing Claude Code from npm...\n" -printf "By continuing, you agree to Anthropic's Terms of Service.\n" -for i in 3 2 1; do printf "\rInstalling in %d..." "$i"; sleep 1; done -printf "\rInstalling now... \n" - -if npm install -g @anthropic-ai/claude-code; then - # Install succeeded — remove this wrapper so /usr/bin/claude takes priority - rm -f /usr/local/bin/claude - exec claude "$@" -else - printf "\nFailed to install Claude Code.\n" >&2 - exit 1 -fi diff --git a/sandbox/docker/validate-image.sh b/sandbox/docker/validate-image.sh new file mode 100755 index 00000000..57f8f689 --- /dev/null +++ b/sandbox/docker/validate-image.sh @@ -0,0 +1,73 @@ +#!/usr/bin/env bash +# Build-time smoke for the sandbox VM image. +# +# Runs INSIDE the freshly-built orcabot-sandbox container (invoked by +# build-images.sh via `docker run` right after `docker build`, BEFORE the rootfs is +# exported/assembled). A non-zero exit aborts the build (build-images.sh runs with +# `set -e`), so a regressed dependency FAILS THE BUILD instead of silently shipping a +# broken VM image that replaces a working one. +# +# It catches the exact classes that silently broke real images, which a plain +# `--version` check does NOT: +# - chromium that installs and reports a version but IMMEDIATE_CRASHes on launch +# (Debian 150.0.7871.46 SIGTRAP) → browser block "Browser failed to start". +# → we actually LAUNCH chromium headless and require its DevTools to answer. +# - claude installed only for root / a per-user launcher that can't resolve for the +# no-home pty-NNN users that run PTYs → "exec format error" / "Failed to install". +# → we run claude (and the other agents) as a no-home, empty-$HOME user. +# +# Deps used (all present in the image): bash, curl, su, useradd, seq. +set -uo pipefail + +FAIL=0 +ok() { echo " OK: $1"; } +bad() { echo " FAIL: $1"; FAIL=1; } + +# A no-own-home, sandbox-group user with a FRESH empty $HOME — mirrors how PTYs run +# agents (pty-NNN system users, gid=sandbox, $HOME set to a fresh session dir). If +# claude were installed per-user (into root's home) this $HOME wouldn't have it, so +# this catches that regression. NOTE: $HOME must NOT be under /tmp — codex refuses to +# create its helper binaries in a temp dir and would false-fail here. +id ptysmoke >/dev/null 2>&1 || useradd --system --no-create-home --gid sandbox ptysmoke 2>/dev/null || true +PTY_HOME=/home/ptysmoke-home +rm -rf "$PTY_HOME"; mkdir -p "$PTY_HOME"; chown ptysmoke "$PTY_HOME" 2>/dev/null || true +as_pty() { su ptysmoke -s /bin/bash -c "HOME=$PTY_HOME $1" 2>&1; } +# First MAJOR.MINOR(.PATCH) token in a --version output (agents prepend banners/warnings). +ver_of() { grep -oE '[0-9]+\.[0-9]+(\.[0-9]+)?' | head -1; } + +echo "[validate-image] 1. chromium LAUNCHES (not just --version)" +CHROME="$(command -v chromium.real || command -v chromium)" +rm -rf /tmp/csmoke +"$CHROME" --headless=new --no-sandbox --disable-gpu \ + --remote-debugging-port=9222 --user-data-dir=/tmp/csmoke about:blank \ + >/tmp/chrome-smoke.log 2>&1 & +CPID=$! +UP=0 +for _ in $(seq 1 25); do + if curl -sf http://127.0.0.1:9222/json/version >/dev/null 2>&1; then UP=1; break; fi + kill -0 "$CPID" 2>/dev/null || break # chromium already exited → crashed on launch + sleep 1 +done +kill "$CPID" 2>/dev/null || true +if [ "$UP" = 1 ]; then + ok "chromium up + DevTools answered: $("$CHROME" --version 2>&1 | head -1)" +else + bad "chromium did NOT come up: $("$CHROME" --version 2>&1 | head -1) — crashes on launch" + echo " --- last chromium output ---"; tail -5 /tmp/chrome-smoke.log | sed 's/^/ /' +fi + +echo "[validate-image] 2. agents run as a no-own-home pty user (claude first — its regression)" +for a in claude opencode gemini codex; do + OUT="$(as_pty "$a --version 2>&1")" + V="$(printf '%s' "$OUT" | ver_of)" + if [ -n "$V" ]; then ok "$a: $V"; else bad "$a: no version — $(printf '%s' "$OUT" | tr '\n' ' ' | cut -c1-100)"; fi +done + +echo +if [ "$FAIL" = 0 ]; then + echo "[validate-image] PASS — image is shippable" + exit 0 +else + echo "[validate-image] FAIL — refusing to assemble/ship this image (a dependency regressed)" + exit 1 +fi diff --git a/sandbox/internal/browser/browser.go b/sandbox/internal/browser/browser.go index 81f5712c..8c8c9665 100644 --- a/sandbox/internal/browser/browser.go +++ b/sandbox/internal/browser/browser.go @@ -1,7 +1,7 @@ // Copyright 2026 Rob Macrae. All rights reserved. // SPDX-License-Identifier: LicenseRef-Proprietary -// REVISION: browser-v9-ready-reprobe +// REVISION: browser-v10-no-sandbox-warning package browser import ( @@ -44,8 +44,8 @@ type Controller struct { processes []*exec.Cmd } -// REVISION: browser-v9-ready-reprobe -const browserRevision = "browser-v9-ready-reprobe" +// REVISION: browser-v10-no-sandbox-warning +const browserRevision = "browser-v10-no-sandbox-warning" func init() { log.Printf("[browser] REVISION: %s loaded at %s", browserRevision, time.Now().Format(time.RFC3339)) @@ -62,7 +62,7 @@ func NewController(workspace string) *Controller { // that redirects, XHR, and subresource loads are subject to the same // CheckAndHold approval flow as the initial navigation URL. // Loopback addresses bypass the proxy so the CDP debug connection is unaffected. -// REVISION: browser-v9-ready-reprobe +// REVISION: browser-v10-no-sandbox-warning func NewControllerWithEgress(workspace string, egressProxyPort int) *Controller { return &Controller{workspace: workspace, egressProxyPort: egressProxyPort} } @@ -151,7 +151,11 @@ func (c *Controller) Start() (Status, error) { webArgs := []string{ "--heartbeat", "30", strconv.Itoa(wsPort), - fmt.Sprintf("localhost:%d", vncPort), + // 127.0.0.1, not "localhost": the VM's /etc/hosts may lack a localhost + // entry, in which case websockify's name resolution of "localhost:" + // fails and it closes every VNC client with 1011 "Failed to connect to + // downstream server". x11vnc listens on 127.0.0.1, so target it directly. + fmt.Sprintf("127.0.0.1:%d", vncPort), } if _, err := os.Stat("/usr/share/novnc/vnc.html"); err == nil { webArgs = append([]string{"--web", "/usr/share/novnc"}, webArgs...) @@ -161,6 +165,11 @@ func (c *Controller) Start() (Status, error) { chromiumArgs := []string{ "--no-sandbox", + // Suppress chromium's yellow "You are using an unsupported command-line + // flag: --no-sandbox" infobar (--disable-infobars does NOT hide this one in + // current chromium). --no-sandbox is required here (chromium runs as root in + // the VM), so the warning is noise that just clutters the browser-block view. + "--test-type", "--disable-dev-shm-usage", "--no-first-run", "--no-default-browser-check", @@ -183,7 +192,7 @@ func (c *Controller) Start() (Status, error) { // Chromium runs as root and is outside the iptables UID range, so without // this every redirect, XHR, and subresource load bypasses kernel egress controls. // Loopback is bypassed so the CDP debug connection on 127.0.0.1 is unaffected. - // REVISION: browser-v9-ready-reprobe + // REVISION: browser-v10-no-sandbox-warning if c.egressProxyPort > 0 { chromiumArgs = append(chromiumArgs, "--proxy-server=http://127.0.0.1:"+strconv.Itoa(c.egressProxyPort), diff --git a/sandbox/internal/sessions/model_selection.go b/sandbox/internal/sessions/model_selection.go index ef331781..5b6d47f9 100644 --- a/sandbox/internal/sessions/model_selection.go +++ b/sandbox/internal/sessions/model_selection.go @@ -74,7 +74,10 @@ func applyOpenRouterEnv(envVars map[string]string, agentType mcp.AgentType, sel if !sel.IsOpenRouter() { return } - brokerURL := fmt.Sprintf("http://localhost:%d/broker/%s/openrouter", brokerPort, sessionID) + // 127.0.0.1, NOT localhost: the broker binds IPv4 only, and Node/undici + // (opencode, droid) resolves "localhost" to IPv6 ::1 first — which the broker + // never answers — hanging the request. Matches the custom-endpoint path below. + brokerURL := fmt.Sprintf("http://127.0.0.1:%d/broker/%s/openrouter", brokerPort, sessionID) switch agentType { case mcp.AgentTypeCodex, mcp.AgentTypeOpenCode, mcp.AgentTypeDroid: @@ -97,7 +100,7 @@ func applyOpenRouterEnv(envVars map[string]string, agentType mcp.AgentType, sel envVars["OPENAI_API_KEY"] = broker.GetDummyValue("openai") log.Printf("[model-selection] agent=%s routing via OpenRouter model=%s broker=%s", agentType, sel.Model, brokerURL) case mcp.AgentTypeClaude: - anthropicBroker := fmt.Sprintf("http://localhost:%d/broker/%s/openrouter-anthropic", brokerPort, sessionID) + anthropicBroker := fmt.Sprintf("http://127.0.0.1:%d/broker/%s/openrouter-anthropic", brokerPort, sessionID) envVars["ANTHROPIC_BASE_URL"] = anthropicBroker // Claude Code only enters API mode (honoring ANTHROPIC_BASE_URL) when // ANTHROPIC_API_KEY looks like a real key — the human-readable "[BROKERED]" @@ -140,7 +143,10 @@ func applyCustomEndpointEnv(envVars map[string]string, agentType mcp.AgentType, if !sel.IsCustom() { return } - brokerURL := fmt.Sprintf("http://localhost:%d/broker/%s/%s", brokerPort, sessionID, customProviderName) + // 127.0.0.1, NOT localhost: the broker binds IPv4 only, and Node/undici (opencode, + // droid) resolves "localhost" to IPv6 ::1 first — which the broker never answers — + // causing a multi-minute retry-storm "hang". Same fix as env.go's broker URLs. + brokerURL := fmt.Sprintf("http://127.0.0.1:%d/broker/%s/%s", brokerPort, sessionID, customProviderName) switch agentType { case mcp.AgentTypeOpenCode, mcp.AgentTypeDroid: envVars["OPENAI_BASE_URL"] = brokerURL @@ -235,7 +241,8 @@ func buildCodexOpenRouterCommand(command string, sel *ModelSelection, sessionID if loc == nil { return command } - brokerURL := fmt.Sprintf("http://localhost:%d/broker/%s/openrouter", brokerPort, sessionID) + // 127.0.0.1, NOT localhost — keep every broker URL IPv4-only (see applyOpenRouterEnv). + brokerURL := fmt.Sprintf("http://127.0.0.1:%d/broker/%s/openrouter", brokerPort, sessionID) flags := []string{ "--model " + shellSingleQuote(sel.Model), "-c " + shellSingleQuote(`model_provider="openrouter"`), diff --git a/sandbox/internal/sessions/model_selection_test.go b/sandbox/internal/sessions/model_selection_test.go index acc8c233..1ef9d0f1 100644 --- a/sandbox/internal/sessions/model_selection_test.go +++ b/sandbox/internal/sessions/model_selection_test.go @@ -24,7 +24,7 @@ func TestBuildCodexOpenRouterCommand(t *testing.T) { for _, want := range []string{ "--model 'deepseek/deepseek-chat'", `model_provider="openrouter"`, - `model_providers.openrouter.base_url="http://localhost:8082/broker/sess123/openrouter"`, + `model_providers.openrouter.base_url="http://127.0.0.1:8082/broker/sess123/openrouter"`, `model_providers.openrouter.env_key="OPENAI_API_KEY"`, `model_providers.openrouter.wire_api="responses"`, "model_context_window=131072", @@ -41,7 +41,7 @@ func TestApplyCustomEndpointEnv_OpenAINative(t *testing.T) { for _, agent := range []mcp.AgentType{mcp.AgentTypeOpenCode, mcp.AgentTypeDroid} { env := map[string]string{} applyCustomEndpointEnv(env, agent, sel, "sess1", 8082, 8086) - if env["OPENAI_BASE_URL"] != "http://localhost:8082/broker/sess1/customprovider" { + if env["OPENAI_BASE_URL"] != "http://127.0.0.1:8082/broker/sess1/customprovider" { t.Errorf("%s OPENAI_BASE_URL = %q", agent, env["OPENAI_BASE_URL"]) } if env["OPENAI_MODEL"] != "llama3.3:70b" { diff --git a/sandbox/internal/sessions/session.go b/sandbox/internal/sessions/session.go index 46c3a402..0366ac92 100644 --- a/sandbox/internal/sessions/session.go +++ b/sandbox/internal/sessions/session.go @@ -669,8 +669,25 @@ func (s *Session) CreatePTYWithOptions(opts CreatePTYOptions) (*PTYInfo, error) if agentType == mcp.AgentTypeClaude { envVars["IS_SANDBOX"] = "1" } - // Make ~ resolve to the session workspace so attached assets are UI-manageable. - envVars["HOME"] = s.workspace.Root() + // HOME resolves to a dedicated dir INSIDE the workspace, not the workspace root + // itself. Keeping ~ under the workspace keeps agent home-dir files self-contained + // and (being dot-hidden) out of the file sidebar, while cwd stays the workspace + // project dir. Crucially cwd must NOT equal HOME: opencode's file picker (fff) + // refuses to run when the project dir IS the home dir ("Can not run certain FFF + // features in a file system root or home directories"), which sent opencode into a + // re-init/crash loop. Agent MCP/config is enforced via HOME-independent overrides + // (GEMINI_CLI_SYSTEM_SETTINGS_PATH, /etc/codex/config.toml, project .mcp.json), so + // the ~-fallback moving here is harmless. + homeDir := filepath.Join(s.workspace.Root(), ".home") + if err := os.MkdirAll(homeDir, 0o2775); err != nil { + log.Printf("[session] failed to create HOME dir %s: %v (falling back to workspace root)", homeDir, err) + homeDir = s.workspace.Root() + } else { + // Force group-write + setgid past umask so the pty-NNN users (gid=sandbox) can + // write, and nested files inherit the sandbox group. + _ = os.Chmod(homeDir, 0o2775) + } + envVars["HOME"] = homeDir // Point agents to the localhost-only MCP server (no auth required) mcpPort := os.Getenv("MCP_LOCAL_PORT") if mcpPort == "" {