From 047fa894930b59842645f45cea41c4a54ed1fdf9 Mon Sep 17 00:00:00 2001 From: QianCyrus <101633534+QianCyrus@users.noreply.github.com> Date: Mon, 28 Sep 2026 01:59:16 +0800 Subject: [PATCH] feat(desktop): add local Cua-S1 4B screenshot actions --- .env.example | 12 +- CONTRIBUTING.md | 3 +- docs/agents.md | 5 + evals/desktop/build_visual_fixture.sh | 19 ++ evals/desktop/visual.md | 45 ++++ evals/desktop/visual_fixture.swift | 86 ++++++++ pyproject.toml | 2 + s1a/agents/desktop.py | 58 ++++- s1a/decision_models/cua_four_b.py | 142 ++++++++++++ s1a/decision_models/factory.py | 8 + s1a/desktop/driver.py | 81 +++++-- s1a/desktop/env.py | 72 +++++-- s1a/env.py | 11 +- s1a/tool/models.py | 5 +- tests/test_decision_models_cua_four_b.py | 97 +++++++++ tests/test_decision_models_factory.py | 7 + tests/test_desktop_visual.py | 263 +++++++++++++++++++++++ uv.lock | 148 ++++++++++--- 18 files changed, 997 insertions(+), 67 deletions(-) create mode 100644 evals/desktop/build_visual_fixture.sh create mode 100644 evals/desktop/visual.md create mode 100644 evals/desktop/visual_fixture.swift create mode 100644 s1a/decision_models/cua_four_b.py create mode 100644 tests/test_decision_models_cua_four_b.py create mode 100644 tests/test_desktop_visual.py diff --git a/.env.example b/.env.example index 58acac1..6dd5977 100644 --- a/.env.example +++ b/.env.example @@ -39,7 +39,17 @@ MODEL_NAME=google/gemini-2.5-flash # CUA_S1_SUBFOLDER=text # the text-only checkpoint; the window is 256 bytes of state # CUA_S1_DEVICE=auto # cpu, cuda, mps -# ---- Cua Driver (the hands of the desktop agents, e.g. calculator; install: https://cua.ai/docs/cua-driver) ---- +# ---- Optional Cua-S1 4B ---- +# uv sync --extra cua-four-b (separate from the Laya demo). +# CUA_S1_VARIANT=4b +# CUA_S1_MODALITY=multimodal +# CUA_S1_BASE_MODEL=Qwen/Qwen3.5-4B # or a local model directory +# CUA_S1_CHECKPOINT=cua-ai/cua-s1-4b-0.2 # or a directory with text/ and multimodal/ +# CUA_S1_DEVICE=mps +# CUA_S1_DTYPE=bfloat16 +# HF_DEACTIVATE_ASYNC_LOAD=1 # synchronous Transformers loading on Apple MPS + +# ---- Cua Driver (the hands of the desktop agents; install: https://cua.ai/docs/cua-driver) ---- # CUA_DRIVER_BIN=/usr/local/bin/cua-driver # when cua-driver is not on PATH # CUA_DRIVER_BIN=C:/path/to/cua-driver.exe # Windows: full path to the installed executable # CUA_DRIVER_PERMISSION_MODE=standard # the driver's mode; bounded needs a capability manifest diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 9af4d53..b0f864a 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -72,8 +72,9 @@ Everything outside `openjiuwen` is an extra. An agent whose extra is missing say | `laya` | laya (torch, transformers) | `--model laya` on every agent and on `decide` and `probe`: Laya in process, no Jev key; the checkpoint downloads into the Hugging Face cache (`HF_HOME`) on first use | | `cua` | cua-s1 (torch), huggingface-hub | `--model cua` on tool and browser agents and on `decide` and `probe`: Cua-S1 Nano in process; the 3 MB checkpoint downloads into the Hugging Face cache (`HF_HOME`) on first use | | `dev` | pytest, pytest-asyncio, ruff, ty | the test suite, `scripts/smoke.sh` and the lint and type checks | +| `cua-four-b` | the `cua` extra, transformers 5, peft, torchvision, pillow | optional local Cua-S1 4B; set `CUA_S1_VARIANT=4b`, and `CUA_S1_MODALITY=text` or `multimodal` | -`uv sync --all-extras` installs all seven. The CLI runs from a checkout; a wheel install (`uv tool install`, +`uv sync --all-extras` installs all eight. The CLI runs from a checkout; a wheel install (`uv tool install`, `pip install`) is unsupported, because the data folders (`evals/2048`, `evals/millionaire`, `evals/labelled`) sit next to the package; the command refuses to start outside a checkout with one line on stderr. Runs, logs and results go under the checkout (`runs/`, `evals/results`), or under `S1A_HOME` when that variable names another diff --git a/docs/agents.md b/docs/agents.md index c7ff39a..5c5e4b5 100644 --- a/docs/agents.md +++ b/docs/agents.md @@ -33,6 +33,11 @@ models behind them: [architecture.md](architecture.md#models). `--app`, `--goal`, `--expect`, `--execute`, `--plan` and `--clear`; `ticket_router` takes `--dataset` and `--batch-size`; `injection_guard` takes `--labelled-set`. The four games take no flag of their own. +`--pixel-target KEY=X,Y` offers named points in screenshot fractions. Local Cua-S1 4B multimodal can select +among them when controls have no accessibility elements. Each click carries its screenshot capture ID. +Install the `cua-four-b` extra and set `CUA_S1_VARIANT=4b`, `CUA_S1_MODALITY=multimodal`. +See [the macOS visual task](../evals/desktop/visual.md) for setup and verification commands. + ## Allrecipes `allrecipes` is the first Allrecipes task of the [WebVoyager](https://github.com/MinorJerry/WebVoyager) task set diff --git a/evals/desktop/build_visual_fixture.sh b/evals/desktop/build_visual_fixture.sh new file mode 100644 index 0000000..5362083 --- /dev/null +++ b/evals/desktop/build_visual_fixture.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash +set -euo pipefail + +app="${1:-/tmp/S1AVisualFixture.app}" +root="$(cd "$(dirname "$0")/../.." && pwd)" +mkdir -p "$app/Contents/MacOS" +swiftc -O "$root/evals/desktop/visual_fixture.swift" -framework AppKit -o "$app/Contents/MacOS/S1AVisualFixture" +cat > "$app/Contents/Info.plist" <<'PLIST' + + + + CFBundleExecutableS1AVisualFixture + CFBundleIdentifierorg.thinkflowlab.s1a.visual-fixture + CFBundleNameS1AVisualFixture + CFBundlePackageTypeAPPL + LSMinimumSystemVersion14.0 + +PLIST +printf '%s\n' "$app" diff --git a/evals/desktop/visual.md b/evals/desktop/visual.md new file mode 100644 index 0000000..0a324b7 --- /dev/null +++ b/evals/desktop/visual.md @@ -0,0 +1,45 @@ +# Screenshot target selection on macOS + +The native test app draws Save and Cancel tiles without accessibility children. Reset swaps their positions. +The agent receives the current screenshot and chooses between two configured points. A successful click +shows Saved; the app also appends the selected tile to a temporary file for an independent check. + +The optional `cua-four-b` extra loads the official Cua-S1 4B scorer locally. Set `CUA_S1_VARIANT=4b` and +`CUA_S1_MODALITY=multimodal` for screenshots, or `CUA_S1_MODALITY=text` for text observations. Existing +`--model cua` runs keep using Nano unless the variant is set. The 4B scorer supports up to 26 choices. + +## Run the visual task + +Requires macOS, `uv`, Swift command-line tools, and Cua Driver with Accessibility and Screen Recording +permissions. Keep the Mac unlocked. Close an older S1A visual fixture before rebuilding it. +Run these commands from the current project directory: + +```bash +S1A_PROJECT_DIR="$(pwd -P)" +uv sync --project "$S1A_PROJECT_DIR" --extra cua-four-b +bash "$S1A_PROJECT_DIR/evals/desktop/build_visual_fixture.sh" /tmp/S1AVisualFixture.app +export CUA_DRIVER_BIN=/Applications/CuaDriver.app/Contents/MacOS/cua-driver +export CUA_S1_VARIANT=4b CUA_S1_MODALITY=multimodal +export CUA_S1_DEVICE=mps CUA_S1_DTYPE=bfloat16 HF_DEACTIVATE_ASYNC_LOAD=1 +export PYTORCH_ENABLE_MPS_FALLBACK=1 +: > /tmp/s1a-visual-target.txt +uv run --project "$S1A_PROJECT_DIR" --no-sync s1a run desktop \ + --model cua --rethink off --episodes 4 --max-steps 1 \ + --app S1AVisualFixture --app-path /tmp/S1AVisualFixture.app --expect Saved \ + --window-title "S1A Visual Fixture" \ + --goal "Click the tile labelled Save in the screenshot" \ + --pixel-target left=0.27,0.51 --pixel-target right=0.73,0.51 \ + --clear Reset --execute +uv run --project "$S1A_PROJECT_DIR" --no-sync python -c \ + 'from pathlib import Path; rows = Path("/tmp/s1a-visual-target.txt").read_text().splitlines(); assert rows == ["Save selected"] * 4, rows; print("Verified all four clicks")' +``` + +The base model and adapter download on first use. `CUA_S1_BASE_MODEL` and `CUA_S1_CHECKPOINT` can point +to downloaded directories. Omit `--execute` to preview a decision without clicking or resetting the app. + +For a fixed-action check, use `--model rule --episodes 1 --plan pixel:right` on a newly launched fixture. +Its first Reset moves Save to the right; later resets alternate sides. `--pixel-target` coordinates are fractions +of the captured window. Every click carries its capture ID so the driver can reject a stale target. + +The temporary selection file and ignored run outputs are generated locally. Screenshot bytes are passed +to the model separately from the serialized observation state. diff --git a/evals/desktop/visual_fixture.swift b/evals/desktop/visual_fixture.swift new file mode 100644 index 0000000..de644de --- /dev/null +++ b/evals/desktop/visual_fixture.swift @@ -0,0 +1,86 @@ +import AppKit +import Foundation + +private let output = URL(fileURLWithPath: "/tmp/s1a-visual-target.txt") + +// The tiles are drawn pixels, deliberately absent from the accessibility tree. +final class TileCanvas: NSView { + var saveOnLeft = true + var selected: ((Bool) -> Void)? + override var isFlipped: Bool { true } + private let tiles = [NSRect(x: 60, y: 140, width: 200, height: 100), + NSRect(x: 340, y: 140, width: 200, height: 100)] + + override func draw(_ dirtyRect: NSRect) { + NSColor.windowBackgroundColor.setFill() + bounds.fill() + let heading: NSString = "Choose the tile labelled Save" + heading.draw(at: NSPoint(x: 60, y: 55), withAttributes: [.font: NSFont.systemFont(ofSize: 24), + .foregroundColor: NSColor.labelColor]) + for (index, tile) in tiles.enumerated() { + NSColor.controlBackgroundColor.setFill() + NSBezierPath(roundedRect: tile, xRadius: 12, yRadius: 12).fill() + let label: NSString = (index == 0) == saveOnLeft ? "Save" : "Cancel" + let attributes: [NSAttributedString.Key: Any] = [.font: NSFont.systemFont(ofSize: 28), + .foregroundColor: NSColor.labelColor] + let size = label.size(withAttributes: attributes) + label.draw(at: NSPoint(x: tile.midX - size.width / 2, y: tile.midY - size.height / 2), + withAttributes: attributes) + } + } + + override func mouseDown(with event: NSEvent) { + let point = convert(event.locationInWindow, from: nil) + if let index = tiles.firstIndex(where: { $0.contains(point) }) { + selected?((index == 0) == saveOnLeft) + } + } +} + +final class VisualFixture: NSObject, NSApplicationDelegate { + private let window = NSWindow(contentRect: NSRect(x: 300, y: 300, width: 600, height: 400), + styleMask: [.titled, .closable, .miniaturizable], backing: .buffered, defer: false) + private let canvas = TileCanvas(frame: NSRect(x: 0, y: 0, width: 600, height: 400)) + private let status = NSTextView(frame: .zero) + + func applicationDidFinishLaunching(_ notification: Notification) { + window.title = "S1A Visual Fixture" + canvas.setAccessibilityElement(false) + status.isEditable = false + status.isSelectable = false + status.drawsBackground = false + status.string = "Waiting" + status.setAccessibilityLabel("Status") + status.frame = NSRect(x: 300, y: 315, width: 230, height: 35) + let reset = NSButton(title: "Reset", target: self, action: #selector(resetTask)) + reset.frame = NSRect(x: 60, y: 315, width: 120, height: 35) + canvas.addSubview(reset) + canvas.addSubview(status) + canvas.selected = { correct in + do { + let previous = (try? String(contentsOf: output, encoding: .utf8)) ?? "" + let result = correct ? "Save selected\n" : "Cancel selected\n" + try (previous + result).write(to: output, atomically: true, encoding: .utf8) + self.status.string = correct ? "Saved" : "Wrong tile" + } catch { + self.status.string = "Write failed" + } + } + window.contentView = canvas + window.makeKeyAndOrderFront(nil) + NSApp.activate(ignoringOtherApps: true) + } + + @objc private func resetTask() { + status.string = "Waiting" + canvas.saveOnLeft.toggle() + canvas.needsDisplay = true + } + + func applicationShouldTerminateAfterLastWindowClosed(_ sender: NSApplication) -> Bool { true } +} + +let app = NSApplication.shared +let delegate = VisualFixture() +app.delegate = delegate +app.run() diff --git a/pyproject.toml b/pyproject.toml index 682ddda..572246d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -45,6 +45,7 @@ cua = [ # Cua-S1 Nano behind --model cua; pinned to the Cua PR that ships the c "cua-s1 @ git+https://github.com/trycua/cua.git@aea61b6eb97e2d8c0f6f71eb804e5769fe910af4#subdirectory=libs/cua-s1/python", "huggingface-hub>=0.24", ] +cua-four-b = ["system1-agents[cua]", "transformers>=5.10.1,<6", "peft>=0.21,<1", "torchvision>=0.15", "pillow>=10"] dev = ["pytest>=8", "pytest-asyncio>=0.24", "ruff>=0.6", "ty>=0.0.83"] [build-system] @@ -75,6 +76,7 @@ include = [ "s1a/agents/alfworld.py", "s1a/agents/blackjack.py", "s1a/decision_models/cua.py", + "s1a/decision_models/cua_four_b.py", "s1a/decision_models/laya.py", ] diff --git a/s1a/agents/desktop.py b/s1a/agents/desktop.py index 1727a03..95d04d3 100644 --- a/s1a/agents/desktop.py +++ b/s1a/agents/desktop.py @@ -18,12 +18,15 @@ import argparse import asyncio +import math +import os import sys +import uuid from contextlib import asynccontextmanager from typing import Any, AsyncIterator from s1a.desktop.driver import CuaDriver, Snapshot, driver_from_env, opened -from s1a.desktop.env import ABSTAIN, DONE, WindowEnv, clickable +from s1a.desktop.env import ABSTAIN, DONE, WindowEnv, clickable, observable from s1a.spec import Budget, Series, ToolAgentSpec RULES = ( @@ -38,7 +41,11 @@ def shows(snapshot: Snapshot, text: str) -> bool: wanted = text.strip() if not wanted: return False - return any((e.value.strip() == wanted or e.label.strip() == wanted) for e in snapshot.elements if not clickable(e)) + return any( + (e.value.strip() == wanted or e.label.strip() == wanted) + for e in snapshot.elements + if observable(e) and not clickable(e) + ) def parse_plan(text: str) -> tuple[tuple[str, ...], ...]: @@ -46,6 +53,21 @@ def parse_plan(text: str) -> tuple[tuple[str, ...], ...]: return tuple(tuple(v.strip() for v in step.split("|")) for step in text.split(",") if step.strip()) +def parse_pixel_targets(entries: list[str]) -> dict[str, tuple[float, float]]: + """Task-defined points in screenshot fractions; the model chooses among these bounded targets.""" + points: dict[str, tuple[float, float]] = {} + for entry in entries: + key, sep, value = entry.partition("=") + try: + x, y = map(float, value.split(",")) + except ValueError as exc: + raise ValueError("--pixel-target requires KEY=X,Y with screenshot fractions") from exc + if not sep or not key.strip() or key in points or not all(math.isfinite(p) and 0 <= p < 1 for p in (x, y)): + raise ValueError("--pixel-target requires unique keys and finite coordinates in [0, 1)") + points[key] = (x, y) + return points + + def plan_rule(plan: tuple[tuple[str, ...], ...]) -> Any: """The baseline: the next button of the plan by how many presses were made, then done.""" @@ -53,7 +75,8 @@ def rule(state: dict[str, Any], candidates: dict[str, str]) -> str: step = len(state["presses"]) if step >= len(plan): return DONE - return next((f"click:{label}" for label in plan[step] if f"click:{label}" in candidates), ABSTAIN) + keys = (label if label.startswith(("click:", "pixel:")) else f"click:{label}" for label in plan[step]) + return next((key for key in keys if key in candidates), ABSTAIN) return rule @@ -78,7 +101,20 @@ async def _session(driver: CuaDriver, app: str) -> AsyncIterator[None]: def make_series(flags: argparse.Namespace) -> Series: - driver = driver_from_env("s1a-desktop") # raises before the series starts when the driver is missing + if flags.app_path and sys.platform != "darwin": + raise ValueError("--app-path is supported only on macOS") + pixel_targets = parse_pixel_targets(flags.pixel_target) + visual_model = ( + flags.model == "cua" + and os.getenv("CUA_S1_VARIANT", "nano") == "4b" + and os.getenv("CUA_S1_MODALITY", "multimodal") == "multimodal" + ) + if pixel_targets and flags.model not in {"rule", "random"} and not visual_model: + raise ValueError( + "--pixel-target requires a screenshot model: use --model cua with CUA_S1_VARIANT=4b and " + "CUA_S1_MODALITY=multimodal, or an explicit rule/random baseline" + ) + driver = driver_from_env(f"s1a-desktop-{uuid.uuid4().hex[:8]}") plan = parse_plan(flags.plan) if flags.plan else () return Series( seeds=range(flags.seed, flags.seed + flags.episodes), @@ -89,8 +125,11 @@ def make_series(flags: argparse.Namespace) -> Series: done_when=lambda snapshot: shows(snapshot, flags.expect), execute=flags.execute, clear_labels=tuple(v.strip() for v in flags.clear.split(",") if v.strip()), + window_title=flags.window_title, + pixel_targets=pixel_targets, + screenshot=visual_model, ), - session=_session(driver, flags.app), + session=_session(driver, flags.app_path or flags.app), baseline=("plan", plan_rule(plan)) if plan else None, annotate=lambda env, episode: None, ) @@ -98,11 +137,20 @@ def make_series(flags: argparse.Namespace) -> Series: def flags(parser: argparse.ArgumentParser) -> None: parser.add_argument("--app", required=True, help="app name (Windows Calculator / Calculator), or a Windows AUMID") + parser.add_argument("--app-path", default="", help="macOS app bundle path to launch; --app identifies its window") + parser.add_argument("--window-title", default="", help="exact title of the app window to use") parser.add_argument("--goal", required=True, help="what to do in the window, read by the model on every turn") parser.add_argument("--expect", required=True, help="the text a display or label shows when the goal is met") parser.add_argument("--execute", action="store_true", help="click for real; without it one decision is planned") parser.add_argument("--plan", default="", help="the rule baseline: button labels in order, | between variants") parser.add_argument("--clear", default="", help="button labels pressed on reset when the window has one") + parser.add_argument( + "--pixel-target", + action="append", + default=[], + metavar="KEY=X,Y", + help="task-defined screenshot point (fractions in [0,1)); repeat for closed visual choices", + ) SPEC = ToolAgentSpec( diff --git a/s1a/decision_models/cua_four_b.py b/s1a/decision_models/cua_four_b.py new file mode 100644 index 0000000..41a1ae5 --- /dev/null +++ b/s1a/decision_models/cua_four_b.py @@ -0,0 +1,142 @@ +# coding: utf-8 +"""Optional Cua-S1 4B adapter: a closed choice over text or one current screenshot.""" + +from __future__ import annotations + +import asyncio +import json +import os +import tempfile +import time +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +from openjiuwen.core.common.exception.codes import StatusCode +from openjiuwen.core.common.exception.errors import build_error + +from s1a.decision_models.base import DecisionModel +from s1a.decision_models.cua import checkpoint_directory, cua_context +from s1a.decision_models.types import ChoiceQuestion, Json, Observation, Question, Reply + + +@dataclass(frozen=True) +class Option: + """The fields read by the official ``cua_s1.four_b.FourBModel``.""" + + element_id: str + role: str + label: str + action: str + entity_id: str | None = None + + +def grounded_option(key: str, description: str | Json, observation: Observation) -> Option: + """Preserve desktop click/fill semantics in the official scorer's input contract.""" + text = description if isinstance(description, str) else json.dumps(description, ensure_ascii=False) + action, _, label = key.partition(":") + if action in {"click", "type"} and isinstance(observation.state, dict): + elements = observation.state.get("elements", []) + matches = [e for e in elements if isinstance(e, dict) and label in (e.get("label"), e.get("identifier"))] + if len(matches) == 1: + element = matches[0] + return Option( + key, + str(element.get("role", "element")).removeprefix("AX"), + label, + "fill" if action == "type" else "click", + text if action == "type" else None, + ) + return Option(key, "action", text, "select") + + +class CuaFourBModel(DecisionModel): + name = "cua" + question_types = frozenset({"choice"}) + deterministic = True + + def __init__(self, scorer: Any, *, model: str, modality: str) -> None: + if modality not in {"text", "multimodal"}: + raise ValueError("CUA_S1_MODALITY must be text or multimodal") + self._scorer = scorer + self._model = model + self._modality = modality + self.supports_images = modality == "multimodal" + + @property + def model(self) -> str: + return self._model + + async def warm(self) -> None: + await asyncio.to_thread(self._scorer.load) + + async def _decide(self, observation: Observation, questions: dict[str, Question]) -> Reply: + if self.supports_images and len(observation.images) != 1: + raise build_error(StatusCode.MODEL_SERVICE_CONFIG_ERROR, error_msg="Cua-S1 4B needs exactly one screenshot") + for question in questions.values(): + assert isinstance(question, ChoiceQuestion) + if len(question.options) > 26: + raise build_error( + StatusCode.MODEL_SERVICE_CONFIG_ERROR, error_msg="Cua-S1 4B supports at most 26 options" + ) + started = time.perf_counter() + try: + answers = await asyncio.to_thread(self._forward, observation, questions) + except (ValueError, RuntimeError, OSError) as exc: + raise build_error( + StatusCode.MODEL_CALL_FAILED, cause=exc, error_msg=f"Cua-S1 4B forward pass failed: {exc}" + ) from exc + return Reply(answers=answers, latency_ms=round((time.perf_counter() - started) * 1000), model=self._model) + + def _forward(self, observation: Observation, questions: dict[str, Question]) -> dict[str, Json]: + # The official scorer accepts a file path. Keep image bytes out of JSON observations and run logs. + with tempfile.TemporaryDirectory(prefix="s1a-cua-") as temporary: + screenshot = None + if self.supports_images: + screenshot = Path(temporary) / "screen.png" + screenshot.write_bytes(observation.images[0].data) + answers = {} + for name, question in questions.items(): + assert isinstance(question, ChoiceQuestion) + options = [grounded_option(key, text, observation) for key, text in question.options.items()] + context = cua_context(observation, question, name) + results = self._scorer.forward( + options, + app=str(observation.state.get("app", "s1a")) if isinstance(observation.state, dict) else "s1a", + # Upstream's multimodal prompt ignores ax_tree; carry the goal and rules in its task field. + task_family=context if self.supports_images else name, + ax_tree=None if self.supports_images else context, + screenshot=screenshot, + modality=self._modality, + ) + probabilities = {result.element_id: float(result.probability) for result in results} + if len(results) != len(options) or set(probabilities) != set(question.options): + raise ValueError("scorer returned missing, duplicate or unknown option keys") + key = max(probabilities, key=probabilities.__getitem__) + answers[name] = {"choice": key, "probabilities": probabilities, "confidence": probabilities[key]} + return answers + + @classmethod + def from_env(cls) -> "CuaFourBModel": + try: + from cua_s1.four_b import FourBModel + import peft # noqa: F401 -- fail before downloading weights when the extra is absent + import torchvision # noqa: F401 + except ImportError as exc: + raise build_error( + StatusCode.MODEL_SERVICE_CONFIG_ERROR, + error_msg="CUA_S1_VARIANT=4b needs: uv sync --extra cua-four-b", + ) from exc + modality = os.getenv("CUA_S1_MODALITY", "multimodal") + if modality not in {"text", "multimodal"}: + raise ValueError("CUA_S1_MODALITY must be text or multimodal") + base = os.getenv("CUA_S1_BASE_MODEL") or "Qwen/Qwen3.5-4B" + adapter = os.getenv("CUA_S1_CHECKPOINT") or "cua-ai/cua-s1-4b-0.2" + scorer = FourBModel( + base_model=base, + lora_adapter_path=checkpoint_directory(adapter, modality), + device=os.getenv("CUA_S1_DEVICE") or "cpu", + dtype=os.getenv("CUA_S1_DTYPE") or "float32", + modality=modality, + ) + return cls(scorer, model=f"{base}+{adapter}/{modality}", modality=modality) diff --git a/s1a/decision_models/factory.py b/s1a/decision_models/factory.py index d81ae45..8280aac 100644 --- a/s1a/decision_models/factory.py +++ b/s1a/decision_models/factory.py @@ -3,9 +3,12 @@ from __future__ import annotations +import os + from s1a.decision_models.base import DecisionModel from s1a.decision_models.baselines import RandomModel, Rule, RuleModel from s1a.decision_models.cua import CuaS1Model +from s1a.decision_models.cua_four_b import CuaFourBModel from s1a.decision_models.jev import JevModel from s1a.decision_models.laya import LayaModel @@ -26,6 +29,11 @@ def build_model(model_name: str, *, seed: int = 0, rule: tuple[str, Rule] | None case "laya": return LayaModel.from_env() # the laya import happens inside case "cua": + variant = os.getenv("CUA_S1_VARIANT", "nano") + if variant == "4b": + return CuaFourBModel.from_env() + if variant != "nano": + raise ValueError("CUA_S1_VARIANT must be nano or 4b") return CuaS1Model.from_env() # the cua_s1 import happens inside case "random": return RandomModel(seed) diff --git a/s1a/desktop/driver.py b/s1a/desktop/driver.py index f160e89..97eb2eb 100644 --- a/s1a/desktop/driver.py +++ b/s1a/desktop/driver.py @@ -7,6 +7,7 @@ from __future__ import annotations import json +import math import os import shutil from contextlib import AsyncExitStack, asynccontextmanager @@ -16,7 +17,9 @@ from mcp import ClientSession from mcp.client.stdio import StdioServerParameters, stdio_client -from mcp.types import TextContent +from mcp.types import ImageContent, TextContent + +from s1a.decision_models.types import Image Json = dict[str, Any] INSTALL_HINT = ( @@ -55,6 +58,15 @@ class Element: value: str token: str | None actions: tuple[str, ...] + enabled: bool = True + + +@dataclass(frozen=True) +class Capture: + capture_id: str + image: Image + width: int + height: int @dataclass(frozen=True) @@ -63,14 +75,16 @@ class Snapshot: snapshot_id: str | None elements: tuple[Element, ...] raw: Json = field(repr=False, compare=False) + capture: Capture | None = None class Driver(Protocol): """What a desktop env needs: find the window, read it, click in it.""" - async def find_window(self, app_name: str) -> Window: ... - async def window_state(self, window: Window) -> Snapshot: ... + async def find_window(self, app_name: str, window_title: str = "") -> Window: ... + async def window_state(self, window: Window, *, screenshot: bool = False) -> Snapshot: ... async def click(self, window: Window, token: str) -> Json: ... + async def click_at(self, window: Window, capture: Capture, x: float, y: float) -> Json: ... class CuaDriver: @@ -121,13 +135,17 @@ async def call(self, tool: str, **args: Any) -> Json: if result.isError: raise DriverError(f"{tool}: {text or 'the driver returned an error'}") if isinstance(result.structuredContent, dict): - return result.structuredContent - try: - payload = json.loads(text) if text else {} - except json.JSONDecodeError as exc: - raise DriverError(f"{tool}: the driver returned no JSON object: {text[:200]!r}") from exc + payload = dict(result.structuredContent) + else: + try: + payload = json.loads(text) if text else {} + except json.JSONDecodeError as exc: + raise DriverError(f"{tool}: the driver returned no JSON object: {text[:200]!r}") from exc if not isinstance(payload, dict): raise DriverError(f"{tool}: the driver returned no JSON object: {text[:200]!r}") + images = [{"data": p.data, "media_type": p.mimeType} for p in result.content if isinstance(p, ImageContent)] + if images: + payload["_images"] = images return payload async def launch_app(self, app_name: str) -> None: @@ -164,7 +182,7 @@ async def _launch_target(self, app_name: str) -> Json: return {"aumid": path[len(prefix) :]} return {"launch_path": path} if path else {"name": app_name} - async def find_window(self, app_name: str) -> Window: + async def find_window(self, app_name: str, window_title: str = "") -> Window: """Find the launched window, or the one on-screen window named ``app_name``; never guess among matches.""" listed = await self.call("list_windows", on_screen_only=True) windows = listed.get("windows") @@ -182,6 +200,8 @@ async def find_window(self, app_name: str) -> Window: matches = [ w for w in windows if str(w.get("app_name") or "").casefold() == wanted and w.get("is_on_screen", True) ] + if window_title: + matches = [w for w in matches if str(w.get("title") or "") == window_title] if len(matches) != 1: titles = [str(w.get("title")) for w in matches] raise DriverError(f"list_windows: {len(matches)} on-screen window(s) of {app_name!r}: {titles}") @@ -189,21 +209,40 @@ async def find_window(self, app_name: str) -> Window: owner = str(window.get("app_name") or app_name) if pinned is not None else app_name return Window(int(window["pid"]), int(window["window_id"]), owner, str(window.get("title") or "")) - async def window_state(self, window: Window) -> Snapshot: + async def window_state(self, window: Window, *, screenshot: bool = False) -> Snapshot: state = await self.call( "get_window_state", pid=window.pid, window_id=window.window_id, session=self._label, include_accessibility_tree=True, - include_screenshot=False, + include_screenshot=screenshot, ) raw_elements = state.get("elements") + if screenshot and str(state.get("degraded_reason", "")).startswith("ax_window_unresolved"): + raise DriverError(f"get_window_state: {state['degraded_reason']}") if not isinstance(raw_elements, list): raise DriverError(f"get_window_state: no elements in the snapshot ({state.get('degradation')!r})") elements = tuple(_element(raw) for raw in raw_elements) snapshot_id = state.get("snapshot_id") - return Snapshot(window, str(snapshot_id) if snapshot_id else None, elements, state) + capture = None + if screenshot: + images = state.pop("_images", []) + width, height = state.get("screenshot_width"), state.get("screenshot_height") + if ( + not state.get("screenshot_frame_valid") + or not state.get("capture_id") + or len(images) != 1 + or not isinstance(width, int) + or not isinstance(height, int) + or width <= 0 + or height <= 0 + ): + raise DriverError("get_window_state: no valid screenshot capture") + capture = Capture( + str(state["capture_id"]), Image.from_base64(images[0]["data"], images[0]["media_type"]), width, height + ) + return Snapshot(window, str(snapshot_id) if snapshot_id else None, elements, state, capture) async def click(self, window: Window, token: str) -> Json: """One background click on a snapshot-bound element; a refused action is an error.""" @@ -214,6 +253,23 @@ async def click(self, window: Window, token: str) -> Json: raise DriverError(f"click: refused ({result.get('escalation')!r})") return result + async def click_at(self, window: Window, capture: Capture, x: float, y: float) -> Json: + """Click a point in the captured window; the driver rejects stale or mismatched captures.""" + if not (math.isfinite(x) and math.isfinite(y) and 0 <= x < capture.width and 0 <= y < capture.height): + raise ValueError("pixel click outside capture bounds") + result = await self.call( + "click", + target=window.target, + capture_id=capture.capture_id, + x=x, + y=y, + delivery_mode="background", + session=self._label, + ) + if result.get("effect") == "refused": + raise DriverError(f"click: refused ({result.get('escalation')!r})") + return result + def _element(raw: Any) -> Element: if not isinstance(raw, dict) or "element_index" not in raw: @@ -226,6 +282,7 @@ def _element(raw: Any) -> Element: value=str(raw.get("value") or ""), token=str(token) if token else None, actions=tuple(str(action) for action in raw.get("actions") or ()), + enabled=raw.get("enabled") is not False, ) diff --git a/s1a/desktop/env.py b/s1a/desktop/env.py index 6328a92..f7d7b02 100644 --- a/s1a/desktop/env.py +++ b/s1a/desktop/env.py @@ -5,7 +5,8 @@ from typing import Any, Callable -from s1a.desktop.driver import Driver, Element, Snapshot, Window +from s1a.desktop.driver import Driver, DriverError, Element, Snapshot, Window +from s1a.decision_models.types import Image DONE = "done" ABSTAIN = "abstain" @@ -14,15 +15,24 @@ ABSTAIN: "No offered click moves the task forward; stop without acting.", } _CLICKABLE_ROLES = ("button", "checkbox", "radiobutton", "menubutton", "link", "popupbutton", "disclosuretriangle") +_MENU_ROLES = ("menu", "menubar", "menubaritem", "menuitem") def clickable(element: Element) -> bool: role = element.role.casefold().removeprefix("ax") + if not element.enabled or not element.label.strip() or role in _MENU_ROLES: + return False return element.token is not None and ( role in _CLICKABLE_ROLES or any("press" in a.casefold() or "click" in a.casefold() for a in element.actions) ) +def observable(element: Element) -> bool: + """Keep named controls and visible values, excluding application menu trees.""" + role = element.role.casefold().removeprefix("ax") + return role not in _MENU_ROLES and bool(element.label.strip() or element.value.strip()) + + class WindowEnv: """The mechanics: bind to the app's window, offer its clickable elements, click one, re-read the window. @@ -40,6 +50,9 @@ def __init__( done_when: Callable[[Snapshot], bool], execute: bool, clear_labels: tuple[str, ...], + window_title: str = "", + pixel_targets: dict[str, tuple[float, float]] | None = None, + screenshot: bool = False, ) -> None: self._driver = driver self._app_name = app_name @@ -47,6 +60,9 @@ def __init__( self._done_when = done_when self._execute = execute self._clear_labels = clear_labels + self._window_title = window_title + self._pixel_targets = dict(pixel_targets or {}) + self._screenshot = screenshot or bool(self._pixel_targets) self._window: Window | None = None self._snapshot: Snapshot | None = None self._keys: dict[str, Element] = {} @@ -55,7 +71,11 @@ def __init__( self._ended: str | None = None async def reset(self) -> None: - self._window = await self._driver.find_window(self._app_name) + self._window = ( + await self._driver.find_window(self._app_name, self._window_title) + if self._window_title + else await self._driver.find_window(self._app_name) + ) self._presses, self._planned, self._ended = [], None, None await self._refresh() clear = next((e for e in self._keys.values() if e.label in self._clear_labels), None) @@ -69,7 +89,13 @@ async def reset(self) -> None: async def _refresh(self) -> None: assert self._window is not None - self._snapshot = await self._driver.window_state(self._window) + self._snapshot = ( + await self._driver.window_state(self._window, screenshot=True) + if self._screenshot + else await self._driver.window_state(self._window) + ) + if self._screenshot and self._snapshot.capture is None: + raise DriverError("visual observations require a valid screenshot capture") self._keys = {} for element in self._snapshot.elements: if clickable(element): @@ -78,12 +104,13 @@ async def _refresh(self) -> None: async def observe(self) -> dict[str, Any]: snapshot = self._require_snapshot() - values = [e.value for e in snapshot.elements if e.value and not clickable(e)] + visible = [e for e in snapshot.elements if observable(e)] + values = [e.value for e in visible if e.value and not clickable(e)] state: dict[str, Any] = { "goal": self._goal, "app": self._app_name, "title": snapshot.window.title, - "elements": [{"role": e.role, "label": e.label, "value": e.value} for e in snapshot.elements], + "elements": [{"role": e.role, "label": e.label, "value": e.value} for e in visible], "presses": list(self._presses), "progress": {"values": values, "presses": len(self._presses)}, } @@ -95,20 +122,41 @@ async def candidates(self) -> dict[str, str]: if self.done: return {} offered = {key: f'{e.role} "{e.label}"' + (f" = {e.value}" if e.value else "") for key, e in self._keys.items()} + offered.update( + { + f"pixel:{key}": f"Click {key}, at {x:.0%} across and {y:.0%} down the attached screenshot" + for key, (x, y) in self._pixel_targets.items() + } + ) return {**offered, **RESERVED} + async def images(self) -> tuple[Image, ...]: + capture = self._require_snapshot().capture + return (capture.image,) if self._screenshot and capture is not None else () + async def step(self, key: str) -> None: if key in RESERVED: self._ended = key return - element = self._keys[key] window = self._require_snapshot().window - if not self._execute: - self._planned = {"key": key, "role": element.role, "label": element.label, "token": element.token} - self._ended = "planned" - return - await self._driver.click(window, self._token(element)) - self._presses.append(element.label) + if key.startswith("pixel:"): + x, y = self._pixel_targets[key.removeprefix("pixel:")] + capture = self._require_snapshot().capture + assert capture is not None + if not self._execute: + self._planned = {"key": key} + self._ended = "planned" + return + await self._driver.click_at(window, capture, x * capture.width, y * capture.height) + self._presses.append(key) + else: + element = self._keys[key] + if not self._execute: + self._planned = {"key": key, "role": element.role, "label": element.label, "token": element.token} + self._ended = "planned" + return + await self._driver.click(window, self._token(element)) + self._presses.append(element.label) await self._refresh() @property diff --git a/s1a/env.py b/s1a/env.py index b3f11cd..3863adb 100644 --- a/s1a/env.py +++ b/s1a/env.py @@ -3,7 +3,16 @@ from __future__ import annotations -from typing import Any, Protocol +from typing import Any, Protocol, runtime_checkable + +from s1a.decision_models.types import Image + + +@runtime_checkable +class VisualEnv(Protocol): + """Optional images from the same observation; kept out of serializable tool state and logs.""" + + async def images(self) -> tuple[Image, ...]: ... class Env(Protocol): diff --git a/s1a/tool/models.py b/s1a/tool/models.py index 1e76e33..3054b05 100644 --- a/s1a/tool/models.py +++ b/s1a/tool/models.py @@ -18,7 +18,7 @@ from openjiuwen.core.foundation.llm import AssistantMessage, AssistantMessageChunk, Model, ToolCall, init_model from s1a.decision_models import DecisionModel, ChoiceQuestion, Observation -from s1a.env import Env +from s1a.env import Env, VisualEnv ACT_TOOL = "act" OBSERVE_TOOL = "observe" @@ -137,9 +137,10 @@ async def _decide(self) -> AssistantMessage: if state.notices: request_state["harness_notices"] = list(state.notices) started = time.perf_counter() + images = await env.images() if isinstance(env, VisualEnv) else () try: decision = await self._decision_model.decide_many( - Observation(request_state), {"pick": ChoiceQuestion(offered, rules=self._rules)} + Observation(request_state, images), {"pick": ChoiceQuestion(offered, rules=self._rules)} ) except BaseError as exc: # any decisions failure ends the episode as BLOCKED, recorded state.error = f"decision failed: {exc}" diff --git a/tests/test_decision_models_cua_four_b.py b/tests/test_decision_models_cua_four_b.py new file mode 100644 index 0000000..09184f0 --- /dev/null +++ b/tests/test_decision_models_cua_four_b.py @@ -0,0 +1,97 @@ +"""Cua-S1 4B reads the current screenshot and preserves the offered option keys.""" + +from pathlib import Path +from types import SimpleNamespace +from unittest import IsolatedAsyncioTestCase + +from openjiuwen.core.common.exception.errors import BaseError + +from s1a.decision_models import ChoiceQuestion, Image, Observation +from s1a.decision_models.cua_four_b import CuaFourBModel + + +class FakeFourB: + def __init__(self) -> None: + self.seen = [] + self.bad = False + + def load(self) -> None: + pass + + def forward(self, options, **kwargs): + self.seen.append( + (options, kwargs, Path(kwargs["screenshot"]).read_bytes() if kwargs.get("screenshot") else None) + ) + return [ + SimpleNamespace( + element_id="unknown" if self.bad else option.element_id, + probability=0.8 / (len(options) - 1) if i else 0.2, + ) + for i, option in reversed(list(enumerate(options))) + ] + + +class TestFourB(IsolatedAsyncioTestCase): + async def test_desktop_options_keep_native_click_and_fill_semantics(self) -> None: + scorer = FakeFourB() + model = CuaFourBModel(scorer, model="test/text", modality="text") + observation = Observation( + { + "app": "Editor", + "elements": [ + {"role": "AXTextField", "label": "Untitled.txt", "identifier": "filename"}, + {"role": "AXButton", "label": "Save"}, + ], + } + ) + await model.decide_many( + observation, + { + "pick": ChoiceQuestion( + { + "type:filename": 'Replace AXTextField "filename" with "report.txt"', + "click:Save": 'AXButton "Save"', + "abstain": "Stop", + } + ) + }, + ) + options = scorer.seen[0][0] + self.assertEqual( + [(o.role, o.action) for o in options], [("TextField", "fill"), ("Button", "click"), ("action", "select")] + ) + self.assertIn("report.txt", options[0].entity_id) + self.assertEqual([o.element_id for o in options], ["type:filename", "click:Save", "abstain"]) + + async def test_multimodal_reads_bytes_and_maps_scores_by_key_not_return_order(self) -> None: + scorer = FakeFourB() + model = CuaFourBModel(scorer, model="test/mm", modality="multimodal") + obs = Observation({"goal": "click Save"}, (Image(b"image bytes"),)) + decision = await model.decide_many( + obs, {"pick": ChoiceQuestion({"pixel:left": "left point", "pixel:right": "right point"})} + ) + self.assertEqual(decision.choice("pick").key, "pixel:right") + self.assertEqual(decision.choice("pick").probabilities, {"pixel:left": 0.2, "pixel:right": 0.8}) + self.assertEqual(scorer.seen[0][2], b"image bytes") + self.assertFalse(Path(scorer.seen[0][1]["screenshot"]).exists()) + scorer.bad = True + with self.assertRaises(BaseError): + await model.decide_many(obs, {"pick": ChoiceQuestion({"a": "A", "b": "B"})}) + + async def test_missing_image_and_too_many_options_fail_before_forward(self) -> None: + scorer = FakeFourB() + model = CuaFourBModel(scorer, model="test/mm", modality="multimodal") + with self.assertRaises(BaseError): + await model.decide_many(Observation({}), {"pick": ChoiceQuestion({"a": "A"})}) + with self.assertRaises(BaseError): + await model.decide_many( + Observation({}, (Image(b"p"),)), {"pick": ChoiceQuestion({str(n): str(n) for n in range(27)})} + ) + self.assertEqual(scorer.seen, []) + + async def test_text_adapter_uses_state_without_requesting_an_image(self) -> None: + scorer = FakeFourB() + model = CuaFourBModel(scorer, model="test/text", modality="text") + await model.decide_many(Observation({"goal": "choose B"}), {"pick": ChoiceQuestion({"a": "A", "b": "B"})}) + self.assertIn("choose B", scorer.seen[0][1]["ax_tree"]) + self.assertIsNone(scorer.seen[0][2]) diff --git a/tests/test_decision_models_factory.py b/tests/test_decision_models_factory.py index 3c85f6d..9c9d245 100644 --- a/tests/test_decision_models_factory.py +++ b/tests/test_decision_models_factory.py @@ -21,11 +21,18 @@ RuleModel, build_model, ) +from s1a.decision_models.cua_four_b import CuaFourBModel KEYS = {"TYPESAFE_API_KEY": "k", "TYPESAFE_API_URL": "", "OPENROUTER_API_KEY": ""} class TestBuildModel(TestCase): + def test_cua_four_b_is_explicit_and_unknown_variants_fail(self) -> None: + with patch.dict(os.environ, {"CUA_S1_VARIANT": "4b"}), patch.object(CuaFourBModel, "from_env") as build: + self.assertIs(build_model("cua"), build.return_value) + with patch.dict(os.environ, {"CUA_S1_VARIANT": "typo"}), self.assertRaises(ValueError): + build_model("cua") + def test_every_name_builds_its_class(self) -> None: with patch.dict(os.environ, KEYS): self.assertIsInstance(build_model("jev"), JevModel) diff --git a/tests/test_desktop_visual.py b/tests/test_desktop_visual.py new file mode 100644 index 0000000..e15b32c --- /dev/null +++ b/tests/test_desktop_visual.py @@ -0,0 +1,263 @@ +"""Screenshot bytes reach the decision model; pixel input remains capture-bound.""" + +import base64 +import json +import os +from unittest import IsolatedAsyncioTestCase, TestCase +from unittest.mock import AsyncMock, patch + +from mcp.types import CallToolResult, ImageContent + +from s1a.agents import desktop +from s1a.agents.desktop import parse_pixel_targets +from s1a.decision_models import Image, ScriptedModel +from s1a.desktop.driver import Capture, DriverError, Snapshot, Window +from s1a.desktop.env import WindowEnv +from s1a.tool.models import EvalState, ToolDecisionModel +from s1a.tool import series +from test_desktop_driver import _driver, _result + +WINDOW = Window(42, 7, "Canvas", "Canvas task") + + +class TestVisualTargets(IsolatedAsyncioTestCase): + async def test_disabled_result_is_readable_and_menus_are_not_click_candidates(self) -> None: + driver, _ = _driver( + _result( + { + "elements": [ + { + "element_index": 1, + "role": "AXButton", + "label": "Saved", + "enabled": False, + "element_token": "status", + "actions": ["AXPress"], + }, + { + "element_index": 2, + "role": "AXMenuItem", + "label": "Recent document", + "element_token": "menu", + "actions": ["AXPress"], + }, + ] + } + ) + ) + snapshot = await driver.window_state(WINDOW) + self.assertTrue(desktop.shows(snapshot, "Saved")) + fake = AsyncMock() + fake.find_window.return_value = WINDOW + fake.window_state.return_value = snapshot + env = WindowEnv(fake, app_name="Canvas", goal="Save", done_when=lambda s: False, execute=False, clear_labels=()) + await env.reset() + self.assertEqual(set(await env.candidates()), {"done", "abstain"}) + self.assertEqual([e["label"] for e in (await env.observe())["elements"]], ["Saved"]) + + async def test_window_title_selects_the_canvas_among_auxiliary_windows(self) -> None: + windows = { + "windows": [ + {"pid": 42, "window_id": 7, "app_name": "Canvas", "title": "Canvas task"}, + {"pid": 42, "window_id": 9, "app_name": "Canvas", "title": "Window"}, + ] + } + driver, _ = _driver(_result(windows), _result(windows)) + self.assertEqual(await driver.find_window("Canvas", "Canvas task"), WINDOW) + with self.assertRaises(DriverError): + await driver.find_window("Canvas", "Other") + + async def test_explicit_app_path_launches_bundle_and_discovers_window_by_app_name(self) -> None: + args = series.parser(desktop.SPEC).parse_args( + [ + "--model", + "rule", + "--rethink", + "off", + "--episodes", + "1", + "--app", + "Canvas", + "--app-path", + "/tmp/Canvas.app", + "--goal", + "click Save", + "--expect", + "Saved", + ] + ) + driver = AsyncMock() + driver.find_window.return_value = WINDOW + driver.window_state.return_value = Snapshot(WINDOW, "s1", (), {}) + with ( + patch.object(desktop, "driver_from_env", return_value=driver), + patch.object(desktop, "launch_app", AsyncMock()) as launch, + patch("sys.platform", "darwin"), + ): + task = desktop.make_series(args) + async with task.session: + await task.env_for(0).reset() + launch.assert_awaited_once_with("/tmp/Canvas.app", driver) + driver.find_window.assert_awaited_once_with("Canvas") + + async def test_multimodal_ax_task_receives_current_window_image(self) -> None: + driver = AsyncMock() + driver.find_window.return_value = WINDOW + + async def snapshot(window, *, screenshot=False): + self.assertTrue(screenshot) + return Snapshot( + window, "s1", (), {}, capture=Capture(str(window.window_id), Image(window.title.encode()), 100, 100) + ) + + driver.window_state.side_effect = snapshot + env = WindowEnv( + driver, + app_name="Canvas", + goal="save", + done_when=lambda s: False, + execute=False, + clear_labels=(), + screenshot=True, + ) + await env.reset() + self.assertEqual(await env.images(), (Image(b"Canvas task"),)) + self.assertEqual(set(await env.candidates()), {"done", "abstain"}) + + async def test_screenshot_goes_to_model_and_selected_point_uses_its_capture(self) -> None: + picture = Image(b"png") + capture = Capture("capture-1", picture, 800, 600) + snapshot = Snapshot(WINDOW, "s1", (), {}, capture=capture) + driver = AsyncMock() + driver.find_window.return_value = WINDOW + driver.window_state.return_value = snapshot + env = WindowEnv( + driver, + app_name="Canvas", + goal="click the tile marked Save", + done_when=lambda s: False, + execute=True, + clear_labels=(), + pixel_targets={"left": (0.25, 0.5), "right": (0.75, 0.5)}, + ) + await env.reset() + model = ScriptedModel(choose="pixel:right") + slot = ToolDecisionModel(env, EvalState(), rules="look at the image", decision_model=model, fallback=None) + msg = await slot._decide() + self.assertEqual(model.calls[0][0].images, (picture,)) + self.assertEqual(json.loads(msg.tool_calls[0].arguments), {"key": "pixel:right"}) + await env.step("pixel:right") + driver.click_at.assert_awaited_once_with(WINDOW, capture, 600.0, 300.0) + self.assertNotIn("png", json.dumps(await env.observe())) + + async def test_dry_run_never_clicks_and_missing_capture_is_an_error(self) -> None: + driver = AsyncMock() + driver.find_window.return_value = WINDOW + driver.window_state.return_value = Snapshot(WINDOW, "s1", (), {}, capture=Capture("c", Image(b"p"), 100, 100)) + env = WindowEnv( + driver, + app_name="Canvas", + goal="click", + done_when=lambda s: False, + execute=False, + clear_labels=(), + pixel_targets={"left": (0.25, 0.5)}, + ) + await env.reset() + await env.step("pixel:left") + driver.click_at.assert_not_called() + self.assertEqual((await env.observe())["planned"]["key"], "pixel:left") + driver.window_state.return_value = Snapshot(WINDOW, "s2", (), {}) + with self.assertRaisesRegex(DriverError, "capture"): + await env.reset() + + async def test_driver_preserves_image_content_and_passes_capture_id_to_pixel_click(self) -> None: + response = CallToolResult( + content=[ImageContent(type="image", data=base64.b64encode(b"png").decode(), mimeType="image/png")], + structuredContent={ + "elements": [], + "capture_id": "c1", + "screenshot_frame_valid": True, + "screenshot_width": 800, + "screenshot_height": 600, + }, + ) + driver, session = _driver( + response, _result({"effect": "confirmed"}), _result({"effect": "refused", "escalation": "stale_capture"}) + ) + snap = await driver.window_state(WINDOW, screenshot=True) + self.assertEqual(snap.capture, Capture("c1", Image(b"png"), 800, 600)) + await driver.click_at(WINDOW, snap.capture, 300, 400) + self.assertEqual(session.calls[-1][1]["capture_id"], "c1") + self.assertEqual(session.calls[-1][1]["target"], WINDOW.target) + with self.assertRaisesRegex(DriverError, "stale_capture"): + await driver.click_at(WINDOW, snap.capture, 300, 400) + with self.assertRaisesRegex(ValueError, "bounds"): + await driver.click_at(WINDOW, snap.capture, 800, 0) + + async def test_unresolved_window_is_a_driver_error_not_a_model_abstention(self) -> None: + driver, _ = _driver( + _result( + {"elements": [], "degraded": True, "degraded_reason": "ax_window_unresolved: no matching AX window"} + ) + ) + with self.assertRaisesRegex(DriverError, "ax_window_unresolved"): + await driver.window_state(WINDOW, screenshot=True) + + +class TestPixelArguments(TestCase): + def test_runs_use_separate_driver_sessions_and_app_paths_are_mac_only(self) -> None: + args = series.parser(desktop.SPEC).parse_args( + [ + "--model", + "rule", + "--rethink", + "off", + "--episodes", + "1", + "--app", + "Canvas", + "--goal", + "click Save", + "--expect", + "Saved", + ] + ) + with patch.object(desktop, "driver_from_env", return_value=AsyncMock()) as factory: + desktop.make_series(args) + desktop.make_series(args) + labels = [call.args[0] for call in factory.call_args_list] + self.assertNotEqual(*labels) + args.app_path = "/tmp/Canvas.app" + with patch("sys.platform", "win32"), self.assertRaisesRegex(ValueError, "macOS"): + desktop.make_series(args) + + def test_text_only_model_cannot_silently_ignore_required_screenshot(self) -> None: + for model in ("laya", "jev", "llm", "cua"): + args = series.parser(desktop.SPEC).parse_args( + [ + "--model", + model, + "--rethink", + "off", + "--episodes", + "1", + "--app", + "Canvas", + "--goal", + "click Save", + "--expect", + "Saved", + "--pixel-target", + "left=0.25,0.5", + ] + ) + with self.subTest(model=model), patch.dict(os.environ, {"CUA_S1_VARIANT": "nano"}): + with self.assertRaisesRegex(ValueError, "screenshot"): + desktop.make_series(args) + + def test_points_are_explicit_unique_and_finite_fractions(self) -> None: + self.assertEqual(parse_pixel_targets(["left=0.25,0.5"]), {"left": (0.25, 0.5)}) + for entries in (["bad"], ["x=nan,0.5"], ["x=1,0"], ["x=-0.1,0"], ["x=0,0", "x=0.5,0.5"]): + with self.subTest(entries=entries), self.assertRaises(ValueError): + parse_pixel_targets(entries) diff --git a/uv.lock b/uv.lock index 52ec63f..1b7f48f 100644 --- a/uv.lock +++ b/uv.lock @@ -13,6 +13,25 @@ resolution-markers = [ "python_full_version < '3.12' and sys_platform != 'emscripten' and sys_platform != 'win32'", ] +[[package]] +name = "accelerate" +version = "1.15.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "huggingface-hub" }, + { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.12'" }, + { name = "numpy", version = "2.5.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, + { name = "packaging" }, + { name = "psutil" }, + { name = "pyyaml" }, + { name = "safetensors" }, + { name = "torch" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/f5/b5/1d3ed029ac71d3f2961346829a268da923698e9fd63f218f78841f216bfd/accelerate-1.15.0.tar.gz", hash = "sha256:5654f8c5eaa0d4fa68b33e287a97765da6849bf6d51dcac874e73fbbddfb6134", size = 422615, upload-time = "2026-09-09T13:04:49.078Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/8a/4c/34f0450479d01195027260da68d8a3880683f1640c3ca5adf64acb3185f1/accelerate-1.15.0-py3-none-any.whl", hash = "sha256:97eacca0b73e45cb867dbf8c5d5d4dc32219544300e0c8992c7334dc2ef33cec", size = 394295, upload-time = "2026-09-09T13:04:47.331Z" }, +] + [[package]] name = "agentdescent" version = "0.4.6" @@ -739,7 +758,7 @@ resolution-markers = [ "python_full_version < '3.12' and sys_platform != 'emscripten' and sys_platform != 'win32'", ] dependencies = [ - { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" } }, + { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.12'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/58/01/1253e6698a07380cd31a736d248a3f2a50a7c88779a1813da27503cadc2a/contourpy-1.3.3.tar.gz", hash = "sha256:083e12155b210502d0bca491432bb04d56dc3432f95a979b429f2848c3dbe880", size = 13466174, upload-time = "2025-07-26T12:03:12.549Z" } wheels = [ @@ -829,7 +848,7 @@ resolution-markers = [ "python_full_version >= '3.12' and python_full_version < '3.14' and sys_platform != 'emscripten' and sys_platform != 'win32'", ] dependencies = [ - { name = "numpy", version = "2.5.3", source = { registry = "https://pypi.org/simple" } }, + { name = "numpy", version = "2.5.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/83/5a/a55177dd22553a277388e8a1b3220e92de91bacb28356cdc73caa240121d/contourpy-1.4.0.tar.gz", hash = "sha256:20156f5a1ac4f8ce02656e39a61e82164a3d359796dc8026f75b062783d500e1", size = 13323726, upload-time = "2026-09-11T19:05:05.808Z" } wheels = [ @@ -990,7 +1009,7 @@ name = "cuda-bindings" version = "13.4.2" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "cuda-pathfinder" }, + { name = "cuda-pathfinder", marker = "sys_platform != 'emscripten' and sys_platform != 'win32'" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/19/6f/e00ffcebcad6405a2326a52612eff98240a67c637dfed516e61b881e2db3/cuda_bindings-13.4.2-cp311-cp311-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:fc0a18f65b26459cd3c49c0250a44b419cdf790cffcb096ff4d46a1973f6f4b7", size = 6484069, upload-time = "2026-09-17T22:05:52.594Z" }, @@ -1023,43 +1042,43 @@ wheels = [ [package.optional-dependencies] cublas = [ - { name = "nvidia-cublas", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, - { name = "nvidia-cuda-nvrtc", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, + { name = "nvidia-cublas", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "nvidia-cuda-nvrtc", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, ] cudart = [ - { name = "nvidia-cuda-runtime", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, + { name = "nvidia-cuda-runtime", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, ] cufft = [ - { name = "nvidia-cufft", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, - { name = "nvidia-nvjitlink", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, + { name = "nvidia-cufft", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "nvidia-nvjitlink", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, ] cufile = [ - { name = "nvidia-cufile", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, + { name = "nvidia-cufile", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, ] cupti = [ - { name = "nvidia-cuda-cupti", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, + { name = "nvidia-cuda-cupti", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, ] curand = [ - { name = "nvidia-curand", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, + { name = "nvidia-curand", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, ] cusolver = [ - { name = "nvidia-cublas", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, - { name = "nvidia-cusolver", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, - { name = "nvidia-cusparse", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, - { name = "nvidia-nvjitlink", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, + { name = "nvidia-cublas", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "nvidia-cusolver", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "nvidia-cusparse", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "nvidia-nvjitlink", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, ] cusparse = [ - { name = "nvidia-cusparse", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, - { name = "nvidia-nvjitlink", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, + { name = "nvidia-cusparse", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "nvidia-nvjitlink", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, ] nvjitlink = [ - { name = "nvidia-nvjitlink", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, + { name = "nvidia-nvjitlink", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, ] nvrtc = [ - { name = "nvidia-cuda-nvrtc", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, + { name = "nvidia-cuda-nvrtc", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, ] nvtx = [ - { name = "nvidia-nvtx", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, + { name = "nvidia-nvtx", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, ] [[package]] @@ -1746,8 +1765,8 @@ name = "httpcore2" version = "2.13.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "h11" }, - { name = "truststore" }, + { name = "h11", marker = "sys_platform != 'emscripten'" }, + { name = "truststore", marker = "sys_platform != 'emscripten'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/15/8c/e925b1c92018abb3a1863ce1549d76d2381e334d21d65d4ac8f65dabd78a/httpcore2-2.13.0.tar.gz", hash = "sha256:2adc8be4fb285fbcd6d894298db3b52c177e74b6674eda3a76bd36be3292a3db", size = 67740, upload-time = "2026-09-14T14:18:04.717Z" } wheels = [ @@ -1838,7 +1857,7 @@ name = "importlib-metadata" version = "9.0.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "zipp" }, + { name = "zipp", marker = "python_full_version < '3.12'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/6f/7e/1e7e8dc30634b93ebb3d58a3dea569ad146e656218d3960ab04f62047b29/importlib_metadata-9.0.1.tar.gz", hash = "sha256:ab830580bc0ef3db61ce8fae716389e5462b67e033018bab6d8f80ef17172f99", size = 59124, upload-time = "2026-08-28T15:30:34.646Z" } wheels = [ @@ -3123,7 +3142,7 @@ name = "nvidia-cublas" version = "13.1.1.3" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "nvidia-cuda-nvrtc" }, + { name = "nvidia-cuda-nvrtc", marker = "sys_platform != 'emscripten' and sys_platform != 'win32'" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/a7/a1/0bd24ee8c8d03adac032fd2909426a00c88f8c57961b1277ded97f91119f/nvidia_cublas-13.1.1.3-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:b7a210458267ac818974c53038fbec2e969d5c99f305ab15c72522fa9f001dd5", size = 542848918, upload-time = "2026-04-08T18:46:22.985Z" }, @@ -3162,7 +3181,7 @@ name = "nvidia-cudnn-cu13" version = "9.24.0.43" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "nvidia-cublas" }, + { name = "nvidia-cublas", marker = "sys_platform != 'emscripten' and sys_platform != 'win32'" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/ca/30/7c257e3d5cb4fecb147b93895c66e29c93f8e76d74b45bb418ff0587c4ec/nvidia_cudnn_cu13-9.24.0.43-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:a6812a554a1ff0413e9c52b84c26c050380649ab9615f9c16bded368ce9f421f", size = 650976863, upload-time = "2026-07-02T16:23:39.248Z" }, @@ -3174,7 +3193,7 @@ name = "nvidia-cufft" version = "12.0.0.61" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "nvidia-nvjitlink" }, + { name = "nvidia-nvjitlink", marker = "sys_platform != 'emscripten' and sys_platform != 'win32'" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/8b/ae/f417a75c0259e85c1d2f83ca4e960289a5f814ed0cea74d18c353d3e989d/nvidia_cufft-12.0.0.61-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:2708c852ef8cd89d1d2068bdbece0aa188813a0c934db3779b9b1faa8442e5f5", size = 214053554, upload-time = "2025-09-04T08:31:38.196Z" }, @@ -3204,9 +3223,9 @@ name = "nvidia-cusolver" version = "12.0.4.66" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "nvidia-cublas" }, - { name = "nvidia-cusparse" }, - { name = "nvidia-nvjitlink" }, + { name = "nvidia-cublas", marker = "sys_platform != 'emscripten' and sys_platform != 'win32'" }, + { name = "nvidia-cusparse", marker = "sys_platform != 'emscripten' and sys_platform != 'win32'" }, + { name = "nvidia-nvjitlink", marker = "sys_platform != 'emscripten' and sys_platform != 'win32'" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/c8/c3/b30c9e935fc01e3da443ec0116ed1b2a009bb867f5324d3f2d7e533e776b/nvidia_cusolver-12.0.4.66-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:02c2457eaa9e39de20f880f4bd8820e6a1cfb9f9a34f820eb12a155aa5bc92d2", size = 223467760, upload-time = "2025-09-04T08:33:04.222Z" }, @@ -3218,7 +3237,7 @@ name = "nvidia-cusparse" version = "12.6.3.3" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "nvidia-nvjitlink" }, + { name = "nvidia-nvjitlink", marker = "sys_platform != 'emscripten' and sys_platform != 'win32'" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/f8/94/5c26f33738ae35276672f12615a64bd008ed5be6d1ebcb23579285d960a9/nvidia_cusparse-12.6.3.3-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:80bcc4662f23f1054ee334a15c72b8940402975e0eab63178fc7e670aa59472c", size = 162155568, upload-time = "2025-09-04T08:33:42.864Z" }, @@ -3608,6 +3627,28 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/a2/9a/07d658e1e7fad860f1c541ab941348125dbdab773be3a0afaf32361866c7/pdfplumber-0.11.10-py3-none-any.whl", hash = "sha256:7741ea81bf165b474b153e6789d10d18e06b6ddcf3ec84289c3ef2fed6802580", size = 60047, upload-time = "2026-06-15T03:31:29.702Z" }, ] +[[package]] +name = "peft" +version = "0.21.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "accelerate" }, + { name = "huggingface-hub" }, + { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.12'" }, + { name = "numpy", version = "2.5.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, + { name = "packaging" }, + { name = "psutil" }, + { name = "pyyaml" }, + { name = "safetensors" }, + { name = "torch" }, + { name = "tqdm" }, + { name = "transformers" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/4f/91/56cc2b1b6824f5a4026750274240f26cd23465fb1a29cd14772d555433f2/peft-0.21.0.tar.gz", hash = "sha256:17f2b5a264439f4cd983c02e95954f2e948e3bcb85f75895ee307f94a55fd28f", size = 978334, upload-time = "2026-09-15T13:34:02.583Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/13/f0/29c37002f5ef5cb5be54890114d5c8f726f0cb5072a8ba0880758c912bfd/peft-0.21.0-py3-none-any.whl", hash = "sha256:b64eb75fd9dece7401c70e675b8d9de024993b70483691b41c62876f0c7809b7", size = 832883, upload-time = "2026-09-15T13:33:59.775Z" }, +] + [[package]] name = "pillow" version = "12.3.0" @@ -3958,6 +3999,34 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/e4/04/d52c7016b04b6c5108f26691f9d33ec82a9b65d041f1a9c771137693d618/protobuf-7.36.2-py3-none-any.whl", hash = "sha256:bdb3a345d48db958e6ce1f18e508beb0cc981d64f24088427549c866cd039f1e", size = 179806, upload-time = "2026-09-17T20:07:58.211Z" }, ] +[[package]] +name = "psutil" +version = "7.2.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/aa/c6/d1ddf4abb55e93cebc4f2ed8b5d6dbad109ecb8d63748dd2b20ab5e57ebe/psutil-7.2.2.tar.gz", hash = "sha256:0746f5f8d406af344fd547f1c8daa5f5c33dbc293bb8d6a16d80b4bb88f59372", size = 493740, upload-time = "2026-01-28T18:14:54.428Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/51/08/510cbdb69c25a96f4ae523f733cdc963ae654904e8db864c07585ef99875/psutil-7.2.2-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:2edccc433cbfa046b980b0df0171cd25bcaeb3a68fe9022db0979e7aa74a826b", size = 130595, upload-time = "2026-01-28T18:14:57.293Z" }, + { url = "https://files.pythonhosted.org/packages/d6/f5/97baea3fe7a5a9af7436301f85490905379b1c6f2dd51fe3ecf24b4c5fbf/psutil-7.2.2-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:e78c8603dcd9a04c7364f1a3e670cea95d51ee865e4efb3556a3a63adef958ea", size = 131082, upload-time = "2026-01-28T18:14:59.732Z" }, + { url = "https://files.pythonhosted.org/packages/37/d6/246513fbf9fa174af531f28412297dd05241d97a75911ac8febefa1a53c6/psutil-7.2.2-cp313-cp313t-manylinux2010_x86_64.manylinux_2_12_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1a571f2330c966c62aeda00dd24620425d4b0cc86881c89861fbc04549e5dc63", size = 181476, upload-time = "2026-01-28T18:15:01.884Z" }, + { url = "https://files.pythonhosted.org/packages/b8/b5/9182c9af3836cca61696dabe4fd1304e17bc56cb62f17439e1154f225dd3/psutil-7.2.2-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:917e891983ca3c1887b4ef36447b1e0873e70c933afc831c6b6da078ba474312", size = 184062, upload-time = "2026-01-28T18:15:04.436Z" }, + { url = "https://files.pythonhosted.org/packages/16/ba/0756dca669f5a9300d0cbcbfae9a4c30e446dfc7440ffe43ded5724bfd93/psutil-7.2.2-cp313-cp313t-win_amd64.whl", hash = "sha256:ab486563df44c17f5173621c7b198955bd6b613fb87c71c161f827d3fb149a9b", size = 139893, upload-time = "2026-01-28T18:15:06.378Z" }, + { url = "https://files.pythonhosted.org/packages/1c/61/8fa0e26f33623b49949346de05ec1ddaad02ed8ba64af45f40a147dbfa97/psutil-7.2.2-cp313-cp313t-win_arm64.whl", hash = "sha256:ae0aefdd8796a7737eccea863f80f81e468a1e4cf14d926bd9b6f5f2d5f90ca9", size = 135589, upload-time = "2026-01-28T18:15:08.03Z" }, + { url = "https://files.pythonhosted.org/packages/81/69/ef179ab5ca24f32acc1dac0c247fd6a13b501fd5534dbae0e05a1c48b66d/psutil-7.2.2-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:eed63d3b4d62449571547b60578c5b2c4bcccc5387148db46e0c2313dad0ee00", size = 130664, upload-time = "2026-01-28T18:15:09.469Z" }, + { url = "https://files.pythonhosted.org/packages/7b/64/665248b557a236d3fa9efc378d60d95ef56dd0a490c2cd37dafc7660d4a9/psutil-7.2.2-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:7b6d09433a10592ce39b13d7be5a54fbac1d1228ed29abc880fb23df7cb694c9", size = 131087, upload-time = "2026-01-28T18:15:11.724Z" }, + { url = "https://files.pythonhosted.org/packages/d5/2e/e6782744700d6759ebce3043dcfa661fb61e2fb752b91cdeae9af12c2178/psutil-7.2.2-cp314-cp314t-manylinux2010_x86_64.manylinux_2_12_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1fa4ecf83bcdf6e6c8f4449aff98eefb5d0604bf88cb883d7da3d8d2d909546a", size = 182383, upload-time = "2026-01-28T18:15:13.445Z" }, + { url = "https://files.pythonhosted.org/packages/57/49/0a41cefd10cb7505cdc04dab3eacf24c0c2cb158a998b8c7b1d27ee2c1f5/psutil-7.2.2-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e452c464a02e7dc7822a05d25db4cde564444a67e58539a00f929c51eddda0cf", size = 185210, upload-time = "2026-01-28T18:15:16.002Z" }, + { url = "https://files.pythonhosted.org/packages/dd/2c/ff9bfb544f283ba5f83ba725a3c5fec6d6b10b8f27ac1dc641c473dc390d/psutil-7.2.2-cp314-cp314t-win_amd64.whl", hash = "sha256:c7663d4e37f13e884d13994247449e9f8f574bc4655d509c3b95e9ec9e2b9dc1", size = 141228, upload-time = "2026-01-28T18:15:18.385Z" }, + { url = "https://files.pythonhosted.org/packages/f2/fc/f8d9c31db14fcec13748d373e668bc3bed94d9077dbc17fb0eebc073233c/psutil-7.2.2-cp314-cp314t-win_arm64.whl", hash = "sha256:11fe5a4f613759764e79c65cf11ebdf26e33d6dd34336f8a337aa2996d71c841", size = 136284, upload-time = "2026-01-28T18:15:19.912Z" }, + { url = "https://files.pythonhosted.org/packages/e7/36/5ee6e05c9bd427237b11b3937ad82bb8ad2752d72c6969314590dd0c2f6e/psutil-7.2.2-cp36-abi3-macosx_10_9_x86_64.whl", hash = "sha256:ed0cace939114f62738d808fdcecd4c869222507e266e574799e9c0faa17d486", size = 129090, upload-time = "2026-01-28T18:15:22.168Z" }, + { url = "https://files.pythonhosted.org/packages/80/c4/f5af4c1ca8c1eeb2e92ccca14ce8effdeec651d5ab6053c589b074eda6e1/psutil-7.2.2-cp36-abi3-macosx_11_0_arm64.whl", hash = "sha256:1a7b04c10f32cc88ab39cbf606e117fd74721c831c98a27dc04578deb0c16979", size = 129859, upload-time = "2026-01-28T18:15:23.795Z" }, + { url = "https://files.pythonhosted.org/packages/b5/70/5d8df3b09e25bce090399cf48e452d25c935ab72dad19406c77f4e828045/psutil-7.2.2-cp36-abi3-manylinux2010_x86_64.manylinux_2_12_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:076a2d2f923fd4821644f5ba89f059523da90dc9014e85f8e45a5774ca5bc6f9", size = 155560, upload-time = "2026-01-28T18:15:25.976Z" }, + { url = "https://files.pythonhosted.org/packages/63/65/37648c0c158dc222aba51c089eb3bdfa238e621674dc42d48706e639204f/psutil-7.2.2-cp36-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b0726cecd84f9474419d67252add4ac0cd9811b04d61123054b9fb6f57df6e9e", size = 156997, upload-time = "2026-01-28T18:15:27.794Z" }, + { url = "https://files.pythonhosted.org/packages/8e/13/125093eadae863ce03c6ffdbae9929430d116a246ef69866dad94da3bfbc/psutil-7.2.2-cp36-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:fd04ef36b4a6d599bbdb225dd1d3f51e00105f6d48a28f006da7f9822f2606d8", size = 148972, upload-time = "2026-01-28T18:15:29.342Z" }, + { url = "https://files.pythonhosted.org/packages/04/78/0acd37ca84ce3ddffaa92ef0f571e073faa6d8ff1f0559ab1272188ea2be/psutil-7.2.2-cp36-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:b58fabe35e80b264a4e3bb23e6b96f9e45a3df7fb7eed419ac0e5947c61e47cc", size = 148266, upload-time = "2026-01-28T18:15:31.597Z" }, + { url = "https://files.pythonhosted.org/packages/b4/90/e2159492b5426be0c1fef7acba807a03511f97c5f86b3caeda6ad92351a7/psutil-7.2.2-cp37-abi3-win_amd64.whl", hash = "sha256:eb7e81434c8d223ec4a219b5fc1c47d0417b12be7ea866e24fb5ad6e84b3d988", size = 137737, upload-time = "2026-01-28T18:15:33.849Z" }, + { url = "https://files.pythonhosted.org/packages/8c/c7/7bb2e321574b10df20cbde462a94e2b71d05f9bbda251ef27d104668306a/psutil-7.2.2-cp37-abi3-win_arm64.whl", hash = "sha256:8c233660f575a5a89e6d4cb65d9f938126312bca76d8fe087b947b3a1aaac9ee", size = 134617, upload-time = "2026-01-28T18:15:36.514Z" }, +] + [[package]] name = "py-key-value-aio" version = "0.4.6" @@ -4845,8 +4914,8 @@ name = "secretstorage" version = "3.5.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "cryptography" }, - { name = "jeepney" }, + { name = "cryptography", marker = "sys_platform != 'emscripten' and sys_platform != 'win32'" }, + { name = "jeepney", marker = "sys_platform != 'emscripten' and sys_platform != 'win32'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/1c/03/e834bcd866f2f8a49a85eaff47340affa3bfa391ee9912a952a1faa68c7b/secretstorage-3.5.0.tar.gz", hash = "sha256:f04b8e4689cbce351744d5537bf6b1329c6fc68f91fa666f60a380edddcd11be", size = 19884, upload-time = "2025-11-23T19:02:53.191Z" } wheels = [ @@ -5179,6 +5248,14 @@ cua = [ { name = "cua-s1" }, { name = "huggingface-hub" }, ] +cua-four-b = [ + { name = "cua-s1" }, + { name = "huggingface-hub" }, + { name = "peft" }, + { name = "pillow" }, + { name = "torchvision" }, + { name = "transformers" }, +] dev = [ { name = "pytest" }, { name = "pytest-asyncio" }, @@ -5204,6 +5281,8 @@ requires-dist = [ { name = "mcp", specifier = ">=1.26" }, { name = "opencv-python-headless", marker = "extra == 'alfworld-visual'", specifier = ">=4.10" }, { name = "openjiuwen", git = "https://github.com/ThinkFlowLab/agent-core?rev=jj-0.1.0" }, + { name = "peft", marker = "extra == 'cua-four-b'", specifier = ">=0.21,<1" }, + { name = "pillow", marker = "extra == 'cua-four-b'", specifier = ">=10" }, { name = "pillow", marker = "extra == 'report'", specifier = ">=10" }, { name = "playwright", marker = "extra == 'report'", specifier = ">=1.45" }, { name = "pytest", marker = "extra == 'dev'", specifier = ">=8" }, @@ -5212,13 +5291,16 @@ requires-dist = [ { name = "rlcard", marker = "extra == 'blackjack'", specifier = ">=1.2" }, { name = "ruff", marker = "extra == 'dev'", specifier = ">=0.6" }, { name = "system1-agents", extras = ["alfworld"], marker = "extra == 'alfworld-visual'" }, + { name = "system1-agents", extras = ["cua"], marker = "extra == 'cua-four-b'" }, { name = "textworld", marker = "extra == 'alfworld'", specifier = ">=1.7" }, { name = "torch", marker = "extra == 'alfworld-visual'", specifier = ">=2" }, { name = "torchvision", marker = "extra == 'alfworld-visual'", specifier = ">=0.15" }, + { name = "torchvision", marker = "extra == 'cua-four-b'", specifier = ">=0.15" }, + { name = "transformers", marker = "extra == 'cua-four-b'", specifier = ">=5.10.1,<6" }, { name = "ty", marker = "extra == 'dev'", specifier = ">=0.0.83" }, { name = "werkzeug", marker = "extra == 'alfworld-visual'", specifier = "==2.0.3" }, ] -provides-extras = ["blackjack", "alfworld", "alfworld-visual", "report", "laya", "cua", "dev"] +provides-extras = ["blackjack", "alfworld", "alfworld-visual", "report", "laya", "cua", "cua-four-b", "dev"] [[package]] name = "tatsu"