[Feat] Add verified text input to the desktop agent

docs/agents.md

@@ -30,9 +30,21 @@ models behind them: [architecture.md](architecture.md#models).
 
 `s1a run <agent> --help` lists every flag with its default. Beyond the shared ones: `flights` and `allrecipes` take `--goal`,
 `--batch on|off`, `--prefetch on|off`, `--goal-values on|off`, `--profile-out` and `--logs-dir`; `desktop` takes
-`--app`, `--goal`, `--expect`, `--execute`, `--plan` and `--clear`; `ticket_router` takes `--dataset` and
+`--app`, `--app-path`, `--window-title`, `--goal`, `--expect`, `--execute`, `--plan`, `--clear`, `--text`,
+`--text-target`, `--text-mode` and `--verify-file`; `ticket_router` takes `--dataset` and
 `--batch-size`; `injection_guard` takes `--labelled-set`. The four games take no flag of their own.
 
+### Desktop text input
+
+`--text` supplies the content for a `type:*` action. `--text-target` selects an exact field label or native
+identifier. The default `--text-mode insert` inserts at the current selection; `replace` sets the whole field.
+Both require a fresh readback before the input is recorded as successful. Chinese and multiline text are
+supported through native field replacement. `--window-title` selects the document when an app has several windows.
+
+`--verify-file` requires the expected window state and a file changed during this episode whose UTF-8 content
+matches `--text`. Without `--execute`, the first decision is only recorded as a plan.
+The [macOS fixture](../evals/desktop/README.md) provides a local Laya demo and a fixed-plan execution check.
+
 ## Allrecipes
 
 `allrecipes` is the first Allrecipes task of the [WebVoyager](https://github.com/MinorJerry/WebVoyager) task set

evals/desktop/README.md

@@ -0,0 +1,31 @@
+# Desktop text input on macOS
+
+The native fixture has a Body field and Save/Clear buttons. Save writes the current text to
+`/tmp/s1a-desktop-fixture.txt`. The agent checks both the visible result and the saved file.
+
+Install Cua Driver and grant macOS Accessibility and Screen Recording permissions. Keep the Mac unlocked
+and avoid interacting with the test window during a run.
+
+## Local Laya
+
+```bash
+bash evals/desktop/run_local.sh --episodes 3
+```
+
+This builds the fixture if missing and uses local Laya to choose the input and Save actions. Set `LAYA_MODEL`
+to a downloaded checkpoint directory, `S1A` to an existing CLI executable, or `CUA_DRIVER_BIN` to the installed
+driver. The script's default command is `uv run --extra laya s1a`. No Jev or chat-model key is needed.
+
+## Fixed-plan execution check
+
+```bash
+bash evals/desktop/build_fixture.sh /tmp/S1ADocumentFixture.app
+uv run s1a run desktop --model rule --rethink off --episodes 1 \
+  --app S1ADocumentFixture --app-path /tmp/S1ADocumentFixture.app \
+  --window-title "S1A Document Fixture" --goal "Enter the text in Body and save it" --expect Saved \
+  --text "本地输入测试" --text-target Body --text-mode replace \
+  --verify-file /tmp/s1a-desktop-fixture.txt --clear Clear --plan 'type:Body,Save' --execute
+```
+
+Omit `--execute` to preview one decision. Close the fixture before rebuilding it. Run outputs stay in the
+ignored `evals/results/` directory; they are not source files.

evals/desktop/build_fixture.sh

@@ -0,0 +1,19 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+app="${1:-/tmp/S1ADocumentFixture.app}"
+root="$(cd "$(dirname "$0")/../.." && pwd)"
+mkdir -p "$app/Contents/MacOS"
+swiftc -O "$root/evals/desktop/fixture.swift" -framework AppKit -o "$app/Contents/MacOS/S1ADocumentFixture"
+cat > "$app/Contents/Info.plist" <<'PLIST'
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
+<plist version="1.0"><dict>
+  <key>CFBundleExecutable</key><string>S1ADocumentFixture</string>
+  <key>CFBundleIdentifier</key><string>org.thinkflowlab.s1a.document-fixture</string>
+  <key>CFBundleName</key><string>S1ADocumentFixture</string>
+  <key>CFBundlePackageType</key><string>APPL</string>
+  <key>LSMinimumSystemVersion</key><string>14.0</string>
+</dict></plist>
+PLIST
+printf '%s\n' "$app"

evals/desktop/fixture.swift

@@ -0,0 +1,76 @@
+import AppKit
+import Foundation
+
+// A small local app for the desktop agent's macOS end-to-end test. The file is
+// written only when the app's Save button is pressed through the GUI.
+private let output = URL(fileURLWithPath: "/tmp/s1a-desktop-fixture.txt")
+
+final class DocumentFixture: NSObject, NSApplicationDelegate {
+    private let window = NSWindow(
+        contentRect: NSRect(x: 300, y: 300, width: 620, height: 440),
+        styleMask: [.titled, .closable, .miniaturizable, .resizable],
+        backing: .buffered,
+        defer: false
+    )
+    private let editor = NSTextView(frame: NSRect(x: 0, y: 0, width: 580, height: 320))
+    private let status = NSTextView(frame: NSRect(x: 210, y: 25, width: 360, height: 22))
+
+    func applicationDidFinishLaunching(_ notification: Notification) {
+        window.title = "S1A Document Fixture"
+        let content = NSView(frame: window.contentView!.bounds)
+        content.autoresizingMask = [.width, .height]
+
+        let scroll = NSScrollView(frame: NSRect(x: 20, y: 75, width: 580, height: 340))
+        scroll.hasVerticalScroller = true
+        scroll.autoresizingMask = [.width, .height]
+        editor.isEditable = true
+        editor.isSelectable = true
+        editor.setAccessibilityLabel("Body")
+        scroll.documentView = editor
+        content.addSubview(scroll)
+
+        let save = NSButton(title: "Save", target: self, action: #selector(saveDocument))
+        save.frame = NSRect(x: 20, y: 20, width: 80, height: 30)
+        content.addSubview(save)
+
+        let clear = NSButton(title: "Clear", target: self, action: #selector(clearDocument))
+        clear.frame = NSRect(x: 110, y: 20, width: 80, height: 30)
+        content.addSubview(clear)
+
+        status.frame = NSRect(x: 210, y: 25, width: 360, height: 22)
+        status.isEditable = false
+        status.isSelectable = false
+        status.drawsBackground = false
+        status.string = "Editing"
+        status.setAccessibilityLabel("Status")
+        content.addSubview(status)
+
+        window.contentView = content
+        window.makeKeyAndOrderFront(nil)
+        NSApp.activate(ignoringOtherApps: true)
+    }
+
+    @objc private func saveDocument() {
+        do {
+            try editor.string.write(to: output, atomically: true, encoding: .utf8)
+            status.string = "Saved"
+        } catch {
+            status.string = "Save failed"
+        }
+    }
+
+    @objc private func clearDocument() {
+        editor.string = ""
+        status.string = "Editing"
+        try? FileManager.default.removeItem(at: output)
+    }
+
+    func applicationShouldTerminateAfterLastWindowClosed(_ sender: NSApplication) -> Bool {
+        true
+    }
+}
+
+let app = NSApplication.shared
+let delegate = DocumentFixture()
+app.delegate = delegate
+app.run()

evals/desktop/run_local.sh

@@ -0,0 +1,20 @@
+#!/usr/bin/env bash
+# Runs the native document demo with local Laya; no chat or Jev calls.
+set -euo pipefail
+root="$(cd "$(dirname "$0")/../.." && pwd)"
+cd "$root"
+export PYTHONPATH="$root${PYTHONPATH:+:$PYTHONPATH}"
+app="${S1A_FIXTURE_APP:-/tmp/S1ADocumentFixture.app}"
+if [[ ! -x "$app/Contents/MacOS/S1ADocumentFixture" ]]; then
+  bash evals/desktop/build_fixture.sh "$app"
+fi
+if [[ -n "${S1A:-}" ]]; then
+  s1a_command=("$S1A")
+else
+  s1a_command=(uv run --extra laya s1a)
+fi
+exec "${s1a_command[@]}" run desktop --model laya --rethink off --episodes 1 \
+  --app S1ADocumentFixture --app-path "$app" --window-title "S1A Document Fixture" \
+  --goal "Enter the task text in Body and save it" --expect Saved \
+  --text "Local Laya agent demo" --text-target Body --text-mode replace --verify-file /tmp/s1a-desktop-fixture.txt \
+  --clear Clear --execute --log "$@"

s1a/agents/desktop.py

@@ -19,17 +19,20 @@
 import argparse
 import asyncio
 import sys
+import uuid
 from contextlib import asynccontextmanager
+from pathlib import Path
 from typing import Any, AsyncIterator
 
-from s1a.desktop.driver import CuaDriver, Snapshot, driver_from_env, opened
-from s1a.desktop.env import ABSTAIN, DONE, WindowEnv, clickable
+from s1a.desktop.driver import CuaDriver, DriverError, Snapshot, driver_from_env, opened
+from s1a.desktop.env import ABSTAIN, WindowEnv, clickable
 from s1a.spec import Budget, Series, ToolAgentSpec
 
 RULES = (
-    "A desktop app window. goal says what to do; elements lists the window's controls with their labels and values; "
-    "presses lists what was clicked so far. Click the one control that moves the goal forward, one click per turn. "
-    "When the window shows the goal's result, pick done. Pick abstain only when no offered click helps."
+    "A desktop app window. goal says what to do; elements lists current controls and values; presses lists past "
+    "actions. Pick one offered click or type action that moves the goal forward. Type uses the task's "
+    "supplied text, never text invented by this decision. After each action the window is observed again. "
+    "The environment stops automatically when the result is verified. Pick abstain only when no offered action helps."
 )
 
 
@@ -47,13 +50,14 @@ def parse_plan(text: str) -> tuple[tuple[str, ...], ...]:
 
 
 def plan_rule(plan: tuple[tuple[str, ...], ...]) -> Any:
-    """The baseline: the next button of the plan by how many presses were made, then done."""
+    """The baseline: the next action of the plan by how many actions were made."""
 
     def rule(state: dict[str, Any], candidates: dict[str, str]) -> str:
         step = len(state["presses"])
         if step >= len(plan):
-            return DONE
-        return next((f"click:{label}" for label in plan[step] if f"click:{label}" in candidates), ABSTAIN)
+            return ABSTAIN
+        keys = (label if label.startswith(("click:", "type:")) else f"click:{label}" for label in plan[step])
+        return next((key for key in keys if key in candidates), ABSTAIN)
 
     return rule
 
@@ -71,43 +75,97 @@ async def launch_app(app: str, driver: CuaDriver) -> None:
 
 
 @asynccontextmanager
-async def _session(driver: CuaDriver, app: str) -> AsyncIterator[None]:
+async def _session(driver: CuaDriver, app: str, *, owner: str, title: str) -> AsyncIterator[None]:
     async with opened(driver):
         await launch_app(app, driver)
+        for attempt in range(20):
+            try:
+                await driver.find_window(owner, title) if title else await driver.find_window(owner)
+                break
+            except DriverError as exc:
+                if not str(exc).startswith("list_windows: 0 on-screen") or attempt == 19:
+                    raise
+                await asyncio.sleep(0.25)
         yield
 
 
 def make_series(flags: argparse.Namespace) -> Series:
-    driver = driver_from_env("s1a-desktop")  # raises before the series starts when the driver is missing
+    if flags.app_path and sys.platform == "win32":
+        raise ValueError("--app-path currently accepts macOS .app bundles only")
     plan = parse_plan(flags.plan) if flags.plan else ()
-    return Series(
-        seeds=range(flags.seed, flags.seed + flags.episodes),
-        env_for=lambda seed: WindowEnv(
+    if flags.verify_file and not flags.text:
+        raise ValueError("--verify-file requires --text")
+
+    def file_version() -> tuple[int, int, int] | None:
+        try:
+            stat = Path(flags.verify_file).stat()
+            return stat.st_ino, stat.st_mtime_ns, stat.st_ctime_ns
+        except OSError:
+            return None
+
+    def finished(snapshot: Snapshot, initial_file: tuple[int, int, int] | None) -> bool:
+        if not shows(snapshot, flags.expect):
+            return False
+        if not flags.verify_file:
+            return True
+        try:
+            return file_version() != initial_file and Path(flags.verify_file).read_text(encoding="utf-8") == flags.text
+        except (OSError, UnicodeError):
+            return False
+
+    driver = driver_from_env(f"s1a-desktop-{uuid.uuid4().hex[:12]}")  # one public session per run/transport
+
+    def env_for(seed: int) -> WindowEnv:
+        initial_file = file_version() if flags.verify_file else None
+        return WindowEnv(
             driver,
             app_name=flags.app,
+            window_title=flags.window_title,
             goal=flags.goal,
-            done_when=lambda snapshot: shows(snapshot, flags.expect),
+            done_when=lambda snapshot: finished(snapshot, initial_file),
             execute=flags.execute,
             clear_labels=tuple(v.strip() for v in flags.clear.split(",") if v.strip()),
-        ),
-        session=_session(driver, flags.app),
+            text=flags.text,
+            text_target=flags.text_target,
+            text_mode=flags.text_mode,
+        )
+
+    return Series(
+        seeds=range(flags.seed, flags.seed + flags.episodes),
+        env_for=env_for,
+        session=_session(driver, flags.app_path or flags.app, owner=flags.app, title=flags.window_title),
         baseline=("plan", plan_rule(plan)) if plan else None,
         annotate=lambda env, episode: None,
     )
 
 
 def flags(parser: argparse.ArgumentParser) -> None:
     parser.add_argument("--app", required=True, help="app name (Windows Calculator / Calculator), or a Windows AUMID")
+    parser.add_argument("--window-title", default="", help="exact window title when the app has multiple windows")
+    parser.add_argument(
+        "--app-path", default="", help="optional macOS .app bundle path; --app remains the window owner name"
+    )
     parser.add_argument("--goal", required=True, help="what to do in the window, read by the model on every turn")
     parser.add_argument("--expect", required=True, help="the text a display or label shows when the goal is met")
-    parser.add_argument("--execute", action="store_true", help="click for real; without it one decision is planned")
-    parser.add_argument("--plan", default="", help="the rule baseline: button labels in order, | between variants")
+    parser.add_argument("--execute", action="store_true", help="act for real; without it one decision is planned")
+    parser.add_argument(
+        "--plan", default="", help="the rule baseline: button labels or action keys in order, | between variants"
+    )
     parser.add_argument("--clear", default="", help="button labels pressed on reset when the window has one")
+    parser.add_argument("--text", default="", help="task-supplied text to enter and verify in an editable element")
+    parser.add_argument("--text-target", default="", help="restrict --text to this exact field label or identifier")
+    parser.add_argument(
+        "--text-mode",
+        choices=("insert", "replace"),
+        default="insert",
+        help="insert at selection, or replace a native field's entire value; both require fresh readback",
+    )
+    parser.add_argument("--verify-file", default="", help="also require this file's UTF-8 content to equal --text")
 
 
 SPEC = ToolAgentSpec(
     name="desktop",
-    description="A Windows or macOS app window through Cua Driver: click controls toward --goal until --expect appears.",
+    description="A Windows or macOS app window through Cua Driver: choose grounded click and type actions.",
     rules=RULES,
     budget=Budget(max_steps=12, timeout_s=90, stall_after=0),
     flags=flags,