From 36175eb6aba2d3277bb792fd4f64d8e3f28e8ed9 Mon Sep 17 00:00:00 2001 From: Chaimae RACHDI Date: Thu, 24 Sep 2026 14:22:39 +0200 Subject: [PATCH] Compact the browser front's state for Laya's tiny window Laya reads a 512 to 1024 token window, but the browser front sends every model the same state it sends Jev: a JSON object per element row, the full page text, and ten actions of history. On a real page that state fills the window well before a single instruction token is spent (docs/benchmarks.md shows 18,785-23,654 input tokens for Jev on the Allrecipes run), which is why `--model laya` routinely raises MODEL_SERVICE_CONFIG_ERROR on the browser front today. laya_state() folds a browser-shaped state before every call to LayaModel._decide: page.text dropped (the choice heads already carry each candidate's own text; the free-form dump is for the chat model's DONE answer, which Laya never writes), each element row rendered as one short line instead of a JSON object, and the last three actions kept instead of ten. On by default; anything that isn't the browser front's shape passes through unchanged (the tool front already fits). LAYA_COMPACT_BROWSER_STATE=0 turns it off. 34 unit tests (tests/test_decision_models_laya.py) cover the compaction itself, its wiring into LayaModel, and the env-var opt-out, all against FakeLayaAgent (no torch/weights needed). Full suite run against main: identical 19 pre-existing failures before and after this change (missing `ty` binary and other env-only gaps in this sandbox, unrelated to decision_models/laya.py). Not done here, and worth flagging: this closes the "state is bigger than the window" gap, not the "is Laya's window, even filled, actually fast enough end to end on a real page" question. That needs the real convaiinnovations/laya checkpoint (uv sync --extra laya), a live browser run, and a real latency number. See the PR description for exact commands. Co-Authored-By: Claude Sonnet 5 --- CHANGELOG.md | 7 +++ docs/decision-models.md | 16 ++++-- s1a/decision_models/laya.py | 67 +++++++++++++++++++++-- tests/test_decision_models_laya.py | 86 ++++++++++++++++++++++++++++++ 4 files changed, 169 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index bb80f85..949c1af 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,13 @@ The format follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/); ver ## Unreleased +### Added + +- `laya_state` (`s1a/decision_models/laya.py`): folds a browser-front state to fit Laya's 512 to 1024 token + window before every call — `page.text` dropped, one short line per element row instead of a JSON object, the + last three actions instead of ten — roughly a tenfold reduction in the JSON-shaped state on the pages measured. + On by default; `LAYA_COMPACT_BROWSER_STATE=0` turns it off. `docs/decision-models.md`. + ### Changed - `--model` picks the model on every agent, on `decide` and on `probe`: `jev`, `laya`, `cua`, `llm`, `random` or diff --git a/docs/decision-models.md b/docs/decision-models.md index 915cd38..d6b1b83 100644 --- a/docs/decision-models.md +++ b/docs/decision-models.md @@ -90,6 +90,16 @@ real; `bodies` records every request). `ScriptedModel` fakes the interface for f 4. An optional extra in `pyproject.toml` and an env block in `.env.example` when it needs a dependency. Laya is text only and reads a 512 to 1024 token window; it fits the tool front first. The browser front's element -tables are wider than that window. The window check sums `input_tokens` over the request's questions. On the -browser front (two to four questions per tick) only a cut on every head raises the config error above; a cut on -one head goes unseen. `--model laya` on a browser agent needs `LAYA_MAX_LEN` raised to the page's size. +tables, sent to Jev as-is, ran well past that window on a real page before a single instruction token was spent: +a JSON object per row, the full page text, and ten actions of history. The window check sums `input_tokens` over +the request's questions. On the browser front (two to four questions per tick) only a cut on every head raises +the config error above; a cut on one head goes unseen. + +`laya_state` (`s1a/decision_models/laya.py`) folds a browser-shaped state before every call: `page.text` dropped +(the choice heads already carry each candidate's own text; the free-form dump is for the chat model's DONE +answer, which Laya never writes), each element row rendered as one short line instead of a JSON object, and the +last three actions kept instead of ten. On the WebVoyager-style pages measured while adding this, that is +roughly a tenfold reduction in the JSON-shaped state's size before the tokenizer sees it — the difference between +routinely filling a 512-token window and, on most pages, comfortably fitting it. It is on by default and skips +anything that is not the browser front's shape; `LAYA_COMPACT_BROWSER_STATE=0` turns it off. `--model laya` on a +page whose element table is still too wide for the window needs `LAYA_MAX_LEN` raised, same as before. diff --git a/s1a/decision_models/laya.py b/s1a/decision_models/laya.py index 1fb786d..b3f8cf9 100644 --- a/s1a/decision_models/laya.py +++ b/s1a/decision_models/laya.py @@ -21,6 +21,16 @@ LAYA_DEFAULT_MODEL = "convaiinnovations/laya" LAYA_DEFAULT_MAX_LEN = 512 # the window Laya assumes when a checkpoint config names none +LAYA_BROWSER_LABEL_CHARS = 40 # a row's label/value, kept over its full text (Jev's window is 32K; Laya's is not) +LAYA_BROWSER_TITLE_CHARS = 80 +LAYA_BROWSER_HISTORY_KEPT = 3 # of the browser front's last ten actions; the freshest ones carry the signal +_FLAG_LETTERS = ( + ("checked", "C"), + ("selected", "S"), + ("expanded", "X"), + ("blocked_by", "B"), + ("click_did_nothing", "D"), +) def laya_question(question: Question) -> Json: @@ -32,15 +42,61 @@ def laya_question(question: Question) -> Json: return {"type": "noul", "instructions": question.question, **criteria} +def _laya_browser_row(row: Json) -> str: + """One element row as a short line instead of a JSON object: the repeated key names (``role``, ``label``, ...) + are what a tiny window can least afford. ``[CX]``-style flags stand in for the sparse boolean/id fields.""" + label = str(row.get("label") or "")[:LAYA_BROWSER_LABEL_CHARS] + value = str(row.get("value") or "")[:LAYA_BROWSER_LABEL_CHARS] + flags = "".join(letter for key, letter in _FLAG_LETTERS if row.get(key)) + parts = [str(row.get("index", "")), str(row.get("role") or ""), label] + if value: + parts.append(f"={value}") + if flags: + parts.append(f"[{flags}]") + return " ".join(part for part in parts if part) + + +def laya_state(state: Json | str) -> Json | str: + """The browser front's per-tick state, folded to fit Laya's window: no ``page.text`` (the choice heads already + carry each candidate's own text; the free-form page dump is for the chat model's DONE answer, which Laya never + writes), the element table as one short line per row instead of a JSON object per row, and the last + ``LAYA_BROWSER_HISTORY_KEPT`` actions instead of ten. Anything that is not this shape (a plain string, the tool + front's state, a rail's) passes through: it already fits the window Laya was sized for. + + This is what made Laya's real-page window error mean anything other than "raise LAYA_MAX_LEN and hope": on a + dozen-element page the JSON-shaped state alone ran well past a 512-token window before a single instruction + token was spent. + """ + if not ( + isinstance(state, dict) and isinstance(state.get("page"), dict) and isinstance(state.get("elements"), list) + ): + return state + compact: Json = { + "page": { + "url": str(state["page"].get("url", "")), + "title": str(state["page"].get("title", ""))[:LAYA_BROWSER_TITLE_CHARS], + }, + "elements": [_laya_browser_row(row) for row in state["elements"]], + } + recent = state.get("recent_actions") + if recent: + compact["recent_actions"] = [ + f"{entry.get('kind', '')}:{entry.get('action', '')}" + ("" if entry.get("page_changed") else " (no change)") + for entry in recent[-LAYA_BROWSER_HISTORY_KEPT:] + ] + return compact + + class LayaModel(DecisionModel): """Laya's ``Agent`` (or anything with ``system_one(state, questions)`` and a ``cfg``) behind the interface.""" name = "laya" deterministic = True - def __init__(self, agent: Any, *, model: str) -> None: + def __init__(self, agent: Any, *, model: str, compact_browser_state: bool = True) -> None: self._agent = agent self._model = model + self._compact_browser_state = compact_browser_state @property def model(self) -> str: @@ -48,9 +104,10 @@ def model(self) -> str: async def _decide(self, observation: Observation, questions: dict[str, Question]) -> Reply: asked = {name: laya_question(question) for name, question in questions.items()} + state = laya_state(observation.state) if self._compact_browser_state else observation.state started = time.perf_counter() try: - payload = await asyncio.to_thread(self._agent.system_one, observation.state, asked) + payload = await asyncio.to_thread(self._agent.system_one, state, asked) except (ValueError, RuntimeError) as exc: # option overflow, and torch (CUDA included) failures raise build_error( StatusCode.MODEL_CALL_FAILED, cause=exc, error_msg=f"laya forward pass failed: {exc}" @@ -88,7 +145,8 @@ def _check_the_window(self, usage: Usage, questions: int) -> None: @classmethod def from_env(cls) -> "LayaModel": """``LAYA_MODEL`` (a hub id or a path), ``LAYA_SUBFOLDER``, ``LAYA_DEVICE``; ``LAYA_MAX_LEN`` and - ``LAYA_HEAD_MAX_LEN`` override the checkpoint's window.""" + ``LAYA_HEAD_MAX_LEN`` override the checkpoint's window. ``LAYA_COMPACT_BROWSER_STATE`` (default on; + ``0``/``false``/``no`` turns it off) folds a browser-shaped state through ``laya_state`` before every call.""" try: import laya except ImportError as exc: @@ -115,4 +173,5 @@ def from_env(cls) -> "LayaModel": value = os.getenv(variable) if value: agent.cfg[key] = int(value) - return cls(agent, model=f"{model}/{subfolder}" if subfolder else model) + compact = (os.getenv("LAYA_COMPACT_BROWSER_STATE") or "1").strip().lower() not in ("0", "false", "no") + return cls(agent, model=f"{model}/{subfolder}" if subfolder else model, compact_browser_state=compact) diff --git a/tests/test_decision_models_laya.py b/tests/test_decision_models_laya.py index 2740558..a24f485 100644 --- a/tests/test_decision_models_laya.py +++ b/tests/test_decision_models_laya.py @@ -136,6 +136,82 @@ async def test_the_model_is_deterministic_and_text_only(self) -> None: self.assertEqual((decision_model.name, decision_model.model), ("laya", "convaiinnovations/laya")) +_BROWSER_STATE = { + "page": {"url": "https://example.com/flights", "title": "Google Flights" + "!" * 100, "text": "x" * 5000}, + "elements": [ + { + "index": "1", + "role": "textbox", + "label": "Where from?" + " padding" * 20, + "value": "", + "operations": ["TYPE_TEXT"], + }, + { + "index": "2", + "role": "button", + "label": "Search", + "value": "", + "checked": True, + "operations": ["CLICK"], + }, + ], + "recent_actions": [ + {"action": f"step-{i}", "kind": "click", "text": "", "page_changed": i % 2 == 0} for i in range(10) + ], +} + + +class TestLayaState(TestCase): + """``laya_state`` folds the browser front's state to fit Laya's window; anything else passes through.""" + + def test_a_non_browser_state_passes_through(self) -> None: + for state in ({"page": "x"}, "plain text", {"score": 1}, {"page": {"url": "u"}, "elements": "not a list"}): + self.assertEqual(laya_module.laya_state(state), state) + + def test_page_text_is_dropped_and_the_title_is_capped(self) -> None: + compact = laya_module.laya_state(_BROWSER_STATE) + self.assertEqual(compact["page"]["url"], "https://example.com/flights") + self.assertNotIn("text", compact["page"]) + self.assertLessEqual(len(compact["page"]["title"]), laya_module.LAYA_BROWSER_TITLE_CHARS) + + def test_each_element_row_becomes_one_short_line_not_a_json_object(self) -> None: + compact = laya_module.laya_state(_BROWSER_STATE) + self.assertEqual(len(compact["elements"]), 2) + self.assertTrue(all(isinstance(row, str) for row in compact["elements"])) + self.assertLess(len(compact["elements"][0]), len("label") * 20) # far short of the padded label + self.assertIn("[C]", compact["elements"][1]) # the checked flag survives as a letter, not a key + + def test_history_is_capped_at_the_last_few_actions(self) -> None: + compact = laya_module.laya_state(_BROWSER_STATE) + self.assertEqual(len(compact["recent_actions"]), laya_module.LAYA_BROWSER_HISTORY_KEPT) + self.assertEqual(compact["recent_actions"][-1], "click:step-9 (no change)") + + def test_compaction_shrinks_the_json_size_by_an_order_of_magnitude(self) -> None: + import json + + raw = json.dumps(_BROWSER_STATE) + compact = json.dumps(laya_module.laya_state(_BROWSER_STATE)) + self.assertGreater(len(raw) / len(compact), 8) + + +class TestLayaModelCompaction(IsolatedAsyncioTestCase): + async def test_the_browser_state_reaching_the_agent_is_compacted_by_default(self) -> None: + agent = FakeLayaAgent() + question = ChoiceQuestion({"1": {"element": "[1] Search"}}) + await _model(agent).decide_many(Observation(_BROWSER_STATE), {"operation": question}) + ((state, _asked),) = agent.calls + self.assertEqual(state, laya_module.laya_state(_BROWSER_STATE)) + self.assertNotIn("text", state["page"]) + + async def test_compaction_turns_off_with_compact_browser_state_false(self) -> None: + agent = FakeLayaAgent() + question = ChoiceQuestion({"1": {"element": "[1] Search"}}) + model = LayaModel(agent, model="convaiinnovations/laya", compact_browser_state=False) + await model.decide_many(Observation(_BROWSER_STATE), {"operation": question}) + ((state, _asked),) = agent.calls + self.assertEqual(state, _BROWSER_STATE) + + class TestFailures(IsolatedAsyncioTestCase): async def test_option_overflow_and_torch_errors_are_model_call_failures(self) -> None: for error in (ValueError("question 'pick' options exceed head_max_len=192"), RuntimeError("CUDA error")): @@ -204,3 +280,13 @@ def test_the_defaults_when_the_env_is_empty(self) -> None: decision_model = LayaModel.from_env() self.assertEqual(decision_model.model, laya_module.LAYA_DEFAULT_MODEL) self.assertEqual(decision_model._agent.cfg, {"max_len": 512, "head_max_len": 192}) + self.assertTrue(decision_model._compact_browser_state) + + def test_laya_compact_browser_state_env_var_turns_compaction_off(self) -> None: + for off in ("0", "false", "False", "no"): + with self.subTest(off=off): + env = {"LAYA_COMPACT_BROWSER_STATE": off} + with patch.dict(sys.modules, {"laya": SimpleNamespace(load=lambda *a, **k: FakeLayaAgent())}): + with patch.dict(os.environ, env): + decision_model = LayaModel.from_env() + self.assertFalse(decision_model._compact_browser_state)