From 59e5de69f9b62136f6cf3cbebe17b66bede90c4e Mon Sep 17 00:00:00 2001 From: Arshad Date: Sun, 27 Sep 2026 15:39:34 +0530 Subject: [PATCH 1/2] feat: condense with Apple's on-device fm, drop Ollama Condensing now runs through macOS 27 `fm` (stdin in, 8k window), with `none` as the fallback. Ollama provider, its model setting and doctor check are removed; the log showed 52 of 52 Ollama condenses failing because the server was rarely running. New default condense prompt keeps numbers and names, never answers a question it is relaying, and drops filler. smoke.sh passes. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01LmQVErtihWNfyrQK91MaRw --- CLAUDE.md | 2 +- controller.html | 4 +-- daemon.py | 79 +++++++++++++++++++----------------------------- field-guide.html | 2 +- install.html | 4 +-- smoke.sh | 4 +-- 6 files changed, 39 insertions(+), 56 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index af315de..8c37e26 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -27,7 +27,7 @@ Repo-local notes for Claude and Codex. - Spoken claims are checked against actual local test commands and outcomes found in Claude/Codex transcripts. Diagnostics shows intent and verification metadata, never prompt or transcript bodies. - Attributable Git diffs are translated into behavioral facts. `.timeline.json` keeps only compact outcomes for session-arc narration and `./voice recap`; it stores no prompt, reply, or source text. - Optional Voice Inbox mode persists prepared replies in `.voice-inbox.json` without speaking them. `I'm back` drains them into one local return briefing. -- Condensing is local-only through Ollama, with `none` as the deterministic fallback. +- Condensing is local-only through Apple's on-device `fm` (macOS 27), with `none` as the deterministic fallback. - Replies over `max_direct_chars` (default 400) get condensed by the configured summarizer; on failure it speaks the first sentence with a short note. - Markdown-table replies send the raw full response to the summarizer first, so the voice gets a conversational table summary instead of row-by-row noise. If summarization fails, a mostly-table reply becomes a spoken invite to look. - Path, URL, and filename tokens are stripped from sentences; diff and traceback lines are dropped whole. diff --git a/controller.html b/controller.html index 4fb667b..b7710bd 100644 --- a/controller.html +++ b/controller.html @@ -639,13 +639,13 @@

let-there-be-voice

// ---------- condense provider ---------- const CONDENSE_PROVIDERS = [ - { name: "ollama", detail: () => cfg.condense_ollama_model || "qwen3.5:4b" }, + { name: "fm", detail: () => "apple on-device" }, { name: "none", detail: () => "skip condensing" }, ]; function renderCondense() { const grid = $("condense-grid"); grid.innerHTML = ""; - const active = cfg.condense_provider || "ollama"; + const active = cfg.condense_provider || "fm"; for (const p of CONDENSE_PROVIDERS) { const card = document.createElement("div"); card.className = "engine-card" + (p.name === active ? " active" : ""); diff --git a/daemon.py b/daemon.py index f13b05c..b9bedc0 100644 --- a/daemon.py +++ b/daemon.py @@ -65,11 +65,11 @@ SAMPLE_RATE = 24000 DEFAULT_CONDENSE_PROMPT = ( - "You are speaking for a coding agent after it has finished a turn. " - "The user hears this out loud, so speak the smallest truthful thing that lets them continue without looking. " - "Lead with the result, blocker, or exact decision required. Mention evidence and uncertainty when they matter. " - "Never discuss summarizing, prompts, the assistant, or the user. " - "If the reply contains a table, summarize the table's meaning conversationally. " + "You are speaking for a coding agent after it finished a turn. The user hears this out loud. " + "Say it in one or two short sentences, like a calm coding partner. " + "Keep the concrete result: numbers and names. " + "Mention a decision or question only if the reply actually asks one, and never answer it yourself. " + "No filler, no praise, no talk about summarizing. If there is a table, say what it means." ) CATALOG_VOICES = ( @@ -304,8 +304,7 @@ def all_voices() -> list: "max_direct_chars": {"default": 400, "kind": "int", "min": 50, "max": 5000}, "idle_exit_s": {"default": 600, "kind": "int", "min": 60, "max": 86400}, "project_window_s": {"default": 600, "kind": "int", "min": 30, "max": 7200}, - "condense_provider": {"default": "ollama", "kind": "condense_provider"}, - "condense_ollama_model": {"default": "qwen3.5:4b", "kind": "str"}, + "condense_provider": {"default": "fm", "kind": "condense_provider"}, "condense_timeout_s": {"default": 30, "kind": "int", "min": 2, "max": 60}, "rate": {"default": 1.0, "kind": "float", "min": 0.5, "max": 3.0}, "volume": {"default": 1.0, "kind": "float", "min": 0.0, "max": 1.0}, @@ -393,10 +392,10 @@ def hook_install_status() -> dict: return status -def ollama_reachable() -> bool: +def fm_available() -> bool: try: - urllib.request.urlopen("http://127.0.0.1:11434/api/tags", timeout=0.5).read(1) - return True + p = subprocess.run(["fm", "available"], capture_output=True, text=True, timeout=5) + return p.returncode == 0 and "unavailable" not in p.stdout except Exception: return False @@ -404,11 +403,11 @@ def ollama_reachable() -> bool: def doctor_snapshot() -> dict: hooks = hook_install_status() provider = config.get("condense_provider", "none") - ollama = ollama_reachable() - condense_ok = provider == "none" or (provider == "ollama" and ollama) + fm = provider == "fm" and fm_available() + condense_ok = provider == "none" or fm condense_detail = { "none": "disabled · local", - "ollama": f"ollama · local · {'reachable' if ollama else 'not running'}", + "fm": f"apple fm · on-device · {'available' if fm else 'unavailable'}", }.get(provider, provider) checks = [ {"name": "daemon", "ok": True, "detail": f"127.0.0.1:{PORT}", "required": True}, @@ -853,7 +852,7 @@ def validate_field(key: str, value): return value if kind == "condense_provider": value = str(value).strip().lower() - if value not in {"ollama", "none"}: + if value not in {"fm", "none"}: raise ValueError(key) return value raise ValueError(key) @@ -998,44 +997,28 @@ def first_sentence(text: str) -> str: return text[: config["max_direct_chars"]].strip() -def condense_via_ollama(text: str, instruction: str | None = None) -> str | None: +def condense_via_fm(text: str, instruction: str | None = None) -> str | None: + # Apple's on-device model (macOS 27 `fm`). Text goes in on stdin so a reply + # starting with "-" can't be read as a flag. The model has an 8k-token window. system_prompt = config["condense_prompt"] if instruction: system_prompt = f"{system_prompt} {instruction}" - body = { - "model": config.get("condense_ollama_model") or "qwen3.5:4b", - "stream": True, - "think": False, - "messages": [ - {"role": "system", "content": system_prompt}, - {"role": "user", "content": text}, - ], - } - payload = json.dumps(body).encode("utf-8") - req = urllib.request.Request( - "http://127.0.0.1:11434/api/chat", - data=payload, - headers={"Content-Type": "application/json"}, - method="POST", - ) try: - chunks = [] - with urllib.request.urlopen( - req, timeout=config["condense_timeout_s"] - ) as resp: - for line in resp: - line = line.strip() - if not line: - continue - data = json.loads(line) - chunks.append((data.get("message") or {}).get("content", "")) - if data.get("done"): - break - return strip_markdown("".join(chunks)) or None + p = subprocess.run( + ["fm", "respond", "--no-stream", "-i", system_prompt], + input=text[:12000], + capture_output=True, + text=True, + timeout=config["condense_timeout_s"], + ) + out = re.sub(r"\x1b\[[0-9;]*m", "", p.stdout).strip() + if p.returncode != 0 or not out: + raise RuntimeError((p.stderr or p.stdout or "empty reply").strip()) + return strip_markdown(out) or None except Exception as exc: append_log( "condense_error", - provider="ollama", + provider="fm", detail=str(exc)[:300], error=type(exc).__name__, ) @@ -1043,11 +1026,11 @@ def condense_via_ollama(text: str, instruction: str | None = None) -> str | None def condense(text: str, instruction: str | None = None) -> str | None: - provider = config.get("condense_provider", "ollama") + provider = config.get("condense_provider", "fm") if provider == "none": return None - if provider == "ollama": - return condense_via_ollama(text, instruction) + if provider == "fm": + return condense_via_fm(text, instruction) append_log("condense_error", provider=provider, detail="unsupported provider") return None diff --git a/field-guide.html b/field-guide.html index 02e0536..425e519 100644 --- a/field-guide.html +++ b/field-guide.html @@ -554,7 +554,7 @@

Long Answers

Short repliesThese can be spoken almost directly, so the answer stays crisp and human.
-
Long repliesThese get condensed into a short spoken summary by local Ollama, so the speaker does not ramble through dense prose. Set the provider to none for the deterministic fallback.
+
Long repliesThese get condensed into a short spoken summary by Apple's on-device model, so the speaker does not ramble through dense prose. Set the provider to none for the deterministic fallback.
Behavior and historyAttributable diffs become behavioral facts, while a compact local timeline connects earlier failures to later fixes. It stores outcomes and evidence, never prompt, reply, or source text.
TablesA reply with a table is never read row by row. The full reply goes to the condenser first, and the voice speaks the short table summary. If that fails, it simply tells you to look.
diff --git a/install.html b/install.html index e451f27..8b5b7e6 100644 --- a/install.html +++ b/install.html @@ -118,7 +118,7 @@

Install ltbv

Optional

-
OllamaCondenses long replies locally. Without it, ltbv uses deterministic fallback speech.
+
Apple fmCondenses long replies on-device (macOS 27, Apple Intelligence on, sudo fm license once). Without it, ltbv uses deterministic fallback speech.
Browser audioLoad ~/.local/share/ltbv/browser-extension as an unpacked Arc or Chrome extension for YouTube ducking.
@@ -138,7 +138,7 @@

Install ltbv

command not foundOpen a new terminal. The installer adds ~/.local/bin to ~/.zprofile.
silentRun ltbv doctor, then check SHUT UP and quiet hours in the controller.
-
Ollama unavailableInstall and start Ollama, or choose none as the condense provider.
+
fm unavailableRun fm available. If it says modelNotReady, the model is still downloading; otherwise choose none as the condense provider.
update refusedThe installed checkout has local changes. Commit or remove them before running ltbv update.

The installer is safe to rerun. Existing Claude and Codex settings are backed up before hook changes.

diff --git a/smoke.sh b/smoke.sh index 2e1a62e..d71f8ef 100755 --- a/smoke.sh +++ b/smoke.sh @@ -236,11 +236,11 @@ if [ "$ERRS" = "1" ] && curl -sf -m 2 "$BASE/health" >/dev/null; then pass "conf # 8. condense provider validates as a closed config set PROVIDER=$(curl -sf -m 2 "$BASE/config" | "$PY" -c 'import json,sys; print(json.load(sys.stdin)["config"]["condense_provider"])') -curl -sf -m 2 -X POST "$BASE/config" -d '{"condense_provider": "ollama"}' >/dev/null +curl -sf -m 2 -X POST "$BASE/config" -d '{"condense_provider": "fm"}' >/dev/null GOT=$(curl -sf -m 2 "$BASE/config" | "$PY" -c 'import json,sys; print(json.load(sys.stdin)["config"]["condense_provider"])') curl -sf -m 2 -X POST "$BASE/config" -d "{\"condense_provider\": \"$PROVIDER\"}" >/dev/null BAD=$(curl -sf -m 2 -X POST "$BASE/config" -d '{"condense_provider": "ramble"}' | "$PY" -c 'import json,sys; print(len(json.load(sys.stdin)["errors"]))') -if [ "$GOT" = "ollama" ] && [ "$BAD" = "1" ]; then pass "condense provider config"; else fail "condense provider config"; fi +if [ "$GOT" = "fm" ] && [ "$BAD" = "1" ]; then pass "condense provider config"; else fail "condense provider config"; fi BROWSER_DUCK=$(curl -sf -m 2 "$BASE/config" | "$PY" -c 'import json,sys; print(str(json.load(sys.stdin)["config"]["browser_youtube_ducking_enabled"]).lower())') curl -sf -m 2 -X POST "$BASE/config" -d '{"browser_youtube_ducking_enabled": true}' >/dev/null From f59bbfd313dea342ac12d2e58a7ab8c366f3f99f Mon Sep 17 00:00:00 2001 From: Arshad Date: Sun, 27 Sep 2026 16:01:26 +0530 Subject: [PATCH 2/2] feat: send long replies raw to the condenser Short replies (max_direct_chars) are spoken as-is minus markdown; longer ones, tables and turns with evidence go raw to fm. Drops the word-budget "lead with the required action" instruction, which turned questions and finished work into spoken orders on a 30-reply side-by-side. The spoken contract still guards trivial success, meta speech and false passes, and now also rejects "I am a coding agent". strip_markdown drops list bullets. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01LmQVErtihWNfyrQK91MaRw --- daemon.py | 109 +++++++++++++++++----------------------------- voice_features.py | 1 + 2 files changed, 41 insertions(+), 69 deletions(-) diff --git a/daemon.py b/daemon.py index b9bedc0..7109a94 100644 --- a/daemon.py +++ b/daemon.py @@ -65,11 +65,11 @@ SAMPLE_RATE = 24000 DEFAULT_CONDENSE_PROMPT = ( - "You are speaking for a coding agent after it finished a turn. The user hears this out loud. " - "Say it in one or two short sentences, like a calm coding partner. " - "Keep the concrete result: numbers and names. " - "Mention a decision or question only if the reply actually asks one, and never answer it yourself. " - "No filler, no praise, no talk about summarizing. If there is a table, say what it means." + "Rewrite this message from a coding agent as one or two short spoken sentences for the user, like a calm coding partner. " + "Use at most 25 words. Keep the concrete result: numbers and names. " + "Mention a decision or question only if the message actually asks one, and never answer it yourself. " + "Plain sentences only: no lists, no symbols, no filler, no praise. Never talk about yourself or about summarizing. " + "If there is a table, say what it means." ) CATALOG_VOICES = ( @@ -978,6 +978,7 @@ def strip_markdown(text: str) -> str: " ", s, ) + s = re.sub(r"^[-•]\s+", "", s) s = re.sub(r"\s+", " ", s).strip() if not s: continue @@ -986,6 +987,7 @@ def strip_markdown(text: str) -> str: text = re.sub(r"`([^`]+)`", r"\1", text) text = re.sub(r"\*\*([^*]+)\*\*", r"\1", text) text = re.sub(r"[*_#>\[\]()]", " ", text) + text = re.sub(r"\s[-•]\s", " ", text) text = re.sub(r"\s+", " ", text).strip() return text @@ -1044,91 +1046,60 @@ def prepare_speech( verification: dict | None = None, temporal: str = "", ) -> tuple[str, dict]: + # Short replies are spoken as-is (minus markdown). Everything longer goes raw + # to the condenser, which reads tables, code and paths better than regex does. + # The spoken contract still guards trivial success, meta speech and false passes. tableless, meta = strip_markdown_tables(raw or "") text = strip_markdown(tableless) meta["_cleaned"] = text diff_info = diff_info or {} + verification = verification or {} diff_summary = str(diff_info.get("summary") or "") budget = adaptive_word_budget(intent, kind, int(diff_info.get("count") or 0)) - meta["intent"] = intent - meta["word_budget"] = budget - meta["diff_files"] = int(diff_info.get("count") or 0) - meta["git_verified"] = diff_info.get("verified") - meta["git_reason"] = diff_info.get("reason") - meta["git_branch_changed"] = bool(diff_info.get("branch_changed")) - meta["git_oversized"] = bool(diff_info.get("oversized")) - verification = verification or {} - meta["request_intent"] = request_summary - meta["verification"] = verification.get("verification", "unknown") - meta["verification_tests"] = verification.get("tests", []) - meta["semantic"] = diff_info.get("semantic") or [] - meta["temporal"] = temporal - instruction = ( - f"Use at most {budget} words. Lead with the blocker or required action. " - "Do not mention implementation chatter." - ) - source = raw + meta.update({ + "intent": intent, + "word_budget": budget, + "diff_files": int(diff_info.get("count") or 0), + "git_verified": diff_info.get("verified"), + "git_reason": diff_info.get("reason"), + "git_branch_changed": bool(diff_info.get("branch_changed")), + "git_oversized": bool(diff_info.get("oversized")), + "request_intent": request_summary, + "verification": verification.get("verification", "unknown"), + "verification_tests": verification.get("tests", []), + "semantic": diff_info.get("semantic") or [], + "temporal": temporal, + }) context = [] if request_summary: context.append(f"Request intent: {request_summary}.") - if verification.get("verification") != "unknown": + if verification.get("verification", "unknown") != "unknown": tests = ", ".join(verification.get("tests") or ["checks"]) context.append(f"Actual tool evidence: {tests} {verification['verification']}.") if temporal: context.append(f"Session arc: {temporal}") if diff_summary: context.append(f"Verified Git evidence: {diff_summary}") - instruction += " Treat the verified Git evidence as factual and mention its material change." - if context: - source = "\n".join(context) + f"\n\nAgent reply:\n{raw}" - - def concise_fallback(value: str) -> str: - generic = len(value.split()) <= 6 and re.search( - r"\b(done|fixed|finished|completed|implemented|ready)\b", value, re.I - ) - if diff_summary and generic: - return trim_to_words(diff_summary, budget) - return trim_to_words(value, budget) + source = "\n".join(context) + f"\n\nAgent reply:\n{raw}" if context else (raw or "") def finish(value: str) -> tuple[str, dict]: - spoken, contract_meta = enforce_spoken_contract( - value, - source, - intent, - budget, - diff_summary, - ) + spoken, contract_meta = enforce_spoken_contract(value, source, intent, budget, diff_summary) meta.update(contract_meta) return spoken, meta - if meta["table_skipped"]: - condensed = condense(source, instruction) - if condensed is not None: - meta["condense"] = "ok" - meta["condense_source"] = "raw_table" - return finish(strip_markdown(condensed)) - meta["condense"] = "failed" - meta["condense_source"] = "raw_table" - if text: - return finish(concise_fallback(first_sentence(text))) - meta["table_fallback"] = True - return finish(TABLE_FALLBACK_TEXT) - if not text: + if not text and not meta["table_skipped"]: return "", meta - adaptive_overflow = config.get("adaptive_brevity_enabled") and len(text.split()) > budget - diff_wants_summary = bool(diff_summary) and len(text.split()) > 6 - temporal_wants_summary = bool(temporal) - if len(text) > config["max_direct_chars"] or adaptive_overflow or diff_wants_summary or temporal_wants_summary: - condensed = condense(source if context else text, instruction) - if condensed is None: - meta["condense"] = "failed" - meta["condense_source"] = "cleaned" - return finish(concise_fallback(first_sentence(text))) - meta["condense"] = "ok" - meta["condense_source"] = "cleaned" - return finish(strip_markdown(condensed)) - meta["condense"] = "not_needed" - return finish(concise_fallback(text)) + if not meta["table_skipped"] and not context and len(text) <= config["max_direct_chars"]: + meta["condense"] = "not_needed" + return finish(text) + condensed = condense(source) + meta["condense_source"] = "raw" + if condensed is None: + meta["condense"] = "failed" + return finish(first_sentence(text) if text else TABLE_FALLBACK_TEXT) + meta["condense"] = "ok" + return finish(strip_markdown(condensed)) + def transcript_last_assistant(path_str: str) -> str: diff --git a/voice_features.py b/voice_features.py index 0a5b26b..b6945fd 100644 --- a/voice_features.py +++ b/voice_features.py @@ -70,6 +70,7 @@ r"\bthis (?:summary|response|reply)\b", r"\bi (?:summarized|condensed)\b", r"\bas an ai\b", + r"\bi am (?:a|an) (?:coding )?(?:agent|assistant|ai)\b", r"\blet me know what you think\b", )