From 55ba1ba79771493b3e25b65a4fc4b8dabc89ca22 Mon Sep 17 00:00:00 2001 From: Ankush Malaker <43288948+AnkushMalaker@users.noreply.github.com> Date: Wed, 5 Aug 2026 18:55:30 +0000 Subject: [PATCH 01/18] test: move mock streaming STT off port 9999 to 8879 9999 is a popular default port for other tools, so the test stack's mock-streaming-stt kept colliding with unrelated services on dev hosts. 8879 is unclaimed by anything common. Port changed end-to-end (container, host publish, healthcheck, configs, robot keywords, verify script). --- backends/advanced/docker-compose-test.yml | 6 ++++-- tests/Dockerfile.mock-streaming-stt | 4 ++-- tests/configs/mock-services.yml | 4 ++-- tests/configs/mock-vibevoice.yml | 2 +- tests/libs/mock_streaming_stt_server.py | 4 ++-- tests/profiles.yml | 2 +- tests/resources/system_keywords.robot | 6 +++--- tests/scripts/verify_mock_servers.py | 2 +- 8 files changed, 16 insertions(+), 14 deletions(-) diff --git a/backends/advanced/docker-compose-test.yml b/backends/advanced/docker-compose-test.yml index 87ba52768..9a59a1715 100644 --- a/backends/advanced/docker-compose-test.yml +++ b/backends/advanced/docker-compose-test.yml @@ -166,10 +166,12 @@ services: build: context: ../.. dockerfile: tests/Dockerfile.mock-streaming-stt + # 8879 on purpose: the old 9999 is a popular default port for other + # tools and collided with unrelated services on a dev host. ports: - - "9999:9999" + - "8879:8879" healthcheck: - test: ["CMD", "python", "-c", "import socket; s=socket.socket(); s.connect(('localhost',9999)); s.close()"] + test: ["CMD", "python", "-c", "import socket; s=socket.socket(); s.connect(('localhost',8879)); s.close()"] interval: 10s timeout: 5s retries: 3 diff --git a/tests/Dockerfile.mock-streaming-stt b/tests/Dockerfile.mock-streaming-stt index 964946f60..970f92dd3 100644 --- a/tests/Dockerfile.mock-streaming-stt +++ b/tests/Dockerfile.mock-streaming-stt @@ -9,7 +9,7 @@ RUN pip install --no-cache-dir websockets COPY tests/libs/mock_streaming_stt_server.py . # Expose WebSocket port -EXPOSE 9999 +EXPOSE 8879 # Run server -CMD ["python", "mock_streaming_stt_server.py", "--host", "0.0.0.0", "--port", "9999"] +CMD ["python", "mock_streaming_stt_server.py", "--host", "0.0.0.0", "--port", "8879"] diff --git a/tests/configs/mock-services.yml b/tests/configs/mock-services.yml index 1e1e47c9c..2d974802d 100644 --- a/tests/configs/mock-services.yml +++ b/tests/configs/mock-services.yml @@ -59,7 +59,7 @@ models: description: Mock STT for testing (batch) model_provider: mock model_type: stt - model_url: http://host.docker.internal:9999 + model_url: http://host.docker.internal:8879 name: mock-stt operations: stt_transcribe: @@ -78,7 +78,7 @@ models: description: Mock STT for testing (streaming) model_provider: mock model_type: stt_stream - model_url: ws://host.docker.internal:9999 + model_url: ws://host.docker.internal:8879 name: mock-stt-stream operations: chunk_header: diff --git a/tests/configs/mock-vibevoice.yml b/tests/configs/mock-vibevoice.yml index 96249b0f8..812c3c8f8 100644 --- a/tests/configs/mock-vibevoice.yml +++ b/tests/configs/mock-vibevoice.yml @@ -69,7 +69,7 @@ models: model_type: stt_stream model_provider: mock api_family: mock - model_url: ws://host.docker.internal:9999 + model_url: ws://host.docker.internal:8879 api_key: mock-key-not-used operations: start: diff --git a/tests/libs/mock_streaming_stt_server.py b/tests/libs/mock_streaming_stt_server.py index b5f548653..0c89cfa49 100755 --- a/tests/libs/mock_streaming_stt_server.py +++ b/tests/libs/mock_streaming_stt_server.py @@ -6,7 +6,7 @@ that match the extraction paths used in the config (e.g., channel.alternatives[0].transcript). Architecture: -- Async WebSocket server on 0.0.0.0:9999 +- Async WebSocket server on 0.0.0.0:8879 - Sends interim results every 10 audio chunks - Sends final results on CloseStream with >2s duration and >5 words (speech detection thresholds) @@ -265,7 +265,7 @@ async def main(host: str, port: int): "--host", default="0.0.0.0", help="Server host (default: 0.0.0.0)" ) parser.add_argument( - "--port", type=int, default=9999, help="Server port (default: 9999)" + "--port", type=int, default=8879, help="Server port (default: 8879)" ) parser.add_argument("--debug", action="store_true", help="Enable debug logging") diff --git a/tests/profiles.yml b/tests/profiles.yml index 7c86ecf19..371dafffa 100644 --- a/tests/profiles.yml +++ b/tests/profiles.yml @@ -2,7 +2,7 @@ # # A profile declares WHICH BACKING SERVICES ARE REAL for a test run. Services # that are not listed as real are served by the in-repo stubs (mock STT on -# :9999/:8765, mock LLM on :11435, the in-process mock speaker client). +# :8879/:8765, mock LLM on :11435, the in-process mock speaker client). # # The Robot suite is IDENTICAL across every profile. There is deliberately no # per-test credential gating: a test is never skipped because a key is absent. diff --git a/tests/resources/system_keywords.robot b/tests/resources/system_keywords.robot index 6fb6b3d48..814e31db4 100644 --- a/tests/resources/system_keywords.robot +++ b/tests/resources/system_keywords.robot @@ -42,12 +42,12 @@ Health Check Start Mock Transcription Server - [Documentation] Start the mock WebSocket transcription server on port 9999 + [Documentation] Start the mock WebSocket transcription server on port 8879 ... Used for testing transcription workflows without external API dependencies. # Start mock server as background process ${handle}= Start Process - ... python3 ${CURDIR}/../scripts/mock_transcription_server.py --host 0.0.0.0 --port 9999 + ... python3 ${CURDIR}/../scripts/mock_transcription_server.py --host 0.0.0.0 --port 8879 ... alias=mock_transcription_server ... stdout=${OUTPUTDIR}/mock_transcription_server.log ... stderr=STDOUT @@ -58,7 +58,7 @@ Start Mock Transcription Server # Wait for server to start Sleep 2s - Log ✅ Started Mock Transcription Server on ws://localhost:9999 + Log ✅ Started Mock Transcription Server on ws://localhost:8879 Stop Mock Transcription Server diff --git a/tests/scripts/verify_mock_servers.py b/tests/scripts/verify_mock_servers.py index 8983e1259..3d485c637 100755 --- a/tests/scripts/verify_mock_servers.py +++ b/tests/scripts/verify_mock_servers.py @@ -140,7 +140,7 @@ async def test_streaming_stt(): """Test mock streaming STT server.""" print("Testing Mock Streaming STT - WebSocket...") try: - async with websockets.connect("ws://localhost:9999") as ws: + async with websockets.connect("ws://localhost:8879") as ws: # Receive initial empty result initial_msg = await ws.recv() initial_data = json.loads(initial_msg) From c4a0fc226082170166179bb4317fdd2fe3580aa9 Mon Sep 17 00:00:00 2001 From: Ankush <43288948+AnkushMalaker@users.noreply.github.com> Date: Thu, 6 Aug 2026 03:29:21 +0530 Subject: [PATCH 02/18] fix(immich): restore UTC on Mongo timestamps before the asset search MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit MongoDB returns the newest DeviceInputItem.captured_at as a naive datetime, so subtracting the 48h overlap window produced a naive bound that Immich read in an unintended zone — silently shifting which assets a scan sees. Normalize through _as_utc() before computing the window. --- .../services/immich_discovery.py | 12 +++++++++--- backends/advanced/tests/test_immich_discovery.py | 8 +++++++- 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/backends/advanced/src/advanced_omi_backend/services/immich_discovery.py b/backends/advanced/src/advanced_omi_backend/services/immich_discovery.py index fd2f69c78..14ad74a76 100644 --- a/backends/advanced/src/advanced_omi_backend/services/immich_discovery.py +++ b/backends/advanced/src/advanced_omi_backend/services/immich_discovery.py @@ -47,6 +47,13 @@ def _asset_time(asset: dict[str, Any]) -> datetime | None: return value if value.tzinfo else value.replace(tzinfo=timezone.utc) +def _as_utc(value: datetime) -> datetime: + """Restore UTC stripped by MongoDB before serializing times for Immich.""" + if value.tzinfo is None: + return value.replace(tzinfo=timezone.utc) + return value.astimezone(timezone.utc) + + def select_candidates( assets: list[dict[str, Any]], limit: int = _DAILY_LIMIT ) -> list[dict[str, Any]]: @@ -105,9 +112,8 @@ async def scan_immich_memories() -> dict[str, Any]: DeviceInputItem.source_id == source_id, sort=[("captured_at", -1)], ) - since = ( - newest.captured_at if newest else utcnow() - timedelta(days=2) - ) - timedelta(hours=48) + newest_at = _as_utc(newest.captured_at) if newest else utcnow() - timedelta(days=2) + since = newest_at - timedelta(hours=48) page: int | None = 1 assets: list[dict[str, Any]] = [] async with httpx.AsyncClient( diff --git a/backends/advanced/tests/test_immich_discovery.py b/backends/advanced/tests/test_immich_discovery.py index c9e7c5800..bdd70ab83 100644 --- a/backends/advanced/tests/test_immich_discovery.py +++ b/backends/advanced/tests/test_immich_discovery.py @@ -1,6 +1,6 @@ from datetime import datetime, timedelta, timezone -from advanced_omi_backend.services.immich_discovery import select_candidates +from advanced_omi_backend.services.immich_discovery import _as_utc, select_candidates def asset(identifier: str, when: datetime, name: str = "photo.jpg"): @@ -29,3 +29,9 @@ def test_candidate_selection_honors_analysis_budget(): start = datetime(2026, 7, 22, tzinfo=timezone.utc) rows = [asset(str(i), start + timedelta(hours=i)) for i in range(20)] assert len(select_candidates(rows)) == 12 + + +def test_mongo_datetime_is_restored_to_utc_for_immich_search(): + stored = datetime(2026, 7, 27, 16, 46, 7) + + assert _as_utc(stored).isoformat() == "2026-07-27T16:46:07+00:00" From c4fc0c332db31390ed2a057886178393b8c8026d Mon Sep 17 00:00:00 2001 From: Ankush <43288948+AnkushMalaker@users.noreply.github.com> Date: Thu, 6 Aug 2026 03:29:36 +0530 Subject: [PATCH 03/18] fix(timeline): treat Immich as a polled source, not a heartbeat agent MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit _effective_source_status() aged every online source out to offline once its last_seen_at went stale, which is right for ScreenPipe capture agents that heartbeat continuously. Chronicle polls Immich on a schedule instead, so its last_seen_at means last successful sync — and a healthy library kept flipping to Disconnected between scans. Exempt the immich provider from the staleness check (an explicit error status still wins), and label it in the Timeline as Connected/Disconnected with synced rather than seen. --- .../routers/modules/device_input_routes.py | 5 ++ .../tests/test_device_input_routes_helpers.py | 63 +++++++++++++++++++ .../advanced/webui/src/pages/Timeline.tsx | 18 ++++-- 3 files changed, 82 insertions(+), 4 deletions(-) create mode 100644 backends/advanced/tests/test_device_input_routes_helpers.py diff --git a/backends/advanced/src/advanced_omi_backend/routers/modules/device_input_routes.py b/backends/advanced/src/advanced_omi_backend/routers/modules/device_input_routes.py index e903568ab..e3f1ae313 100644 --- a/backends/advanced/src/advanced_omi_backend/routers/modules/device_input_routes.py +++ b/backends/advanced/src/advanced_omi_backend/routers/modules/device_input_routes.py @@ -83,6 +83,11 @@ def _effective_source_status( ) -> str: if source.status != "online": return source.status + # Immich is polled by Chronicle on a schedule; it does not send the frequent + # heartbeats expected from live ScreenPipe capture agents. Its last_seen_at + # therefore means "last successful sync", not "last heartbeat". + if source.provider == "immich": + return "online" checked_at = _as_utc(now or utcnow()) if source.last_seen_at is None: return "offline" diff --git a/backends/advanced/tests/test_device_input_routes_helpers.py b/backends/advanced/tests/test_device_input_routes_helpers.py new file mode 100644 index 000000000..feb279b86 --- /dev/null +++ b/backends/advanced/tests/test_device_input_routes_helpers.py @@ -0,0 +1,63 @@ +from datetime import datetime, timedelta, timezone +from types import SimpleNamespace + +from advanced_omi_backend.routers.modules.device_input_routes import ( + _effective_source_status, + _utc_iso, +) + + +def test_utc_iso_marks_naive_mongo_datetimes_as_utc(): + assert _utc_iso(datetime(2026, 7, 24, 16, 0, 23, 834000)) == ( + "2026-07-24T16:00:23.834000Z" + ) + + +def test_utc_iso_converts_aware_datetimes_to_utc(): + india = timezone(timedelta(hours=5, minutes=30)) + assert _utc_iso(datetime(2026, 7, 24, 21, 30, tzinfo=india)) == ( + "2026-07-24T16:00:00Z" + ) + + +def test_online_source_becomes_offline_when_heartbeat_is_stale(): + now = datetime(2026, 7, 24, 16, 5, tzinfo=timezone.utc) + source = SimpleNamespace( + provider="screenpipe", + status="online", + last_seen_at=datetime(2026, 7, 24, 16, 2, tzinfo=timezone.utc), + ) + + assert _effective_source_status(source, now) == "offline" + + +def test_recent_source_remains_online(): + now = datetime(2026, 7, 24, 16, 5, tzinfo=timezone.utc) + source = SimpleNamespace( + provider="screenpipe", + status="online", + last_seen_at=datetime(2026, 7, 24, 16, 4, 30, tzinfo=timezone.utc), + ) + + assert _effective_source_status(source, now) == "online" + + +def test_immich_source_uses_last_seen_as_sync_time_not_heartbeat(): + now = datetime(2026, 7, 31, 16, 5, tzinfo=timezone.utc) + source = SimpleNamespace( + provider="immich", + status="online", + last_seen_at=datetime(2026, 7, 24, 16, 2, tzinfo=timezone.utc), + ) + + assert _effective_source_status(source, now) == "online" + + +def test_immich_source_preserves_explicit_error_status(): + source = SimpleNamespace( + provider="immich", + status="error", + last_seen_at=datetime(2026, 7, 24, 16, 2, tzinfo=timezone.utc), + ) + + assert _effective_source_status(source) == "error" diff --git a/backends/advanced/webui/src/pages/Timeline.tsx b/backends/advanced/webui/src/pages/Timeline.tsx index 2c9121ee8..ef71f26ff 100644 --- a/backends/advanced/webui/src/pages/Timeline.tsx +++ b/backends/advanced/webui/src/pages/Timeline.tsx @@ -1,7 +1,7 @@ import { useEffect, useMemo, useState } from 'react' import { useMutation, useQuery } from '@tanstack/react-query' import { Activity, AppWindow, ArrowDownUp, CalendarDays, Copy, Image, Link2, Monitor, RefreshCw } from 'lucide-react' -import { deviceInputApi, DeviceInputItem } from '../services/api' +import { deviceInputApi, DeviceInputItem, DeviceInputSource } from '../services/api' import { timeAgo } from '../utils/timeAgo' import { Button, Card, IconButton } from '../components/ui' @@ -120,8 +120,18 @@ function formatDuration(item: DeviceInputItem) { return `${minutes}m ${seconds % 60}s` } -function sourceStatusLabel(status: string) { - return status.charAt(0).toUpperCase() + status.slice(1) +function sourceStatusLabel(source: DeviceInputSource) { + if (source.provider === 'immich') { + if (source.status === 'online') return 'Connected' + if (source.status === 'offline') return 'Disconnected' + } + return source.status.charAt(0).toUpperCase() + source.status.slice(1) +} + +function sourceSeenLabel(source: DeviceInputSource) { + if (!source.last_seen_at) return '' + const event = source.provider === 'immich' ? 'synced' : 'seen' + return ` · ${event} ${timeAgo(source.last_seen_at)}` } function observationReviewLabel(curation: DeviceInputItem['curation']) { @@ -221,7 +231,7 @@ export default function Timeline() {
{source.name}
{source.provider} · {source.platform}
-
{sourceStatusLabel(source.status)}{source.last_seen_at ? ` · seen ${timeAgo(source.last_seen_at)}` : ''}
+
{sourceStatusLabel(source)}{sourceSeenLabel(source)}
))} From 4a0d5af94e6de2dbe8652fc6642b7d6fb00760a4 Mon Sep 17 00:00:00 2001 From: Ankush <43288948+AnkushMalaker@users.noreply.github.com> Date: Thu, 6 Aug 2026 03:30:43 +0530 Subject: [PATCH 04/18] feat(memory): deterministic person identity and merge service MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Merging duplicate People notes was previously an LLM-mediated vault_tools path; the actual mutation now lives in a deterministic, previewable service that both the memory agent and external clients share. - person_merge.py: PersonMergeService — locked merge of metadata, facts, media, and backlinks, with a plan token so apply executes exactly the previewed plan (409 on stale state) and file-level rollback on failure. - person_identity.py: conservative duplicate-person suggestions from name/alias/context evidence, plus a symmetric distinct_from annotation in the People frontmatter that blocks future merge previews. - person_merge_actions.py + memory_routes.py: /api/memories/people/ suggestions, identity, merge/preview, and merge endpoints; preview and apply are separate steps and nothing merges automatically. - vault_tools.py: the agent rename-collision path delegates to the same PersonMergeService instead of its own migrate-and-delete logic. - audit: new obsidian_action cause (human_external actor) so explicit semantic actions are distinguishable from Syncthing sync edits. - agent/__init__.py: lazy exports so deterministic helpers can import section_edit without initializing the LLM agents. --- .../models/memory_audit.py | 3 +- .../routers/modules/memory_routes.py | 109 +++- .../services/memory/agent/__init__.py | 49 +- .../services/memory/agent/vault_tools.py | 25 +- .../services/memory/audit.py | 5 +- .../services/memory/person_identity.py | 354 +++++++++++++ .../services/memory/person_merge.py | 464 ++++++++++++++++++ .../services/memory/person_merge_actions.py | 122 +++++ .../services/memory/vault_templates.py | 1 + .../advanced/tests/test_person_identity.py | 226 +++++++++ backends/advanced/tests/test_person_merge.py | 236 +++++++++ docs/backend/memories.md | 3 + 12 files changed, 1562 insertions(+), 35 deletions(-) create mode 100644 backends/advanced/src/advanced_omi_backend/services/memory/person_identity.py create mode 100644 backends/advanced/src/advanced_omi_backend/services/memory/person_merge.py create mode 100644 backends/advanced/src/advanced_omi_backend/services/memory/person_merge_actions.py create mode 100644 backends/advanced/tests/test_person_identity.py create mode 100644 backends/advanced/tests/test_person_merge.py diff --git a/backends/advanced/src/advanced_omi_backend/models/memory_audit.py b/backends/advanced/src/advanced_omi_backend/models/memory_audit.py index a259f0b22..f3e6b6ee3 100644 --- a/backends/advanced/src/advanced_omi_backend/models/memory_audit.py +++ b/backends/advanced/src/advanced_omi_backend/models/memory_audit.py @@ -39,7 +39,8 @@ class MemoryAuditEntry(Document): None, description="Why the memory changed (provenance), one of MemoryCause: " "auto_extraction, memory_replay, memory_rebuild, transcript_reprocess, " - "speaker_reprocess, annotation_apply, obsidian_sync, delete_all. " + "speaker_reprocess, annotation_apply, obsidian_sync, obsidian_action, " + "delete_all. " "See services/memory/audit.py.", ) strategy: Optional[str] = Field( diff --git a/backends/advanced/src/advanced_omi_backend/routers/modules/memory_routes.py b/backends/advanced/src/advanced_omi_backend/routers/modules/memory_routes.py index ebd99e8eb..f31d4983c 100644 --- a/backends/advanced/src/advanced_omi_backend/routers/modules/memory_routes.py +++ b/backends/advanced/src/advanced_omi_backend/routers/modules/memory_routes.py @@ -5,13 +5,23 @@ """ import logging -from typing import Optional +from typing import Literal, Optional -from fastapi import APIRouter, Body, Depends, Query +from fastapi import APIRouter, Body, Depends, HTTPException, Query from pydantic import BaseModel from advanced_omi_backend.auth import current_active_user, current_superuser from advanced_omi_backend.controllers import memory_controller +from advanced_omi_backend.services.memory.person_merge import ( + PersonMergeError, + PersonMergeStale, +) +from advanced_omi_backend.services.memory.person_merge_actions import ( + apply_person_merge, + get_person_suggestions, + preview_person_merge, + set_people_distinct, +) from advanced_omi_backend.users import User logger = logging.getLogger(__name__) @@ -26,6 +36,37 @@ class AddMemoryRequest(BaseModel): source_id: Optional[str] = None +class PersonMergePreviewRequest(BaseModel): + """Local state supplied by an Obsidian or automation client.""" + + source_name: str + target_name: str + source_hash: Optional[str] = None + target_hash: Optional[str] = None + + +class PersonMergeApplyRequest(BaseModel): + """Apply the exact server-side plan previously shown to the user.""" + + source_name: str + target_name: str + plan_token: str + + +class PersonIdentityDecisionRequest(BaseModel): + """A durable user decision about whether two person notes are distinct.""" + + person_a: str + person_b: str + decision: Literal["distinct", "clear_distinct"] + revision: Optional[str] = None + + +def _person_merge_http_error(error: PersonMergeError) -> HTTPException: + status = 409 if isinstance(error, PersonMergeStale) else 422 + return HTTPException(status_code=status, detail=str(error)) + + @router.get("") async def get_memories( current_user: User = Depends(current_active_user), @@ -109,6 +150,70 @@ async def add_memory( ) +@router.post("/people/merge/preview") +async def preview_people_merge( + request: PersonMergePreviewRequest, + current_user: User = Depends(current_active_user), +): + """Preview a deterministic person merge without changing the vault.""" + try: + return await preview_person_merge( + current_user.user_id, + request.source_name, + request.target_name, + request.source_hash, + request.target_hash, + ) + except PersonMergeError as error: + raise _person_merge_http_error(error) from error + + +@router.post("/people/merge") +async def merge_people( + request: PersonMergeApplyRequest, + current_user: User = Depends(current_active_user), +): + """Apply a previously previewed deterministic person merge.""" + try: + return await apply_person_merge( + current_user.user_id, + request.source_name, + request.target_name, + request.plan_token, + ) + except PersonMergeError as error: + raise _person_merge_http_error(error) from error + + +@router.get("/people/suggestions") +async def get_people_suggestions( + limit: int = Query(default=20, ge=1, le=100), + current_user: User = Depends(current_active_user), +): + """Return deterministic duplicate-person candidates for user review.""" + return { + "suggestions": await get_person_suggestions(current_user.user_id, limit), + } + + +@router.post("/people/identity") +async def set_people_identity( + request: PersonIdentityDecisionRequest, + current_user: User = Depends(current_active_user), +): + """Persist or clear a symmetric distinct-person annotation.""" + try: + return await set_people_distinct( + current_user.user_id, + request.person_a, + request.person_b, + distinct=request.decision == "distinct", + revision=request.revision, + ) + except PersonMergeError as error: + raise _person_merge_http_error(error) from error + + @router.delete("/{memory_id}") async def delete_memory( memory_id: str, current_user: User = Depends(current_active_user) diff --git a/backends/advanced/src/advanced_omi_backend/services/memory/agent/__init__.py b/backends/advanced/src/advanced_omi_backend/services/memory/agent/__init__.py index 806522efa..f710cb442 100644 --- a/backends/advanced/src/advanced_omi_backend/services/memory/agent/__init__.py +++ b/backends/advanced/src/advanced_omi_backend/services/memory/agent/__init__.py @@ -1,22 +1,31 @@ -"""Chronicle memory agent: a tool-calling agent that maintains the markdown vault.""" +"""Chronicle memory agent package. -from .codex_agent import CodexMemoryAgent, codex_executor_available -from .memory_agent import MemoryAgent, MemoryAgentResult, search_vault -from .vault_tools import ( - VAULT_SEARCH_TOOL_SCHEMAS, - VAULT_TOOL_SCHEMAS, - VaultToolError, - VaultTools, -) +Public symbols are loaded lazily so low-level deterministic vault helpers can import +``agent.section_edit`` without initializing the LLM agents (or creating cycles with +``vault_tools``). +""" -__all__ = [ - "CodexMemoryAgent", - "codex_executor_available", - "MemoryAgent", - "MemoryAgentResult", - "search_vault", - "VaultTools", - "VaultToolError", - "VAULT_TOOL_SCHEMAS", - "VAULT_SEARCH_TOOL_SCHEMAS", -] +from importlib import import_module + +_EXPORTS = { + "CodexMemoryAgent": (".codex_agent", "CodexMemoryAgent"), + "codex_executor_available": (".codex_agent", "codex_executor_available"), + "MemoryAgent": (".memory_agent", "MemoryAgent"), + "MemoryAgentResult": (".memory_agent", "MemoryAgentResult"), + "search_vault": (".memory_agent", "search_vault"), + "VaultTools": (".vault_tools", "VaultTools"), + "VaultToolError": (".vault_tools", "VaultToolError"), + "VAULT_TOOL_SCHEMAS": (".vault_tools", "VAULT_TOOL_SCHEMAS"), + "VAULT_SEARCH_TOOL_SCHEMAS": (".vault_tools", "VAULT_SEARCH_TOOL_SCHEMAS"), +} + +__all__ = list(_EXPORTS) + + +def __getattr__(name: str): + if name not in _EXPORTS: + raise AttributeError(name) + module_name, attribute = _EXPORTS[name] + value = getattr(import_module(module_name, __name__), attribute) + globals()[name] = value + return value diff --git a/backends/advanced/src/advanced_omi_backend/services/memory/agent/vault_tools.py b/backends/advanced/src/advanced_omi_backend/services/memory/agent/vault_tools.py index c54488e01..acbfe029b 100644 --- a/backends/advanced/src/advanced_omi_backend/services/memory/agent/vault_tools.py +++ b/backends/advanced/src/advanced_omi_backend/services/memory/agent/vault_tools.py @@ -28,6 +28,7 @@ from pathlib import Path from typing import Any, Dict, Iterator, List +from ..person_merge import PersonMergeService from ..vault_lock import VaultLockTimeout, vault_note_lock from ..vault_scaffold import write_category from .edit_engine import Edit, EditError, apply_edits @@ -406,20 +407,22 @@ def rename_person(self, old_name: str, new_name: str) -> str: # keeps its final content and the merge never loses facts unrecorded. old_content = old_fp.read_text(encoding="utf-8") if new_fp.exists(): - # Merge case — a plain move would clobber the target. Migrate the old - # note's facts into the target *before* deleting it (non-lossy by - # construction — never rely on a follow-up edit_note that may not come), - # rewrite backlinks, then remove the old note. - migrated = self._migrate_person_facts(old_content, new_fp, old_rel) - n = self._rewrite_backlinks_python(old_name, new_name) - old_fp.unlink() - self.touched.add(new_rel) + # Merge case — delegate to the same deterministic operation exposed to + # Obsidian and automation clients. The caller already owns the vault + # lock, so use its locked implementation directly. + service = PersonMergeService(self.root) + preview = service.preview(old_name, new_name) + result = service.apply_preview_locked(preview) + for rel, after in result.after.items(): + if after is not None: + self.touched.add(rel) self._record_removal(old_rel, new_rel, old_content) return ( f"'{new_name}' already existed — merged into People/{new_name}.md: " - f"migrated {migrated} fact bullet(s), rewrote {n} backlink(s), and " - f"deleted People/{old_name}.md. Review People/{new_name}.md and use " - f"edit_note to de-duplicate any overlapping facts." + f"migrated {preview.facts_to_add} fact bullet(s), skipped " + f"{preview.duplicate_facts_skipped} duplicate(s), rewrote " + f"{preview.backlink_occurrences} backlink(s), added '{old_name}' as " + f"an alias, and deleted People/{old_name}.md." ) self.touched.add(new_rel) if self._notesmd: diff --git a/backends/advanced/src/advanced_omi_backend/services/memory/audit.py b/backends/advanced/src/advanced_omi_backend/services/memory/audit.py index 9961e9a07..d4d6ac2d6 100644 --- a/backends/advanced/src/advanced_omi_backend/services/memory/audit.py +++ b/backends/advanced/src/advanced_omi_backend/services/memory/audit.py @@ -53,6 +53,7 @@ class MemoryCause(str, Enum): SPEAKER_REPROCESS = "speaker_reprocess" # re-ran diarization ANNOTATION_APPLY = "annotation_apply" # user applied annotation corrections OBSIDIAN_SYNC = "obsidian_sync" # inbound human edit via Syncthing + OBSIDIAN_ACTION = "obsidian_action" # explicit semantic action from Obsidian DELETE_ALL = "delete_all" # bulk vault wipe @@ -73,6 +74,7 @@ class UpdateStrategy(str, Enum): MemoryCause.SPEAKER_REPROCESS: "reprocess", MemoryCause.ANNOTATION_APPLY: "reprocess", MemoryCause.OBSIDIAN_SYNC: "human", + MemoryCause.OBSIDIAN_ACTION: "human", MemoryCause.DELETE_ALL: "bulk", } @@ -84,6 +86,7 @@ class UpdateStrategy(str, Enum): MemoryCause.SPEAKER_REPROCESS: "Speaker reprocess", MemoryCause.ANNOTATION_APPLY: "Annotation applied", MemoryCause.OBSIDIAN_SYNC: "Human · Obsidian", + MemoryCause.OBSIDIAN_ACTION: "Obsidian action", MemoryCause.DELETE_ALL: "Bulk delete", } @@ -132,7 +135,7 @@ def actor_for(cause: Optional[str], agent_mode: bool, operation: Optional[str]) if agent_mode: return "agent" c = _as_cause(cause) - if c == MemoryCause.OBSIDIAN_SYNC: + if c in (MemoryCause.OBSIDIAN_SYNC, MemoryCause.OBSIDIAN_ACTION): return "human_external" if c == MemoryCause.AUTO_EXTRACTION: return "system" diff --git a/backends/advanced/src/advanced_omi_backend/services/memory/person_identity.py b/backends/advanced/src/advanced_omi_backend/services/memory/person_identity.py new file mode 100644 index 000000000..06fbd37d1 --- /dev/null +++ b/backends/advanced/src/advanced_omi_backend/services/memory/person_identity.py @@ -0,0 +1,354 @@ +"""Deterministic duplicate-person suggestions and durable identity annotations.""" + +import json +import re +import uuid +from dataclasses import dataclass +from datetime import date +from difflib import SequenceMatcher +from itertools import combinations +from pathlib import Path +from typing import Any, Optional + +from .person_merge import ( + PersonMergeError, + PersonMergeStale, + _as_list, + _atomic_write, + _join_frontmatter, + _linked_person_names, + _resolve_person, + _section_bullets, + _sha256, + _split_frontmatter, +) +from .vault_lock import VaultLockTimeout, vault_note_lock + +_TOKEN_RE = re.compile(r"[a-z0-9]+") +_LINK_RE = re.compile(r"\[\[([^\]|#]+)") +_CONVERSATION_RE = re.compile(r"Conversations/([0-9a-f-]{36})", re.IGNORECASE) +_PHOTO_RE = re.compile(r"_media/([^\]|]+)", re.IGNORECASE) +_IGNORED_CONTEXT_LINKS = { + "people", + "conversations.base", + "conversations", +} + + +@dataclass +class PersonRecord: + name: str + path: str + text: str + content_hash: str + aliases: set[str] + distinct_from: set[str] + org: str + role: str + links: set[str] + conversations: set[str] + photos: set[str] + snippets: list[str] + + +@dataclass +class IdentityChangeResult: + action_id: str + person_a: str + person_b: str + decision: str + changed_paths: list[str] + before: dict[str, str] + after: dict[str, str] + + def to_dict(self) -> dict[str, Any]: + return { + "action_id": self.action_id, + "person_a": self.person_a, + "person_b": self.person_b, + "decision": self.decision, + "changed_paths": self.changed_paths, + } + + +def _normalise_name(name: str) -> str: + return "".join(_TOKEN_RE.findall(name.casefold())) + + +def _tokens(name: str) -> set[str]: + return set(_TOKEN_RE.findall(name.casefold())) + + +def _edit_distance(left: str, right: str) -> int: + left = _normalise_name(left) + right = _normalise_name(right) + row = list(range(len(right) + 1)) + for index, left_char in enumerate(left, 1): + next_row = [index] + for right_index, right_char in enumerate(right, 1): + next_row.append( + min( + next_row[-1] + 1, + row[right_index] + 1, + row[right_index - 1] + (left_char != right_char), + ) + ) + row = next_row + return row[-1] + + +def _plain_value(value: Any) -> str: + return str(value).strip().casefold() if value else "" + + +def _aliases(value: Any) -> set[str]: + result: set[str] = set() + for item in _as_list(value): + result.update(_linked_person_names(item)) + return result + + +def _record(path: Path, root: Path) -> PersonRecord: + text = path.read_text(encoding="utf-8") + frontmatter, _ = _split_frontmatter(text) + links = { + link.strip().casefold() + for link in _LINK_RE.findall(text) + if link.strip().casefold() not in _IGNORED_CONTEXT_LINKS + and not link.startswith(("../", "Conversations/")) + } + snippets = [ + bullet.strip().lstrip("-").strip() + for bullet in _section_bullets(text, "About")[:2] + ] + return PersonRecord( + name=path.stem, + path=path.relative_to(root).as_posix(), + text=text, + content_hash=_sha256(text), + aliases=_aliases(frontmatter.get("aliases")), + distinct_from=_linked_person_names(frontmatter.get("distinct_from")), + org=_plain_value(frontmatter.get("org")), + role=_plain_value(frontmatter.get("role")), + links=links, + conversations=set(_CONVERSATION_RE.findall(text)), + photos={photo.casefold() for photo in _PHOTO_RE.findall(text)}, + snippets=snippets, + ) + + +def _pair_revision(left: PersonRecord, right: PersonRecord) -> str: + payload = { + "people": sorted( + [(left.path, left.content_hash), (right.path, right.content_hash)] + ) + } + return _sha256(json.dumps(payload, sort_keys=True, separators=(",", ":"))) + + +def _score_pair(left: PersonRecord, right: PersonRecord) -> tuple[int, list[str]]: + left_name = _normalise_name(left.name) + right_name = _normalise_name(right.name) + shorter = min(len(left_name), len(right_name)) + distance = _edit_distance(left.name, right.name) + similarity = SequenceMatcher(None, left_name, right_name).ratio() + score = 0 + identity_signal = False + reasons: list[str] = [] + + if right.name.casefold() in left.aliases or left.name.casefold() in right.aliases: + score += 100 + identity_signal = True + reasons.append("one name is already an alias of the other") + elif left_name == right_name: + score += 90 + identity_signal = True + reasons.append("names match after normalization") + + shared_photos = left.photos & right.photos + if shared_photos: + score += 90 + identity_signal = True + reasons.append("same person photo") + elif left.photos and right.photos: + score -= 25 + + if shorter >= 4 and distance == 1: + score += 45 + identity_signal = True + reasons.append("names differ by one character") + elif shorter >= 6 and distance == 2: + score += 25 + identity_signal = True + reasons.append("names differ by two characters") + + if similarity >= 0.88: + score += 25 + identity_signal = True + reasons.append("very similar spelling") + elif similarity >= 0.80: + score += 15 + identity_signal = True + reasons.append("similar spelling") + + left_tokens = _tokens(left.name) + right_tokens = _tokens(right.name) + if ( + left_tokens + and right_tokens + and left_tokens != right_tokens + and (left_tokens < right_tokens or right_tokens < left_tokens) + and shorter >= 4 + ): + score += 25 + identity_signal = True + reasons.append("one name appears to be a fuller form") + + shared_links = left.links & right.links + if shared_links: + score += min(18, len(shared_links) * 6) + reasons.append(f"shared context in {len(shared_links)} linked note(s)") + + shared_conversations = left.conversations & right.conversations + if shared_conversations: + score += min(24, len(shared_conversations) * 12) + reasons.append(f"same source conversation ({len(shared_conversations)})") + + if left.org and left.org == right.org: + score += 15 + reasons.append("same organization") + if left.role and left.role == right.role: + score += 8 + reasons.append("same role") + if not identity_signal: + return 0, [] + return score, reasons + + +class PersonIdentityService: + """Read identity candidates and write symmetric distinct-person decisions.""" + + def __init__(self, root: Path): + self.root = Path(root) + + def suggestions(self, limit: int = 20, min_score: int = 40) -> list[dict[str, Any]]: + people_dir = self.root / "People" + if not people_dir.is_dir(): + return [] + records = [_record(path, self.root) for path in sorted(people_dir.glob("*.md"))] + suggestions = [] + for left, right in combinations(records, 2): + if ( + right.name.casefold() in left.distinct_from + or left.name.casefold() in right.distinct_from + ): + continue + score, reasons = _score_pair(left, right) + if score < min_score: + continue + suggestions.append( + { + "pair_id": _sha256( + "\0".join(sorted([left.name.casefold(), right.name.casefold()])) + )[:16], + "revision": _pair_revision(left, right), + "score": score, + "reasons": reasons, + "person_a": { + "name": left.name, + "path": left.path, + "hash": left.content_hash, + "snippets": left.snippets, + }, + "person_b": { + "name": right.name, + "path": right.path, + "hash": right.content_hash, + "snippets": right.snippets, + }, + } + ) + suggestions.sort( + key=lambda item: ( + -item["score"], + item["person_a"]["name"].casefold(), + item["person_b"]["name"].casefold(), + ) + ) + return suggestions[:limit] + + def set_distinct( + self, + person_a: str, + person_b: str, + *, + distinct: bool, + revision: Optional[str] = None, + ) -> IdentityChangeResult: + try: + with vault_note_lock(self.root.name): + return self._set_distinct_locked( + person_a, person_b, distinct=distinct, revision=revision + ) + except VaultLockTimeout as exc: + raise PersonMergeError("The vault is busy. Retry shortly.") from exc + + def _set_distinct_locked( + self, + person_a: str, + person_b: str, + *, + distinct: bool, + revision: Optional[str], + ) -> IdentityChangeResult: + path_a = _resolve_person(self.root, person_a) + path_b = _resolve_person(self.root, person_b) + if path_a == path_b: + raise PersonMergeError( + "A person cannot be marked distinct from themselves." + ) + record_a = _record(path_a, self.root) + record_b = _record(path_b, self.root) + if revision and revision != _pair_revision(record_a, record_b): + raise PersonMergeStale( + "One of these people changed after the suggestion was shown. Review again." + ) + + new_a = self._update_distinct(record_a, record_b.name, distinct) + new_b = self._update_distinct(record_b, record_a.name, distinct) + before = {record_a.path: record_a.text, record_b.path: record_b.text} + after = {record_a.path: new_a, record_b.path: new_b} + changed = [path for path in after if after[path] != before[path]] + try: + for path in changed: + _atomic_write(self.root / path, after[path]) + except Exception: + for path, content in before.items(): + _atomic_write(self.root / path, content) + raise + return IdentityChangeResult( + action_id=str(uuid.uuid4()), + person_a=record_a.name, + person_b=record_b.name, + decision="distinct" if distinct else "clear_distinct", + changed_paths=sorted(changed), + before=before, + after=after, + ) + + def _update_distinct( + self, record: PersonRecord, other_name: str, distinct: bool + ) -> str: + frontmatter, body = _split_frontmatter(record.text) + values = _as_list(frontmatter.get("distinct_from")) + filtered = [ + value + for value in values + if other_name.casefold() not in _linked_person_names(value) + ] + if distinct: + filtered.append(f"[[{other_name}]]") + frontmatter["distinct_from"] = filtered + if "updated" in frontmatter: + frontmatter["updated"] = date.today().isoformat() + return _join_frontmatter(frontmatter, body) diff --git a/backends/advanced/src/advanced_omi_backend/services/memory/person_merge.py b/backends/advanced/src/advanced_omi_backend/services/memory/person_merge.py new file mode 100644 index 000000000..07681ce49 --- /dev/null +++ b/backends/advanced/src/advanced_omi_backend/services/memory/person_merge.py @@ -0,0 +1,464 @@ +"""Deterministic, transactional-ish person-note merges for the Chronicle vault. + +Identity resolution is intentionally outside this module: a human or agent decides +that two notes describe the same person. This module only executes the mechanical +operation with fixed rules, a preview token, the per-user vault lock, and rollback on +ordinary failures. +""" + +import hashlib +import io +import json +import os +import re +import tempfile +import uuid +from dataclasses import dataclass, field +from datetime import date +from pathlib import Path +from typing import Any, Optional + +from ruamel.yaml import YAML + +from .agent.section_edit import SectionEditError, apply_section_edit +from .vault_lock import VaultLockTimeout, vault_note_lock + +_H2_RE = re.compile(r"^##\s+(.+?)\s*$", re.MULTILINE) +_MERGE_SECTIONS = ("About", "Mentions") +_SCALAR_IDENTITY_FIELDS = ("org", "role", "relationship", "location") + + +class PersonMergeError(Exception): + """A person merge cannot be previewed or applied safely.""" + + +class PersonMergeStale(PersonMergeError): + """The vault changed after the caller read or previewed it.""" + + +@dataclass(frozen=True) +class MetadataConflict: + field: str + source_value: Any + target_value: Any + + def to_dict(self) -> dict[str, Any]: + return { + "field": self.field, + "source_value": self.source_value, + "target_value": self.target_value, + } + + +@dataclass +class PersonMergePreview: + source_name: str + target_name: str + source_path: str + target_path: str + source_hash: str + target_hash: str + plan_token: str + facts_to_add: int + duplicate_facts_skipped: int + backlink_files: list[str] + backlink_occurrences: int + metadata_conflicts: list[MetadataConflict] + _source_text: str = field(repr=False) + _target_text: str = field(repr=False) + _before: dict[str, str] = field(repr=False) + + def to_dict(self) -> dict[str, Any]: + return { + "source_name": self.source_name, + "target_name": self.target_name, + "source_path": self.source_path, + "target_path": self.target_path, + "source_hash": self.source_hash, + "target_hash": self.target_hash, + "plan_token": self.plan_token, + "facts_to_add": self.facts_to_add, + "duplicate_facts_skipped": self.duplicate_facts_skipped, + "backlink_files": self.backlink_files, + "backlink_occurrences": self.backlink_occurrences, + "metadata_conflicts": [item.to_dict() for item in self.metadata_conflicts], + } + + +@dataclass +class PersonMergeResult: + action_id: str + preview: PersonMergePreview + changed_paths: list[str] + before: dict[str, str] + after: dict[str, Optional[str]] + + def to_dict(self) -> dict[str, Any]: + return { + "action_id": self.action_id, + **self.preview.to_dict(), + "changed_paths": self.changed_paths, + } + + +def _sha256(text: str) -> str: + return hashlib.sha256(text.encode("utf-8")).hexdigest() + + +def _validate_name(name: str) -> str: + cleaned = name.strip() + if ( + not cleaned + or cleaned in (".", "..") + or Path(cleaned).name != cleaned + or "/" in cleaned + or "\\" in cleaned + ): + raise PersonMergeError( + "Person names must be plain note titles without slashes." + ) + return cleaned + + +def _resolve_person(root: Path, name: str) -> Path: + people = root / "People" + wanted = f"{_validate_name(name)}.md".casefold() + if not people.is_dir(): + raise PersonMergeError("The vault has no People folder.") + matches = [path for path in people.glob("*.md") if path.name.casefold() == wanted] + if not matches: + raise PersonMergeError(f"People/{name}.md does not exist.") + if len(matches) > 1: + raise PersonMergeError(f"Multiple case-variant notes match People/{name}.md.") + return matches[0] + + +def _split_frontmatter(text: str) -> tuple[dict[str, Any], str]: + if not text.startswith("---\n"): + raise PersonMergeError("Person note is missing YAML frontmatter.") + end = text.find("\n---\n", 4) + if end < 0: + raise PersonMergeError("Person note has malformed YAML frontmatter.") + yaml = YAML(typ="rt") + data = yaml.load(text[4:end]) or {} + if not isinstance(data, dict): + raise PersonMergeError("Person note frontmatter must be a mapping.") + return data, text[end + 5 :] + + +def _join_frontmatter(data: dict[str, Any], body: str) -> str: + yaml = YAML(typ="rt") + yaml.preserve_quotes = True + yaml.indent(mapping=2, sequence=4, offset=2) + stream = io.StringIO() + yaml.dump(data, stream) + return f"---\n{stream.getvalue()}---\n{body.lstrip()}" + + +def _as_list(value: Any) -> list[Any]: + if value is None or value == "": + return [] + return list(value) if isinstance(value, list) else [value] + + +def _union_values(*collections: list[Any]) -> list[Any]: + result: list[Any] = [] + seen: set[str] = set() + for collection in collections: + for value in collection: + marker = str(value).strip().casefold() + if marker and marker not in seen: + result.append(value) + seen.add(marker) + return result + + +def _linked_person_names(value: Any) -> set[str]: + """Return case-folded person titles from a frontmatter link/list value.""" + names: set[str] = set() + for item in _as_list(value): + text = str(item).strip() + match = re.fullmatch(r"\[\[(?:People/)?([^\]|#]+)(?:[|#][^\]]*)?\]\]", text) + title = match.group(1) if match else text + if title: + names.add(title.strip().casefold()) + return names + + +def _section_bullets(content: str, heading: str) -> list[str]: + wanted = heading.casefold() + found = False + result: list[str] = [] + for line in content.splitlines(): + match = _H2_RE.match(line.rstrip()) + if match: + found = match.group(1).casefold() == wanted + continue + stripped = line.strip() + if found and stripped.startswith("-") and stripped.lstrip("-").strip(): + result.append(line.rstrip()) + return result + + +def _normalise_bullet(line: str) -> str: + return " ".join(line.lstrip().lstrip("-").split()).casefold() + + +def _media_embeds(body: str) -> list[str]: + prologue = body.split("\n## ", 1)[0] + return [ + line.strip() for line in prologue.splitlines() if line.strip().startswith("![[") + ] + + +def _add_media_embeds(body: str, embeds: list[str]) -> str: + missing = [embed for embed in embeds if embed not in body] + if not missing: + return body + block = "\n".join(missing) + "\n" + return block + body.lstrip() + + +def _link_pattern(source_name: str) -> re.Pattern[str]: + return re.compile( + rf"(?P\[\[(?:People/)?)({re.escape(source_name)})" + rf"(?P(?:[#|][^\]]*)?\]\])", + re.IGNORECASE, + ) + + +def _rewrite_links(text: str, source_name: str, target_name: str) -> tuple[str, int]: + pattern = _link_pattern(source_name) + return pattern.subn( + lambda match: f"{match.group('prefix')}{target_name}{match.group('suffix')}", + text, + ) + + +def _atomic_write(path: Path, content: str) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + fd, temporary = tempfile.mkstemp(prefix=f".{path.name}.", dir=path.parent) + try: + with os.fdopen(fd, "w", encoding="utf-8") as handle: + handle.write(content) + handle.flush() + os.fsync(handle.fileno()) + os.replace(temporary, path) + except Exception: + try: + os.unlink(temporary) + except FileNotFoundError: + pass + raise + + +class PersonMergeService: + """Preview and apply one deterministic person merge inside a vault root.""" + + def __init__(self, root: Path): + self.root = Path(root) + + def preview( + self, + source_name: str, + target_name: str, + *, + expected_source_hash: Optional[str] = None, + expected_target_hash: Optional[str] = None, + ) -> PersonMergePreview: + source = _resolve_person(self.root, source_name) + target = _resolve_person(self.root, target_name) + if source == target: + raise PersonMergeError("Source and target resolve to the same person note.") + + source_text = source.read_text(encoding="utf-8") + target_text = target.read_text(encoding="utf-8") + source_hash = _sha256(source_text) + target_hash = _sha256(target_text) + if expected_source_hash and expected_source_hash != source_hash: + raise PersonMergeStale("The source note differs from the server copy.") + if expected_target_hash and expected_target_hash != target_hash: + raise PersonMergeStale("The target note differs from the server copy.") + + source_frontmatter, _ = _split_frontmatter(source_text) + target_frontmatter, _ = _split_frontmatter(target_text) + if target.stem.casefold() in _linked_person_names( + source_frontmatter.get("distinct_from") + ) or source.stem.casefold() in _linked_person_names( + target_frontmatter.get("distinct_from") + ): + raise PersonMergeError( + f"{source.stem} and {target.stem} are marked as separate people. " + "Clear that identity annotation before merging them." + ) + conflicts = [] + for key in _SCALAR_IDENTITY_FIELDS: + source_value = source_frontmatter.get(key) + target_value = target_frontmatter.get(key) + if source_value and target_value and source_value != target_value: + conflicts.append(MetadataConflict(key, source_value, target_value)) + + facts_to_add = 0 + duplicate_facts = 0 + for heading in _MERGE_SECTIONS: + existing = { + _normalise_bullet(line) + for line in _section_bullets(target_text, heading) + } + for bullet in _section_bullets(source_text, heading): + if _normalise_bullet(bullet) in existing: + duplicate_facts += 1 + else: + facts_to_add += 1 + existing.add(_normalise_bullet(bullet)) + + backlink_files: list[str] = [] + backlink_occurrences = 0 + before: dict[str, str] = { + source.relative_to(self.root).as_posix(): source_text, + target.relative_to(self.root).as_posix(): target_text, + } + pattern = _link_pattern(source.stem) + for path in sorted(self.root.rglob("*.md")): + if path == source: + continue + text = path.read_text(encoding="utf-8") + count = len(pattern.findall(text)) + if count: + rel = path.relative_to(self.root).as_posix() + backlink_files.append(rel) + backlink_occurrences += count + before[rel] = text + + token_payload = { + "source": source.relative_to(self.root).as_posix(), + "target": target.relative_to(self.root).as_posix(), + "files": {path: _sha256(text) for path, text in sorted(before.items())}, + } + plan_token = _sha256( + json.dumps(token_payload, sort_keys=True, separators=(",", ":")) + ) + return PersonMergePreview( + source_name=source.stem, + target_name=target.stem, + source_path=source.relative_to(self.root).as_posix(), + target_path=target.relative_to(self.root).as_posix(), + source_hash=source_hash, + target_hash=target_hash, + plan_token=plan_token, + facts_to_add=facts_to_add, + duplicate_facts_skipped=duplicate_facts, + backlink_files=backlink_files, + backlink_occurrences=backlink_occurrences, + metadata_conflicts=conflicts, + _source_text=source_text, + _target_text=target_text, + _before=before, + ) + + def apply( + self, source_name: str, target_name: str, plan_token: str + ) -> PersonMergeResult: + try: + with vault_note_lock(self.root.name): + preview = self.preview(source_name, target_name) + if preview.plan_token != plan_token: + raise PersonMergeStale( + "The vault changed after this merge was previewed. Preview it again." + ) + return self.apply_preview_locked(preview) + except VaultLockTimeout as exc: + raise PersonMergeError( + "The vault is busy. Retry the merge shortly." + ) from exc + + def apply_preview_locked(self, preview: PersonMergePreview) -> PersonMergeResult: + """Apply a preview while the caller already holds the per-user vault lock.""" + source = self.root / preview.source_path + target = self.root / preview.target_path + before = dict(preview._before) + after: dict[str, Optional[str]] = {} + changed: list[str] = [] + try: + merged = self._merge_person_notes(preview) + merged, _ = _rewrite_links(merged, preview.source_name, preview.target_name) + _atomic_write(target, merged) + after[preview.target_path] = merged + changed.append(preview.target_path) + + for rel in preview.backlink_files: + if rel == preview.target_path: + continue + rewritten, count = _rewrite_links( + before[rel], preview.source_name, preview.target_name + ) + if count: + _atomic_write(self.root / rel, rewritten) + after[rel] = rewritten + changed.append(rel) + + source.unlink() + after[preview.source_path] = None + changed.append(preview.source_path) + except Exception: + for rel, content in before.items(): + _atomic_write(self.root / rel, content) + raise + + return PersonMergeResult( + action_id=str(uuid.uuid4()), + preview=preview, + changed_paths=sorted(set(changed)), + before=before, + after=after, + ) + + def _merge_person_notes(self, preview: PersonMergePreview) -> str: + source_frontmatter, source_body = _split_frontmatter(preview._source_text) + target_frontmatter, target_body = _split_frontmatter(preview._target_text) + + target_frontmatter["categories"] = _union_values( + _as_list(target_frontmatter.get("categories")), + _as_list(source_frontmatter.get("categories")), + ) + target_frontmatter["aliases"] = _union_values( + _as_list(target_frontmatter.get("aliases")), + _as_list(source_frontmatter.get("aliases")), + [preview.source_name], + ) + source_distinct = [ + value + for value in _as_list(source_frontmatter.get("distinct_from")) + if preview.target_name.casefold() not in _linked_person_names(value) + ] + target_frontmatter["distinct_from"] = _union_values( + _as_list(target_frontmatter.get("distinct_from")), source_distinct + ) + for key in _SCALAR_IDENTITY_FIELDS: + if not target_frontmatter.get(key) and source_frontmatter.get(key): + target_frontmatter[key] = source_frontmatter[key] + if "updated" in target_frontmatter: + target_frontmatter["updated"] = date.today().isoformat() + + target_body = _add_media_embeds(target_body, _media_embeds(source_body)) + for heading in _MERGE_SECTIONS: + existing = { + _normalise_bullet(line) + for line in _section_bullets(target_body, heading) + } + additions = [] + for bullet in _section_bullets(source_body, heading): + marker = _normalise_bullet(bullet) + if marker not in existing: + additions.append(bullet) + existing.add(marker) + if additions: + try: + target_body = apply_section_edit( + target_body, heading, "\n".join(additions), "append" + ) + except SectionEditError as exc: + raise PersonMergeError( + f"Target person note is missing its {heading} section." + ) from exc + return _join_frontmatter(target_frontmatter, target_body) diff --git a/backends/advanced/src/advanced_omi_backend/services/memory/person_merge_actions.py b/backends/advanced/src/advanced_omi_backend/services/memory/person_merge_actions.py new file mode 100644 index 000000000..4bc5c312c --- /dev/null +++ b/backends/advanced/src/advanced_omi_backend/services/memory/person_merge_actions.py @@ -0,0 +1,122 @@ +"""Authenticated-service orchestration for deterministic person merges.""" + +import asyncio + +from .audit import MemoryCause, memory_provenance, record_vault_change +from .person_identity import IdentityChangeResult, PersonIdentityService +from .person_merge import PersonMergeResult, PersonMergeService +from .vault_manager import ConvDocVaultManager + + +def _service(user_id: str) -> PersonMergeService: + return PersonMergeService(ConvDocVaultManager().user_root(user_id)) + + +def _identity_service(user_id: str) -> PersonIdentityService: + return PersonIdentityService(ConvDocVaultManager().user_root(user_id)) + + +async def get_person_suggestions(user_id: str, limit: int = 20) -> list[dict]: + return await asyncio.to_thread(_identity_service(user_id).suggestions, limit) + + +async def set_people_distinct( + user_id: str, + person_a: str, + person_b: str, + *, + distinct: bool, + revision: str | None = None, +) -> dict: + result = await asyncio.to_thread( + _identity_service(user_id).set_distinct, + person_a, + person_b, + distinct=distinct, + revision=revision, + ) + await _record_identity_audit(user_id, result) + return result.to_dict() + + +async def preview_person_merge( + user_id: str, + source_name: str, + target_name: str, + source_hash: str | None = None, + target_hash: str | None = None, +) -> dict: + preview = await asyncio.to_thread( + _service(user_id).preview, + source_name, + target_name, + expected_source_hash=source_hash, + expected_target_hash=target_hash, + ) + return preview.to_dict() + + +async def apply_person_merge( + user_id: str, source_name: str, target_name: str, plan_token: str +) -> dict: + result = await asyncio.to_thread( + _service(user_id).apply, source_name, target_name, plan_token + ) + await _record_merge_audit(user_id, result) + return result.to_dict() + + +async def _record_merge_audit(user_id: str, result: PersonMergeResult) -> None: + action_id = result.action_id + source_path = result.preview.source_path + target_path = result.preview.target_path + with memory_provenance(MemoryCause.OBSIDIAN_ACTION): + for path in result.changed_paths: + before = result.before.get(path) + after = result.after.get(path) + if path == source_path: + await record_vault_change( + user_id=user_id, + operation="rename", + note_path=path, + before=before, + after=None, + summary=f"merged into {target_path}", + action_id=action_id, + new_path=target_path, + ) + continue + await record_vault_change( + user_id=user_id, + operation="update", + note_path=path, + before=before, + after=after, + summary=( + f"person merge {result.preview.source_name} → " + f"{result.preview.target_name}" + ), + action_id=action_id, + source_path=source_path, + target_path=target_path, + ) + + +async def _record_identity_audit(user_id: str, result: IdentityChangeResult) -> None: + with memory_provenance(MemoryCause.OBSIDIAN_ACTION): + for path in result.changed_paths: + await record_vault_change( + user_id=user_id, + operation="update", + note_path=path, + before=result.before[path], + after=result.after[path], + summary=( + f"identity decision: {result.person_a} and {result.person_b} are " + f"{'separate people' if result.decision == 'distinct' else 'no longer marked separate'}" + ), + action_id=result.action_id, + identity_decision=result.decision, + person_a=result.person_a, + person_b=result.person_b, + ) diff --git a/backends/advanced/src/advanced_omi_backend/services/memory/vault_templates.py b/backends/advanced/src/advanced_omi_backend/services/memory/vault_templates.py index 763d9d7e1..b876d1c82 100644 --- a/backends/advanced/src/advanced_omi_backend/services/memory/vault_templates.py +++ b/backends/advanced/src/advanced_omi_backend/services/memory/vault_templates.py @@ -51,6 +51,7 @@ categories: - "[[People]]" aliases: [] +distinct_from: [] org: role: relationship: diff --git a/backends/advanced/tests/test_person_identity.py b/backends/advanced/tests/test_person_identity.py new file mode 100644 index 000000000..8544b038e --- /dev/null +++ b/backends/advanced/tests/test_person_identity.py @@ -0,0 +1,226 @@ +"""Duplicate suggestions and durable distinct-person identity decisions.""" + +import contextlib + +import pytest +from ruamel.yaml import YAML + +from advanced_omi_backend.services.memory import person_identity, person_merge_actions +from advanced_omi_backend.services.memory.person_identity import PersonIdentityService +from advanced_omi_backend.services.memory.person_merge import ( + PersonMergeError, + PersonMergeService, + PersonMergeStale, +) + + +def _person( + name: str, + *, + aliases: list[str] | None = None, + distinct_from: list[str] | None = None, + org: str = "", + topic: str = "", + conversation: str = "", + photo: str = "", +) -> str: + aliases_yaml = ( + "\n" + "\n".join(f" - {value}" for value in aliases) if aliases else " []" + ) + distinct_yaml = ( + "\n" + "\n".join(f' - "[[{value}]]"' for value in distinct_from) + if distinct_from + else " []" + ) + image = f"![[../_media/{photo}|200]]\n" if photo else "" + context = f" Discussed [[{topic}]]." if topic else "" + mention = ( + f"- Met in [[Conversations/{conversation}|Conversation]].\n" + if conversation + else "- Mentioned once.\n" + ) + return ( + "---\n" + 'categories:\n - "[[People]]"\n' + f"aliases:{aliases_yaml}\n" + f"distinct_from:{distinct_yaml}\n" + f"org: {org}\n" + "role:\nrelationship:\nlocation:\n" + "created: 2026-08-01\nupdated: 2026-08-01\n" + "---\n" + f"{image}" + "## About\n" + f"- Information about {name}.{context}\n\n" + "## Conversations\n![[Conversations.base#Person]]\n\n" + "## Mentions\n" + f"{mention}" + ) + + +def _metadata(path) -> dict: + text = path.read_text(encoding="utf-8") + end = text.index("\n---\n", 4) + return YAML(typ="safe").load(text[4:end]) + + +@pytest.fixture +def vault(tmp_path): + people = tmp_path / "People" + people.mkdir() + (people / "Sabi.md").write_text( + _person( + "Sabi", + org="Acme", + topic="Model Training", + conversation="11111111-1111-1111-1111-111111111111", + ), + encoding="utf-8", + ) + (people / "Sabri.md").write_text( + _person( + "Sabri", + org="Acme", + topic="Model Training", + conversation="11111111-1111-1111-1111-111111111111", + ), + encoding="utf-8", + ) + (people / "Robert.md").write_text( + _person("Robert", aliases=["Bob"]), encoding="utf-8" + ) + (people / "Bob.md").write_text(_person("Bob"), encoding="utf-8") + (people / "Alice.md").write_text(_person("Alice"), encoding="utf-8") + (people / "Carlos.md").write_text( + _person( + "Carlos", + topic="Shared Project", + conversation="22222222-2222-2222-2222-222222222222", + ), + encoding="utf-8", + ) + (people / "Diana.md").write_text( + _person( + "Diana", + topic="Shared Project", + conversation="22222222-2222-2222-2222-222222222222", + ), + encoding="utf-8", + ) + return tmp_path + + +def test_suggestions_combine_name_alias_and_context_evidence(vault): + suggestions = PersonIdentityService(vault).suggestions() + by_pair = { + frozenset((item["person_a"]["name"], item["person_b"]["name"])): item + for item in suggestions + } + + sabi = by_pair[frozenset(("Sabi", "Sabri"))] + assert sabi["score"] >= 100 + assert "names differ by one character" in sabi["reasons"] + assert "same organization" in sabi["reasons"] + assert any( + reason.startswith("same source conversation") for reason in sabi["reasons"] + ) + assert sabi["revision"] + + robert = by_pair[frozenset(("Robert", "Bob"))] + assert "one name is already an alias of the other" in robert["reasons"] + assert frozenset(("Alice", "Bob")) not in by_pair + assert frozenset(("Carlos", "Diana")) not in by_pair + + +def test_distinct_decision_is_symmetric_and_removes_suggestion(vault, monkeypatch): + monkeypatch.setattr( + person_identity, "vault_note_lock", lambda _user: contextlib.nullcontext() + ) + service = PersonIdentityService(vault) + suggestion = next( + item + for item in service.suggestions() + if {item["person_a"]["name"], item["person_b"]["name"]} == {"Sabi", "Sabri"} + ) + + result = service.set_distinct( + "Sabi", "Sabri", distinct=True, revision=suggestion["revision"] + ) + + assert result.decision == "distinct" + assert set(result.changed_paths) == {"People/Sabi.md", "People/Sabri.md"} + assert _metadata(vault / "People/Sabi.md")["distinct_from"] == ["[[Sabri]]"] + assert _metadata(vault / "People/Sabri.md")["distinct_from"] == ["[[Sabi]]"] + assert not any( + {item["person_a"]["name"], item["person_b"]["name"]} == {"Sabi", "Sabri"} + for item in service.suggestions() + ) + + +def test_distinct_decision_blocks_merge_until_cleared(vault, monkeypatch): + monkeypatch.setattr( + person_identity, "vault_note_lock", lambda _user: contextlib.nullcontext() + ) + identity = PersonIdentityService(vault) + identity.set_distinct("Sabi", "Sabri", distinct=True) + + with pytest.raises(PersonMergeError, match="marked as separate people"): + PersonMergeService(vault).preview("Sabi", "Sabri") + + identity.set_distinct("Sabi", "Sabri", distinct=False) + preview = PersonMergeService(vault).preview("Sabi", "Sabri") + assert preview.source_name == "Sabi" + + +def test_distinct_decision_rejects_stale_suggestion(vault, monkeypatch): + monkeypatch.setattr( + person_identity, "vault_note_lock", lambda _user: contextlib.nullcontext() + ) + service = PersonIdentityService(vault) + suggestion = next( + item + for item in service.suggestions() + if {item["person_a"]["name"], item["person_b"]["name"]} == {"Sabi", "Sabri"} + ) + path = vault / "People/Sabi.md" + path.write_text(path.read_text(encoding="utf-8") + "\nChanged.\n", encoding="utf-8") + + with pytest.raises(PersonMergeStale, match="changed after the suggestion"): + service.set_distinct( + "Sabi", "Sabri", distinct=True, revision=suggestion["revision"] + ) + + +def test_existing_one_sided_annotation_is_respected(vault): + path = vault / "People/Sabi.md" + path.write_text( + _person("Sabi", distinct_from=["Sabri"]), + encoding="utf-8", + ) + suggestions = PersonIdentityService(vault).suggestions() + assert not any( + {item["person_a"]["name"], item["person_b"]["name"]} == {"Sabi", "Sabri"} + for item in suggestions + ) + with pytest.raises(PersonMergeError, match="marked as separate people"): + PersonMergeService(vault).preview("Sabi", "Sabri") + + +async def test_identity_decision_audits_both_notes(vault, monkeypatch): + monkeypatch.setattr( + person_identity, "vault_note_lock", lambda _user: contextlib.nullcontext() + ) + result = PersonIdentityService(vault).set_distinct("Sabi", "Sabri", distinct=True) + entries = [] + + async def capture(**kwargs): + entries.append(kwargs) + + monkeypatch.setattr(person_merge_actions, "record_vault_change", capture) + await person_merge_actions._record_identity_audit("user-1", result) + + assert {entry["note_path"] for entry in entries} == { + "People/Sabi.md", + "People/Sabri.md", + } + assert {entry["identity_decision"] for entry in entries} == {"distinct"} + assert {entry["action_id"] for entry in entries} == {result.action_id} diff --git a/backends/advanced/tests/test_person_merge.py b/backends/advanced/tests/test_person_merge.py new file mode 100644 index 000000000..729875738 --- /dev/null +++ b/backends/advanced/tests/test_person_merge.py @@ -0,0 +1,236 @@ +"""Deterministic person-merge behavior shared by API, Obsidian, and agents.""" + +import contextlib +import hashlib + +import pytest +from ruamel.yaml import YAML + +from advanced_omi_backend.services.memory import person_merge, person_merge_actions +from advanced_omi_backend.services.memory.audit import ( + MemoryCause, + actor_for, + source_label_for, +) +from advanced_omi_backend.services.memory.person_merge import ( + PersonMergeService, + PersonMergeStale, +) + + +def _person( + name: str, + about: list[str], + mentions: list[str], + *, + aliases: list[str] | None = None, + distinct_from: list[str] | None = None, + org: str = "", + role: str = "", + photo: str = "", +) -> str: + alias_lines = "\n".join(f" - {alias}" for alias in aliases or []) or "[]" + if aliases: + alias_value = f"\n{alias_lines}" + else: + alias_value = " []" + distinct_lines = ( + "\n" + "\n".join(f' - "[[{name}]]"' for name in distinct_from) + if distinct_from + else " []" + ) + image = f"![[../_media/{photo}|200]]\n" if photo else "" + return ( + "---\n" + 'categories:\n - "[[People]]"\n' + f"aliases:{alias_value}\n" + f"distinct_from:{distinct_lines}\n" + f"org: {org}\n" + f"role: {role}\n" + "relationship:\nlocation:\ncreated: 2026-07-28\nupdated: 2026-07-28\n" + "---\n" + f"{image}" + "## About\n" + + "\n".join(f"- {fact}" for fact in about) + + "\n\n## Conversations\n![[Conversations.base#Person]]\n\n## Mentions\n" + + "\n".join(f"- {mention}" for mention in mentions) + + "\n" + ) + + +def _frontmatter(text: str) -> dict: + end = text.index("\n---\n", 4) + return YAML(typ="safe").load(text[4:end]) + + +@pytest.fixture +def vault(tmp_path): + people = tmp_path / "People" + conversations = tmp_path / "Conversations" + people.mkdir() + conversations.mkdir() + (people / "Amay.md").write_text( + _person( + "Amay", + ["Owns the radar pipeline.", "Shared fact."], + ["2026-07-28 — Planned work."], + aliases=["A. May"], + distinct_from=["Carol"], + org="Acme", + role="Engineer", + photo="amay.jpg", + ), + encoding="utf-8", + ) + (people / "Amey.md").write_text( + _person( + "Amey", + ["Discussed model parsing.", "Shared fact."], + ["2026-07-28 — Discussed metrics."], + aliases=["A Mehta"], + org="Acme", + role="Lead", + photo="amey.jpg", + ), + encoding="utf-8", + ) + (conversations / "one.md").write_text( + 'people:\n - "[[Amay]]"\n- [[Amay]] owns this.\n', encoding="utf-8" + ) + (conversations / "two.md").write_text( + "See [[People/Amay|Amay from work]].\n", encoding="utf-8" + ) + return tmp_path + + +def test_preview_is_read_only_and_reports_complete_plan(vault): + service = PersonMergeService(vault) + source_before = (vault / "People/Amay.md").read_text(encoding="utf-8") + + preview = service.preview("amay", "AMEY") + + assert preview.source_name == "Amay" + assert preview.target_name == "Amey" + assert preview.facts_to_add == 2 + assert preview.duplicate_facts_skipped == 1 + assert preview.backlink_files == ["Conversations/one.md", "Conversations/two.md"] + assert preview.backlink_occurrences == 3 + assert [conflict.field for conflict in preview.metadata_conflicts] == ["role"] + assert (vault / "People/Amay.md").read_text(encoding="utf-8") == source_before + + +def test_apply_merges_metadata_facts_media_and_backlinks(vault, monkeypatch): + monkeypatch.setattr( + person_merge, "vault_note_lock", lambda _user: contextlib.nullcontext() + ) + service = PersonMergeService(vault) + preview = service.preview("Amay", "Amey") + + result = service.apply("Amay", "Amey", preview.plan_token) + + assert not (vault / "People/Amay.md").exists() + merged = (vault / "People/Amey.md").read_text(encoding="utf-8") + metadata = _frontmatter(merged) + assert metadata["aliases"] == ["A Mehta", "A. May", "Amay"] + assert metadata["distinct_from"] == ["[[Carol]]"] + assert metadata["org"] == "Acme" + assert metadata["role"] == "Lead" + assert "Owns the radar pipeline" in merged + assert merged.count("Shared fact") == 1 + assert "amey.jpg" in merged and "amay.jpg" in merged + assert "[[Amay]]" not in (vault / "Conversations/one.md").read_text( + encoding="utf-8" + ) + assert "[[People/Amey|Amay from work]]" in ( + vault / "Conversations/two.md" + ).read_text(encoding="utf-8") + assert set(result.changed_paths) == { + "People/Amay.md", + "People/Amey.md", + "Conversations/one.md", + "Conversations/two.md", + } + + +def test_apply_rejects_a_stale_preview(vault, monkeypatch): + monkeypatch.setattr( + person_merge, "vault_note_lock", lambda _user: contextlib.nullcontext() + ) + service = PersonMergeService(vault) + preview = service.preview("Amay", "Amey") + target = vault / "People/Amey.md" + target.write_text( + target.read_text(encoding="utf-8") + "\nNew edit.\n", encoding="utf-8" + ) + + with pytest.raises(PersonMergeStale, match="Preview it again"): + service.apply("Amay", "Amey", preview.plan_token) + + +def test_preview_rejects_a_local_copy_that_is_not_synced(vault): + local_hash = hashlib.sha256(b"older local note").hexdigest() + with pytest.raises(PersonMergeStale, match="source note differs"): + PersonMergeService(vault).preview( + "Amay", "Amey", expected_source_hash=local_hash + ) + + +def test_apply_rolls_back_files_when_a_write_fails(vault, monkeypatch): + monkeypatch.setattr( + person_merge, "vault_note_lock", lambda _user: contextlib.nullcontext() + ) + service = PersonMergeService(vault) + preview = service.preview("Amay", "Amey") + before = { + path.relative_to(vault).as_posix(): path.read_text(encoding="utf-8") + for path in vault.rglob("*.md") + } + real_write = person_merge._atomic_write + failed = False + + def fail_once(path, content): + nonlocal failed + if not failed and path.name == "one.md": + failed = True + raise OSError("simulated write failure") + real_write(path, content) + + monkeypatch.setattr(person_merge, "_atomic_write", fail_once) + with pytest.raises(OSError, match="simulated"): + service.apply("Amay", "Amey", preview.plan_token) + + after = { + path.relative_to(vault).as_posix(): path.read_text(encoding="utf-8") + for path in vault.rglob("*.md") + } + assert after == before + + +async def test_merge_audit_covers_every_changed_note(vault, monkeypatch): + monkeypatch.setattr( + person_merge, "vault_note_lock", lambda _user: contextlib.nullcontext() + ) + service = PersonMergeService(vault) + preview = service.preview("Amay", "Amey") + result = service.apply("Amay", "Amey", preview.plan_token) + entries = [] + + async def capture(**kwargs): + entries.append(kwargs) + + monkeypatch.setattr(person_merge_actions, "record_vault_change", capture) + await person_merge_actions._record_merge_audit("user-1", result) + + assert {entry["note_path"] for entry in entries} == set(result.changed_paths) + assert {entry["action_id"] for entry in entries} == {result.action_id} + source = next(entry for entry in entries if entry["note_path"] == "People/Amay.md") + assert source["operation"] == "rename" + assert source["after"] is None + assert source["new_path"] == "People/Amey.md" + + +def test_obsidian_action_provenance_is_human(): + assert source_label_for(MemoryCause.OBSIDIAN_ACTION, False, "update") == ( + "Obsidian action" + ) + assert actor_for(MemoryCause.OBSIDIAN_ACTION, False, "update") == "human_external" diff --git a/docs/backend/memories.md b/docs/backend/memories.md index bb70780f5..2de25c63b 100644 --- a/docs/backend/memories.md +++ b/docs/backend/memories.md @@ -94,6 +94,9 @@ Chat is always **agentic / tool-calling**. The chat LLM is given a `search_memor ## API Endpoints - `GET /api/memories/search?query={query}&limit={limit}` — runs the agentic vault search and returns the synthesized answer plus the notes the read agent consulted. +- `GET /api/memories/people/suggestions` — ranks conservative deterministic duplicate-person candidates for review; it never merges automatically. +- `POST /api/memories/people/identity` — records or clears a symmetric `distinct_from` decision in two People notes, with optional stale-revision protection. +- `POST /api/memories/people/merge/preview` and `POST /api/memories/people/merge` — preview and apply a locked deterministic merge. A `distinct_from` decision blocks preview. - Other `/api/memories/*` management endpoints operate over the vault notes. ## Vault sync to Obsidian (separate feature) From b5dc4749d688dcf06239bdefd0f6b00296b11b9c Mon Sep 17 00:00:00 2001 From: Ankush <43288948+AnkushMalaker@users.noreply.github.com> Date: Thu, 6 Aug 2026 03:31:05 +0530 Subject: [PATCH 05/18] feat(obsidian): Chronicle Companion plugin for semantic vault actions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A deliberately small Obsidian community plugin (extras/obsidian-chronicle) for explicit maintenance of a synced Chronicle vault. It runs no LLM and never mutates Markdown itself: it gathers intent plus local file hashes, previews the operation through the backend person-merge API, and submits the exact approved plan. Stale local state surfaces as the same HTTP 409 conflict the server uses for its own plan tokens. First action: Merge current person… from a People/ note, with a preview of fact/link counts and metadata conflicts before confirming. Docs: docs/obsidian-companion.md, linked from docs/README.md and the memory-system page. --- docs/README.md | 1 + docs/backend/memories.md | 5 + docs/obsidian-companion.md | 63 ++ extras/obsidian-chronicle/.gitignore | 2 + extras/obsidian-chronicle/esbuild.config.mjs | 23 + extras/obsidian-chronicle/main.js | 446 +++++++++++++ extras/obsidian-chronicle/main.ts | 569 ++++++++++++++++ extras/obsidian-chronicle/manifest.json | 9 + extras/obsidian-chronicle/package-lock.json | 656 +++++++++++++++++++ extras/obsidian-chronicle/package.json | 16 + extras/obsidian-chronicle/styles.css | 119 ++++ extras/obsidian-chronicle/tsconfig.json | 21 + extras/obsidian-chronicle/versions.json | 3 + 13 files changed, 1933 insertions(+) create mode 100644 docs/obsidian-companion.md create mode 100644 extras/obsidian-chronicle/.gitignore create mode 100644 extras/obsidian-chronicle/esbuild.config.mjs create mode 100644 extras/obsidian-chronicle/main.js create mode 100644 extras/obsidian-chronicle/main.ts create mode 100644 extras/obsidian-chronicle/manifest.json create mode 100644 extras/obsidian-chronicle/package-lock.json create mode 100644 extras/obsidian-chronicle/package.json create mode 100644 extras/obsidian-chronicle/styles.css create mode 100644 extras/obsidian-chronicle/tsconfig.json create mode 100644 extras/obsidian-chronicle/versions.json diff --git a/docs/README.md b/docs/README.md index f6260f31c..f72b5b81d 100644 --- a/docs/README.md +++ b/docs/README.md @@ -21,6 +21,7 @@ day-to-day operation, and use [AGENTS.md](../AGENTS.md) for development conventi - [Authentication](backend/auth.md): user identity, JWTs, and protected endpoints - [Memory system](backend/memories.md): agentic Markdown vault and retrieval +- [Obsidian companion](obsidian-companion.md): deterministic vault maintenance from Obsidian and agent skills - [Audio durability](backend/audio-durability.md): raw-audio write path and its state machine - [Data archive and memory rebuild](backend/data-archive.md): full export/import and clean vault reconstruction - [Plugin configuration](backend/plugin-configuration.md): configuration and secret boundaries diff --git a/docs/backend/memories.md b/docs/backend/memories.md index 2de25c63b..23d607822 100644 --- a/docs/backend/memories.md +++ b/docs/backend/memories.md @@ -103,6 +103,11 @@ Chat is always **agentic / tool-calling**. The chat LLM is given a `search_memor The vault is designed to be edited and viewed directly. The optional **vault sync** feature (in the cross-platform desktop tray, `extras/chronicle-tray/`) syncs `data/conversation_docs/` to an Obsidian vault via Syncthing, so you can browse and hand-edit your memory notes in Obsidian. Human edits made in Obsidian sync back into the vault. This sync is independent of the memory provider itself — the vault on the backend remains the source of truth. +The optional [Chronicle Companion](../obsidian-companion.md) plugin adds explicit, +deterministic maintenance actions such as merging duplicate people. The UI previews and +confirms the action, while the backend performs the locked mutation; no LLM participates +in execution. + ## What was removed For historical context, the previous architecture used **FalkorDB** hybrid search (vector + BM25 + entity-graph BFS over ConvDoc/ConvChunk/ConvEntity nodes and a knowledge graph), plus alternative providers (OpenMemory MCP, Graphiti) and Qdrant/Mem0 vector storage. **All of these have been removed.** There is now a single `chronicle` provider backed entirely by the Markdown vault; the `falkordb` container and `FALKORDB_*` environment variables no longer exist. diff --git a/docs/obsidian-companion.md b/docs/obsidian-companion.md new file mode 100644 index 000000000..1ae226b86 --- /dev/null +++ b/docs/obsidian-companion.md @@ -0,0 +1,63 @@ +# Obsidian companion + +Chronicle Companion is a deliberately small Obsidian community plugin for semantic +maintenance of a synced Chronicle vault. It does not run an LLM and does not mutate +Markdown directly. The plugin gathers intent and local file revisions, previews the +operation through Chronicle, and submits the exact approved plan to the backend. + +## Person merge + +Open a note directly under `People/`, then run **Chronicle: Merge current person…** +from the command palette. Select the canonical person, review the fact/link counts and +metadata conflicts, and confirm the merge. Syncthing delivers the backend-authored files +back to Obsidian. + +The backend applies fixed rules: + +- retain the target as the canonical note; +- add the source name and aliases to the target aliases; +- union categories, copy identity metadata into empty target fields, and retain target + values when both sides conflict; +- retain media embeds and merge unique `About`/`Mentions` bullets; +- rewrite direct and `People/`-qualified wikilinks; +- delete the source only after the other writes succeed; +- rollback ordinary write failures and record every changed path under one action ID. + +Preview returns a token derived from all affected server-side files. Apply recomputes the +plan under Chronicle's per-user vault lock and rejects a stale token with HTTP 409. +Obsidian also supplies hashes of the local source and target notes, so it receives the +same conflict when Syncthing has not delivered a local edit to the backend yet. + +## Duplicate review + +Run **Chronicle: Review possible duplicate people…** from the command palette. Chronicle +uses conservative deterministic signals—name/alias similarity or a shared photo—and may +add shared conversation, link, organization, or role context to rank the candidates. +Context alone never creates a suggestion, and no suggestion is merged automatically. + +Each card supports three outcomes: + +- **Same person…** selects the canonical name and opens the ordinary merge preview; +- **Separate people** writes symmetric `distinct_from` wikilinks into both People notes, + audits the changes, suppresses the candidate, and blocks an accidental later merge; +- **Not sure** hides that exact candidate revision only in the local plugin settings. A + change to either note gives the candidate a new revision, allowing it to surface again. + +The backend rejects a stale separate-person decision if either note changed after the +suggestion was shown. + +## Configuration + +Build from `extras/obsidian-chronicle/` with `npm install && npm run build`. Install +`main.js`, `manifest.json`, and `styles.css` under +`.obsidian/plugins/chronicle-companion/`. + +In Obsidian settings, configure the Chronicle HTTPS address and select a long-lived, +revocable Chronicle API key through Obsidian SecretStorage. The plugin stores only the +secret's name in its own `data.json`. + +## Automation skill + +The shared `chronicle-merge-person` Agent Skill uses the same suggestion, identity, and +merge endpoints. It never edits vault files itself. This keeps identity discovery +optionally agentic while keeping every mutation deterministic. diff --git a/extras/obsidian-chronicle/.gitignore b/extras/obsidian-chronicle/.gitignore new file mode 100644 index 000000000..dce19d84d --- /dev/null +++ b/extras/obsidian-chronicle/.gitignore @@ -0,0 +1,2 @@ +node_modules/ +*.map diff --git a/extras/obsidian-chronicle/esbuild.config.mjs b/extras/obsidian-chronicle/esbuild.config.mjs new file mode 100644 index 000000000..e01d0b660 --- /dev/null +++ b/extras/obsidian-chronicle/esbuild.config.mjs @@ -0,0 +1,23 @@ +import esbuild from "esbuild"; +import process from "process"; +import builtins from "builtin-modules"; + +const production = process.argv[2] === "production"; +const context = await esbuild.context({ + entryPoints: ["main.ts"], + bundle: true, + external: ["obsidian", "electron", "@codemirror/autocomplete", "@codemirror/collab", "@codemirror/commands", "@codemirror/language", "@codemirror/lint", "@codemirror/search", "@codemirror/state", "@codemirror/view", "@lezer/common", "@lezer/highlight", "@lezer/lr", ...builtins], + format: "cjs", + target: "es2018", + logLevel: "info", + sourcemap: production ? false : "inline", + treeShaking: true, + outfile: "main.js" +}); + +if (production) { + await context.rebuild(); + await context.dispose(); +} else { + await context.watch(); +} diff --git a/extras/obsidian-chronicle/main.js b/extras/obsidian-chronicle/main.js new file mode 100644 index 000000000..29cdb7420 --- /dev/null +++ b/extras/obsidian-chronicle/main.js @@ -0,0 +1,446 @@ +var __defProp = Object.defineProperty; +var __getOwnPropDesc = Object.getOwnPropertyDescriptor; +var __getOwnPropNames = Object.getOwnPropertyNames; +var __hasOwnProp = Object.prototype.hasOwnProperty; +var __export = (target, all) => { + for (var name in all) + __defProp(target, name, { get: all[name], enumerable: true }); +}; +var __copyProps = (to, from, except, desc) => { + if (from && typeof from === "object" || typeof from === "function") { + for (let key of __getOwnPropNames(from)) + if (!__hasOwnProp.call(to, key) && key !== except) + __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable }); + } + return to; +}; +var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod); + +// main.ts +var main_exports = {}; +__export(main_exports, { + default: () => ChronicleCompanionPlugin +}); +module.exports = __toCommonJS(main_exports); +var import_obsidian = require("obsidian"); +var DEFAULT_SETTINGS = { + baseUrl: "", + secretName: "", + dismissedSuggestions: [] +}; +async function sha256(text) { + const bytes = new TextEncoder().encode(text); + const digest = await crypto.subtle.digest("SHA-256", bytes); + return Array.from(new Uint8Array(digest)).map((byte) => byte.toString(16).padStart(2, "0")).join(""); +} +function personName(file) { + var _a; + return ((_a = file.parent) == null ? void 0 : _a.path) === "People" ? file.basename : null; +} +var ChronicleCompanionPlugin = class extends import_obsidian.Plugin { + constructor() { + super(...arguments); + this.settings = DEFAULT_SETTINGS; + } + async onload() { + this.settings = Object.assign({}, DEFAULT_SETTINGS, await this.loadData()); + await this.migrateBootstrapApiKey(); + this.addSettingTab(new ChronicleSettingTab(this.app, this)); + this.addCommand({ + id: "merge-current-person", + name: "Merge current person\u2026", + checkCallback: (checking) => { + const source = this.app.workspace.getActiveFile(); + if (!source || !personName(source)) return false; + if (!checking) new PersonPickerModal(this, source).open(); + return true; + } + }); + this.addCommand({ + id: "review-duplicate-people", + name: "Review possible duplicate people\u2026", + callback: () => void this.reviewDuplicatePeople() + }); + } + async saveSettings() { + await this.saveData(this.settings); + } + async migrateBootstrapApiKey() { + if (!this.settings.bootstrapApiKey) return; + const secretName = "chronicle-companion-api-key"; + this.app.secretStorage.setSecret(secretName, this.settings.bootstrapApiKey); + this.settings.secretName = secretName; + delete this.settings.bootstrapApiKey; + await this.saveSettings(); + } + apiKey() { + if (!this.settings.secretName) { + throw new Error("Choose a Chronicle API key in the plugin settings."); + } + const value = this.app.secretStorage.getSecret(this.settings.secretName); + if (!value) throw new Error("The selected Chronicle API key is empty."); + return value; + } + endpoint(path) { + const base = this.settings.baseUrl.trim().replace(/\/+$/, ""); + if (!base) throw new Error("Set the Chronicle server URL in the plugin settings."); + return `${base}${path}`; + } + async previewMerge(source, target) { + const [sourceText, targetText] = await Promise.all([ + this.app.vault.read(source), + this.app.vault.read(target) + ]); + const response = await (0, import_obsidian.requestUrl)({ + url: this.endpoint("/api/memories/people/merge/preview"), + method: "POST", + headers: { + Authorization: `Bearer ${this.apiKey()}`, + "Content-Type": "application/json" + }, + body: JSON.stringify({ + source_name: source.basename, + target_name: target.basename, + source_hash: await sha256(sourceText), + target_hash: await sha256(targetText) + }) + }); + return response.json; + } + async applyMerge(preview) { + await (0, import_obsidian.requestUrl)({ + url: this.endpoint("/api/memories/people/merge"), + method: "POST", + headers: { + Authorization: `Bearer ${this.apiKey()}`, + "Content-Type": "application/json" + }, + body: JSON.stringify({ + source_name: preview.source_name, + target_name: preview.target_name, + plan_token: preview.plan_token + }) + }); + } + async getSuggestions() { + const response = await (0, import_obsidian.requestUrl)({ + url: this.endpoint("/api/memories/people/suggestions?limit=30"), + method: "GET", + headers: { Authorization: `Bearer ${this.apiKey()}` } + }); + return response.json.suggestions; + } + async markSeparate(suggestion) { + await (0, import_obsidian.requestUrl)({ + url: this.endpoint("/api/memories/people/identity"), + method: "POST", + headers: { + Authorization: `Bearer ${this.apiKey()}`, + "Content-Type": "application/json" + }, + body: JSON.stringify({ + person_a: suggestion.person_a.name, + person_b: suggestion.person_b.name, + decision: "distinct", + revision: suggestion.revision + }) + }); + } + async dismissSuggestion(suggestion) { + const dismissal = `${suggestion.pair_id}:${suggestion.revision}`; + this.settings.dismissedSuggestions = [ + .../* @__PURE__ */ new Set([...this.settings.dismissedSuggestions, dismissal]) + ].slice(-500); + await this.saveSettings(); + } + async reviewDuplicatePeople() { + new import_obsidian.Notice("Looking for possible duplicate people\u2026"); + try { + const suggestions = await this.getSuggestions(); + const dismissed = new Set(this.settings.dismissedSuggestions); + const pending = suggestions.filter( + (suggestion) => !dismissed.has(`${suggestion.pair_id}:${suggestion.revision}`) + ); + if (!pending.length) { + new import_obsidian.Notice("No new possible duplicate people found."); + return; + } + new DuplicateReviewModal(this, pending).open(); + } catch (error) { + new import_obsidian.Notice(`Could not load suggestions: ${errorMessage(error)}`, 8e3); + } + } +}; +var DuplicateReviewModal = class extends import_obsidian.Modal { + constructor(plugin, suggestions) { + super(plugin.app); + this.index = 0; + this.acting = false; + this.plugin = plugin; + this.suggestions = suggestions; + } + onOpen() { + this.render(); + } + render() { + this.contentEl.empty(); + const suggestion = this.suggestions[this.index]; + if (!suggestion) { + this.close(); + new import_obsidian.Notice("Duplicate review complete."); + return; + } + this.setTitle("Possible duplicate people"); + const header = this.contentEl.createDiv({ cls: "chronicle-identity-header" }); + header.createEl("p", { + cls: "chronicle-merge-help", + text: `Suggestion ${this.index + 1} of ${this.suggestions.length} \xB7 confidence ${suggestion.score}` + }); + const navigation = header.createDiv({ cls: "chronicle-identity-navigation" }); + new import_obsidian.ButtonComponent(navigation).setButtonText("Back").setDisabled(this.index === 0).onClick(() => this.move(-1)); + new import_obsidian.ButtonComponent(navigation).setButtonText("Next").setDisabled(this.index === this.suggestions.length - 1).onClick(() => this.move(1)); + const comparison = this.contentEl.createDiv({ + cls: "chronicle-identity-comparison" + }); + this.addPerson(comparison, suggestion.person_a); + this.addPerson(comparison, suggestion.person_b); + const evidence = this.contentEl.createDiv({ cls: "chronicle-identity-evidence" }); + evidence.createDiv({ + cls: "chronicle-identity-evidence-title", + text: "Why Chronicle suggested this" + }); + const reasons = evidence.createEl("ul"); + for (const reason of suggestion.reasons) reasons.createEl("li", { text: reason }); + this.contentEl.createEl("p", { + cls: "chronicle-merge-help", + text: "Separate people is a durable vault annotation. Not sure hides only this version of the suggestion." + }); + const actions = this.contentEl.createDiv({ cls: "chronicle-identity-actions" }); + new import_obsidian.ButtonComponent(actions).setButtonText("Not sure").onClick(() => void this.dismiss()); + new import_obsidian.ButtonComponent(actions).setButtonText("Separate people").onClick(() => void this.separate()); + new import_obsidian.ButtonComponent(actions).setButtonText("Same person\u2026").setCta().onClick(() => { + this.close(); + new CanonicalPersonModal(this.plugin, suggestion).open(); + }); + } + move(offset) { + const nextIndex = this.index + offset; + if (nextIndex < 0 || nextIndex >= this.suggestions.length) return; + this.index = nextIndex; + this.render(); + } + removeCurrent() { + this.suggestions.splice(this.index, 1); + if (this.index >= this.suggestions.length) this.index = this.suggestions.length - 1; + this.render(); + } + addPerson(parent, person) { + const card = parent.createDiv({ cls: "chronicle-identity-person" }); + card.createEl("h3", { text: person.name }); + card.createEl("div", { cls: "chronicle-merge-path", text: person.path }); + if (person.snippets.length) { + const snippets = card.createEl("ul"); + for (const snippet of person.snippets.slice(0, 3)) { + snippets.createEl("li", { text: snippet }); + } + } + const open = card.createEl("button", { text: "Open note" }); + open.addEventListener("click", () => void this.openNote(person.path)); + } + async openNote(path) { + const file = this.app.vault.getAbstractFileByPath(path); + if (!(file instanceof import_obsidian.TFile)) { + new import_obsidian.Notice(`Could not find ${path}. The vault may still be syncing.`); + return; + } + await this.app.workspace.getLeaf("tab").openFile(file); + } + async dismiss() { + if (this.acting) return; + this.acting = true; + await this.plugin.dismissSuggestion(this.suggestions[this.index]); + this.acting = false; + this.removeCurrent(); + } + async separate() { + if (this.acting) return; + this.acting = true; + const suggestion = this.suggestions[this.index]; + try { + await this.plugin.markSeparate(suggestion); + new import_obsidian.Notice(`${suggestion.person_a.name} and ${suggestion.person_b.name} marked as separate.`); + this.acting = false; + this.removeCurrent(); + } catch (error) { + this.acting = false; + new import_obsidian.Notice(`Could not save decision: ${errorMessage(error)}`, 8e3); + } + } +}; +var CanonicalPersonModal = class extends import_obsidian.Modal { + constructor(plugin, suggestion) { + super(plugin.app); + this.plugin = plugin; + this.suggestion = suggestion; + } + onOpen() { + const { person_a: personA, person_b: personB } = this.suggestion; + this.setTitle("Which name should Chronicle keep?"); + this.contentEl.createEl("p", { + cls: "chronicle-merge-help", + text: "The other name is retained as an alias, and links are rewritten after a final preview." + }); + const actions = this.contentEl.createDiv({ cls: "chronicle-canonical-actions" }); + new import_obsidian.ButtonComponent(actions).setButtonText(`Keep ${personA.name}`).setCta().onClick(() => void this.choose(personB, personA)); + new import_obsidian.ButtonComponent(actions).setButtonText(`Keep ${personB.name}`).setCta().onClick(() => void this.choose(personA, personB)); + } + async choose(sourcePerson, targetPerson) { + const source = this.app.vault.getAbstractFileByPath(sourcePerson.path); + const target = this.app.vault.getAbstractFileByPath(targetPerson.path); + if (!(source instanceof import_obsidian.TFile) || !(target instanceof import_obsidian.TFile)) { + new import_obsidian.Notice("One of the person notes is not available yet. Wait for vault sync and retry."); + return; + } + this.close(); + new import_obsidian.Notice("Checking that the Chronicle vault is in sync\u2026"); + try { + const preview = await this.plugin.previewMerge(source, target); + new MergePreviewModal(this.plugin, preview).open(); + } catch (error) { + new import_obsidian.Notice(`Could not preview merge: ${errorMessage(error)}`, 8e3); + } + } +}; +var PersonPickerModal = class extends import_obsidian.Modal { + constructor(plugin, source) { + super(plugin.app); + this.people = []; + this.plugin = plugin; + this.source = source; + } + onOpen() { + this.setTitle(`Merge ${this.source.basename} into\u2026`); + this.people = this.app.vault.getMarkdownFiles().filter((file) => personName(file) && file.path !== this.source.path).sort((left, right) => left.basename.localeCompare(right.basename)); + const search = new import_obsidian.SearchComponent(this.contentEl); + search.setPlaceholder("Find the canonical person"); + search.inputEl.setAttr("aria-label", "Find the canonical person"); + search.onChange((query) => this.renderPeople(query)); + this.listEl = this.contentEl.createDiv({ cls: "chronicle-merge-list" }); + this.renderPeople(""); + search.inputEl.focus(); + } + renderPeople(query) { + this.listEl.empty(); + const normalized = query.trim().toLocaleLowerCase(); + const matches = this.people.filter( + (file) => file.basename.toLocaleLowerCase().includes(normalized) + ); + for (const file of matches) { + const button = this.listEl.createEl("button", { + cls: "chronicle-merge-person clickable-icon", + text: file.basename + }); + button.setAttr("aria-label", `Merge ${this.source.basename} into ${file.basename}`); + button.addEventListener("click", () => void this.choose(file)); + } + if (!matches.length) { + this.listEl.createDiv({ cls: "chronicle-merge-help", text: "No matching people." }); + } + } + async choose(target) { + this.close(); + new import_obsidian.Notice("Checking that the Chronicle vault is in sync\u2026"); + try { + const preview = await this.plugin.previewMerge(this.source, target); + new MergePreviewModal(this.plugin, preview).open(); + } catch (error) { + new import_obsidian.Notice(`Could not preview merge: ${errorMessage(error)}`, 8e3); + } + } +}; +var MergePreviewModal = class extends import_obsidian.Modal { + constructor(plugin, preview) { + super(plugin.app); + this.applying = false; + this.plugin = plugin; + this.preview = preview; + } + onOpen() { + this.setTitle(`Merge ${this.preview.source_name} into ${this.preview.target_name}?`); + this.contentEl.createEl("p", { + cls: "chronicle-merge-path", + text: `${this.preview.source_path} \u2192 ${this.preview.target_path}` + }); + const summary = this.contentEl.createDiv({ cls: "chronicle-merge-summary" }); + this.addStat(summary, String(this.preview.facts_to_add), "facts added"); + this.addStat(summary, String(this.preview.backlink_occurrences), "links rewritten"); + this.addStat(summary, String(this.preview.backlink_files.length), "notes updated"); + this.addStat( + summary, + String(this.preview.duplicate_facts_skipped), + "duplicates skipped" + ); + if (this.preview.metadata_conflicts.length) { + this.contentEl.createEl("p", { + text: "The canonical note keeps these existing values:" + }); + const conflicts = this.contentEl.createEl("ul", { cls: "chronicle-merge-conflicts" }); + for (const conflict of this.preview.metadata_conflicts) { + conflicts.createEl("li", { + text: `${conflict.field}: ${String(conflict.target_value)} (discarding ${String(conflict.source_value)})` + }); + } + } + this.contentEl.createEl("p", { + cls: "chronicle-merge-help", + text: `\u201C${this.preview.source_name}\u201D will be retained as an alias. Chronicle will audit every changed note.` + }); + const actions = this.contentEl.createDiv({ cls: "chronicle-merge-actions" }); + new import_obsidian.ButtonComponent(actions).setButtonText("Cancel").onClick(() => this.close()); + new import_obsidian.ButtonComponent(actions).setButtonText("Merge person").setWarning().onClick(() => void this.apply()); + } + addStat(parent, value, label) { + const stat = parent.createDiv({ cls: "chronicle-merge-stat" }); + stat.createSpan({ cls: "chronicle-merge-stat-value", text: value }); + stat.createSpan({ cls: "chronicle-merge-stat-label", text: label }); + } + async apply() { + if (this.applying) return; + this.applying = true; + try { + await this.plugin.applyMerge(this.preview); + this.close(); + new import_obsidian.Notice( + `Merged ${this.preview.source_name} into ${this.preview.target_name}. Waiting for vault sync.`, + 8e3 + ); + } catch (error) { + this.applying = false; + new import_obsidian.Notice(`Merge failed: ${errorMessage(error)}`, 8e3); + } + } +}; +var ChronicleSettingTab = class extends import_obsidian.PluginSettingTab { + constructor(app, plugin) { + super(app, plugin); + this.plugin = plugin; + } + display() { + this.containerEl.empty(); + new import_obsidian.Setting(this.containerEl).setName("Chronicle server").setDesc("HTTPS address reachable from this device").addText( + (text) => text.setPlaceholder("https://chronicle.example.com").setValue(this.plugin.settings.baseUrl).onChange(async (value) => { + this.plugin.settings.baseUrl = value.trim(); + await this.plugin.saveSettings(); + }) + ); + new import_obsidian.Setting(this.containerEl).setName("Chronicle API key").setDesc("Select or create a long-lived Chronicle API key in Obsidian SecretStorage").addComponent( + (element) => new import_obsidian.SecretComponent(this.app, element).setValue(this.plugin.settings.secretName).onChange(async (value) => { + this.plugin.settings.secretName = value; + await this.plugin.saveSettings(); + }) + ); + } +}; +function errorMessage(error) { + return error instanceof Error ? error.message : String(error); +} diff --git a/extras/obsidian-chronicle/main.ts b/extras/obsidian-chronicle/main.ts new file mode 100644 index 000000000..5a8d15b74 --- /dev/null +++ b/extras/obsidian-chronicle/main.ts @@ -0,0 +1,569 @@ +import { + App, + ButtonComponent, + Modal, + Notice, + Plugin, + PluginSettingTab, + SearchComponent, + SecretComponent, + Setting, + TFile, + requestUrl, +} from "obsidian"; + +interface ChronicleSettings { + baseUrl: string; + secretName: string; + dismissedSuggestions: string[]; + bootstrapApiKey?: string; +} + +interface SuggestedPerson { + name: string; + path: string; + hash: string; + snippets: string[]; +} + +interface PersonSuggestion { + pair_id: string; + revision: string; + score: number; + reasons: string[]; + person_a: SuggestedPerson; + person_b: SuggestedPerson; +} + +interface MetadataConflict { + field: string; + source_value: unknown; + target_value: unknown; +} + +interface MergePreview { + source_name: string; + target_name: string; + source_path: string; + target_path: string; + plan_token: string; + facts_to_add: number; + duplicate_facts_skipped: number; + backlink_files: string[]; + backlink_occurrences: number; + metadata_conflicts: MetadataConflict[]; +} + +const DEFAULT_SETTINGS: ChronicleSettings = { + baseUrl: "", + secretName: "", + dismissedSuggestions: [], +}; + +async function sha256(text: string): Promise { + const bytes = new TextEncoder().encode(text); + const digest = await crypto.subtle.digest("SHA-256", bytes); + return Array.from(new Uint8Array(digest)) + .map((byte) => byte.toString(16).padStart(2, "0")) + .join(""); +} + +function personName(file: TFile): string | null { + return file.parent?.path === "People" ? file.basename : null; +} + +export default class ChronicleCompanionPlugin extends Plugin { + settings: ChronicleSettings = DEFAULT_SETTINGS; + + async onload(): Promise { + this.settings = Object.assign({}, DEFAULT_SETTINGS, await this.loadData()); + await this.migrateBootstrapApiKey(); + this.addSettingTab(new ChronicleSettingTab(this.app, this)); + this.addCommand({ + id: "merge-current-person", + name: "Merge current person…", + checkCallback: (checking) => { + const source = this.app.workspace.getActiveFile(); + if (!source || !personName(source)) return false; + if (!checking) new PersonPickerModal(this, source).open(); + return true; + }, + }); + this.addCommand({ + id: "review-duplicate-people", + name: "Review possible duplicate people…", + callback: () => void this.reviewDuplicatePeople(), + }); + } + + async saveSettings(): Promise { + await this.saveData(this.settings); + } + + private async migrateBootstrapApiKey(): Promise { + if (!this.settings.bootstrapApiKey) return; + const secretName = "chronicle-companion-api-key"; + this.app.secretStorage.setSecret(secretName, this.settings.bootstrapApiKey); + this.settings.secretName = secretName; + delete this.settings.bootstrapApiKey; + await this.saveSettings(); + } + + private apiKey(): string { + if (!this.settings.secretName) { + throw new Error("Choose a Chronicle API key in the plugin settings."); + } + const value = this.app.secretStorage.getSecret(this.settings.secretName); + if (!value) throw new Error("The selected Chronicle API key is empty."); + return value; + } + + private endpoint(path: string): string { + const base = this.settings.baseUrl.trim().replace(/\/+$/, ""); + if (!base) throw new Error("Set the Chronicle server URL in the plugin settings."); + return `${base}${path}`; + } + + async previewMerge(source: TFile, target: TFile): Promise { + const [sourceText, targetText] = await Promise.all([ + this.app.vault.read(source), + this.app.vault.read(target), + ]); + const response = await requestUrl({ + url: this.endpoint("/api/memories/people/merge/preview"), + method: "POST", + headers: { + Authorization: `Bearer ${this.apiKey()}`, + "Content-Type": "application/json", + }, + body: JSON.stringify({ + source_name: source.basename, + target_name: target.basename, + source_hash: await sha256(sourceText), + target_hash: await sha256(targetText), + }), + }); + return response.json as MergePreview; + } + + async applyMerge(preview: MergePreview): Promise { + await requestUrl({ + url: this.endpoint("/api/memories/people/merge"), + method: "POST", + headers: { + Authorization: `Bearer ${this.apiKey()}`, + "Content-Type": "application/json", + }, + body: JSON.stringify({ + source_name: preview.source_name, + target_name: preview.target_name, + plan_token: preview.plan_token, + }), + }); + } + + async getSuggestions(): Promise { + const response = await requestUrl({ + url: this.endpoint("/api/memories/people/suggestions?limit=30"), + method: "GET", + headers: { Authorization: `Bearer ${this.apiKey()}` }, + }); + return (response.json as { suggestions: PersonSuggestion[] }).suggestions; + } + + async markSeparate(suggestion: PersonSuggestion): Promise { + await requestUrl({ + url: this.endpoint("/api/memories/people/identity"), + method: "POST", + headers: { + Authorization: `Bearer ${this.apiKey()}`, + "Content-Type": "application/json", + }, + body: JSON.stringify({ + person_a: suggestion.person_a.name, + person_b: suggestion.person_b.name, + decision: "distinct", + revision: suggestion.revision, + }), + }); + } + + async dismissSuggestion(suggestion: PersonSuggestion): Promise { + const dismissal = `${suggestion.pair_id}:${suggestion.revision}`; + this.settings.dismissedSuggestions = [ + ...new Set([...this.settings.dismissedSuggestions, dismissal]), + ].slice(-500); + await this.saveSettings(); + } + + private async reviewDuplicatePeople(): Promise { + new Notice("Looking for possible duplicate people…"); + try { + const suggestions = await this.getSuggestions(); + const dismissed = new Set(this.settings.dismissedSuggestions); + const pending = suggestions.filter( + (suggestion) => !dismissed.has(`${suggestion.pair_id}:${suggestion.revision}`), + ); + if (!pending.length) { + new Notice("No new possible duplicate people found."); + return; + } + new DuplicateReviewModal(this, pending).open(); + } catch (error) { + new Notice(`Could not load suggestions: ${errorMessage(error)}`, 8000); + } + } +} + +class DuplicateReviewModal extends Modal { + private readonly plugin: ChronicleCompanionPlugin; + private readonly suggestions: PersonSuggestion[]; + private index = 0; + private acting = false; + + constructor(plugin: ChronicleCompanionPlugin, suggestions: PersonSuggestion[]) { + super(plugin.app); + this.plugin = plugin; + this.suggestions = suggestions; + } + + onOpen(): void { + this.render(); + } + + private render(): void { + this.contentEl.empty(); + const suggestion = this.suggestions[this.index]; + if (!suggestion) { + this.close(); + new Notice("Duplicate review complete."); + return; + } + + this.setTitle("Possible duplicate people"); + const header = this.contentEl.createDiv({ cls: "chronicle-identity-header" }); + header.createEl("p", { + cls: "chronicle-merge-help", + text: `Suggestion ${this.index + 1} of ${this.suggestions.length} · confidence ${suggestion.score}`, + }); + const navigation = header.createDiv({ cls: "chronicle-identity-navigation" }); + new ButtonComponent(navigation) + .setButtonText("Back") + .setDisabled(this.index === 0) + .onClick(() => this.move(-1)); + new ButtonComponent(navigation) + .setButtonText("Next") + .setDisabled(this.index === this.suggestions.length - 1) + .onClick(() => this.move(1)); + const comparison = this.contentEl.createDiv({ + cls: "chronicle-identity-comparison", + }); + this.addPerson(comparison, suggestion.person_a); + this.addPerson(comparison, suggestion.person_b); + + const evidence = this.contentEl.createDiv({ cls: "chronicle-identity-evidence" }); + evidence.createDiv({ + cls: "chronicle-identity-evidence-title", + text: "Why Chronicle suggested this", + }); + const reasons = evidence.createEl("ul"); + for (const reason of suggestion.reasons) reasons.createEl("li", { text: reason }); + + this.contentEl.createEl("p", { + cls: "chronicle-merge-help", + text: "Separate people is a durable vault annotation. Not sure hides only this version of the suggestion.", + }); + const actions = this.contentEl.createDiv({ cls: "chronicle-identity-actions" }); + new ButtonComponent(actions) + .setButtonText("Not sure") + .onClick(() => void this.dismiss()); + new ButtonComponent(actions) + .setButtonText("Separate people") + .onClick(() => void this.separate()); + new ButtonComponent(actions) + .setButtonText("Same person…") + .setCta() + .onClick(() => { + this.close(); + new CanonicalPersonModal(this.plugin, suggestion).open(); + }); + } + + private move(offset: number): void { + const nextIndex = this.index + offset; + if (nextIndex < 0 || nextIndex >= this.suggestions.length) return; + this.index = nextIndex; + this.render(); + } + + private removeCurrent(): void { + this.suggestions.splice(this.index, 1); + if (this.index >= this.suggestions.length) this.index = this.suggestions.length - 1; + this.render(); + } + + private addPerson(parent: HTMLElement, person: SuggestedPerson): void { + const card = parent.createDiv({ cls: "chronicle-identity-person" }); + card.createEl("h3", { text: person.name }); + card.createEl("div", { cls: "chronicle-merge-path", text: person.path }); + if (person.snippets.length) { + const snippets = card.createEl("ul"); + for (const snippet of person.snippets.slice(0, 3)) { + snippets.createEl("li", { text: snippet }); + } + } + const open = card.createEl("button", { text: "Open note" }); + open.addEventListener("click", () => void this.openNote(person.path)); + } + + private async openNote(path: string): Promise { + const file = this.app.vault.getAbstractFileByPath(path); + if (!(file instanceof TFile)) { + new Notice(`Could not find ${path}. The vault may still be syncing.`); + return; + } + await this.app.workspace.getLeaf("tab").openFile(file); + } + + private async dismiss(): Promise { + if (this.acting) return; + this.acting = true; + await this.plugin.dismissSuggestion(this.suggestions[this.index]); + this.acting = false; + this.removeCurrent(); + } + + private async separate(): Promise { + if (this.acting) return; + this.acting = true; + const suggestion = this.suggestions[this.index]; + try { + await this.plugin.markSeparate(suggestion); + new Notice(`${suggestion.person_a.name} and ${suggestion.person_b.name} marked as separate.`); + this.acting = false; + this.removeCurrent(); + } catch (error) { + this.acting = false; + new Notice(`Could not save decision: ${errorMessage(error)}`, 8000); + } + } +} + +class CanonicalPersonModal extends Modal { + private readonly plugin: ChronicleCompanionPlugin; + private readonly suggestion: PersonSuggestion; + + constructor(plugin: ChronicleCompanionPlugin, suggestion: PersonSuggestion) { + super(plugin.app); + this.plugin = plugin; + this.suggestion = suggestion; + } + + onOpen(): void { + const { person_a: personA, person_b: personB } = this.suggestion; + this.setTitle("Which name should Chronicle keep?"); + this.contentEl.createEl("p", { + cls: "chronicle-merge-help", + text: "The other name is retained as an alias, and links are rewritten after a final preview.", + }); + const actions = this.contentEl.createDiv({ cls: "chronicle-canonical-actions" }); + new ButtonComponent(actions) + .setButtonText(`Keep ${personA.name}`) + .setCta() + .onClick(() => void this.choose(personB, personA)); + new ButtonComponent(actions) + .setButtonText(`Keep ${personB.name}`) + .setCta() + .onClick(() => void this.choose(personA, personB)); + } + + private async choose(sourcePerson: SuggestedPerson, targetPerson: SuggestedPerson): Promise { + const source = this.app.vault.getAbstractFileByPath(sourcePerson.path); + const target = this.app.vault.getAbstractFileByPath(targetPerson.path); + if (!(source instanceof TFile) || !(target instanceof TFile)) { + new Notice("One of the person notes is not available yet. Wait for vault sync and retry."); + return; + } + this.close(); + new Notice("Checking that the Chronicle vault is in sync…"); + try { + const preview = await this.plugin.previewMerge(source, target); + new MergePreviewModal(this.plugin, preview).open(); + } catch (error) { + new Notice(`Could not preview merge: ${errorMessage(error)}`, 8000); + } + } +} + +class PersonPickerModal extends Modal { + private readonly plugin: ChronicleCompanionPlugin; + private readonly source: TFile; + private people: TFile[] = []; + private listEl!: HTMLDivElement; + + constructor(plugin: ChronicleCompanionPlugin, source: TFile) { + super(plugin.app); + this.plugin = plugin; + this.source = source; + } + + onOpen(): void { + this.setTitle(`Merge ${this.source.basename} into…`); + this.people = this.app.vault + .getMarkdownFiles() + .filter((file) => personName(file) && file.path !== this.source.path) + .sort((left, right) => left.basename.localeCompare(right.basename)); + + const search = new SearchComponent(this.contentEl); + search.setPlaceholder("Find the canonical person"); + search.inputEl.setAttr("aria-label", "Find the canonical person"); + search.onChange((query) => this.renderPeople(query)); + this.listEl = this.contentEl.createDiv({ cls: "chronicle-merge-list" }); + this.renderPeople(""); + search.inputEl.focus(); + } + + private renderPeople(query: string): void { + this.listEl.empty(); + const normalized = query.trim().toLocaleLowerCase(); + const matches = this.people.filter((file) => + file.basename.toLocaleLowerCase().includes(normalized), + ); + for (const file of matches) { + const button = this.listEl.createEl("button", { + cls: "chronicle-merge-person clickable-icon", + text: file.basename, + }); + button.setAttr("aria-label", `Merge ${this.source.basename} into ${file.basename}`); + button.addEventListener("click", () => void this.choose(file)); + } + if (!matches.length) { + this.listEl.createDiv({ cls: "chronicle-merge-help", text: "No matching people." }); + } + } + + private async choose(target: TFile): Promise { + this.close(); + new Notice("Checking that the Chronicle vault is in sync…"); + try { + const preview = await this.plugin.previewMerge(this.source, target); + new MergePreviewModal(this.plugin, preview).open(); + } catch (error) { + new Notice(`Could not preview merge: ${errorMessage(error)}`, 8000); + } + } +} + +class MergePreviewModal extends Modal { + private readonly plugin: ChronicleCompanionPlugin; + private readonly preview: MergePreview; + private applying = false; + + constructor(plugin: ChronicleCompanionPlugin, preview: MergePreview) { + super(plugin.app); + this.plugin = plugin; + this.preview = preview; + } + + onOpen(): void { + this.setTitle(`Merge ${this.preview.source_name} into ${this.preview.target_name}?`); + this.contentEl.createEl("p", { + cls: "chronicle-merge-path", + text: `${this.preview.source_path} → ${this.preview.target_path}`, + }); + const summary = this.contentEl.createDiv({ cls: "chronicle-merge-summary" }); + this.addStat(summary, String(this.preview.facts_to_add), "facts added"); + this.addStat(summary, String(this.preview.backlink_occurrences), "links rewritten"); + this.addStat(summary, String(this.preview.backlink_files.length), "notes updated"); + this.addStat( + summary, + String(this.preview.duplicate_facts_skipped), + "duplicates skipped", + ); + + if (this.preview.metadata_conflicts.length) { + this.contentEl.createEl("p", { + text: "The canonical note keeps these existing values:", + }); + const conflicts = this.contentEl.createEl("ul", { cls: "chronicle-merge-conflicts" }); + for (const conflict of this.preview.metadata_conflicts) { + conflicts.createEl("li", { + text: `${conflict.field}: ${String(conflict.target_value)} (discarding ${String(conflict.source_value)})`, + }); + } + } + + this.contentEl.createEl("p", { + cls: "chronicle-merge-help", + text: `“${this.preview.source_name}” will be retained as an alias. Chronicle will audit every changed note.`, + }); + const actions = this.contentEl.createDiv({ cls: "chronicle-merge-actions" }); + new ButtonComponent(actions).setButtonText("Cancel").onClick(() => this.close()); + new ButtonComponent(actions) + .setButtonText("Merge person") + .setWarning() + .onClick(() => void this.apply()); + } + + private addStat(parent: HTMLElement, value: string, label: string): void { + const stat = parent.createDiv({ cls: "chronicle-merge-stat" }); + stat.createSpan({ cls: "chronicle-merge-stat-value", text: value }); + stat.createSpan({ cls: "chronicle-merge-stat-label", text: label }); + } + + private async apply(): Promise { + if (this.applying) return; + this.applying = true; + try { + await this.plugin.applyMerge(this.preview); + this.close(); + new Notice( + `Merged ${this.preview.source_name} into ${this.preview.target_name}. Waiting for vault sync.`, + 8000, + ); + } catch (error) { + this.applying = false; + new Notice(`Merge failed: ${errorMessage(error)}`, 8000); + } + } +} + +class ChronicleSettingTab extends PluginSettingTab { + private readonly plugin: ChronicleCompanionPlugin; + + constructor(app: App, plugin: ChronicleCompanionPlugin) { + super(app, plugin); + this.plugin = plugin; + } + + display(): void { + this.containerEl.empty(); + new Setting(this.containerEl) + .setName("Chronicle server") + .setDesc("HTTPS address reachable from this device") + .addText((text) => + text + .setPlaceholder("https://chronicle.example.com") + .setValue(this.plugin.settings.baseUrl) + .onChange(async (value) => { + this.plugin.settings.baseUrl = value.trim(); + await this.plugin.saveSettings(); + }), + ); + new Setting(this.containerEl) + .setName("Chronicle API key") + .setDesc("Select or create a long-lived Chronicle API key in Obsidian SecretStorage") + .addComponent((element) => + new SecretComponent(this.app, element) + .setValue(this.plugin.settings.secretName) + .onChange(async (value) => { + this.plugin.settings.secretName = value; + await this.plugin.saveSettings(); + }), + ); + } +} + +function errorMessage(error: unknown): string { + return error instanceof Error ? error.message : String(error); +} diff --git a/extras/obsidian-chronicle/manifest.json b/extras/obsidian-chronicle/manifest.json new file mode 100644 index 000000000..821706840 --- /dev/null +++ b/extras/obsidian-chronicle/manifest.json @@ -0,0 +1,9 @@ +{ + "id": "chronicle-companion", + "name": "Chronicle Companion", + "version": "0.1.0", + "minAppVersion": "1.11.4", + "description": "Safe, deterministic maintenance actions for a synced Chronicle memory vault.", + "author": "Chronicle", + "isDesktopOnly": false +} diff --git a/extras/obsidian-chronicle/package-lock.json b/extras/obsidian-chronicle/package-lock.json new file mode 100644 index 000000000..17a20352c --- /dev/null +++ b/extras/obsidian-chronicle/package-lock.json @@ -0,0 +1,656 @@ +{ + "name": "chronicle-obsidian-companion", + "version": "0.1.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "chronicle-obsidian-companion", + "version": "0.1.0", + "devDependencies": { + "@types/node": "^24.0.0", + "builtin-modules": "^5.0.0", + "esbuild": "^0.25.0", + "obsidian": "latest", + "typescript": "^5.8.0" + } + }, + "node_modules/@codemirror/state": { + "version": "6.5.0", + "resolved": "https://registry.npmjs.org/@codemirror/state/-/state-6.5.0.tgz", + "integrity": "sha512-MwBHVK60IiIHDcoMet78lxt6iw5gJOGSbNbOIVBHWVXIH4/Nq1+GQgLLGgI1KlnN86WDXsPudVaqYHKBIx7Eyw==", + "dev": true, + "license": "MIT", + "peer": true, + "dependencies": { + "@marijn/find-cluster-break": "^1.0.0" + } + }, + "node_modules/@codemirror/view": { + "version": "6.38.6", + "resolved": "https://registry.npmjs.org/@codemirror/view/-/view-6.38.6.tgz", + "integrity": "sha512-qiS0z1bKs5WOvHIAC0Cybmv4AJSkAXgX5aD6Mqd2epSLlVJsQl8NG23jCVouIgkh4All/mrbdsf2UOLFnJw0tw==", + "dev": true, + "license": "MIT", + "peer": true, + "dependencies": { + "@codemirror/state": "^6.5.0", + "crelt": "^1.0.6", + "style-mod": "^4.1.0", + "w3c-keyname": "^2.2.4" + } + }, + "node_modules/@esbuild/aix-ppc64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.25.12.tgz", + "integrity": "sha512-Hhmwd6CInZ3dwpuGTF8fJG6yoWmsToE+vYgD4nytZVxcu1ulHpUQRAB1UJ8+N1Am3Mz4+xOByoQoSZf4D+CpkA==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "aix" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/android-arm": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm/-/android-arm-0.25.12.tgz", + "integrity": "sha512-VJ+sKvNA/GE7Ccacc9Cha7bpS8nyzVv0jdVgwNDaR4gDMC/2TTRc33Ip8qrNYUcpkOHUT5OZ0bUcNNVZQ9RLlg==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/android-arm64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm64/-/android-arm64-0.25.12.tgz", + "integrity": "sha512-6AAmLG7zwD1Z159jCKPvAxZd4y/VTO0VkprYy+3N2FtJ8+BQWFXU+OxARIwA46c5tdD9SsKGZ/1ocqBS/gAKHg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/android-x64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/android-x64/-/android-x64-0.25.12.tgz", + "integrity": "sha512-5jbb+2hhDHx5phYR2By8GTWEzn6I9UqR11Kwf22iKbNpYrsmRB18aX/9ivc5cabcUiAT/wM+YIZ6SG9QO6a8kg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/darwin-arm64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-arm64/-/darwin-arm64-0.25.12.tgz", + "integrity": "sha512-N3zl+lxHCifgIlcMUP5016ESkeQjLj/959RxxNYIthIg+CQHInujFuXeWbWMgnTo4cp5XVHqFPmpyu9J65C1Yg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/darwin-x64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-x64/-/darwin-x64-0.25.12.tgz", + "integrity": "sha512-HQ9ka4Kx21qHXwtlTUVbKJOAnmG1ipXhdWTmNXiPzPfWKpXqASVcWdnf2bnL73wgjNrFXAa3yYvBSd9pzfEIpA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/freebsd-arm64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-arm64/-/freebsd-arm64-0.25.12.tgz", + "integrity": "sha512-gA0Bx759+7Jve03K1S0vkOu5Lg/85dou3EseOGUes8flVOGxbhDDh/iZaoek11Y8mtyKPGF3vP8XhnkDEAmzeg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/freebsd-x64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-x64/-/freebsd-x64-0.25.12.tgz", + "integrity": "sha512-TGbO26Yw2xsHzxtbVFGEXBFH0FRAP7gtcPE7P5yP7wGy7cXK2oO7RyOhL5NLiqTlBh47XhmIUXuGciXEqYFfBQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-arm": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm/-/linux-arm-0.25.12.tgz", + "integrity": "sha512-lPDGyC1JPDou8kGcywY0YILzWlhhnRjdof3UlcoqYmS9El818LLfJJc3PXXgZHrHCAKs/Z2SeZtDJr5MrkxtOw==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-arm64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm64/-/linux-arm64-0.25.12.tgz", + "integrity": "sha512-8bwX7a8FghIgrupcxb4aUmYDLp8pX06rGh5HqDT7bB+8Rdells6mHvrFHHW2JAOPZUbnjUpKTLg6ECyzvas2AQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-ia32": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ia32/-/linux-ia32-0.25.12.tgz", + "integrity": "sha512-0y9KrdVnbMM2/vG8KfU0byhUN+EFCny9+8g202gYqSSVMonbsCfLjUO+rCci7pM0WBEtz+oK/PIwHkzxkyharA==", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-loong64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/linux-loong64/-/linux-loong64-0.25.12.tgz", + "integrity": "sha512-h///Lr5a9rib/v1GGqXVGzjL4TMvVTv+s1DPoxQdz7l/AYv6LDSxdIwzxkrPW438oUXiDtwM10o9PmwS/6Z0Ng==", + "cpu": [ + "loong64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-mips64el": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/linux-mips64el/-/linux-mips64el-0.25.12.tgz", + "integrity": "sha512-iyRrM1Pzy9GFMDLsXn1iHUm18nhKnNMWscjmp4+hpafcZjrr2WbT//d20xaGljXDBYHqRcl8HnxbX6uaA/eGVw==", + "cpu": [ + "mips64el" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-ppc64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ppc64/-/linux-ppc64-0.25.12.tgz", + "integrity": "sha512-9meM/lRXxMi5PSUqEXRCtVjEZBGwB7P/D4yT8UG/mwIdze2aV4Vo6U5gD3+RsoHXKkHCfSxZKzmDssVlRj1QQA==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-riscv64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/linux-riscv64/-/linux-riscv64-0.25.12.tgz", + "integrity": "sha512-Zr7KR4hgKUpWAwb1f3o5ygT04MzqVrGEGXGLnj15YQDJErYu/BGg+wmFlIDOdJp0PmB0lLvxFIOXZgFRrdjR0w==", + "cpu": [ + "riscv64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-s390x": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/linux-s390x/-/linux-s390x-0.25.12.tgz", + "integrity": "sha512-MsKncOcgTNvdtiISc/jZs/Zf8d0cl/t3gYWX8J9ubBnVOwlk65UIEEvgBORTiljloIWnBzLs4qhzPkJcitIzIg==", + "cpu": [ + "s390x" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-x64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/linux-x64/-/linux-x64-0.25.12.tgz", + "integrity": "sha512-uqZMTLr/zR/ed4jIGnwSLkaHmPjOjJvnm6TVVitAa08SLS9Z0VM8wIRx7gWbJB5/J54YuIMInDquWyYvQLZkgw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/netbsd-arm64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/netbsd-arm64/-/netbsd-arm64-0.25.12.tgz", + "integrity": "sha512-xXwcTq4GhRM7J9A8Gv5boanHhRa/Q9KLVmcyXHCTaM4wKfIpWkdXiMog/KsnxzJ0A1+nD+zoecuzqPmCRyBGjg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "netbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/netbsd-x64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/netbsd-x64/-/netbsd-x64-0.25.12.tgz", + "integrity": "sha512-Ld5pTlzPy3YwGec4OuHh1aCVCRvOXdH8DgRjfDy/oumVovmuSzWfnSJg+VtakB9Cm0gxNO9BzWkj6mtO1FMXkQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "netbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/openbsd-arm64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-arm64/-/openbsd-arm64-0.25.12.tgz", + "integrity": "sha512-fF96T6KsBo/pkQI950FARU9apGNTSlZGsv1jZBAlcLL1MLjLNIWPBkj5NlSz8aAzYKg+eNqknrUJ24QBybeR5A==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/openbsd-x64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-x64/-/openbsd-x64-0.25.12.tgz", + "integrity": "sha512-MZyXUkZHjQxUvzK7rN8DJ3SRmrVrke8ZyRusHlP+kuwqTcfWLyqMOE3sScPPyeIXN/mDJIfGXvcMqCgYKekoQw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/openharmony-arm64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/openharmony-arm64/-/openharmony-arm64-0.25.12.tgz", + "integrity": "sha512-rm0YWsqUSRrjncSXGA7Zv78Nbnw4XL6/dzr20cyrQf7ZmRcsovpcRBdhD43Nuk3y7XIoW2OxMVvwuRvk9XdASg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openharmony" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/sunos-x64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/sunos-x64/-/sunos-x64-0.25.12.tgz", + "integrity": "sha512-3wGSCDyuTHQUzt0nV7bocDy72r2lI33QL3gkDNGkod22EsYl04sMf0qLb8luNKTOmgF/eDEDP5BFNwoBKH441w==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "sunos" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/win32-arm64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/win32-arm64/-/win32-arm64-0.25.12.tgz", + "integrity": "sha512-rMmLrur64A7+DKlnSuwqUdRKyd3UE7oPJZmnljqEptesKM8wx9J8gx5u0+9Pq0fQQW8vqeKebwNXdfOyP+8Bsg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/win32-ia32": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/win32-ia32/-/win32-ia32-0.25.12.tgz", + "integrity": "sha512-HkqnmmBoCbCwxUKKNPBixiWDGCpQGVsrQfJoVGYLPT41XWF8lHuE5N6WhVia2n4o5QK5M4tYr21827fNhi4byQ==", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/win32-x64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/win32-x64/-/win32-x64-0.25.12.tgz", + "integrity": "sha512-alJC0uCZpTFrSL0CCDjcgleBXPnCrEAhTBILpeAp7M/OFgoqtAetfBzX0xM00MUsVVPpVjlPuMbREqnZCXaTnA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@marijn/find-cluster-break": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/@marijn/find-cluster-break/-/find-cluster-break-1.0.3.tgz", + "integrity": "sha512-FY+MKLBoTsLNJF/eLWaOsXGdz6uh3Iu1axjPf6TUq92IYumcTcXWHoS747JARLkcdlJ/Waiaxc5wQfFO8jC6NA==", + "dev": true, + "license": "MIT", + "peer": true + }, + "node_modules/@types/codemirror": { + "version": "5.60.8", + "resolved": "https://registry.npmjs.org/@types/codemirror/-/codemirror-5.60.8.tgz", + "integrity": "sha512-VjFgDF/eB+Aklcy15TtOTLQeMjTo07k7KAjql8OK5Dirr7a6sJY4T1uVBDuTVG9VEmn1uUsohOpYnVfgC6/jyw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/tern": "*" + } + }, + "node_modules/@types/estree": { + "version": "1.0.9", + "resolved": "https://registry.npmjs.org/@types/estree/-/estree-1.0.9.tgz", + "integrity": "sha512-GhdPgy1el4/ImP05X05Uw4cw2/M93BCUmnEvWZNStlCzEKME4Fkk+YpoA5OiHNQmoS7Cafb8Xa3Pya8m1Qrzeg==", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/node": { + "version": "24.13.3", + "resolved": "https://registry.npmjs.org/@types/node/-/node-24.13.3.tgz", + "integrity": "sha512-Dh8vAsV36ig5wa9OX4pXvMc9D3Veibfw2wix0CUwYODLD8nkj9UsLjASr49nPg+2eKzxhBV+v7L8pXvT4e639Q==", + "dev": true, + "license": "MIT", + "dependencies": { + "undici-types": "~7.18.0" + } + }, + "node_modules/@types/tern": { + "version": "0.23.9", + "resolved": "https://registry.npmjs.org/@types/tern/-/tern-0.23.9.tgz", + "integrity": "sha512-ypzHFE/wBzh+BlH6rrBgS5I/Z7RD21pGhZ2rltb/+ZrVM1awdZwjx7hE5XfuYgHWk9uvV5HLZN3SloevCAp3Bw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/estree": "*" + } + }, + "node_modules/builtin-modules": { + "version": "5.3.0", + "resolved": "https://registry.npmjs.org/builtin-modules/-/builtin-modules-5.3.0.tgz", + "integrity": "sha512-hMQUl2bUFG339QygPM97E+mc8OY1IAchORZxm4a/frcYwKzozMzRVDBwHW0NjOqGElLm2O37AVQE8ikxlZHrMQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=18.20" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/crelt": { + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/crelt/-/crelt-1.0.7.tgz", + "integrity": "sha512-aK6BbWfhf4U/wCcLHKPJl/xa6VkVstRaPywWtMKGwuOLc/wZTyQYuoxgvZnNsBvv7Kg3YTBQYYBCggcviQczuA==", + "dev": true, + "license": "MIT", + "peer": true + }, + "node_modules/esbuild": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.25.12.tgz", + "integrity": "sha512-bbPBYYrtZbkt6Os6FiTLCTFxvq4tt3JKall1vRwshA3fdVztsLAatFaZobhkBC8/BrPetoa0oksYoKXoG4ryJg==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "bin": { + "esbuild": "bin/esbuild" + }, + "engines": { + "node": ">=18" + }, + "optionalDependencies": { + "@esbuild/aix-ppc64": "0.25.12", + "@esbuild/android-arm": "0.25.12", + "@esbuild/android-arm64": "0.25.12", + "@esbuild/android-x64": "0.25.12", + "@esbuild/darwin-arm64": "0.25.12", + "@esbuild/darwin-x64": "0.25.12", + "@esbuild/freebsd-arm64": "0.25.12", + "@esbuild/freebsd-x64": "0.25.12", + "@esbuild/linux-arm": "0.25.12", + "@esbuild/linux-arm64": "0.25.12", + "@esbuild/linux-ia32": "0.25.12", + "@esbuild/linux-loong64": "0.25.12", + "@esbuild/linux-mips64el": "0.25.12", + "@esbuild/linux-ppc64": "0.25.12", + "@esbuild/linux-riscv64": "0.25.12", + "@esbuild/linux-s390x": "0.25.12", + "@esbuild/linux-x64": "0.25.12", + "@esbuild/netbsd-arm64": "0.25.12", + "@esbuild/netbsd-x64": "0.25.12", + "@esbuild/openbsd-arm64": "0.25.12", + "@esbuild/openbsd-x64": "0.25.12", + "@esbuild/openharmony-arm64": "0.25.12", + "@esbuild/sunos-x64": "0.25.12", + "@esbuild/win32-arm64": "0.25.12", + "@esbuild/win32-ia32": "0.25.12", + "@esbuild/win32-x64": "0.25.12" + } + }, + "node_modules/moment": { + "version": "2.29.4", + "resolved": "https://registry.npmjs.org/moment/-/moment-2.29.4.tgz", + "integrity": "sha512-5LC9SOxjSc2HF6vO2CyuTDNivEdoz2IvyJJGj6X8DJ0eFyfszE0QiEd+iXmBvUP3WHxSjFH/vIsA0EN00cgr8w==", + "dev": true, + "license": "MIT", + "engines": { + "node": "*" + } + }, + "node_modules/obsidian": { + "version": "1.13.1", + "resolved": "https://registry.npmjs.org/obsidian/-/obsidian-1.13.1.tgz", + "integrity": "sha512-qtTEA2pmhJzhuhJqzbBFRYhpIOqvW+krDYjtFynv66KbxBbumHBlsJfWw3I4jtnK/6fZwbQhCrmmDdRwXmX56w==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/codemirror": "5.60.8", + "moment": "2.29.4" + }, + "peerDependencies": { + "@codemirror/state": "6.5.0", + "@codemirror/view": "6.38.6" + } + }, + "node_modules/style-mod": { + "version": "4.1.3", + "resolved": "https://registry.npmjs.org/style-mod/-/style-mod-4.1.3.tgz", + "integrity": "sha512-i/n8VsZydrugj3Iuzll8+x/00GH2vnYsk1eomD8QiRrSAeW6ItbCQDtfXCeJHd0iwiNagqjQkvpvREEPtW3IoQ==", + "dev": true, + "license": "MIT", + "peer": true + }, + "node_modules/typescript": { + "version": "5.9.3", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.9.3.tgz", + "integrity": "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==", + "dev": true, + "license": "Apache-2.0", + "bin": { + "tsc": "bin/tsc", + "tsserver": "bin/tsserver" + }, + "engines": { + "node": ">=14.17" + } + }, + "node_modules/undici-types": { + "version": "7.18.2", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-7.18.2.tgz", + "integrity": "sha512-AsuCzffGHJybSaRrmr5eHr81mwJU3kjw6M+uprWvCXiNeN9SOGwQ3Jn8jb8m3Z6izVgknn1R0FTCEAP2QrLY/w==", + "dev": true, + "license": "MIT" + }, + "node_modules/w3c-keyname": { + "version": "2.2.8", + "resolved": "https://registry.npmjs.org/w3c-keyname/-/w3c-keyname-2.2.8.tgz", + "integrity": "sha512-dpojBhNsCNN7T82Tm7k26A6G9ML3NkhDsnw9n/eoxSRlVBB4CEtIQ/KTCLI2Fwf3ataSXRhYFkQi3SlnFwPvPQ==", + "dev": true, + "license": "MIT", + "peer": true + } + } +} diff --git a/extras/obsidian-chronicle/package.json b/extras/obsidian-chronicle/package.json new file mode 100644 index 000000000..783611f5c --- /dev/null +++ b/extras/obsidian-chronicle/package.json @@ -0,0 +1,16 @@ +{ + "name": "chronicle-obsidian-companion", + "version": "0.1.0", + "private": true, + "scripts": { + "build": "node esbuild.config.mjs production", + "dev": "node esbuild.config.mjs" + }, + "devDependencies": { + "@types/node": "^24.0.0", + "builtin-modules": "^5.0.0", + "esbuild": "^0.25.0", + "obsidian": "latest", + "typescript": "^5.8.0" + } +} diff --git a/extras/obsidian-chronicle/styles.css b/extras/obsidian-chronicle/styles.css new file mode 100644 index 000000000..6daca1a86 --- /dev/null +++ b/extras/obsidian-chronicle/styles.css @@ -0,0 +1,119 @@ +.chronicle-merge-summary { + display: grid; + grid-template-columns: repeat(2, minmax(0, 1fr)); + gap: var(--size-4-2) var(--size-4-4); + margin: var(--size-4-4) 0; +} + +.chronicle-merge-stat { + min-width: 0; +} + +.chronicle-merge-stat-value { + display: block; + color: var(--text-normal); + font-size: var(--font-ui-large); + font-weight: var(--font-semibold); +} + +.chronicle-merge-stat-label, +.chronicle-merge-path, +.chronicle-merge-help { + color: var(--text-muted); + font-size: var(--font-ui-small); +} + +.chronicle-merge-list { + max-height: 18rem; + overflow-y: auto; + margin-top: var(--size-4-3); +} + +.chronicle-merge-person { + width: 100%; + padding: var(--size-4-3); + border-radius: var(--radius-s); + text-align: left; +} + +.chronicle-merge-person:hover, +.chronicle-merge-person:focus-visible { + background: var(--background-modifier-hover); +} + +.chronicle-merge-conflicts { + margin: var(--size-4-3) 0; + padding-left: var(--size-4-5); + color: var(--text-warning); +} + +.chronicle-merge-actions { + display: flex; + justify-content: flex-end; + gap: var(--size-4-2); + margin-top: var(--size-4-5); +} + +.chronicle-identity-comparison { + display: grid; + grid-template-columns: repeat(2, minmax(0, 1fr)); + gap: var(--size-4-3); + margin: var(--size-4-4) 0; +} + +.chronicle-identity-header, +.chronicle-identity-navigation { + display: flex; + align-items: center; + gap: var(--size-4-2); +} + +.chronicle-identity-header { + justify-content: space-between; +} + +.chronicle-identity-header p { + margin: 0; +} + +.chronicle-identity-person { + min-width: 0; + padding: var(--size-4-3); + border: 1px solid var(--background-modifier-border); + border-radius: var(--radius-m); +} + +.chronicle-identity-person h3 { + margin: 0 0 var(--size-4-1); +} + +.chronicle-identity-person ul, +.chronicle-identity-evidence ul { + padding-left: var(--size-4-5); +} + +.chronicle-identity-evidence { + padding: var(--size-4-3); + border-radius: var(--radius-s); + background: var(--background-secondary); +} + +.chronicle-identity-evidence-title { + font-weight: var(--font-semibold); +} + +.chronicle-identity-actions, +.chronicle-canonical-actions { + display: flex; + flex-wrap: wrap; + justify-content: flex-end; + gap: var(--size-4-2); + margin-top: var(--size-4-5); +} + +@media (max-width: 480px) { + .chronicle-merge-summary, + .chronicle-identity-comparison { + grid-template-columns: 1fr; + } +} diff --git a/extras/obsidian-chronicle/tsconfig.json b/extras/obsidian-chronicle/tsconfig.json new file mode 100644 index 000000000..f795b0678 --- /dev/null +++ b/extras/obsidian-chronicle/tsconfig.json @@ -0,0 +1,21 @@ +{ + "compilerOptions": { + "baseUrl": ".", + "inlineSourceMap": true, + "inlineSources": true, + "module": "ESNext", + "target": "ES2018", + "allowJs": false, + "noImplicitAny": true, + "strictNullChecks": true, + "noImplicitReturns": true, + "noUnusedLocals": true, + "skipLibCheck": true, + "moduleResolution": "node", + "importHelpers": true, + "downlevelIteration": true, + "esModuleInterop": true, + "lib": ["DOM", "ES2018"] + }, + "include": ["**/*.ts"] +} diff --git a/extras/obsidian-chronicle/versions.json b/extras/obsidian-chronicle/versions.json new file mode 100644 index 000000000..7e3c9468c --- /dev/null +++ b/extras/obsidian-chronicle/versions.json @@ -0,0 +1,3 @@ +{ + "0.1.0": "1.11.4" +} From f2f7f572c4bec33306ca63042e73af95a8561a20 Mon Sep 17 00:00:00 2001 From: Ankush Malaker <43288948+AnkushMalaker@users.noreply.github.com> Date: Wed, 5 Aug 2026 22:03:53 +0000 Subject: [PATCH 06/18] build: drop the unused deepgram extra Deepgram is called over its HTTP/WebSocket APIs directly; nothing imports deepgram-sdk anymore. Remove the extra from both backends' pyproject/locks and the --extra deepgram flags from every Dockerfile stage. --- backends/advanced/Dockerfile | 4 ++-- backends/advanced/Dockerfile.k8s | 6 +++--- backends/advanced/pyproject.toml | 4 ---- backends/advanced/uv.lock | 22 +--------------------- backends/simple/Dockerfile | 2 +- 5 files changed, 7 insertions(+), 31 deletions(-) diff --git a/backends/advanced/Dockerfile b/backends/advanced/Dockerfile index 4841b6d46..e1f8380de 100644 --- a/backends/advanced/Dockerfile +++ b/backends/advanced/Dockerfile @@ -22,7 +22,7 @@ COPY pyproject.toml uv.lock ./ # Build a project .venv from the lockfile (includes git deps like Graphiti) RUN --mount=type=cache,target=/root/.cache/uv \ - uv sync --frozen --no-dev --extra deepgram --extra galileo --extra benchmark --no-install-project + uv sync --frozen --no-dev --extra galileo --extra benchmark --no-install-project # ============================================ @@ -138,7 +138,7 @@ WORKDIR /app COPY --from=ghcr.io/astral-sh/uv:latest /uv /uvx /bin/ COPY pyproject.toml uv.lock ./ RUN --mount=type=cache,target=/root/.cache/uv \ - uv sync --frozen --extra deepgram --extra galileo --extra benchmark --group test --no-install-project && \ + uv sync --frozen --extra galileo --extra benchmark --group test --no-install-project && \ rm /bin/uv /bin/uvx ENV VIRTUAL_ENV=/app/.venv diff --git a/backends/advanced/Dockerfile.k8s b/backends/advanced/Dockerfile.k8s index 6500ccf53..6f55affb5 100644 --- a/backends/advanced/Dockerfile.k8s +++ b/backends/advanced/Dockerfile.k8s @@ -23,12 +23,12 @@ COPY uv.lock . RUN mkdir -p src/advanced_omi_backend COPY src/advanced_omi_backend/__init__.py src/advanced_omi_backend/ -# Install dependencies using uv with deepgram extra +# Install dependencies using uv # Use cache mount for BuildKit, fallback for legacy builds # RUN --mount=type=cache,target=/root/.cache/uv \ -# uv sync --extra deepgram +# uv sync # Fallback for legacy Docker builds (CI compatibility) -RUN uv sync --extra deepgram +RUN uv sync # Copy all application code COPY . . diff --git a/backends/advanced/pyproject.toml b/backends/advanced/pyproject.toml index 66f519b86..86a585713 100644 --- a/backends/advanced/pyproject.toml +++ b/backends/advanced/pyproject.toml @@ -40,10 +40,6 @@ dependencies = [ ] [project.optional-dependencies] -deepgram = [ - "deepgram-sdk>=4.0.0", -] - local-audio = [ "easy-audio-interfaces[local-audio]>=0.7.1", ] diff --git a/backends/advanced/uv.lock b/backends/advanced/uv.lock index cdb3bae7d..9b2959762 100644 --- a/backends/advanced/uv.lock +++ b/backends/advanced/uv.lock @@ -57,9 +57,6 @@ benchmark = [ { name = "huggingface-hub" }, { name = "ijson" }, ] -deepgram = [ - { name = "deepgram-sdk" }, -] galileo = [ { name = "galileo" }, { name = "opentelemetry-exporter-otlp" }, @@ -92,7 +89,6 @@ test = [ requires-dist = [ { name = "aiohttp", specifier = ">=3.8.0" }, { name = "croniter", specifier = ">=1.3.0" }, - { name = "deepgram-sdk", marker = "extra == 'deepgram'", specifier = ">=4.0.0" }, { name = "easy-audio-interfaces", specifier = ">=0.7.1" }, { name = "easy-audio-interfaces", extras = ["local-audio"], marker = "extra == 'local-audio'", specifier = ">=0.7.1" }, { name = "en-core-web-sm", url = "https://github.com/explosion/spacy-models/releases/download/en_core_web_sm-3.8.0/en_core_web_sm-3.8.0-py3-none-any.whl" }, @@ -129,7 +125,7 @@ requires-dist = [ { name = "websockets", specifier = ">=12.0" }, { name = "wyoming", specifier = ">=1.6.1" }, ] -provides-extras = ["deepgram", "local-audio", "galileo", "benchmark"] +provides-extras = ["local-audio", "galileo", "benchmark"] [package.metadata.requires-dev] dev = [ @@ -989,22 +985,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/89/94/b7ff6279e642b014cd4aef4d914b9fca3917c2c9c35df49db062023cbdfc/dbus_fast-3.1.2-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:1d7cc1315586e4c50875c9a2d56b9ad2e056ec75e2f27c43cd80392f72d0f6e3", size = 1623709, upload_time = "2025-11-17T03:49:59.571Z" }, ] -[[package]] -name = "deepgram-sdk" -version = "5.3.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "httpx" }, - { name = "pydantic" }, - { name = "pydantic-core" }, - { name = "typing-extensions" }, - { name = "websockets" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/2d/9c/4529cc5818e9305ac9be3c24545249ad57418cbc3736c3f1c0a8397b59f5/deepgram_sdk-5.3.0.tar.gz", hash = "sha256:4e682a53f64c26dc49d8fd70865eae1e98236d313870d1bcf5f107f125e53793", size = 148179, upload_time = "2025-11-03T15:24:02.436Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/58/e2/cda09edad156199cc9e330533f6b72cb5276c0d476cab7f1744be7ffa16e/deepgram_sdk-5.3.0-py3-none-any.whl", hash = "sha256:431418fdffbd93cdf6a78a168984e3df3cb696818ced1cfc52ce336e0bc6a7fe", size = 390669, upload_time = "2025-11-03T15:24:01.078Z" }, -] - [[package]] name = "diskcache" version = "5.6.3" diff --git a/backends/simple/Dockerfile b/backends/simple/Dockerfile index 7842a3120..af388719a 100644 --- a/backends/simple/Dockerfile +++ b/backends/simple/Dockerfile @@ -18,7 +18,7 @@ COPY pyproject.toml . # Install dependencies using uv RUN --mount=type=cache,target=/root/.cache/uv \ - uv sync --group deepgram + uv sync # Set up the working directory From 03416fe26d5cbcb8f8e15b37e2f3968ccab4b47c Mon Sep 17 00:00:00 2001 From: Ankush Malaker <43288948+AnkushMalaker@users.noreply.github.com> Date: Wed, 5 Aug 2026 22:05:20 +0000 Subject: [PATCH 07/18] feat(compose): canonical compose-stack doc, MagicDNS-first DNS pin, doctor check MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit docs/backend/compose-stack.md becomes the canonical reference for the advanced backend stack; docker-compose.yml's long inline explanations are slimmed to pointers at its anchors. The functional half: pinning only public resolvers in x-public-dns broke every *.ts.net lookup inside containers — MagicDNS names are served solely by 100.100.100.100, so services addressed by tailnet name (remote ASR, Immich) failed with "Name or service not known" while public and container names kept working. List Tailscale's resolver first; public resolvers stay as fallbacks for tailnet-less hosts. services.py doctor grows check_container_magicdns — resolving this node's own MagicDNS name from inside a container — because check_container_dns passes on public upstreams while every tailnet name fails, so both probes are needed. Documented in podman.md alongside the aardvark caveat, with the asymmetric host-vs-container diagnosis recipe. --- backends/advanced/docker-compose.yml | 134 ++++------ docs/README.md | 1 + docs/backend/compose-stack.md | 190 ++++++++++++++ docs/podman.md | 12 + .../chronicle-setup/chronicle_setup/checks.py | 47 ++++ extras/chronicle-setup/tests/test_checks.py | 76 ++++++ extras/chronicle-setup/uv.lock | 237 ++++++++++++++++++ 7 files changed, 616 insertions(+), 81 deletions(-) create mode 100644 docs/backend/compose-stack.md create mode 100644 extras/chronicle-setup/uv.lock diff --git a/backends/advanced/docker-compose.yml b/backends/advanced/docker-compose.yml index c886402cc..f22e10b63 100644 --- a/backends/advanced/docker-compose.yml +++ b/backends/advanced/docker-compose.yml @@ -1,10 +1,12 @@ -# Explicit DNS upstreams. On a network with DNS enabled these are handed to the -# engine's embedded resolver (aardvark under Podman) as *upstream* servers — the -# container's resolv.conf still points at the resolver, so container-name lookups -# are unaffected. Without them, aardvark falls back to the host's resolver, and -# aardvark 1.4.0 stops forwarding external queries for good after one transient -# upstream failure while still answering container names. See docs/podman.md. +# The advanced backend stack. What each service is for, and why the non-obvious +# settings below are the way they are: docs/backend/compose-stack.md +# Drive it with ./start.sh / ./stop.sh / ./restart.sh, not by hand. + +# Explicit DNS upstreams; Tailscale's resolver first. Without these, aardvark +# (Podman) can silently stop forwarding external queries; with only public ones, +# *.ts.net stops resolving. See docs/backend/compose-stack.md#dns-pinning-x-public-dns x-public-dns: &public-dns + - 100.100.100.100 - 1.1.1.1 - 8.8.8.8 @@ -16,33 +18,29 @@ services: dockerfile: Dockerfile target: prod # Use prod stage without test dependencies args: - # git describe of the checkout (exported by services.py before builds; - # CI sets it to the release tag) — reported by the /version endpoint. + # git describe, exported by services.py; must match the other services + # sharing this image tag. See compose-stack.md#the-shared-backend-image CHRONICLE_BUILD_VERSION: ${CHRONICLE_BUILD_VERSION:-dev} ports: - "8000:8000" dns: *public-dns env_file: - .env + # Shared mounts (src/config/plugins are bind-mounted, so code changes need a + # restart, not a rebuild): compose-stack.md#shared-mounts volumes: - - ./src:/app/src # Mount source code for development + - ./src:/app/src - ./benchmark:/app/benchmark # LongMemEval benchmark harness (Phase A+) - ./data/audio_chunks:/app/audio_chunks - ./data/debug_dir:/app/debug_dir - ./data:/app/data - - ../../config:/app/config # Mount entire config directory (includes config.yml, defaults.yml, plugins.yml) - - ../../plugins:/app/plugins # External plugins directory - - ../../discovery.py:/app/discovery.py:ro # Service discovery module - # Mount the DIRECTORY, not the socket file. Bind-mounting a unix socket pins - # an inode, and systemd's RuntimeDirectory=tailscale deletes and recreates - # /run/tailscale on every tailscaled restart — leaving the container holding a - # deleted socket that refuses every connection until it is restarted. Pair with - # RuntimeDirectoryPreserve=yes (installed by services.py) so the directory - # itself also survives. See docs/ssl-certificates.md. - - /var/run/tailscale:/var/run/tailscale:ro # Tailscale socket dir for minidisc - # Codex CLI auth for the optional codex memory-agent executor. The wizard points - # CODEX_HOME_DIR at the host's ~/.codex; rw because codex rotates its tokens. - - ${CODEX_HOME_DIR:-./data/codex-home}:/codex-home + - ../../config:/app/config + - ../../plugins:/app/plugins + - ../../discovery.py:/app/discovery.py:ro + # The DIRECTORY, not the socket file — a mounted socket goes stale on every + # tailscaled restart. compose-stack.md#the-tailscale-socket-directory + - /var/run/tailscale:/var/run/tailscale:ro + - ${CODEX_HOME_DIR:-./data/codex-home}:/codex-home # Codex CLI auth (rw: it rotates tokens) environment: - CODEX_HOME=/codex-home - PARAKEET_ASR_URL=${PARAKEET_ASR_URL} @@ -57,18 +55,12 @@ services: - CORS_ORIGINS=http://localhost:5173,http://localhost:8000,http://192.168.1.153:5173,http://192.168.1.153:8000,https://localhost:5173,https://localhost:8000,https://100.105.225.45,https://localhost - REDIS_URL=redis://redis:6379/0 - MONGODB_URI=mongodb://mongo:27017 - # Vault sync broker -> server Syncthing REST API (internal docker network) + # Service endpoints — what each is for: compose-stack.md#chronicle-backend - VAULT_SYNC_SYNCTHING_URL=http://vault-syncthing:8384 - VAULT_SYNC_API_KEY=${VAULT_SYNC_API_KEY:-} - VAULT_SYNC_ADDRESS=${VAULT_SYNC_ADDRESS:-} - # Wake-word data-collection proxy -> standalone wakeword-service (chronicle-network) - WAKEWORD_SERVICE_URL=${WAKEWORD_SERVICE_URL:-http://chronicle-wakeword-service:8770} - # TTS service (kitten/etc) for spoken replies on the device. Empty → the backend - # discovers chronicle-tts on the Tailnet (set explicitly by the wizard for a - # local/own/pinned endpoint). - - CHRONICLE_TTS_URL=${CHRONICLE_TTS_URL:-} - # Host service-manager agent (start/stop services from the WebUI). - # Token comes from .env via env_file (auto-generated by services.py). + - CHRONICLE_TTS_URL=${CHRONICLE_TTS_URL:-} # empty → discover chronicle-tts on the Tailnet - SERVICE_MANAGER_URL=${SERVICE_MANAGER_URL:-http://host.docker.internal:8775} depends_on: mongo: @@ -85,13 +77,9 @@ services: start_period: 5s restart: unless-stopped - # Unified Worker Container - # No CUDA needed for chronicle-backend and workers, workers only orchestrate jobs and call external services - # Runs all workers in a single container for efficiency: - # - 6 RQ workers (transcription, memory, default queues) - # - 1 Audio persistence worker (audio queue) - # - 1+ Stream workers (conditional based on config.yml - Deepgram/Parakeet) - # Uses Python orchestrator for process management, health monitoring, and self-healing + # The whole worker fleet in one container (RQ workers + audio persistence + + # stream consumers), supervised by worker_orchestrator.py. No CUDA — workers + # orchestrate jobs and call external services. compose-stack.md#workers workers: image: ${CHRONICLE_REGISTRY:-}chronicle-backend:${CHRONICLE_TAG:-latest} build: @@ -105,24 +93,19 @@ services: dns: *public-dns env_file: - .env + # Same shared mounts as chronicle-backend: compose-stack.md#shared-mounts volumes: - ./src:/app/src - ./worker_orchestrator.py:/app/worker_orchestrator.py - - ./worker_healthcheck.py:/app/worker_healthcheck.py # Container healthcheck probe + - ./worker_healthcheck.py:/app/worker_healthcheck.py - ./data/audio_chunks:/app/audio_chunks - ./data:/app/data - - ../../config:/app/config # Mount entire config directory (includes config.yml, defaults.yml, plugins.yml) - - ../../plugins:/app/plugins # External plugins directory - - ../../discovery.py:/app/discovery.py:ro # Service discovery module - # Mount the DIRECTORY, not the socket file. Bind-mounting a unix socket pins - # an inode, and systemd's RuntimeDirectory=tailscale deletes and recreates - # /run/tailscale on every tailscaled restart — leaving the container holding a - # deleted socket that refuses every connection until it is restarted. Pair with - # RuntimeDirectoryPreserve=yes (installed by services.py) so the directory - # itself also survives. See docs/ssl-certificates.md. - - /var/run/tailscale:/var/run/tailscale:ro # Tailscale socket dir for minidisc - # Codex CLI auth for the optional codex memory-agent executor (memory jobs run here). - - ${CODEX_HOME_DIR:-./data/codex-home}:/codex-home + - ../../config:/app/config + - ../../plugins:/app/plugins + - ../../discovery.py:/app/discovery.py:ro + # The DIRECTORY, not the socket file. compose-stack.md#the-tailscale-socket-directory + - /var/run/tailscale:/var/run/tailscale:ro + - ${CODEX_HOME_DIR:-./data/codex-home}:/codex-home # Codex CLI auth (memory jobs run here) environment: - CODEX_HOME=/codex-home - PARAKEET_ASR_URL=${PARAKEET_ASR_URL} @@ -131,14 +114,12 @@ services: - HA_TOKEN=${HA_TOKEN} - REDIS_URL=redis://redis:6379/0 - MONGODB_URI=mongodb://mongo:27017 - # Worker orchestrator configuration (optional - defaults shown) + # Orchestrator tunables (optional - defaults shown): compose-stack.md#workers - WORKER_CHECK_INTERVAL=${WORKER_CHECK_INTERVAL:-10} - MIN_RQ_WORKERS=${MIN_RQ_WORKERS:-6} - WORKER_STARTUP_GRACE_PERIOD=${WORKER_STARTUP_GRACE_PERIOD:-30} - WORKER_SHUTDOWN_TIMEOUT=${WORKER_SHUTDOWN_TIMEOUT:-30} - # TTS service (kitten/etc) for spoken replies on the device — dispatcher runs here. - # Empty → discover chronicle-tts on the Tailnet (wizard sets it for local/pinned). - - CHRONICLE_TTS_URL=${CHRONICLE_TTS_URL:-} + - CHRONICLE_TTS_URL=${CHRONICLE_TTS_URL:-} # TTS dispatcher runs here too extra_hosts: - "host.docker.internal:host-gateway" # Access host services depends_on: @@ -147,9 +128,8 @@ services: mongo: condition: service_healthy restart: unless-stopped - # Probe the actual work, not just the process: fails if the RQ worker fleet - # shrank below MIN_RQ_WORKERS or a stream-consumer heartbeat went stale - # (wedged-but-alive). start_period covers orchestrator startup + worker boot. + # Probes the actual work, not the process — catches wedged-but-alive workers. + # compose-stack.md#workers healthcheck: test: ["CMD", "python", "worker_healthcheck.py"] interval: 30s @@ -157,11 +137,8 @@ services: start_period: 90s retries: 3 - # Annotation Cron Scheduler - # Runs periodic jobs for AI-powered annotation suggestions: - # - Daily: Surface potential errors in transcripts/memories - # - Weekly: Fine-tune error detection models using user feedback - # Set DEV_MODE=true in .env for 1-minute intervals (testing) + # Periodic AI-annotation jobs (daily error surfacing, weekly fine-tuning). + # DEV_MODE=true gives 1-minute intervals. compose-stack.md#annotation-cron annotation-cron: image: ${CHRONICLE_REGISTRY:-}chronicle-backend:${CHRONICLE_TAG:-latest} dns: *public-dns @@ -189,10 +166,8 @@ services: profiles: - annotation # Optional profile - enable with: docker compose --profile annotation up - # Lightweight intent-router microservice: classifies a voice command as a - # home-automation request vs a general agent/chat query (sub-ms Model2Vec + - # logreg). Kept out of the backend image so the ML deps don't bloat it. The - # Home Assistant plugin calls it at http://intent-router:8791/classify. + # Classifies a voice command as home-automation vs general agent/chat query. + # Own image so its ML deps stay out of the backend. compose-stack.md#intent-router intent-router: build: context: ../../extras/intent-router @@ -217,9 +192,8 @@ services: retries: 3 start_period: 25s - # Caddy reverse proxy - provides HTTPS for microphone access - # Access at: https://localhost (accepts self-signed cert warning) - # Only starts when HTTPS is configured (Caddyfile exists) + # HTTPS for the dashboard + Langfuse (browsers require it for mic access). + # Starts only under the https profile. See docs/ssl-certificates.md caddy: image: caddy:2-alpine dns: *public-dns @@ -239,9 +213,8 @@ services: profiles: - https - # WebUI with hot reload — source is volume-mounted, changes appear instantly - # without rebuilds. This is the only webui (home use); served on :5173 and - # fronted by Caddy for HTTPS. + # The only WebUI: Vite dev server on :5173 with hot reload, fronted by Caddy + # for HTTPS. compose-stack.md#webui-dev webui-dev: build: context: ./webui @@ -282,8 +255,8 @@ services: - "6379:6379" # Avoid conflict with dev on 6379 volumes: - ./data/redis_data:/data - # Redis is the raw-audio WAL. ACK XADD only after the append is fsynced so a - # host crash cannot erase the last second of already-accepted audio. + # Redis is the raw-audio WAL — fsync every append, or a host crash erases + # audio the system already accepted. docs/backend/audio-durability.md command: redis-server --appendonly yes --appendfsync always restart: unless-stopped healthcheck: @@ -292,19 +265,16 @@ services: timeout: 3s retries: 5 - # Vault sync - Syncthing instance that shares each user's Obsidian vault - # (data/conversation_docs/{user_id}) with their Mac so it can be opened in Obsidian. - # Configured exclusively by the backend's /api/vault-sync broker - not by hand. - # REST API stays on the internal docker network (reachable as vault-syncthing:8384); - # only the sync protocol port 22000 is published (reach it over Tailscale). - # Enable with: docker compose --profile vault-sync up -d + # Shares each user's vault (data/conversation_docs/{user_id}) to Obsidian. + # Configured exclusively by the backend's /api/vault-sync broker, never by + # hand. compose-stack.md#vault-syncthing vault-syncthing: image: syncthing/syncthing:latest dns: *public-dns container_name: chronicle-vault-syncthing hostname: chronicle-vault-syncthing environment: - - STGUIADDRESS=0.0.0.0:8384 # REST API reachable from the backend container + - STGUIADDRESS=0.0.0.0:8384 # REST API — internal network only - STGUIAPIKEY=${VAULT_SYNC_API_KEY} # backend authenticates to Syncthing with this - PUID=${VAULT_SYNC_PUID:-0} # match conversation_docs file ownership - PGID=${VAULT_SYNC_PGID:-0} @@ -319,6 +289,8 @@ services: profiles: - vault-sync + # Optional in-container tailnet membership; most deployments run Tailscale on + # the host instead. compose-stack.md#tailscale tailscale: image: tailscale/tailscale:latest container_name: advanced-tailscale diff --git a/docs/README.md b/docs/README.md index f6260f31c..04b6e3dab 100644 --- a/docs/README.md +++ b/docs/README.md @@ -19,6 +19,7 @@ day-to-day operation, and use [AGENTS.md](../AGENTS.md) for development conventi ## Backend +- [Compose stack](backend/compose-stack.md): the backend's containers, shared mounts, and profiles - [Authentication](backend/auth.md): user identity, JWTs, and protected endpoints - [Memory system](backend/memories.md): agentic Markdown vault and retrieval - [Audio durability](backend/audio-durability.md): raw-audio write path and its state machine diff --git a/docs/backend/compose-stack.md b/docs/backend/compose-stack.md new file mode 100644 index 000000000..66b6f6037 --- /dev/null +++ b/docs/backend/compose-stack.md @@ -0,0 +1,190 @@ +# The advanced backend compose stack + +Reference for `backends/advanced/docker-compose.yml` — what each service is for and +why the non-obvious settings are the way they are. The compose file itself carries +only short pointers back here. + +Do not drive this file by hand for day-to-day operation: `./start.sh` / `./stop.sh` / +`./restart.sh` route through `services.py`, which selects the container engine +(`container_engine: docker|podman` in `config/config.yml`), exports build args, and +activates the right profiles. See [init-system.md](../init-system.md) and +[podman.md](../podman.md). + +## Services + +| Service | Profile | Published ports | Role | +|---|---|---|---| +| `chronicle-backend` | — | 8000 | FastAPI API, WebSocket ingestion, plugin host | +| `workers` | — | — | RQ workers + audio persistence + stream consumers | +| `annotation-cron` | `annotation` | — | periodic annotation/error-detection jobs | +| `intent-router` | — | 8791 | voice-command classifier (home automation vs agent) | +| `caddy` | `https` | 80, 443, 3443 | HTTPS reverse proxy (dashboard + Langfuse) | +| `webui-dev` | — | 5173 | React dashboard, Vite dev server with hot reload | +| `mongo` | — | 27017 | conversations, chunks, chat, annotations | +| `redis` | — | 6379 | audio WAL + RQ job queues | +| `vault-syncthing` | `vault-sync` | 22000/tcp+udp, 21027/udp | shares each user's vault to their Obsidian | +| `tailscale` | `tailscale` | — | in-container tailnet membership (rarely needed; the host usually runs Tailscale) | + +The stack joins the external `chronicle-network` bridge so other Chronicle compose +projects (speaker recognition, ASR, wake word) can reach it by container name. + +## The shared backend image + +`chronicle-backend`, `workers`, and `annotation-cron` are the *same image* under the +same tag, differing only in `command`. Their `build.args` must therefore stay +identical — in particular `CHRONICLE_BUILD_VERSION`, which `services.py` exports from +`git describe` before a build (CI sets it to the release tag) and which the backend +reports from `/version`. A mismatch means whichever service builds last wins, and the +reported version silently belongs to another build. + +All three build the `prod` stage, which omits test dependencies. + +## Shared mounts + +The three backend containers mount the same set: + +| Mount | Why | +|---|---| +| `./src → /app/src` | source is bind-mounted, so backend code changes need a **restart**, not a rebuild | +| `../../config → /app/config` | whole config directory: `config.yml`, `defaults.yml`, `plugins.yml` | +| `../../plugins → /app/plugins` | external plugins, discovered at startup | +| `../../discovery.py` | service-discovery module (read-only) | +| `./data`, `./data/audio_chunks`, `./data/debug_dir` | audio, vault, and debug artifacts on the host | +| `./benchmark` (backend only) | LongMemEval benchmark harness | +| `${CODEX_HOME_DIR:-./data/codex-home} → /codex-home` | Codex CLI auth for the optional codex memory-agent executor; the wizard points it at the host's `~/.codex`, and it is read-write because Codex rotates its tokens | + +Dependency or Dockerfile changes still require a rebuild. + +### The Tailscale socket directory + +Both backend containers mount `/var/run/tailscale` — the **directory**, not the +socket file inside it — so minidisc can talk to the host `tailscaled`. + +Bind-mounting the socket file pins an inode, and systemd's +`RuntimeDirectory=tailscale` deletes and recreates `/run/tailscale` on every +`tailscaled` restart. The container is then left holding a deleted socket that +refuses every connection until it is restarted. Mounting the directory pairs with the +`RuntimeDirectoryPreserve=yes` drop-in that `services.py` installs, so the directory +survives too. Full failure modes and diagnosis in +[ssl-certificates.md](../ssl-certificates.md). + +## DNS pinning (`x-public-dns`) + +Every service that makes outbound calls gets explicit `dns:` upstreams. On a +DNS-enabled network these are handed to the engine's embedded resolver (aardvark +under Podman) as *upstream* servers; the container's `resolv.conf` still points at +that resolver, so container-name lookups are unaffected. + +Without them, aardvark falls back to the host resolver, and aardvark 1.4.0 can stop +forwarding external queries permanently after one transient upstream failure — while +still answering container names, so every health check stays green and nothing can +reach the internet. + +`100.100.100.100` is listed **first** on purpose. Pinning only public resolvers makes +every Tailscale MagicDNS name (`*.ts.net`) unresolvable inside containers, which is +how services on other tailnet nodes are addressed (an Immich library, a remote ASR +box). Tailscale's resolver answers public names too, and `1.1.1.1` / `8.8.8.8` remain +as fallbacks for hosts with no tailnet. See +[podman.md](../podman.md#caveats) for the diagnosis recipe. + +## Service notes + +### `chronicle-backend` + +Serves the API and WebSocket ingestion. `extra_hosts: host.docker.internal:host-gateway` +gives it the host network, which is how it reaches the node agent and any +host-side/tailnet services. Notable environment: + +| Variable | Purpose | +|---|---| +| `VAULT_SYNC_SYNCTHING_URL` | Syncthing REST API on the internal network (`vault-syncthing:8384`) | +| `WAKEWORD_SERVICE_URL` | wake-word data-collection proxy to the standalone service on `chronicle-network` | +| `CHRONICLE_TTS_URL` | TTS endpoint for spoken device replies; empty means discover `chronicle-tts` on the Tailnet, and the wizard sets it explicitly for a local or pinned endpoint | +| `SERVICE_MANAGER_URL` | host node agent (`:8775`) that the WebUI System page drives; its token comes from `.env` | + +Health is `/readiness`, so dependents wait for service dependencies, not just a +listening port. + +### `workers` + +One container running the whole worker fleet through `worker_orchestrator.py`, which +handles process supervision, health monitoring, and self-healing: + +- 6 RQ workers (transcription, memory, default queues) +- 1 audio-persistence worker (audio queue) +- 1+ stream workers, conditional on the `stt_stream` provider in `config.yml` + +No CUDA: the backend and workers only orchestrate jobs and call external services. + +Tunables (defaults shown): `WORKER_CHECK_INTERVAL=10`, `MIN_RQ_WORKERS=6`, +`WORKER_STARTUP_GRACE_PERIOD=30`, `WORKER_SHUTDOWN_TIMEOUT=30`. + +The healthcheck (`worker_healthcheck.py`) probes the actual work rather than the +process: it fails if the RQ fleet shrank below `MIN_RQ_WORKERS` or a stream-consumer +heartbeat went stale — the wedged-but-alive case a process check misses. Its +`start_period: 90s` covers orchestrator startup plus worker boot. + +### `annotation-cron` + +Periodic jobs for AI-assisted annotation: daily passes that surface potential errors +in transcripts and memories, weekly fine-tuning of the error-detection models from +user feedback. Set `DEV_MODE=true` in `.env` for 1-minute intervals when testing. + +Optional; enable with the `annotation` profile. + +### `intent-router` + +Classifies a voice command as a home-automation request versus a general agent/chat +query (sub-millisecond Model2Vec + logistic regression). It lives in its own image so +the ML dependencies do not bloat the backend image; the Home Assistant plugin calls +`http://intent-router:8791/classify`. `../../extras/intent-router` is mounted live, so +a retrained classifier takes effect without a rebuild. + +### `caddy` + +HTTPS termination for the dashboard and Langfuse — required for browser microphone +access over the network. Starts only under the `https` profile, which the wizard +enables once a `Caddyfile` exists. Certificate modes, Tailscale certs, and renewal are +covered in [ssl-certificates.md](../ssl-certificates.md). + +### `webui-dev` + +The only WebUI. Source is volume-mounted and served by the Vite dev server on `:5173` +with hot reload, so frontend changes appear without a rebuild; Caddy fronts it for +HTTPS. `VITE_ALLOWED_HOSTS` must list the hostnames used to reach it. + +### `redis` + +Both the RQ broker and the raw-audio write-ahead log. It runs with +`--appendonly yes --appendfsync always` so an `XADD` is acknowledged only after the +append is fsynced — otherwise a host crash can erase the last second of audio the +system already accepted. See [audio-durability.md](audio-durability.md). + +### `vault-syncthing` + +A Syncthing instance that shares each user's vault (`data/conversation_docs/{user_id}`) +so it can be opened in Obsidian. It is configured **exclusively** by the backend's +`/api/vault-sync` broker, never by hand. + +Its REST API stays on the internal network (`vault-syncthing:8384`, authenticated with +`VAULT_SYNC_API_KEY`); only the sync protocol port 22000 is published, reached over +Tailscale. `VAULT_SYNC_PUID`/`PGID` must match the ownership of the +`conversation_docs` files. Enable with the `vault-sync` profile. + +### `tailscale` + +Optional in-container tailnet membership, under the `tailscale` profile. Most +deployments run Tailscale on the host instead and reach it through +`host.docker.internal` plus the mounted socket directory. + +## Profiles + +```bash +# services.py handles these; the raw equivalents are: +docker compose --profile https up -d # Caddy / HTTPS +docker compose --profile vault-sync up -d # Obsidian vault sharing +docker compose --profile annotation up -d # annotation cron +docker compose --profile tailscale up -d # in-container Tailscale +``` + +Under Podman substitute `podman-compose`. diff --git a/docs/podman.md b/docs/podman.md index a380dba04..ceb6fd61f 100644 --- a/docs/podman.md +++ b/docs/podman.md @@ -173,6 +173,18 @@ internally by querying `podman ps` scoped to the compose project label. checks for it; the node agent's watchdog repairs it by churning a throwaway container to force a reload. + Pinning only *public* resolvers trades one outage for another: MagicDNS names + are served solely by `100.100.100.100`, so `*.ts.net` stops resolving in + containers (public and container names keep working) and anything addressed by + tailnet name fails with `[Errno -2] Name or service not known`. Hence + `100.100.100.100` first — see + [compose-stack.md](backend/compose-stack.md#dns-pinning-x-public-dns). Same + asymmetric diagnosis: + ```bash + getent hosts ..ts.net # host: works + podman exec getent hosts ..ts.net # container: fails + ``` + - **Docker Desktop auto-start.** On Windows, Docker Desktop relaunches on login and its `restart: unless-stopped` containers reclaim host ports (27017/6379/…) via the WSL relay, blocking Podman. Uninstall it (or disable login-start + `compose down` diff --git a/extras/chronicle-setup/chronicle_setup/checks.py b/extras/chronicle-setup/chronicle_setup/checks.py index 50dd5430a..100222d37 100644 --- a/extras/chronicle-setup/chronicle_setup/checks.py +++ b/extras/chronicle-setup/chronicle_setup/checks.py @@ -179,6 +179,52 @@ def check_container_dns(ctx: CheckContext) -> CheckResult: ) +def check_container_magicdns(ctx: CheckContext) -> CheckResult: + """Resolve this node's own MagicDNS name from inside a container. + + Not implied by ``check_container_dns``: public upstreams answer public names + while every ``*.ts.net`` name fails, so both probes are needed. See + docs/backend/compose-stack.md#dns-pinning-x-public-dns. + """ + cid = "container_magicdns" + title = "Container can resolve Tailscale MagicDNS" + + if shutil.which(ctx.engine) is None: + return CheckResult(cid, title, NOT_APPLICABLE, f"{ctx.engine} not installed") + + status = _tailscale_status() + if status is None or status.get("BackendState") != "Running": + return CheckResult(cid, title, NOT_APPLICABLE, "Tailscale not running") + + probe = ((status.get("Self") or {}).get("DNSName") or "").rstrip(".") + if not probe: + return CheckResult(cid, title, NOT_APPLICABLE, "no MagicDNS name for this node") + + container = _first_running(ctx.engine, ctx.dns_containers) + if container is None: + return CheckResult(cid, title, NOT_APPLICABLE, "no probe container is running") + + result = _run([ctx.engine, "exec", container, "getent", "hosts", probe], timeout=20) + if result is None: + return CheckResult(cid, title, NOT_APPLICABLE, "could not exec into container") + if result.returncode == _CMD_NOT_FOUND: + return CheckResult(cid, title, NOT_APPLICABLE, "getent absent in image") + if result.returncode == 0 and result.stdout.strip(): + return CheckResult(cid, title, OK, f"{container} resolved {probe}") + + return CheckResult( + cid, + title, + FAIL, + f"{container} cannot resolve {probe}", + remedy=( + "List 100.100.100.100 first in the compose file's `dns:` upstreams " + "(x-public-dns), then recreate the containers — `dns:` is applied at " + "create time, so a plain restart will not pick it up." + ), + ) + + def check_tailscale_login(ctx: CheckContext) -> CheckResult: """Whether tailscaled is actually logged in. @@ -493,6 +539,7 @@ def repair_tailscale_operator(user: Optional[str]) -> bool: ALL_CHECKS: Tuple[Callable[[CheckContext], CheckResult], ...] = ( check_container_dns, + check_container_magicdns, check_tailscale_login, check_tailscale_key_expiry, check_tailscale_operator, diff --git a/extras/chronicle-setup/tests/test_checks.py b/extras/chronicle-setup/tests/test_checks.py index d27afbb19..809af6877 100644 --- a/extras/chronicle-setup/tests/test_checks.py +++ b/extras/chronicle-setup/tests/test_checks.py @@ -165,6 +165,82 @@ def test_dns_not_applicable_when_getent_absent(monkeypatch, have_all_binaries): assert result.status == NOT_APPLICABLE +# -------------------------------------------------------------------- container MagicDNS + + +def test_magicdns_ok_when_container_resolves_the_node(monkeypatch, have_all_binaries): + install( + monkeypatch, + FakeRunner( + [ + (["status"], (0, STATUS_RUNNING, "")), + RUNNING_CONTAINER, + (["getent"], (0, "100.83.66.30 kraken.parrot-census.ts.net\n", "")), + ] + ), + ) + result = checks.check_container_magicdns( + CheckContext(engine="podman", dns_containers=("backend",)) + ) + assert result.status == OK + + +def test_magicdns_fails_when_only_public_upstreams_are_pinned( + monkeypatch, have_all_binaries +): + """The regression: public DNS resolves, so check_container_dns stays green.""" + runner = install( + monkeypatch, + FakeRunner( + [ + (["status"], (0, STATUS_RUNNING, "")), + RUNNING_CONTAINER, + (["getent", "api.openai.com"], (0, "162.159.140.245 x\n", "")), + (["getent", "ts.net"], (2, "", "")), + ] + ), + ) + ctx = CheckContext(engine="podman", dns_containers=("backend",), network="net") + + assert checks.check_container_dns(ctx).status == OK + + result = checks.check_container_magicdns(ctx) + assert result.status == FAIL + assert "kraken.parrot-census.ts.net" in result.detail + assert "100.100.100.100" in (result.remedy or "") + # Probed by the node's own name, trailing dot stripped. + assert runner.argv_containing("kraken.parrot-census.ts.net") + + +def test_magicdns_not_applicable_without_a_tailnet(monkeypatch, have_all_binaries): + """A host with no tailnet is not misreported as broken.""" + install( + monkeypatch, + FakeRunner([(["status"], (0, STATUS_LOGGED_OUT, "")), RUNNING_CONTAINER]), + ) + result = checks.check_container_magicdns( + CheckContext(engine="podman", dns_containers=("backend",)) + ) + assert result.status == NOT_APPLICABLE + + +def test_magicdns_not_applicable_when_getent_absent(monkeypatch, have_all_binaries): + install( + monkeypatch, + FakeRunner( + [ + (["status"], (0, STATUS_RUNNING, "")), + RUNNING_CONTAINER, + (["getent"], (127, "", "not found")), + ] + ), + ) + result = checks.check_container_magicdns( + CheckContext(engine="podman", dns_containers=("backend",)) + ) + assert result.status == NOT_APPLICABLE + + # ------------------------------------------------------------------------ tailscale diff --git a/extras/chronicle-setup/uv.lock b/extras/chronicle-setup/uv.lock new file mode 100644 index 000000000..9719a2bcc --- /dev/null +++ b/extras/chronicle-setup/uv.lock @@ -0,0 +1,237 @@ +version = 1 +revision = 2 +requires-python = ">=3.9" +resolution-markers = [ + "python_full_version >= '3.10'", + "python_full_version < '3.10'", +] + +[[package]] +name = "certifi" +version = "2026.7.22" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/a3/c2/24167ea9858356b47a87a50d39908bfdb72ceeefe0041586e704e5376b3a/certifi-2026.7.22.tar.gz", hash = "sha256:741e2c3b351ddf169a738da9f2c048608ff7f2c5cc02f1ebc6b118bb090d5d55", size = 138112, upload_time = "2026-07-22T03:35:12.644Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0b/a7/71ac2cff56fec219ed242bb11b8efb69fcc4bec75db06fb7bfe35de520e6/certifi-2026.7.22-py3-none-any.whl", hash = "sha256:62f22742b58a1a33014a2b6b706588a8d7e2a88ae7bd1a6ebe8c992928483775", size = 136983, upload_time = "2026-07-22T03:35:11.276Z" }, +] + +[[package]] +name = "charset-normalizer" +version = "3.4.9" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/bd/2a/23f34ec9d04624958e137efdc394888716353190e75f25dd22c7a2c7a8aa/charset_normalizer-3.4.9.tar.gz", hash = "sha256:673611bbd43f0810bec0b0f028ddeaaa501190339cac411f347ac76917c3ae7b", size = 152439, upload_time = "2026-07-07T14:34:58.454Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ad/81/8e983840c6e5b93b33c2ba81aa3d52c2e42f0e9a690ce7607a2e61da4a5c/charset_normalizer-3.4.9-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:cd6280cf040f233bd7d3407b743b4b4c74f70e8e1c4199cb112a62c941c0772a", size = 322240, upload_time = "2026-07-07T14:32:36.236Z" }, + { url = "https://files.pythonhosted.org/packages/de/d1/b4319dc3229d8272fba305e206fc0a148e2de8d4087917ce62ae6382f359/charset_normalizer-3.4.9-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:aa99adc8f081b475a12843953db36831eaf83ec33eb46a90629ca6a5de45a616", size = 216475, upload_time = "2026-07-07T14:32:38.142Z" }, + { url = "https://files.pythonhosted.org/packages/80/33/6c99c1b3e6b8bf730e1bc809b9a2608f224145069114c479a2e9e1494346/charset_normalizer-3.4.9-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c1225416b463483160e4af85d5fc3a9690ccb53fd4b1865a6437825f5ede3209", size = 238670, upload_time = "2026-07-07T14:32:39.658Z" }, + { url = "https://files.pythonhosted.org/packages/7f/f4/ffbb83546e1f198ecc70ecd372b65cf2b50f9068b380abd67640f17a8e18/charset_normalizer-3.4.9-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:16d10d789dd9bcca1173c95af82c58433122564b7bc39385124be735a35cbe99", size = 233476, upload_time = "2026-07-07T14:32:41.155Z" }, + { url = "https://files.pythonhosted.org/packages/e8/5f/b98b8da398637b551e427e7be922bdec19177dc54d6811dcdaa503f23aac/charset_normalizer-3.4.9-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9bb41182d93ea91f60b4bc8fbf4c820c69ef8a12ab2d917f3f1834f1acad07e8", size = 223817, upload_time = "2026-07-07T14:32:42.592Z" }, + { url = "https://files.pythonhosted.org/packages/36/31/a276bb2e66243072a3fd06fdcab9cbb61a305b02143d70d2bda21d888fa8/charset_normalizer-3.4.9-cp310-cp310-manylinux_2_31_armv7l.whl", hash = "sha256:bcf74c1df76758a395bf0af608c04c82257523f55c9868b334f06270d0f2112b", size = 207974, upload_time = "2026-07-07T14:32:44.258Z" }, + { url = "https://files.pythonhosted.org/packages/5e/be/7ee4453d7e88dfbc4104ccd34900b9f2c7c17dac22881865fe0e82424a25/charset_normalizer-3.4.9-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:b5314963fce9b0b12743891de876e724997864ee22aa496f903f426c7e2fa5b2", size = 221655, upload_time = "2026-07-07T14:32:45.64Z" }, + { url = "https://files.pythonhosted.org/packages/1d/85/181c652953eb5276d198f375b1dd641047392050098100a3a02d6534f657/charset_normalizer-3.4.9-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:e9701d0049d92c16703a42771b98d560b95248949f23f8cf7b4eddd201814fb9", size = 219229, upload_time = "2026-07-07T14:32:47.376Z" }, + { url = "https://files.pythonhosted.org/packages/0c/e7/aaf6da33fc9f4691cda8f7efbc9f69179d3d39ec8a4799baf273ee1d8db0/charset_normalizer-3.4.9-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:65a7ff3f705e57d392f7261b6d0550fe137c3019477431f1c355e0db0a7d3e15", size = 209704, upload_time = "2026-07-07T14:32:48.855Z" }, + { url = "https://files.pythonhosted.org/packages/63/01/f2fb3bd3a73be48b173ee0c6aa8d2497af97d5663a8c4c4b491de4c62f7a/charset_normalizer-3.4.9-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:79580094b00d1789d1f93ea55bc43cb2f611910c72235b7657f3482ddcc1b22d", size = 226243, upload_time = "2026-07-07T14:32:50.239Z" }, + { url = "https://files.pythonhosted.org/packages/c4/02/c57a22739fe05246b0b5783b3bfb6afaac4eebb46f3ececdfb2f048f780e/charset_normalizer-3.4.9-cp310-cp310-win32.whl", hash = "sha256:432786d3561e69aeeae6c7e8648964ce0ad05736120135601f87ac26b9c83381", size = 150935, upload_time = "2026-07-07T14:32:51.676Z" }, + { url = "https://files.pythonhosted.org/packages/37/8d/ca39a7559a4797505530d084fd3a49a2c959efbbbff146302fb7be4e3b35/charset_normalizer-3.4.9-cp310-cp310-win_amd64.whl", hash = "sha256:8c041122946b7ba21bb32c45b1aa57b1be35527690aeb3c5c234521085632eee", size = 162314, upload_time = "2026-07-07T14:32:53.193Z" }, + { url = "https://files.pythonhosted.org/packages/01/da/a44bd7a13d426e69e4894557106cd58669097bfad4a8681123b618fbfc5d/charset_normalizer-3.4.9-cp310-cp310-win_arm64.whl", hash = "sha256:375b83ed0aecfce76c16d198fbc21f3b11b337d68662bea0a995046682a11419", size = 153075, upload_time = "2026-07-07T14:32:54.554Z" }, + { url = "https://files.pythonhosted.org/packages/0b/e3/85ec501f206fb049259288c1f3506e53876937fb00edb47009348e66756b/charset_normalizer-3.4.9-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:0e94703ec9684807f20cfb5eed95c70f67f2a8f21ad620146d7b5a13677b93e5", size = 317075, upload_time = "2026-07-07T14:32:56.021Z" }, + { url = "https://files.pythonhosted.org/packages/c3/69/2a5385192e67175f7d8bd5ce4f57c24bc956439adeae5c13a99aa28a53d1/charset_normalizer-3.4.9-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2a441ea71902098ffe78c5abe6c494f44160b4af614ed16c3d9a3b1d17fd8ee2", size = 213837, upload_time = "2026-07-07T14:32:57.78Z" }, + { url = "https://files.pythonhosted.org/packages/b3/46/03ddc7da576d814fe0a36dd1f0fd3258e95404b4b2e3c026b7923d7e133f/charset_normalizer-3.4.9-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:304b13570067b2547562e308af560b3963857b1fa90bd6afd978130130fe2d6a", size = 235503, upload_time = "2026-07-07T14:32:59.205Z" }, + { url = "https://files.pythonhosted.org/packages/4e/6e/de0229a7ef40f6f9d28a837eebf4ec47bdca5dab4e900c84f22919af636a/charset_normalizer-3.4.9-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:4773092f8019072343a7447203308b176e10199920eb02d6195e81bbb3274c29", size = 229944, upload_time = "2026-07-07T14:33:00.803Z" }, + { url = "https://files.pythonhosted.org/packages/a5/34/49b9060e8418b14fb5cba9cf6bfb383111e2538a03a1fb18e66a95aeb3d5/charset_normalizer-3.4.9-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:04ce310cb89c15df659582aee80a0603788732a5e017d5bd5c81158106ce249c", size = 221276, upload_time = "2026-07-07T14:33:02.199Z" }, + { url = "https://files.pythonhosted.org/packages/44/95/80282cce0fae9c3061203d723ee87da996aed79679e65d8935050ee7ca1f/charset_normalizer-3.4.9-cp311-cp311-manylinux_2_31_armv7l.whl", hash = "sha256:c0323c9daef75ef2e5083624b4585018a0c9d5e3b40f607eed81a311270b934b", size = 205260, upload_time = "2026-07-07T14:33:03.698Z" }, + { url = "https://files.pythonhosted.org/packages/0c/74/2f62c8821b969ea3bd67cc2e6976834f48ca5d12664d2559ebcd9bcfbed7/charset_normalizer-3.4.9-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:871ff67ea1aad4dfd91736464934d56b32dac49f9fbe16cddba36198a7b3a0db", size = 217786, upload_time = "2026-07-07T14:33:05.12Z" }, + { url = "https://files.pythonhosted.org/packages/d9/8d/feabb82cb49fcad14515b1d7d1ca4787b0da7fc723a212bf89bc9e0fac52/charset_normalizer-3.4.9-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:67830fc78e67501f47bb950471b2dcb9b35b140084429318e862895a8e89c993", size = 216798, upload_time = "2026-07-07T14:33:06.629Z" }, + { url = "https://files.pythonhosted.org/packages/a5/ff/c946d63bc3786d5b84d960b0f7ab7e25b828486a946b5aa997625bcaf6a6/charset_normalizer-3.4.9-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:3d92613ec25e43b05f042302531ec0f00b8445190e43325880cbd6ab7c2581da", size = 206429, upload_time = "2026-07-07T14:33:08.006Z" }, + { url = "https://files.pythonhosted.org/packages/af/ba/5e5007c370702f85d2ef75791fac7943ed41e080364a673b20142e430e3e/charset_normalizer-3.4.9-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:280081916dc341820640489a66e4696049401ef1cf6dd672f672e70ad915aca3", size = 223066, upload_time = "2026-07-07T14:33:09.783Z" }, + { url = "https://files.pythonhosted.org/packages/83/d5/9096aa3cf532dfad237861544eb47a0f20d5adbf1039760fed8eaae935d9/charset_normalizer-3.4.9-cp311-cp311-win32.whl", hash = "sha256:ac351b3b8014eead140e77e9717e2992c6bbe30b63bc3422422eb84865412e3d", size = 150456, upload_time = "2026-07-07T14:33:11.217Z" }, + { url = "https://files.pythonhosted.org/packages/ed/a1/e29995109e455dc8eff8d0fac6ae509be39561318a7cfeac5d33ad029213/charset_normalizer-3.4.9-cp311-cp311-win_amd64.whl", hash = "sha256:6366a16e1a25018694d6a5d784d09b046edc9eac40ea2b54065c3052672516a1", size = 161410, upload_time = "2026-07-07T14:33:12.743Z" }, + { url = "https://files.pythonhosted.org/packages/4f/8d/1569f4d0032d6ba2a4fe4591c35bf87868c600c41a71eb5c2e1ffa8464c2/charset_normalizer-3.4.9-cp311-cp311-win_arm64.whl", hash = "sha256:1d22856ffbe153a602df38e4a5464f0b748a54002e0d69ac6d2ad0a197cc99ec", size = 152649, upload_time = "2026-07-07T14:33:14.173Z" }, + { url = "https://files.pythonhosted.org/packages/70/4a/ecbd131485c07fcdfad54e28946d513e3da22ef3b4bd854dcafae54ec739/charset_normalizer-3.4.9-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:45b0cc4e3556cd875e09102988d1ab8356c998b596c9fced84547c8138b487a0", size = 319300, upload_time = "2026-07-07T14:33:15.666Z" }, + { url = "https://files.pythonhosted.org/packages/ec/96/5d9364e3342d69f3a045e1777bc47c85c383e6e9466d561b33fdb419d1f9/charset_normalizer-3.4.9-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9b2aff1c7b3884512b9512c3eaadd9bab39fb45042ffaaa1dd08ff2b9f8109d9", size = 215802, upload_time = "2026-07-07T14:33:17.031Z" }, + { url = "https://files.pythonhosted.org/packages/4b/4c/5361f9aa7f2cb58d94f2ab831b3d493f69efb1d239654b4744e3c09527cb/charset_normalizer-3.4.9-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:9104ed0bd76a429d46f9ec0dbc9b08ad1d2dcdf2b00a5a0daa1c145329b35b44", size = 237171, upload_time = "2026-07-07T14:33:18.576Z" }, + { url = "https://files.pythonhosted.org/packages/50/78/ce342ca4ff30b2eb49fe6d9578df85974f90c67d294113e94efdd9664cbd/charset_normalizer-3.4.9-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:7b86a2b16095d250c6f58b3d9b2eee6f4147754344f3dab0922f7c9bf7d226c9", size = 233075, upload_time = "2026-07-07T14:33:20.084Z" }, + { url = "https://files.pythonhosted.org/packages/01/c4/4fa4c8b3097a11f3c5f09a35b72ed6855fb1d332469504962ab7bafcc702/charset_normalizer-3.4.9-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5e226f6218febc71f6c1fc2fafb91c226f75bdc1d8fb12d66823716e891608fd", size = 224256, upload_time = "2026-07-07T14:33:21.747Z" }, + { url = "https://files.pythonhosted.org/packages/87/3a/ad914516df7e358a81aae018caa5e0470ba827fa6d763b1d2e87d920a5f6/charset_normalizer-3.4.9-cp312-cp312-manylinux_2_31_armv7l.whl", hash = "sha256:90c44bc373b7687f6948b693cceaea1348ae0975d7474746559494468e3c1d84", size = 208784, upload_time = "2026-07-07T14:33:23.313Z" }, + { url = "https://files.pythonhosted.org/packages/d7/74/3c12f9755717dfe5c5c87da63f35d765fa0c00382ec26bf23f7fae34f2ba/charset_normalizer-3.4.9-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:9cdef90ae47919cae358d8ab15797a800ed41da7aba5d72419fb510729e2ed4b", size = 219928, upload_time = "2026-07-07T14:33:24.814Z" }, + { url = "https://files.pythonhosted.org/packages/33/9a/895095b83e7907abd6d3d99aad3a38ad0d9686cc186cb0c94c24320fe63e/charset_normalizer-3.4.9-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:60f44ade2cf573dad7a277e6f8ca9a51a21dda572b13bd7d8539bb3cd5dbedde", size = 218489, upload_time = "2026-07-07T14:33:26.42Z" }, + { url = "https://files.pythonhosted.org/packages/a1/34/ef5c05f412f42520d7709b7d3784d19640839eb7366ded1755511585429f/charset_normalizer-3.4.9-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:a1786910334ed46ab1dd73222f2cd1e05c2c3bb39f6dddb4f8b36fc382058a39", size = 210267, upload_time = "2026-07-07T14:33:27.952Z" }, + { url = "https://files.pythonhosted.org/packages/83/dc/9b29fa4412b318bf3bfea985c35d67eb55e04b59a7c3f2237168b0e0be6f/charset_normalizer-3.4.9-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:03d07803992c6c7bbc976327f34b18b6160327fc81cb82c9d504720ac0be3b62", size = 226030, upload_time = "2026-07-07T14:33:29.397Z" }, + { url = "https://files.pythonhosted.org/packages/0e/42/6dbc00b8cd16011691203e33570fa42ed5746599a2e878112d16eab403a3/charset_normalizer-3.4.9-cp312-cp312-win32.whl", hash = "sha256:78841cccf1af7b40f6f716338d50c0902dbe88d9f800b3c973b7a9a0a693a642", size = 151185, upload_time = "2026-07-07T14:33:30.781Z" }, + { url = "https://files.pythonhosted.org/packages/80/cc/f920afd1a23c58ccd53c1d36085a71893a4737ff5e66e0371efab6809850/charset_normalizer-3.4.9-cp312-cp312-win_amd64.whl", hash = "sha256:4b3dac63058cc36820b0dd072f89898604e2d39686fe05321729d00d8ac185a0", size = 162557, upload_time = "2026-07-07T14:33:32.176Z" }, + { url = "https://files.pythonhosted.org/packages/f0/e6/0386d43a261ff4e4b30c5857af7df877254b46bec7b9d1b74b6bf969a90b/charset_normalizer-3.4.9-cp312-cp312-win_arm64.whl", hash = "sha256:78fa18e436a1a0e58dbd7e02fc4473f3f32cceb12df9dfca542d075961c307d2", size = 152665, upload_time = "2026-07-07T14:33:33.711Z" }, + { url = "https://files.pythonhosted.org/packages/b2/06/97ec2aeae780b31d742b6352218b43841a6871e2564578ca522dce4a45c3/charset_normalizer-3.4.9-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:440eede837960000d74978f0eba527be106b5b9aee0daf779d395276ed0b0614", size = 317688, upload_time = "2026-07-07T14:33:35.408Z" }, + { url = "https://files.pythonhosted.org/packages/d0/39/8ff066c672434225f8d25f8b739f992af250944392173dcc88362681c9bf/charset_normalizer-3.4.9-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:21e764fd1e70b6a3e205a0e46f3051701f98a8cb3fad66eeb80e48bb502f8698", size = 214982, upload_time = "2026-07-07T14:33:36.996Z" }, + { url = "https://files.pythonhosted.org/packages/92/8f/3a47a3667c83c2df9483d91644c6c107de3bf8874aa1793da9d3012eb986/charset_normalizer-3.4.9-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:e4fd89cc178bced6ad29cb3e6dd4aa63fa5017c3524dbd0b25998fb64a87cc8b", size = 236460, upload_time = "2026-07-07T14:33:38.536Z" }, + { url = "https://files.pythonhosted.org/packages/f1/60/b22cdbee7e4013dab8b0d7647fc6181120fbbbc8f7025c226d15bd5a47fc/charset_normalizer-3.4.9-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:bd47ba7fc3ca94896759ea0109775132d3e7ab921fbf54038e1bab2e46c313c9", size = 232003, upload_time = "2026-07-07T14:33:40.059Z" }, + { url = "https://files.pythonhosted.org/packages/ea/f8/72eb13dcabe7257035cea8aefd922caad2f110d252bf9f67c4c2ca763aee/charset_normalizer-3.4.9-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:84fd18bcc17526fc2b3c1af7d2b9217d32c9c04448c16ec693b9b4f1985c3d33", size = 223149, upload_time = "2026-07-07T14:33:41.631Z" }, + { url = "https://files.pythonhosted.org/packages/b0/3e/faee8f9de92b14ee1198e9163252bb15efee7301b31256a3b6d9ebfdd0dd/charset_normalizer-3.4.9-cp313-cp313-manylinux_2_31_armv7l.whl", hash = "sha256:5b10cd92fc5c498b35a8635df6d5a100207f88b63a4dc1de7ef9a548e1e2cd63", size = 207901, upload_time = "2026-07-07T14:33:43.209Z" }, + { url = "https://files.pythonhosted.org/packages/3a/25/45f30093ae27dd7b92a793b61882a38685f993700113ca36e0c9c14965e1/charset_normalizer-3.4.9-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a4fbdde9dd4a9ce5fd52c2b3a347bb50cc89483ef783f1cb00d408c13f7a96c0", size = 219176, upload_time = "2026-07-07T14:33:44.725Z" }, + { url = "https://files.pythonhosted.org/packages/48/18/c8f397329c35e32f6a837e488986f4ae03bd2abebc453b48714991630c2f/charset_normalizer-3.4.9-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:416c229f77e5ea25b3dfd4b582f8d73d7e43c22320302b9ab128a2d3a0b38efe", size = 217356, upload_time = "2026-07-07T14:33:46.192Z" }, + { url = "https://files.pythonhosted.org/packages/86/7e/5ce0bba863470fd1902d5e5843968951bddf38abe4742fc97116ef4598b3/charset_normalizer-3.4.9-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:75286256590a6320cf106a0d28970d3560aad9ee09aa7b34fb40524792436d35", size = 209614, upload_time = "2026-07-07T14:33:47.705Z" }, + { url = "https://files.pythonhosted.org/packages/6c/ef/2473d3c4d869155be4af1191111d59c4d5c4e0173026f7e85b176e23bf65/charset_normalizer-3.4.9-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:69b157c5d3292bcd443faca052f3096f637f1e074b98212a933c074ae23dc3b8", size = 224991, upload_time = "2026-07-07T14:33:49.238Z" }, + { url = "https://files.pythonhosted.org/packages/d0/a3/53ddae3db108a088156aa8ddfafd411ebbc1340f48c5573f697b27f69a39/charset_normalizer-3.4.9-cp313-cp313-win32.whl", hash = "sha256:51307f5c71007673a2bf8232ad973483d281e74cb99c8c5a990af1eefa6277d9", size = 150622, upload_time = "2026-07-07T14:33:50.711Z" }, + { url = "https://files.pythonhosted.org/packages/e8/ef/6953a77c7cf2c2ff9998e6f575ab3e380119f100223381565a4f94c1f836/charset_normalizer-3.4.9-cp313-cp313-win_amd64.whl", hash = "sha256:fe2c7201c642b7c308f1675355ad7ff7b66acfe3541625efe5a3ad38f29d6115", size = 161947, upload_time = "2026-07-07T14:33:52.197Z" }, + { url = "https://files.pythonhosted.org/packages/6e/fb/d560d1d1555debbfe7849d9cac6145c1b537709d79576bf22557ed803b82/charset_normalizer-3.4.9-cp313-cp313-win_arm64.whl", hash = "sha256:611057cc5d5c0afc743ba8be6bd828c17e0aaa8643f9d0a9b9bb7dea80eb8012", size = 152594, upload_time = "2026-07-07T14:33:53.486Z" }, + { url = "https://files.pythonhosted.org/packages/7e/8d/496817fa0944239ecae662dd57ea765cfeaec6a735f9f025d4b7b72e7143/charset_normalizer-3.4.9-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:0327fcd59a935777d83410750c50600ee9571af2846f71ce40f25b13da1ef380", size = 317253, upload_time = "2026-07-07T14:33:54.994Z" }, + { url = "https://files.pythonhosted.org/packages/2b/f9/ef4a69ea338ad3c0deceea0f5f7d2380ae8b52132b06d652cb0d2cd86706/charset_normalizer-3.4.9-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8a79d9f4d8001473a30c163556b3c3bfebec837495a412dde78b51672f6134f9", size = 215898, upload_time = "2026-07-07T14:33:56.334Z" }, + { url = "https://files.pythonhosted.org/packages/8c/e7/5ddfd76fc061eb52de219658a4aa431cbacadf0a0219c8854f00da50d289/charset_normalizer-3.4.9-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:33bdcc2a32c0a0e861f60841a512c8acc658c87c2ac59d89e3a46dacf7d866e4", size = 236718, upload_time = "2026-07-07T14:33:57.9Z" }, + { url = "https://files.pythonhosted.org/packages/49/ba/768fa3f36048d81c477a0ce61f813bc1454d80917ccfe550abd9f44f5e24/charset_normalizer-3.4.9-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:f840ed6d8ecba8255df8c42b87fadeda98ddfc6eeec05e2dc66e26d46dd6f58a", size = 232519, upload_time = "2026-07-07T14:33:59.811Z" }, + { url = "https://files.pythonhosted.org/packages/f4/c4/b3e049d2aa3766180c78507110543d9d50894cc97f57de543f1be521dcdc/charset_normalizer-3.4.9-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c25fe15c70c59eb7c5ce8c06a1f3fa1da0ecc5ea1e7a5922c40fd2fa9b0d5046", size = 223143, upload_time = "2026-07-07T14:34:01.517Z" }, + { url = "https://files.pythonhosted.org/packages/19/79/55c32d06d76ae4feafe053f061f3e3ab70bcf19f4007797ce8c3efda7830/charset_normalizer-3.4.9-cp314-cp314-manylinux_2_31_armv7l.whl", hash = "sha256:f7fb7d750cfa0a070d2c24e831fd3481019a60dd317ea2b39acbcebc08b6ed81", size = 206742, upload_time = "2026-07-07T14:34:03.04Z" }, + { url = "https://files.pythonhosted.org/packages/10/e0/47c079dd82d217c807479cd59ffd30af56307ea31c108b75758970459ad3/charset_normalizer-3.4.9-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:4d1c96a7a18b9690a4d46df09e3e3382406ae3213727cd1019ebade1c4a81917", size = 219191, upload_time = "2026-07-07T14:34:04.657Z" }, + { url = "https://files.pythonhosted.org/packages/42/ab/b9bc2e77d6b44a7e46ef62ec5cac1c9a6ba7b9135a5d560f002696ec9995/charset_normalizer-3.4.9-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:a4cfde78a9f2880208d16a93b795726a3017d5977e08d1e162a7a31322479c41", size = 218328, upload_time = "2026-07-07T14:34:06.115Z" }, + { url = "https://files.pythonhosted.org/packages/f1/78/c9c71d599f5aa2d42bcdd35cbbd46d7f535351a57e40ff7d8e5a7e219401/charset_normalizer-3.4.9-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:d4d6fcde76f94f5cb9e43e9e9a61f16dacefd228cbbf6f1a09bd9b219a92f1a1", size = 207406, upload_time = "2026-07-07T14:34:07.554Z" }, + { url = "https://files.pythonhosted.org/packages/f6/39/c914445c321a845097ce4f6ac7de9a18228a77b766272125a1ce00d851eb/charset_normalizer-3.4.9-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:898f0e9068ca27d37f8e83a5b962821df851532e6c4a7d615c1c033f9da6eedf", size = 225157, upload_time = "2026-07-07T14:34:09.061Z" }, + { url = "https://files.pythonhosted.org/packages/9b/f2/c0d4b8508565a36bc5c624e88ed297f5b0b1095011034d7f5b83a69908b5/charset_normalizer-3.4.9-cp314-cp314-win32.whl", hash = "sha256:c1c948747b03be832dceed96ca815cef7360de9aa19d37c730f8e3f6101aca48", size = 151095, upload_time = "2026-07-07T14:34:10.901Z" }, + { url = "https://files.pythonhosted.org/packages/49/fd/a1d26144398c67486422a72bf5812cda22cb4ccfcd95a290fb41ceb4b8e2/charset_normalizer-3.4.9-cp314-cp314-win_amd64.whl", hash = "sha256:16b65ea0f2465b6fb52aa22de5eca612aa964ddfec00a912e26f4656cbef890b", size = 162796, upload_time = "2026-07-07T14:34:12.47Z" }, + { url = "https://files.pythonhosted.org/packages/20/95/d75e82f8ce9fd323ebf059c16c9aadefb22a1ecde13b7840b35835e4886c/charset_normalizer-3.4.9-cp314-cp314-win_arm64.whl", hash = "sha256:40a126142a56b2dfc0aacbad1de8310cbf60da7656db0e6b16eebd48e3e93519", size = 153334, upload_time = "2026-07-07T14:34:14.044Z" }, + { url = "https://files.pythonhosted.org/packages/00/5e/17398df3a139985ba9d11ed072531986f408c8fca952835ef1ab1820c02b/charset_normalizer-3.4.9-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:609b3ba8fcc0fb5ab7af00719d0fb6ad0cb518e48e7712d12fd68f1327951198", size = 338848, upload_time = "2026-07-07T14:34:15.688Z" }, + { url = "https://files.pythonhosted.org/packages/cd/91/7253a32e86b7e1d1239b1b36ba6dd0f021a21107ab33054b53119cc083b9/charset_normalizer-3.4.9-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:51447e9aa2684679af07ca5021c3db526e0284347ebf4ffcec1154c3350cfe32", size = 223022, upload_time = "2026-07-07T14:34:17.248Z" }, + { url = "https://files.pythonhosted.org/packages/cb/32/2e64bd2be10e89c61e57ebe6a93fd98ae88eb7ebe414b5121f22c96c69eb/charset_normalizer-3.4.9-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:cc1b0fff8ead343dae06305f954eb8468ba0ec1a97881f42489d198e4ce3c632", size = 241590, upload_time = "2026-07-07T14:34:18.813Z" }, + { url = "https://files.pythonhosted.org/packages/3d/ef/d96ec496cfea0c21db43b0ad03891308b02388d054cc902cf0e5a1ad6a88/charset_normalizer-3.4.9-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:fa36ec09ef71d158186bc79e359ff5fdd6e7996fe8ab638f00d6b93139ba4fcf", size = 239584, upload_time = "2026-07-07T14:34:20.52Z" }, + { url = "https://files.pythonhosted.org/packages/d4/ce/9af95f7876194bd7a14e3dfe4a4de2e0bff02666a3910d72beafd06cc297/charset_normalizer-3.4.9-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:df115d4d83168fdf2cae48ef1ff6d1cb4c466364e30861b37121de0f3bf1b990", size = 230224, upload_time = "2026-07-07T14:34:22.189Z" }, + { url = "https://files.pythonhosted.org/packages/52/94/af74dde74a3996bd959c350709bfe50e297823d70a8c1cbd54b838880863/charset_normalizer-3.4.9-cp314-cp314t-manylinux_2_31_armv7l.whl", hash = "sha256:f86c6358749bd4fda175388691e3ba8c46e24c5347d0afd20f9b7edfc9faf07d", size = 212667, upload_time = "2026-07-07T14:34:23.857Z" }, + { url = "https://files.pythonhosted.org/packages/ee/f0/f1c4fe746c395922961b5916ed1d7d6e7d4c84851d19ed43cc89980ec953/charset_normalizer-3.4.9-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:32286a2c8d167e897177b673176c1e3e00d4057caf5d2b64eef9a3666b03018e", size = 227179, upload_time = "2026-07-07T14:34:25.586Z" }, + { url = "https://files.pythonhosted.org/packages/e4/56/6c745619ac397e8871e2bcd3cea1eec86b877488f33888b3aef5c3ed506e/charset_normalizer-3.4.9-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:83aed2c10721ddd90f68140685391b50811a880af20654c59af6b6c66c40513c", size = 225372, upload_time = "2026-07-07T14:34:27.212Z" }, + { url = "https://files.pythonhosted.org/packages/78/ad/98aae8630ac71f16711968e38a5acfecce41b778bf2f0312851020f565a8/charset_normalizer-3.4.9-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:cd6c3d4b783c556fa00bf540854e42f135e2f256abd29669fcd0da0f2dec79c2", size = 215222, upload_time = "2026-07-07T14:34:28.774Z" }, + { url = "https://files.pythonhosted.org/packages/f7/40/9593d54209765207a7f11073c06494c1721e4ca4a0a426c597679bf7f91e/charset_normalizer-3.4.9-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:ee2f2a527e3c1a6e6411eb4209642e138b544a2d72fe5d0d76daf77b24063534", size = 231958, upload_time = "2026-07-07T14:34:30.345Z" }, + { url = "https://files.pythonhosted.org/packages/b1/27/693ee5e8a18191eb38647360c51cd505013e2bd3b366aa43fd5344c21e3c/charset_normalizer-3.4.9-cp314-cp314t-win32.whl", hash = "sha256:0d861473f743244d349b50f850d10eb87aeb22bbdcc8e64f79273c94af5a8226", size = 155580, upload_time = "2026-07-07T14:34:31.884Z" }, + { url = "https://files.pythonhosted.org/packages/80/3f/bd97d3d9c613013d07cb7733d299385b41df37f0471310f5a73dc359f0b8/charset_normalizer-3.4.9-cp314-cp314t-win_amd64.whl", hash = "sha256:9b8e0f3107e2200b76f6054de99016eac3ee6762713587b36baaa7e4bd2ae177", size = 167620, upload_time = "2026-07-07T14:34:33.438Z" }, + { url = "https://files.pythonhosted.org/packages/3d/c6/eee9dca4439b1061f76373f06ea855678cc4a64c1c3c90b50e479edbb8eb/charset_normalizer-3.4.9-cp314-cp314t-win_arm64.whl", hash = "sha256:19ac87f93086ce37b86e098888555c4b4bc48102279bae3350098c0ed664b501", size = 158037, upload_time = "2026-07-07T14:34:35.018Z" }, + { url = "https://files.pythonhosted.org/packages/a6/ec/81e22253f4b7091eca6515bb3da5e45d05a663f7f567bb745695dc60f892/charset_normalizer-3.4.9-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:253a4a220747e8b5faf57ec320c4f5efb0cef05f647420bf267143ec15dba10a", size = 306122, upload_time = "2026-07-07T14:34:36.607Z" }, + { url = "https://files.pythonhosted.org/packages/c8/53/a8c042eb9eee4716f4d42a0f5a571eb32a09ec429be9fb0b8b9d765393ba/charset_normalizer-3.4.9-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:68ce9f4d6b26d5ccbf7fd4459bf75f74a0a146677ebba80597df60cbdb20e6f4", size = 206284, upload_time = "2026-07-07T14:34:38.166Z" }, + { url = "https://files.pythonhosted.org/packages/14/cb/1db8b96547ee3186cd2dd7f2e59dd560a9b80748f3604171f3c153d62811/charset_normalizer-3.4.9-cp39-cp39-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:58150c9f9b9a552505912d182ccdf26f6396fb6094816ceebcbb20eecabaed94", size = 226837, upload_time = "2026-07-07T14:34:39.77Z" }, + { url = "https://files.pythonhosted.org/packages/6a/05/c94d5cd23396289c54c93b02e0273b4dd8921641d9968c4828caf9bbaad9/charset_normalizer-3.4.9-cp39-cp39-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:df7276909358e5635ae203673ab7e509ddd224225a8d6b0790bf13eb2bde1cc5", size = 222199, upload_time = "2026-07-07T14:34:41.391Z" }, + { url = "https://files.pythonhosted.org/packages/6d/46/79847edd07244a4a2d443c6655a7b6ee94203c21539414b059f32713c357/charset_normalizer-3.4.9-cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3c09a49d6cde137258beb3d551994a2927fd35ad5cf96aed573f61bbd67c5f84", size = 214344, upload_time = "2026-07-07T14:34:42.986Z" }, + { url = "https://files.pythonhosted.org/packages/ec/b4/ef5a49b2e77c00deb43bb3256592b115ba9e4346016e82c516b8d215bf68/charset_normalizer-3.4.9-cp39-cp39-manylinux_2_31_armv7l.whl", hash = "sha256:231ddcbb35e2ff8973e1365db41fe0572662893b99a05deb183b68ad4c0c8bd4", size = 199988, upload_time = "2026-07-07T14:34:44.685Z" }, + { url = "https://files.pythonhosted.org/packages/3d/ca/ad1d7c7d3077dab873f539d3e1d083c0845a762cb0bafdfbe3ef93add598/charset_normalizer-3.4.9-cp39-cp39-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:920079c3f7456fa213e0829ed2073aaa727fd39d889ead5b4f35d0de5460d04f", size = 211908, upload_time = "2026-07-07T14:34:46.227Z" }, + { url = "https://files.pythonhosted.org/packages/ed/61/710738687f90d01c06a04ed52d6ca1e62dd9b1d8cc2567098167c4691034/charset_normalizer-3.4.9-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:0fa1aec2d32bcc03c8fa0f6f1712caad1adc38509f31142112e5c9daf5b9c833", size = 209320, upload_time = "2026-07-07T14:34:47.753Z" }, + { url = "https://files.pythonhosted.org/packages/5f/c0/6eec7bdabe6cbbcc274ec04596f6d93865751a0541d33d60d1ce179bd372/charset_normalizer-3.4.9-cp39-cp39-musllinux_1_2_armv7l.whl", hash = "sha256:ad41ba96094304aa090f5a30cb6e4fb3b3f1c264c523394b4c39bbacc4dc92ba", size = 200980, upload_time = "2026-07-07T14:34:49.362Z" }, + { url = "https://files.pythonhosted.org/packages/eb/78/59344ff9a4a7b5f6530bf7bec2c980047cc42c3a616596cdbd8cb5c1a1af/charset_normalizer-3.4.9-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:43b9e366a31fdd1c87d0eb08f579b4a82b723ea54338f040d6b4e518a026ea29", size = 216545, upload_time = "2026-07-07T14:34:50.98Z" }, + { url = "https://files.pythonhosted.org/packages/17/6d/bff78a4bacc4891bc63ec5bdc6776d8c85e47fab93d0d5f6223068fad0a4/charset_normalizer-3.4.9-cp39-cp39-win32.whl", hash = "sha256:93d59d504b230e83c7a843251681959a0b6a9cd76f6e146ce1b8a80eb8739af9", size = 146256, upload_time = "2026-07-07T14:34:52.509Z" }, + { url = "https://files.pythonhosted.org/packages/a2/55/86048bde1c9d0352940bd7b87d825091a52aef67d01cde6c6f7342c5b552/charset_normalizer-3.4.9-cp39-cp39-win_amd64.whl", hash = "sha256:ddf4af30b417d9fe16481e9b81c27ab2a7cde1ff7ba3e85653b02db7d145dc7b", size = 156413, upload_time = "2026-07-07T14:34:54.117Z" }, + { url = "https://files.pythonhosted.org/packages/28/e9/9fb6099b868c82a40698a748ae0fbd4f31ccc13844c176a07158ba2abbfd/charset_normalizer-3.4.9-cp39-cp39-win_arm64.whl", hash = "sha256:476743fe6dfe14a2da12e3ac79125dc84a3b2cf8094369a47a1529b0cd8549fe", size = 147887, upload_time = "2026-07-07T14:34:55.51Z" }, + { url = "https://files.pythonhosted.org/packages/98/2b/f97f1c193fb855c345d678f5077d6926034db0722df74c8f057020e05a25/charset_normalizer-3.4.9-py3-none-any.whl", hash = "sha256:68e5f26a1ad57ded6d1cfb85331d1c1a195314756471d97758c48498bb4dcdf5", size = 64538, upload_time = "2026-07-07T14:34:56.993Z" }, +] + +[[package]] +name = "chronicle-setup" +version = "0.1.0" +source = { editable = "." } +dependencies = [ + { name = "python-dotenv", version = "1.2.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" }, + { name = "python-dotenv", version = "1.2.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" }, + { name = "requests", version = "2.32.5", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" }, + { name = "requests", version = "2.34.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" }, + { name = "ruamel-yaml" }, +] + +[package.metadata] +requires-dist = [ + { name = "python-dotenv", specifier = ">=1.0.0" }, + { name = "requests", specifier = ">=2.31.0" }, + { name = "ruamel-yaml", specifier = ">=0.18.0" }, +] + +[[package]] +name = "idna" +version = "3.18" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/cd/63/9496c57188a2ee585e0f1db071d75089a11e98aa86eb99d9d7618fc1edce/idna-3.18.tar.gz", hash = "sha256:ffb385a7e039654cef1ab9ef32c6fafe283c0c0467bba1d9029738ce4a14a848", size = 196711, upload_time = "2026-06-02T14:34:07.794Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/1e/5e/d4e9f1a599fb8e573b7b87160658329fbf28d19eac2718f51fc3def3aa5a/idna-3.18-py3-none-any.whl", hash = "sha256:7f952cbe720b688055e3f87de14f5c3e5fdaa8bc3928985c4077ca689de849a2", size = 65455, upload_time = "2026-06-02T14:34:06.319Z" }, +] + +[[package]] +name = "python-dotenv" +version = "1.2.1" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version < '3.10'", +] +sdist = { url = "https://files.pythonhosted.org/packages/f0/26/19cadc79a718c5edbec86fd4919a6b6d3f681039a2f6d66d14be94e75fb9/python_dotenv-1.2.1.tar.gz", hash = "sha256:42667e897e16ab0d66954af0e60a9caa94f0fd4ecf3aaf6d2d260eec1aa36ad6", size = 44221, upload_time = "2025-10-26T15:12:10.434Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/14/1b/a298b06749107c305e1fe0f814c6c74aea7b2f1e10989cb30f544a1b3253/python_dotenv-1.2.1-py3-none-any.whl", hash = "sha256:b81ee9561e9ca4004139c6cbba3a238c32b03e4894671e181b671e8cb8425d61", size = 21230, upload_time = "2025-10-26T15:12:09.109Z" }, +] + +[[package]] +name = "python-dotenv" +version = "1.2.2" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version >= '3.10'", +] +sdist = { url = "https://files.pythonhosted.org/packages/82/ed/0301aeeac3e5353ef3d94b6ec08bbcabd04a72018415dcb29e588514bba8/python_dotenv-1.2.2.tar.gz", hash = "sha256:2c371a91fbd7ba082c2c1dc1f8bf89ca22564a087c2c287cd9b662adde799cf3", size = 50135, upload_time = "2026-03-01T16:00:26.196Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0b/d7/1959b9648791274998a9c3526f6d0ec8fd2233e4d4acce81bbae76b44b2a/python_dotenv-1.2.2-py3-none-any.whl", hash = "sha256:1d8214789a24de455a8b8bd8ae6fe3c6b69a5e3d64aa8a8e5d68e694bbcb285a", size = 22101, upload_time = "2026-03-01T16:00:25.09Z" }, +] + +[[package]] +name = "requests" +version = "2.32.5" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version < '3.10'", +] +dependencies = [ + { name = "certifi", marker = "python_full_version < '3.10'" }, + { name = "charset-normalizer", marker = "python_full_version < '3.10'" }, + { name = "idna", marker = "python_full_version < '3.10'" }, + { name = "urllib3", version = "2.6.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/c9/74/b3ff8e6c8446842c3f5c837e9c3dfcfe2018ea6ecef224c710c85ef728f4/requests-2.32.5.tar.gz", hash = "sha256:dbba0bac56e100853db0ea71b82b4dfd5fe2bf6d3754a8893c3af500cec7d7cf", size = 134517, upload_time = "2025-08-18T20:46:02.573Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/1e/db/4254e3eabe8020b458f1a747140d32277ec7a271daf1d235b70dc0b4e6e3/requests-2.32.5-py3-none-any.whl", hash = "sha256:2462f94637a34fd532264295e186976db0f5d453d1cdd31473c85a6a161affb6", size = 64738, upload_time = "2025-08-18T20:46:00.542Z" }, +] + +[[package]] +name = "requests" +version = "2.34.2" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version >= '3.10'", +] +dependencies = [ + { name = "certifi", marker = "python_full_version >= '3.10'" }, + { name = "charset-normalizer", marker = "python_full_version >= '3.10'" }, + { name = "idna", marker = "python_full_version >= '3.10'" }, + { name = "urllib3", version = "2.7.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/ac/c3/e2a2b89f2d3e2179abd6d00ebd70bff6273f37fb3e0cc209f48b39d00cbf/requests-2.34.2.tar.gz", hash = "sha256:f288924cae4e29463698d6d60bc6a4da69c89185ad1e0bcc4104f584e960b9ed", size = 142856, upload_time = "2026-05-14T19:25:27.735Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a0/f4/c67b0b3f1b9245e8d266f0f112c500d50e5b4e83cb6f3b71b6528104182a/requests-2.34.2-py3-none-any.whl", hash = "sha256:2a0d60c172f83ac6ab31e4554906c0f3b3588d37b5cb939b1c061f4907e278e0", size = 73075, upload_time = "2026-05-14T19:25:26.443Z" }, +] + +[[package]] +name = "ruamel-yaml" +version = "0.19.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/c7/3b/ebda527b56beb90cb7652cb1c7e4f91f48649fbcd8d2eb2fb6e77cd3329b/ruamel_yaml-0.19.1.tar.gz", hash = "sha256:53eb66cd27849eff968ebf8f0bf61f46cdac2da1d1f3576dd4ccee9b25c31993", size = 142709, upload_time = "2026-01-02T16:50:31.84Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b8/0c/51f6841f1d84f404f92463fc2b1ba0da357ca1e3db6b7fbda26956c3b82a/ruamel_yaml-0.19.1-py3-none-any.whl", hash = "sha256:27592957fedf6e0b62f281e96effd28043345e0e66001f97683aa9a40c667c93", size = 118102, upload_time = "2026-01-02T16:50:29.201Z" }, +] + +[[package]] +name = "urllib3" +version = "2.6.3" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version < '3.10'", +] +sdist = { url = "https://files.pythonhosted.org/packages/c7/24/5f1b3bdffd70275f6661c76461e25f024d5a38a46f04aaca912426a2b1d3/urllib3-2.6.3.tar.gz", hash = "sha256:1b62b6884944a57dbe321509ab94fd4d3b307075e0c2eae991ac71ee15ad38ed", size = 435556, upload_time = "2026-01-07T16:24:43.925Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/39/08/aaaad47bc4e9dc8c725e68f9d04865dbcb2052843ff09c97b08904852d84/urllib3-2.6.3-py3-none-any.whl", hash = "sha256:bf272323e553dfb2e87d9bfd225ca7b0f467b919d7bbd355436d3fd37cb0acd4", size = 131584, upload_time = "2026-01-07T16:24:42.685Z" }, +] + +[[package]] +name = "urllib3" +version = "2.7.0" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version >= '3.10'", +] +sdist = { url = "https://files.pythonhosted.org/packages/53/0c/06f8b233b8fd13b9e5ee11424ef85419ba0d8ba0b3138bf360be2ff56953/urllib3-2.7.0.tar.gz", hash = "sha256:231e0ec3b63ceb14667c67be60f2f2c40a518cb38b03af60abc813da26505f4c", size = 433602, upload_time = "2026-05-07T16:13:18.596Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7f/3e/5db95bcf282c52709639744ca2a8b149baccf648e39c8cc87553df9eae0c/urllib3-2.7.0-py3-none-any.whl", hash = "sha256:9fb4c81ebbb1ce9531cce37674bbc6f1360472bc18ca9a553ede278ef7276897", size = 131087, upload_time = "2026-05-07T16:13:17.151Z" }, +] From fbaa56e580c7a6775989752c7bd71a7f7724a9f9 Mon Sep 17 00:00:00 2001 From: Ankush Malaker <43288948+AnkushMalaker@users.noreply.github.com> Date: Wed, 5 Aug 2026 22:05:29 +0000 Subject: [PATCH 08/18] fix(setup): install chronicle-setup editable so source edits take effect MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit uv run --with-requirements caches the environment it builds and reuses it while the requirements text is unchanged, so a non-editable path dependency stays frozen at the sources it was first built from — edits to chronicle_setup/ were silently ignored by the wizard, every init.py, and services.py doctor, which kept running a stale wheel from ~/.cache/uv. Measured on uv 0.6.16, neither `uv cache clean` nor cache-keys invalidates it; only --reinstall-package does. An editable install links to the source tree instead. Rationale recorded in init-system.md with a probe command. --- docs/init-system.md | 21 +++++++++++++++++++++ setup-requirements.txt | 5 ++++- 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/docs/init-system.md b/docs/init-system.md index f8defd685..bbd99fa56 100644 --- a/docs/init-system.md +++ b/docs/init-system.md @@ -306,6 +306,27 @@ cd extras/asr-services && docker compose up --build -d - **Backend**: `setup-requirements.txt` (rich>=13.0.0, pyyaml>=6.0.0) - **Extras**: No additional setup dependencies required +### Why `chronicle-setup` is installed editable + +`setup-requirements.txt` lists the shared package as `-e ./extras/chronicle-setup`. +The `-e` is load-bearing: `uv run --with-requirements` caches the environment it +builds and reuses it while this file's text is unchanged, so a non-editable path +dependency stays frozen at the sources it was first built from. Edits to +`chronicle_setup/` are then silently ignored by the wizard, every `init.py`, and +`services.py doctor`, which keep running a stale wheel from `~/.cache/uv`. + +Measured on uv 0.6.16: neither `uv cache clean chronicle-setup` nor a +`[tool.uv] cache-keys` entry invalidates it — only `--reinstall-package`, which +every invocation would have to remember. Editable installs link to the source +tree instead. If a change to this package appears to have no effect, check that +the `-e` is still there: + +```bash +uv run --with-requirements setup-requirements.txt \ + python -c "import chronicle_setup, sys; print(chronicle_setup.__file__)" +# want the repo path, not a ~/.cache/uv/archive-* path +``` + ## Troubleshooting ### Common Issues diff --git a/setup-requirements.txt b/setup-requirements.txt index 184cdc013..ff191af08 100644 --- a/setup-requirements.txt +++ b/setup-requirements.txt @@ -3,7 +3,10 @@ # The relative path below is resolved by uv against the *working directory*, not # against this file, so setup commands must be run from the repository root: # uv run --with-requirements setup-requirements.txt python extras//init.py -./extras/chronicle-setup +# +# Keep the `-e`: without it, edits to chronicle_setup/ are silently ignored. +# See docs/init-system.md#why-chronicle-setup-is-installed-editable +-e ./extras/chronicle-setup rich>=13.0.0 python-dotenv>=1.0.0 requests>=2.31.0 From d9c25d6581d2ec9dc96d2c792c0dcb759100b883 Mon Sep 17 00:00:00 2001 From: Ankush Malaker <43288948+AnkushMalaker@users.noreply.github.com> Date: Wed, 5 Aug 2026 22:05:41 +0000 Subject: [PATCH 09/18] feat(memory): codex quota yielding and token-usage telemetry MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The codex executor shares one account-wide weekly budget with the user's interactive Codex sessions, and Chronicle's background recording is the cheaper consumer to give up: a yielded run still records the conversation via the direct (metered API) memory agent, while a blocked interactive session is stuck for days. - codex_quota.py reads account/rateLimits/read; memory.codex. max_used_percent (default 80) is the share Chronicle may consume before yielding, limit_id selects the metered bucket for non-default models. The probe fails OPEN — it optimises ahead of Codex's own limit error, so an unreadable quota must not stop memory extraction. - Token usage is summed from turn.completed events (the only place the CLI reports cost) into MemoryAgentResult.usage and emitted as a child codex_turn LLM span: current Langfuse drops usage from invoke_agent spans, so the parent span only mirrors it for filtering. - Quota snapshot attributes ride on the agent span for observability. --- .../services/memory/agent/codex_agent.py | 165 +++++++++++- .../services/memory/agent/codex_quota.py | 189 +++++++++++++ .../services/memory/agent/memory_agent.py | 4 + .../advanced/tests/test_codex_executor.py | 255 +++++++++++++++++- config/defaults.yml | 10 + 5 files changed, 615 insertions(+), 8 deletions(-) create mode 100644 backends/advanced/src/advanced_omi_backend/services/memory/agent/codex_quota.py diff --git a/backends/advanced/src/advanced_omi_backend/services/memory/agent/codex_agent.py b/backends/advanced/src/advanced_omi_backend/services/memory/agent/codex_agent.py index 0d23fa53c..3de224378 100644 --- a/backends/advanced/src/advanced_omi_backend/services/memory/agent/codex_agent.py +++ b/backends/advanced/src/advanced_omi_backend/services/memory/agent/codex_agent.py @@ -29,11 +29,13 @@ import os import shutil import tempfile +import time from datetime import datetime, timezone from pathlib import Path from typing import Dict, List, Optional from ..vault_templates import CONVERSATION_TEMPLATE, PERSON_TEMPLATE, TOPIC_TEMPLATE +from . import codex_quota from .memory_agent import MemoryAgentResult, _for_prompt, _get_prompt logger = logging.getLogger("memory_service.agent.codex") @@ -228,6 +230,22 @@ async def run( ) binary = detail + quota_payload, quota_block = await asyncio.to_thread( + self._check_quota, conversation_id + ) + if quota_block: + from .memory_agent import MemoryAgent + + return await MemoryAgent(self.root).run( + transcript, + conversation_id, + date=date, + duration_minutes=duration_minutes, + title=title, + vault_summary=vault_summary, + guidance=guidance, + ) + date = date or datetime.now(timezone.utc).isoformat() system_prompt = await _get_prompt( CODEX_AGENT_SYSTEM_PROMPT_ID, @@ -279,6 +297,9 @@ async def run( "chronicle.memory.executor": "codex", "chronicle.memory.sandbox_mode": sandbox_mode, "chronicle.memory.transcript_chars": len(transcript), + **codex_quota.quota_span_attributes( + quota_payload, str(settings.get("limit_id") or "") + ), "langfuse.observation.input": json.dumps( { "conversation_id": conversation_id, @@ -316,6 +337,11 @@ async def run( ) span.set_attribute("chronicle.memory.error_count", len(result.errors)) span.set_attribute("chronicle.memory.truncated", result.truncated) + # Mirrored onto the agent span for at-a-glance filtering; the + # ingestable copy lives on the child codex_turn span (see + # _record_usage_span for why it cannot live here). + for key, value in result.usage.items(): + span.set_attribute(f"chronicle.memory.usage.{key}", value) span.set_attribute( "langfuse.observation.output", json.dumps( @@ -401,6 +427,7 @@ def _run_locked( model or "default", timeout, ) + started_ns = time.time_ns() try: proc = subprocess.run( cmd, @@ -428,8 +455,11 @@ def _run_locked( except OSError as e: errors.append(f"codex exec failed to start: {e}") - command_count, turn_count, event_errors = self._parse_events(stdout) + ended_ns = time.time_ns() + + command_count, turn_count, event_errors, usage = self._parse_events(stdout) errors.extend(event_errors) + self._record_usage_span(usage, model, started_ns, ended_ns) summary = "" try: @@ -458,22 +488,109 @@ def _run_locked( tool_calls=command_count, removed=removed, errors=errors, + usage=usage, truncated=failed, ) logger.info( "codex agent done: conv=%s turns=%d commands=%d touched=%d removed=%d " - "errors=%d%s — %s", + "errors=%d tokens=in:%d/cached:%d/out:%d%s — %s", conversation_id, result.rounds, command_count, len(touched), len(removed), len(errors), + usage.get("input_tokens", 0), + usage.get("input_cached_tokens", 0), + usage.get("output_tokens", 0), " (FAILED)" if failed else "", summary[:160], ) return result + @staticmethod + def _check_quota(conversation_id: str) -> tuple[Optional[dict], bool]: + """Return the quota snapshot and whether this run should yield the budget. + + Chronicle's background recording shares one account-wide weekly budget with + the user's interactive Codex sessions, and is the cheaper consumer to give + up: a yielded run still records the conversation via the direct (metered + API) executor, while a blocked interactive session is stuck for days. + + Fails OPEN — an unreadable quota yields ``False`` and the run proceeds. The + probe is an optimisation over Codex's own limit error, not a correctness + gate, so a broken probe must not stop memory extraction entirely. + """ + settings = _codex_settings() + threshold = settings.get("max_used_percent") + if threshold is None: + return None, False + try: + threshold = int(threshold) + except (TypeError, ValueError): + logger.warning("ignoring non-numeric memory.codex.max_used_percent") + return None, False + + limit_id = str(settings.get("limit_id") or "") + payload = codex_quota.read_rate_limits() + used = codex_quota.bucket_used_percent(payload, limit_id) + if used is None: + logger.debug("codex quota unknown for conv=%s; proceeding", conversation_id) + return payload, False + if used < threshold: + return payload, False + + logger.warning( + "codex quota %d%% used (>= %d%% budget for Chronicle) — recording conv=%s " + "via the direct memory agent instead, leaving the remainder for " + "interactive use", + used, + threshold, + conversation_id, + ) + return payload, True + + @staticmethod + def _record_usage_span( + usage: Dict[str, int], model: str, started_ns: int, ended_ns: int + ) -> None: + """Emit the model call as a child LLM span carrying the run's token usage. + + Deliberately NOT on the parent ``codex_memory_agent`` span: current Langfuse + drops usage from spans whose ``gen_ai.operation.name`` is ``invoke_agent`` or + ``agent_step`` (it assumes the agent span duplicates usage from child + model-call spans) and would ingest the tokens as zero without erroring. Older + Langfuse — including 3.x — has no such guard, so putting usage on the agent + span works today and silently breaks on upgrade. A child model-call span is + correct under both. + + Created after the subprocess returns, since usage is only known then, with + explicit timestamps so it still spans the real call window. + """ + if not usage: + return + try: + from advanced_omi_backend.observability.otel_setup import get_tracer + + tracer = get_tracer("chronicle.memory.codex") + if tracer is None: + return + attributes = { + "openinference.span.kind": "LLM", + "gen_ai.operation.name": "chat", + "gen_ai.system": "openai", + "gen_ai.provider.name": "openai_codex_cli", + "gen_ai.request.model": model or "codex-default", + "gen_ai.response.model": model or "codex-default", + **{f"gen_ai.usage.{k}": v for k, v in usage.items()}, + } + span = tracer.start_span( + "codex_turn", attributes=attributes, start_time=started_ns + ) + span.end(end_time=ended_ns) + except Exception: # noqa: BLE001 - telemetry must never fail the run + logger.debug("failed to record codex usage span", exc_info=True) + def _snapshot(self) -> Dict[str, str]: """Vault-relative ``*.md`` contents (same shape the provider's audit diff uses).""" snapshot: Dict[str, str] = {} @@ -489,11 +606,17 @@ def _snapshot(self) -> Dict[str, str]: return snapshot @staticmethod - def _parse_events(stdout: str) -> tuple[int, int, List[str]]: - """Tolerantly scan the ``--json`` JSONL stream for counts and errors.""" + def _parse_events(stdout: str) -> tuple[int, int, List[str], Dict[str, int]]: + """Tolerantly scan the ``--json`` JSONL stream for counts, errors, and usage. + + ``turn.completed`` carries the turn's token ``usage``; it is the only place + the CLI reports what a run actually cost, so it is summed across turns and + translated into Langfuse's usage-detail key names. + """ commands = 0 turns = 0 errors: List[str] = [] + usage: Dict[str, int] = {} for line in stdout.splitlines(): line = line.strip() if not line.startswith("{"): @@ -509,10 +632,42 @@ def _parse_events(stdout: str) -> tuple[int, int, List[str]]: commands += 1 elif etype == "turn.completed": turns += 1 + for key, value in CodexMemoryAgent._turn_usage(event).items(): + usage[key] = usage.get(key, 0) + value elif etype == "turn.failed": turns += 1 failure = event.get("error") or {} errors.append(f"codex turn failed: {failure.get('message', failure)}") elif etype == "error": errors.append(f"codex error: {event.get('message', event)}") - return commands, turns, errors + return commands, turns, errors, usage + + @staticmethod + def _turn_usage(event: dict) -> Dict[str, int]: + """Map one ``turn.completed`` event's ``usage`` to Langfuse usage details. + + Tolerant by design: the CLI's field names are not a stable contract, so an + absent or oddly-shaped block yields ``{}`` rather than failing the run. + """ + raw = event.get("usage") + if not isinstance(raw, dict): + return {} + # Codex reports cached input tokens *inside* input_tokens, which is what + # Langfuse's normaliser assumes (it derives uncached input as + # input_tokens - input_cached_tokens), so both pass through unchanged. + # Caveat on Langfuse 3.x: it instead adds the two, so the rollup `usage.input` + # and `total` over-count cached tokens there. `usageDetails.input` is right + # on both. + field_map = { + "input_tokens": "input_tokens", + "cached_input_tokens": "input_cached_tokens", + "output_tokens": "output_tokens", + "reasoning_output_tokens": "output_reasoning_tokens", + } + usage: Dict[str, int] = {} + for source, target in field_map.items(): + value = raw.get(source) + if isinstance(value, bool) or not isinstance(value, (int, float)): + continue + usage[target] = int(value) + return usage diff --git a/backends/advanced/src/advanced_omi_backend/services/memory/agent/codex_quota.py b/backends/advanced/src/advanced_omi_backend/services/memory/agent/codex_quota.py new file mode 100644 index 000000000..3d3e34cd9 --- /dev/null +++ b/backends/advanced/src/advanced_omi_backend/services/memory/agent/codex_quota.py @@ -0,0 +1,189 @@ +"""Read the ChatGPT subscription's Codex quota, so background runs can yield to it. + +Codex's rate limit is an account-wide weekly budget shared with the user's own +interactive sessions. Chronicle's background vault recording is the lower-priority +consumer of it: a memory extraction that is skipped still lands via the direct +(metered API) executor, whereas an interactive session that hits the wall is simply +blocked for days. So the agent checks headroom before spawning ``codex exec``. + +The numbers come from the CLI itself rather than from parsing its error text: the +``codex app-server`` JSON-RPC surface exposes ``account/rateLimits/read``, which +returns per-bucket ``usedPercent`` / ``resetsAt`` / ``windowDurationMins``. That is +the same source the TUI's own usage display reads. +""" + +import contextlib +import json +import logging +import os +import shutil +import subprocess +import threading +import time +from typing import Dict, Optional + +logger = logging.getLogger("memory_service.agent.codex.quota") + +# The app-server is spawned per read and killed as soon as the response arrives; the +# cache keeps that off the hot path when conversations close in quick succession. +_CACHE_TTL_SECONDS = 120 +_READ_TIMEOUT_SECONDS = 20 + +_cache_lock = threading.Lock() +_cached: Optional[tuple[float, Optional[dict]]] = None + + +def read_rate_limits( + *, timeout: int = _READ_TIMEOUT_SECONDS, use_cache: bool = True +) -> Optional[dict]: + """Return the ``account/rateLimits/read`` payload, or ``None`` if unavailable. + + ``None`` means "could not determine" (no binary, no auth, timeout, protocol + change) and is deliberately distinct from a snapshot reporting 100% used. + Callers must not treat it as exhausted — see :func:`bucket_used_percent`. + """ + global _cached + if use_cache: + with _cache_lock: + if _cached and (time.monotonic() - _cached[0]) < _CACHE_TTL_SECONDS: + return _cached[1] + + payload = _read_uncached(timeout) + with _cache_lock: + _cached = (time.monotonic(), payload) + return payload + + +def _read_uncached(timeout: int) -> Optional[dict]: + binary = shutil.which(os.environ.get("CODEX_BINARY", "codex")) + if not binary: + return None + + proc = None + try: + proc = subprocess.Popen( + [binary, "app-server"], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.DEVNULL, + text=True, + bufsize=1, + ) + requests = ( + json.dumps( + { + "jsonrpc": "2.0", + "id": 1, + "method": "initialize", + "params": { + "clientInfo": { + "name": "chronicle-memory", + "title": "Chronicle memory agent", + "version": "1", + } + }, + } + ) + + "\n" + + json.dumps( + { + "jsonrpc": "2.0", + "id": 2, + "method": "account/rateLimits/read", + "params": {}, + } + ) + + "\n" + ) + assert proc.stdin is not None and proc.stdout is not None + proc.stdin.write(requests) + proc.stdin.flush() + + # The stream interleaves unsolicited notifications with responses, so read + # until the id=2 reply appears or the deadline passes. + deadline = time.monotonic() + timeout + result: Optional[dict] = None + while time.monotonic() < deadline: + line = proc.stdout.readline() + if not line: + break + try: + message = json.loads(line) + except json.JSONDecodeError: + continue + if message.get("id") == 2: + result = message.get("result") + break + if result is None: + logger.debug("codex app-server returned no rate-limit response in time") + return result + except Exception as e: # noqa: BLE001 — a quota probe must never break recording + logger.debug("could not read codex rate limits (%s)", e) + return None + finally: + if proc is not None: + proc.kill() + # Best-effort reap; the kill above is what actually ends the server. + with contextlib.suppress(Exception): + proc.wait(timeout=5) + + +def bucket_used_percent(payload: Optional[dict], limit_id: str = "") -> Optional[int]: + """Percent of the weekly Codex budget already spent, or ``None`` if unknown. + + ``limit_id`` selects a specific bucket from ``rateLimitsByLimitId`` (models are + metered against different buckets — e.g. ``codex`` vs ``codex_bengalfox`` for + Spark — and the account may have one exhausted while another is untouched). + Empty selects the payload's own backward-compatible single-bucket view. + """ + if not isinstance(payload, dict): + return None + snapshot = None + if limit_id: + by_id = payload.get("rateLimitsByLimitId") + if isinstance(by_id, dict): + snapshot = by_id.get(limit_id) + if snapshot is None: + # An unknown limit_id must not silently fall back to a different + # bucket's headroom — that would gate on the wrong budget. + logger.warning("codex rate-limit bucket %r not in payload", limit_id) + return None + else: + snapshot = payload.get("rateLimits") + if not isinstance(snapshot, dict): + return None + primary = snapshot.get("primary") + if not isinstance(primary, dict): + return None + used = primary.get("usedPercent") + if isinstance(used, bool) or not isinstance(used, (int, float)): + return None + return int(used) + + +def quota_span_attributes( + payload: Optional[dict], limit_id: str = "" +) -> Dict[str, object]: + """Flatten a snapshot into span attributes (empty when nothing is known).""" + used = bucket_used_percent(payload, limit_id) + if used is None: + return {} + attributes: Dict[str, object] = {"chronicle.memory.quota.used_percent": used} + snapshot = ( + (payload or {}).get("rateLimitsByLimitId", {}).get(limit_id) + if limit_id + else (payload or {}).get("rateLimits") + ) + if isinstance(snapshot, dict): + primary = snapshot.get("primary") + if isinstance(primary, dict): + for source, target in ( + ("resetsAt", "chronicle.memory.quota.resets_at"), + ("windowDurationMins", "chronicle.memory.quota.window_minutes"), + ): + value = primary.get(source) + if isinstance(value, int) and not isinstance(value, bool): + attributes[target] = value + if snapshot.get("limitId"): + attributes["chronicle.memory.quota.limit_id"] = snapshot["limitId"] + return attributes diff --git a/backends/advanced/src/advanced_omi_backend/services/memory/agent/memory_agent.py b/backends/advanced/src/advanced_omi_backend/services/memory/agent/memory_agent.py index 4365af5d7..d3aeb4fd7 100644 --- a/backends/advanced/src/advanced_omi_backend/services/memory/agent/memory_agent.py +++ b/backends/advanced/src/advanced_omi_backend/services/memory/agent/memory_agent.py @@ -185,6 +185,10 @@ class MemoryAgentResult: # so a note disappearing is never invisible in the ledger. removed: List[dict] = field(default_factory=list) errors: List[str] = field(default_factory=list) + # Token counts for the run, keyed as Langfuse usage details (``input_tokens``, + # ``output_tokens``, ``input_cached_tokens``, ``output_reasoning_tokens``). Empty + # when the executor reports none. + usage: Dict[str, int] = field(default_factory=dict) truncated: bool = ( False # loop ended on a truncated/empty LLM response, not a deliberate finish ) diff --git a/backends/advanced/tests/test_codex_executor.py b/backends/advanced/tests/test_codex_executor.py index 26eb8e9d0..3b7b9826e 100644 --- a/backends/advanced/tests/test_codex_executor.py +++ b/backends/advanced/tests/test_codex_executor.py @@ -1,12 +1,17 @@ """Codex CLI memory-agent executor: selection, filesystem-diff auditing, failure paths.""" import contextlib +import json import subprocess from types import SimpleNamespace import pytest -from advanced_omi_backend.services.memory.agent import codex_agent, memory_agent +from advanced_omi_backend.services.memory.agent import ( + codex_agent, + codex_quota, + memory_agent, +) from advanced_omi_backend.services.memory.agent.codex_agent import CodexMemoryAgent from advanced_omi_backend.services.memory.agent.memory_agent import ( MemoryAgent, @@ -59,7 +64,9 @@ def test_agent_class_falls_back_when_codex_unavailable(monkeypatch): # --------------------------------------------------------------------------- -def _fake_codex_run(vault_root, *, summary="Recorded the conversation.", returncode=0): +def _fake_codex_run( + vault_root, *, summary="Recorded the conversation.", returncode=0, usage=None +): """A subprocess.run stand-in that mimics one codex exec editing the vault.""" def fake_run(cmd, **kwargs): @@ -72,9 +79,13 @@ def fake_run(cmd, **kwargs): last_msg = cmd[cmd.index("--output-last-message") + 1] with open(last_msg, "w") as f: f.write(summary) + turn = {"type": "turn.completed"} + if usage is not None: + turn["usage"] = usage stdout = ( '{"type":"item.completed","item":{"item_type":"command_execution"}}\n' - '{"type":"turn.completed"}\n' + + json.dumps(turn) + + "\n" ) return SimpleNamespace(returncode=returncode, stdout=stdout, stderr="") @@ -132,6 +143,244 @@ def failing_run(cmd, **kwargs): assert result.touched == [] # nothing was written +# --------------------------------------------------------------------------- +# Token usage +# --------------------------------------------------------------------------- + + +def test_parse_events_sums_turn_usage(): + stdout = ( + '{"type":"turn.completed","usage":{"input_tokens":1200,' + '"cached_input_tokens":900,"output_tokens":40}}\n' + '{"type":"turn.completed","usage":{"input_tokens":300,' + '"cached_input_tokens":100,"output_tokens":10,' + '"reasoning_output_tokens":7}}\n' + ) + + _, turns, _, usage = CodexMemoryAgent._parse_events(stdout) + + assert turns == 2 + assert usage == { + "input_tokens": 1500, + "input_cached_tokens": 1000, + "output_tokens": 50, + "output_reasoning_tokens": 7, + } + + +@pytest.mark.parametrize( + "event", + [ + {"type": "turn.completed"}, # older CLI: no usage block at all + {"type": "turn.completed", "usage": None}, + {"type": "turn.completed", "usage": "unexpected"}, + {"type": "turn.completed", "usage": {"input_tokens": "many"}}, + ], +) +def test_turn_usage_tolerates_missing_or_odd_shapes(event): + """The CLI's field names are not a stable contract; usage must never break a run.""" + assert CodexMemoryAgent._turn_usage(event) == {} + + +@pytest.mark.asyncio +async def test_run_reports_usage_from_the_json_stream(tmp_path, monkeypatch, unlocked): + root = _seed_vault(tmp_path) + monkeypatch.setattr( + codex_agent, "codex_executor_available", lambda: (True, "/usr/bin/codex") + ) + monkeypatch.setattr( + subprocess, + "run", + _fake_codex_run(root, usage={"input_tokens": 8000, "output_tokens": 120}), + ) + + result = await CodexMemoryAgent(root).run("a real transcript", "conv1") + + assert result.usage == {"input_tokens": 8000, "output_tokens": 120} + + +def test_usage_span_is_a_child_not_the_agent_span(monkeypatch): + """Langfuse drops usage on ``invoke_agent`` spans, so it must ride a child LLM span. + + Pins why usage lives on ``codex_turn``: moved onto ``codex_memory_agent`` + (``gen_ai.operation.name: invoke_agent``), current Langfuse's OTEL processor + ingests the tokens as zero without erroring anywhere. Langfuse 3.x has no such + guard, so that regression would not show up on an older deployment. + """ + recorded = {} + + class _Span: + def end(self, end_time=None): + recorded["end_time"] = end_time + + class _Tracer: + def start_span(self, name, attributes=None, start_time=None): + recorded.update(name=name, attributes=attributes, start_time=start_time) + return _Span() + + monkeypatch.setattr( + "advanced_omi_backend.observability.otel_setup.get_tracer", + lambda _name: _Tracer(), + ) + + CodexMemoryAgent._record_usage_span( + {"input_tokens": 10, "input_cached_tokens": 4}, "gpt-5.6-terra", 111, 222 + ) + + assert recorded["name"] == "codex_turn" + attrs = recorded["attributes"] + assert attrs["gen_ai.operation.name"] == "chat" # NOT invoke_agent + assert attrs["gen_ai.usage.input_tokens"] == 10 + assert attrs["gen_ai.usage.input_cached_tokens"] == 4 + # Explicit timestamps: the span is created only after the subprocess returns. + assert (recorded["start_time"], recorded["end_time"]) == (111, 222) + + +def test_no_usage_emits_no_span(monkeypatch): + def _boom(_name): + raise AssertionError("tracer must not be built when there is no usage") + + monkeypatch.setattr( + "advanced_omi_backend.observability.otel_setup.get_tracer", _boom + ) + + CodexMemoryAgent._record_usage_span({}, "gpt-5.6-terra", 1, 2) + + +# --------------------------------------------------------------------------- +# Quota guard +# --------------------------------------------------------------------------- + +# Verbatim shape of a real `account/rateLimits/read` reply (codex-cli 0.144.4), +# trimmed to the fields the guard reads. Two buckets, one exhausted, one untouched. +REAL_RATE_LIMITS = { + "rateLimits": { + "limitId": "codex", + "primary": { + "usedPercent": 100, + "windowDurationMins": 10080, + "resetsAt": 1785612921, + }, + "secondary": None, + "planType": "prolite", + "rateLimitReachedType": "rate_limit_reached", + }, + "rateLimitsByLimitId": { + "codex": { + "limitId": "codex", + "primary": { + "usedPercent": 100, + "windowDurationMins": 10080, + "resetsAt": 1785612921, + }, + }, + "codex_bengalfox": { + "limitId": "codex_bengalfox", + "limitName": "GPT-5.3-Codex-Spark", + "primary": { + "usedPercent": 0, + "windowDurationMins": 10080, + "resetsAt": 1785798988, + }, + }, + }, +} + + +def test_bucket_used_percent_reads_the_default_and_named_buckets(): + assert codex_quota.bucket_used_percent(REAL_RATE_LIMITS) == 100 + assert codex_quota.bucket_used_percent(REAL_RATE_LIMITS, "codex") == 100 + # A different model's bucket can be untouched while the default is exhausted. + assert codex_quota.bucket_used_percent(REAL_RATE_LIMITS, "codex_bengalfox") == 0 + + +def test_unknown_bucket_is_unknown_not_the_default_bucket(): + """Must not silently gate on some other budget's headroom.""" + assert codex_quota.bucket_used_percent(REAL_RATE_LIMITS, "codex_nope") is None + + +@pytest.mark.parametrize("payload", [None, {}, {"rateLimits": {"primary": {}}}]) +def test_bucket_used_percent_unknown_shapes(payload): + assert codex_quota.bucket_used_percent(payload) is None + + +def test_quota_span_attributes_carry_window_and_reset(): + attrs = codex_quota.quota_span_attributes(REAL_RATE_LIMITS) + assert attrs["chronicle.memory.quota.used_percent"] == 100 + assert attrs["chronicle.memory.quota.window_minutes"] == 10080 + assert attrs["chronicle.memory.quota.resets_at"] == 1785612921 + assert codex_quota.quota_span_attributes(None) == {} + + +@pytest.mark.parametrize( + "settings,used,expect_block", + [ + ({"max_used_percent": 80}, 100, True), # over budget -> yield + ({"max_used_percent": 80}, 80, True), # at budget -> yield + ({"max_used_percent": 80}, 79, False), + ({}, 100, False), # unconfigured -> guard off + ({"max_used_percent": None}, 100, False), + ({"max_used_percent": "abc"}, 100, False), # unparseable -> guard off + ({"max_used_percent": 80}, None, False), # unreadable -> fail OPEN + ], +) +def test_quota_guard_decision(monkeypatch, settings, used, expect_block): + monkeypatch.setattr(codex_agent, "_codex_settings", lambda: settings) + monkeypatch.setattr(codex_quota, "read_rate_limits", lambda **_: {"stub": True}) + monkeypatch.setattr(codex_quota, "bucket_used_percent", lambda *_a, **_k: used) + + _, blocked = CodexMemoryAgent._check_quota("conv1") + + assert blocked is expect_block + + +@pytest.mark.asyncio +async def test_exhausted_quota_records_via_direct_agent_instead( + tmp_path, monkeypatch, unlocked +): + """Yielding must still record the conversation, not drop it.""" + root = _seed_vault(tmp_path) + monkeypatch.setattr( + codex_agent, "codex_executor_available", lambda: (True, "/usr/bin/codex") + ) + monkeypatch.setattr( + codex_agent, "_codex_settings", lambda: {"max_used_percent": 80} + ) + monkeypatch.setattr(codex_quota, "read_rate_limits", lambda **_: REAL_RATE_LIMITS) + + def _no_subprocess(*_a, **_k): + raise AssertionError("codex must not be spawned when over budget") + + monkeypatch.setattr(subprocess, "run", _no_subprocess) + + delegated = {} + + class _Direct: + def __init__(self, root, *a, **kw): + delegated["constructed"] = True + # The yield is a budget decision, not a failed run: it must NOT use the + # note-guarantee recovery path's forced fallback LLM. + delegated["force_fallback"] = kw.get("force_fallback", False) + + async def run(self, transcript, conversation_id, **kwargs): + delegated["conversation_id"] = conversation_id + return MemoryAgentResult( + conversation_id=conversation_id, + rounds=1, + touched=["Conversations/conv1.md"], + summary="recorded by the direct agent", + ) + + monkeypatch.setattr(memory_agent, "MemoryAgent", _Direct) + + result = await CodexMemoryAgent(root).run("a real transcript", "conv1") + + assert delegated["constructed"] is True + assert delegated["force_fallback"] is False + assert result.touched == ["Conversations/conv1.md"] + assert not result.truncated + + @pytest.mark.asyncio async def test_run_unavailable_executor_returns_truncated(tmp_path, monkeypatch): root = _seed_vault(tmp_path) diff --git a/config/defaults.yml b/config/defaults.yml index eac76dc09..267e66c41 100644 --- a/config/defaults.yml +++ b/config/defaults.yml @@ -570,6 +570,16 @@ memory: # danger-full-access instead. sandbox_mode: workspace-write timeout_seconds: 900 # per-run wall clock before the run is failed + # Share of the ChatGPT weekly Codex budget Chronicle may consume before it + # yields. At/over this, background recording falls back to the direct (metered + # API) executor so the remainder stays available for interactive Codex use. + # Read live from `account/rateLimits/read`; null disables the check. An + # unreadable quota fails open (the run proceeds). + max_used_percent: 80 + # Which metered bucket to check. Models meter against different buckets — e.g. + # `codex` vs `codex_bengalfox` (GPT-5.3-Codex-Spark) — so set this when `model` + # above draws on a non-default bucket. Empty uses the account's default view. + limit_id: "" extraction: enabled: true prompt: | From 79cb89413888ed958e90fdcdd5510f84d2d4fb93 Mon Sep 17 00:00:00 2001 From: Ankush Malaker <43288948+AnkushMalaker@users.noreply.github.com> Date: Wed, 5 Aug 2026 22:05:52 +0000 Subject: [PATCH 10/18] fix(transcription): decide stream resume causally, not by entry age alone MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Discovery self-heals a stale transcription:complete flag by re-attaching when the stream looks alive. Recency alone cannot answer that question: finalize_session flushes residual audio and appends the end marker as its last act, so at the exact moment the flag is set the newest entry is milliseconds old — a closing session is indistinguishable from a resuming one by age. Guessing wrong is expensive in one direction: clearing the flag revokes the handshake open_conversation_job waits on and no replacement ever arrives, stalling the conversation for the job's full 30s wait. _session_resumed now consults two causal facts first — the session must still be ACTIVE (the producer appends inside a WATCH/MULTI conditioned on active status, so a departed session can never write again) and the tail must not carry the end marker (appended strictly before the consumer can set the flag). Only then does recency answer the question it is actually good at: whether audio is flowing right now. Errors decline to re-attach, costing streaming transcription rather than corrupting the handshake. --- .../transcription/streaming_consumer.py | 94 +++++++--- .../tests/test_streaming_resume_probe.py | 160 ++++++++++++++++++ 2 files changed, 229 insertions(+), 25 deletions(-) create mode 100644 backends/advanced/tests/test_streaming_resume_probe.py diff --git a/backends/advanced/src/advanced_omi_backend/services/transcription/streaming_consumer.py b/backends/advanced/src/advanced_omi_backend/services/transcription/streaming_consumer.py index b80c7dc05..c1e33ef17 100644 --- a/backends/advanced/src/advanced_omi_backend/services/transcription/streaming_consumer.py +++ b/backends/advanced/src/advanced_omi_backend/services/transcription/streaming_consumer.py @@ -61,6 +61,15 @@ # ride out brief network blips (producer emits chunks every 0.25s when healthy). STREAM_IDLE_TIMEOUT_SECONDS = 300 +# How recently a still-ACTIVE session must have appended for its stream to count as +# resumed rather than merely quiet. A healthy producer emits a chunk every 0.25s. +STREAM_RESUME_MAX_AGE_SECONDS = 10.0 + +# Entries read from the tail when probing for the producer's end marker. The marker is +# the last thing finalize_session appends, so 1 would normally do; a small window keeps +# the probe correct if a chunk raced in behind it. +STREAM_TAIL_PROBE_ENTRIES = 5 + def _is_connection_error(e: Exception) -> bool: """Check if exception indicates WebSocket connection death.""" @@ -285,28 +294,61 @@ async def discover_streams(self) -> list[str]: return streams - async def _stream_has_fresh_entries( - self, stream_name: str, max_age_seconds: float = 10.0 - ) -> bool: - """True if the stream's newest entry is younger than ``max_age_seconds``. - - Used to distinguish a genuinely-finished stream from one a reconnecting - device has resumed writing to. A live producer emits a chunk every ~0.25s, - so a last entry within 10s means audio is actively flowing. Redis stream IDs - are ``-``, so the timestamp comes free from the entry id — no need - to decode the payload. Errors return False (treat as not-fresh → safe skip). + async def _session_resumed(self, stream_name: str, session_id: str) -> bool: + """True if a stream that carries a completion flag is still being written to. + + Answering this wrong in the permissive direction is expensive: clearing the + flag revokes the handshake ``open_conversation_job`` is blocked on, and no + replacement signal ever arrives, so the conversation stalls for that job's + full 30s wait before finishing without it. + + Recency alone cannot answer it. ``finalize_session`` flushes the residual + audio and appends the end marker as its *last* act, so at the exact moment + the flag is set the newest entry is milliseconds old — a closing session is + indistinguishable from a resuming one by age. Two causal facts decide it + instead: + + - **Session status.** ``producer._append_owned_message`` appends inside a + WATCH/MULTI whose precondition is ``status == "active"``, so a session that + has left ACTIVE can never receive another entry. Its stream is frozen, and + whatever sits at the tail is its own closing flush. + - **The end marker.** It is appended (while still ACTIVE) strictly before the + consumer can read it and set the flag, so its presence proves the producer + finished even if the FINALIZING status write has not landed yet. + + Only when neither says "finished" does recency get to speak, and there it + answers the question it is actually good at: whether audio is flowing now, or + the consumer gave up on a stream that has been silent for a long time. + + Errors return False — declining to re-attach costs a resumed session its + streaming transcription, but wrongly re-attaching corrupts the handshake for + every conversation on the session. """ try: - entries = await self.redis_client.xrevrange(stream_name, count=1) + if await self.store.get_status(session_id) != SessionStatus.ACTIVE: + return False + + entries = await self.redis_client.xrevrange( + stream_name, count=STREAM_TAIL_PROBE_ENTRIES + ) if not entries: return False + if any( + fields.get(b"end_marker") or fields.get("end_marker") + for _, fields in entries + ): + return False + + # Redis stream ids are ``-``, so age comes free from the id. entry_id = entries[0][0] if isinstance(entry_id, bytes): entry_id = entry_id.decode() entry_ms = int(entry_id.split("-")[0]) - return (time.time() * 1000 - entry_ms) < (max_age_seconds * 1000) + return (time.time() * 1000 - entry_ms) < ( + STREAM_RESUME_MAX_AGE_SECONDS * 1000 + ) except Exception as e: # noqa: BLE001 — best-effort liveness probe - logger.debug(f"Freshness check failed for {stream_name}: {e}") + logger.debug(f"Resume probe failed for {stream_name}: {e}") return False async def setup_consumer_group(self, stream_name: str): @@ -1200,19 +1242,21 @@ async def start_consuming(self, heartbeat_name: str | None = None): session_id = stream_name.replace("audio:stream:", "") completion_key = f"transcription:complete:{session_id}" if await self.redis_client.exists(completion_key): - # session_id is stable across reconnects, so the flag may be - # stale: a device reconnected onto the same stream after the - # prior connection's provider stream closed. Producer.init_session - # clears the flag on (re)connect, but a backend-only restart - # leaves THIS worker's old process_stream task alive, and it can - # set the flag (idle-timeout exit) AFTER init_session cleared it - # — re-poisoning the resumed stream until the 5-min TTL. - # Self-heal: if fresh audio is flowing into a "completed" stream, - # the session resumed — drop the flag and re-attach. - if await self._stream_has_fresh_entries(stream_name): + # The flag can outlive the provider stream it describes: a + # process_stream task that exits on its idle heartbeat sets it + # while the session is still ACTIVE, and the device may resume + # sending afterwards. Discovery would then skip that live stream + # until the 5-min TTL, starving it of transcription. + # + # Self-heal, but only for a session that can still produce. The + # flag is also the handshake open_conversation_job waits on, so + # clearing it for a session that has finished stalls that job for + # its full 30s wait (see _session_resumed). + if await self._session_resumed(stream_name, session_id): logger.info( - f"Stream {stream_name} marked complete but has fresh " - f"audio — session resumed, clearing flag and re-attaching" + f"Stream {stream_name} marked complete but its session " + f"is still active and producing — clearing flag and " + f"re-attaching" ) await self.redis_client.delete(completion_key) else: diff --git a/backends/advanced/tests/test_streaming_resume_probe.py b/backends/advanced/tests/test_streaming_resume_probe.py new file mode 100644 index 000000000..2f75c55ff --- /dev/null +++ b/backends/advanced/tests/test_streaming_resume_probe.py @@ -0,0 +1,160 @@ +"""Regression tests for the streaming consumer's "did this session resume?" probe. + +``transcription:complete:{session_id}`` does double duty: it stops the discovery +loop from re-attaching a second provider connection to a stream it already +finished, *and* it is the handshake ``open_conversation_job`` blocks on before +reading the final transcript. Clearing it therefore has a cost the discovery loop +cannot see — no replacement signal is ever produced, so the conversation job waits +out its full 30s timeout and finishes without the streaming result. + +That is what CI run 30884816710 hit. The probe used to ask only whether the +stream's newest entry was recent, but ``finalize_session`` flushes the residual +audio and appends the end marker as its last act, so at the moment the flag is set +the newest entry is milliseconds old. A closing session looked exactly like a +resuming one, and the discovery loop cleared the flag 122ms after it was written: + + 06:52:29,006 end_reason determined: websocket_disconnect + 06:52:29,128 marked complete but has fresh audio — clearing flag + 06:52:59,062 Timed out waiting for streaming completion signal (waited 30s) + +The probe now decides on causal state instead: a session that has left ACTIVE can +never append again (``producer._append_owned_message`` appends inside a WATCH/MULTI +whose precondition is ``status == "active"``), and an end marker in the stream +proves the producer finished even if the FINALIZING write has not landed yet. +""" + +import time + +import pytest +from fakeredis import aioredis as fake_aioredis + +import advanced_omi_backend.services.transcription.streaming_consumer as sc_module +from advanced_omi_backend.services.audio_stream.session_store import SessionStore +from advanced_omi_backend.services.transcription.streaming_consumer import ( + StreamingTranscriptionConsumer, +) + +pytestmark = pytest.mark.unit + +SESSION_ID = "989f33-plugin-tes-b43abe11e4a640f58c7f2ca8eee2aa20" +STREAM = f"audio:stream:{SESSION_ID}" + + +class _StubProvider: + """The consumer resolves a provider in __init__; nothing here calls it.""" + + capabilities: list[str] = [] + + +@pytest.fixture +def consumer(monkeypatch): + redis = fake_aioredis.FakeRedis() + monkeypatch.setattr( + sc_module, "get_transcription_provider", lambda mode: _StubProvider() + ) + return StreamingTranscriptionConsumer(redis_client=redis), redis + + +async def _append_chunk(redis, *, age_seconds: float = 0.0, end_marker: bool = False): + """Append one WAL entry, stamped ``age_seconds`` in the past. + + Redis stream ids are ``-`` and the probe reads the age straight off + the id, so an explicit id makes staleness deterministic without sleeping. + """ + entry_id = f"{int((time.time() - age_seconds) * 1000)}-*" + fields = {b"audio_data": b"\x00" * 8000, b"session_id": SESSION_ID.encode()} + if end_marker: + fields = {b"audio_data": b"", b"end_marker": b"true", b"chunk_id": b"END"} + await redis.xadd(STREAM, fields, id=entry_id) + + +async def _finalize_like_producer(redis, *, write_end_marker: bool = True): + """Replay ``finalize_session``: flush residual audio, end marker, then status.""" + await _append_chunk(redis) + if write_end_marker: + await _append_chunk(redis, end_marker=True) + await SessionStore(redis).mark_finalizing(SESSION_ID, "websocket_disconnect") + + +async def test_finalized_session_is_not_resumed_despite_a_fresh_tail(consumer): + """The CI failure: finalize's own closing writes must not read as a resume.""" + c, redis = consumer + await SessionStore(redis).set_status_active(SESSION_ID) + await _finalize_like_producer(redis) + + assert await c._session_resumed(STREAM, SESSION_ID) is False + + +async def test_end_marker_blocks_reattach_before_the_status_write_lands(consumer): + """The marker is appended strictly before the flag can exist, so it decides. + + ``finalize_session`` appends the marker while the session is still ACTIVE and + only then calls ``mark_finalizing``. A consumer that reads the marker and sets + the completion flag inside that window would otherwise see status=active. + """ + c, redis = consumer + await SessionStore(redis).set_status_active(SESSION_ID) + await _append_chunk(redis) + await _append_chunk(redis, end_marker=True) + + assert await c._session_resumed(STREAM, SESSION_ID) is False + + +async def test_end_marker_is_found_behind_a_late_chunk(consumer): + """A chunk racing in behind the marker must not hide it from the tail probe.""" + c, redis = consumer + await SessionStore(redis).set_status_active(SESSION_ID) + await _append_chunk(redis, end_marker=True) + await _append_chunk(redis) + + assert await c._session_resumed(STREAM, SESSION_ID) is False + + +async def test_finalized_session_without_an_end_marker_is_not_resumed(consumer): + """A backend restart loses the producer buffer, so finalize writes no marker. + + Status is then the only evidence the session is over — and it is enough. + """ + c, redis = consumer + await SessionStore(redis).set_status_active(SESSION_ID) + await _finalize_like_producer(redis, write_end_marker=False) + + assert await c._session_resumed(STREAM, SESSION_ID) is False + + +async def test_active_session_still_producing_is_resumed(consumer): + """The self-heal this probe exists for: an idle-exited task must re-attach. + + ``process_stream`` sets the completion flag when its idle heartbeat fires, but + the device may resume afterwards. Without re-attaching, that live stream gets + no transcription until the flag's 5-minute TTL expires. + """ + c, redis = consumer + await SessionStore(redis).set_status_active(SESSION_ID) + await _append_chunk(redis) + + assert await c._session_resumed(STREAM, SESSION_ID) is True + + +async def test_active_session_gone_quiet_is_not_resumed(consumer): + """No audio for a long while — re-attaching would just churn provider sockets.""" + c, redis = consumer + await SessionStore(redis).set_status_active(SESSION_ID) + await _append_chunk(redis, age_seconds=60.0) + + assert await c._session_resumed(STREAM, SESSION_ID) is False + + +async def test_session_without_a_hash_is_not_resumed(consumer): + """No session hash means no producer, whatever the stream still holds.""" + c, redis = consumer + await _append_chunk(redis) + + assert await c._session_resumed(STREAM, SESSION_ID) is False + + +async def test_empty_stream_is_not_resumed(consumer): + c, redis = consumer + await SessionStore(redis).set_status_active(SESSION_ID) + + assert await c._session_resumed(STREAM, SESSION_ID) is False From 4f85776e32cc899c92d55ed00f4950d0466ad652 Mon Sep 17 00:00:00 2001 From: Ankush Malaker <43288948+AnkushMalaker@users.noreply.github.com> Date: Wed, 5 Aug 2026 22:06:03 +0000 Subject: [PATCH 11/18] feat(observability): report the node agent itself going unreachable MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The agent is the only component that sees host-level faults — dead container DNS, a logged-out Tailscale, a stale socket mount — so silently returning when it cannot be reached meant the exact failure mode the health poller exists to catch produced no signal at all. Record the reachable/unreachable transition as a system event (warning on loss, info resolution on return) keyed under one incident, like every other state this poller tracks, so a persistent outage does not spam the ledger. --- .../services/observability/health_poller.py | 41 ++++++++++++++++++- 1 file changed, 40 insertions(+), 1 deletion(-) diff --git a/backends/advanced/src/advanced_omi_backend/services/observability/health_poller.py b/backends/advanced/src/advanced_omi_backend/services/observability/health_poller.py index f8578e0f7..6f2dba314 100644 --- a/backends/advanced/src/advanced_omi_backend/services/observability/health_poller.py +++ b/backends/advanced/src/advanced_omi_backend/services/observability/health_poller.py @@ -45,6 +45,9 @@ # Redis keys for last-known state. _HEALTH_KEY = "system:health:last" # hash: "{node}/{service}" -> health +# Reserved field in that hash; "local/..." and "{node}/..." keys can't collide with it. +_AGENT_KEY = "node-agent" +_AGENT_INCIDENT = "node-agent-unreachable" _SEEN_FAILED_KEY = "system:health:seen_failed_jobs" # set of job ids _CONFIG_SEEN_KEY = "system:health:config_issues" # set of issue keys _WORKER_HEALTH_FIELD = "internal/workers-fleet" @@ -121,8 +124,44 @@ def _bad_severity(health: str | None, detail: str) -> str | None: async def _poll_external_services(redis) -> None: data = await get_external_services() + + # An unreachable agent is itself a reportable fault, not merely an absence of + # data. The agent runs natively on the host and is the only thing that sees + # host-level faults — dead container DNS, a logged-out Tailscale, a stale + # socket mount. Returning silently here meant the failure mode this poller + # exists to catch produced no signal at all. Reported as a transition, like + # every other state below, so a persistent outage does not spam the ledger. + agent_state = "reachable" if data.get("available") else "unreachable" + prev_agent = await redis.hget(_HEALTH_KEY, _AGENT_KEY) + if prev_agent != agent_state: + await redis.hset(_HEALTH_KEY, _AGENT_KEY, agent_state) + if agent_state == "unreachable": + await record_event( + severity="warning", + category="service", + source="node-agent", + title="Node agent unreachable", + detail=( + "The service manager could not be reached " + f"({data.get('reason') or 'unknown'}). Host-level checks are not " + "running, so DNS, Tailscale and certificate faults will go " + "unreported until it returns." + ), + incident_key=_AGENT_INCIDENT, + ) + else: + await record_event( + severity="info", + category="service", + source="node-agent", + title="Node agent reachable", + detail="The service manager is responding again.", + incident_key=_AGENT_INCIDENT, + resolves_incident=True, + ) + if not data.get("available"): - return # agent unreachable/unconfigured → unknown, don't fabricate transitions + return # no per-service data to reconcile; don't fabricate transitions for svc in data.get("services", []) or []: if not svc.get("enabled", True): From 81c7b26bd81757dd15c79feacdf77c55ff2026d0 Mon Sep 17 00:00:00 2001 From: Ankush Malaker <43288948+AnkushMalaker@users.noreply.github.com> Date: Wed, 5 Aug 2026 22:06:03 +0000 Subject: [PATCH 12/18] refactor(curation): use the shared vault_media.promote_image_bytes observation_curation carried its own copy of the content-addressed _media promotion that vault_media now owns (person photos introduced the shared one). Delete the private copy and call the shared helper for both Immich originals and retained ScreenPipe frames. --- .../services/observation_curation.py | 29 ++----------------- 1 file changed, 3 insertions(+), 26 deletions(-) diff --git a/backends/advanced/src/advanced_omi_backend/services/observation_curation.py b/backends/advanced/src/advanced_omi_backend/services/observation_curation.py index b6b657905..f86c2a34f 100644 --- a/backends/advanced/src/advanced_omi_backend/services/observation_curation.py +++ b/backends/advanced/src/advanced_omi_backend/services/observation_curation.py @@ -24,6 +24,7 @@ codex_executor_available, ) from advanced_omi_backend.services.memory.vault_manager import ConvDocVaultManager +from advanced_omi_backend.services.memory.vault_media import promote_image_bytes logger = logging.getLogger(__name__) @@ -392,30 +393,6 @@ async def _immich_image( return response.content, content_type -def _promote_image_bytes(data: bytes, content_type: str, root: Path) -> tuple[str, str]: - if not data or not content_type: - raise ValueError("cannot promote empty image data") - suffixes = { - "image/jpeg": ".jpg", - "image/png": ".png", - "image/webp": ".webp", - "image/heic": ".heic", - "image/avif": ".avif", - } - suffix = suffixes.get(content_type) - if suffix is None: - raise ValueError("unsupported vault image type") - digest = hashlib.sha256(data).hexdigest() - media_dir = root / "_media" - media_dir.mkdir(parents=True, exist_ok=True) - target = media_dir / f"{digest}{suffix}" - if not target.exists(): - temporary = target.with_suffix(target.suffix + ".part") - temporary.write_bytes(data) - os.replace(temporary, target) - return target.relative_to(root).as_posix(), digest - - def _write_media_provenance( root: Path, digest: str, @@ -566,7 +543,7 @@ async def apply_curation_decision( raise ValueError("agent selected an invalid Immich candidate") asset_id = immich_item.metadata.get("asset_id") data, content_type = await _immich_image(str(asset_id), "original") - promoted, digest = _promote_image_bytes(data, content_type, root) + promoted, digest = promote_image_bytes(data, content_type, root) immich_item.promoted_path = promoted immich_item.content_hash = digest immich_item.state = "promoted" @@ -582,7 +559,7 @@ async def apply_curation_decision( elif retain_image: if not item.media_data or not item.media_content_type: raise ValueError("selected ScreenPipe image is unavailable") - promoted, digest = _promote_image_bytes( + promoted, digest = promote_image_bytes( item.media_data, item.media_content_type, root ) item.content_hash = digest From 370051e2ce8fff1eefada19d2ae83115acd071c1 Mon Sep 17 00:00:00 2001 From: Ankush Malaker <43288948+AnkushMalaker@users.noreply.github.com> Date: Wed, 5 Aug 2026 22:06:14 +0000 Subject: [PATCH 13/18] feat(conversations): search by conversation ID, sturdier ID copy MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Pasting a conversation ID from a log or system event into the search box now finds the conversation: an `id` category (over conversation_id) joins title/summary/speakers in the search endpoint, enabled by default and individually toggleable in the field dropdown. On the detail page the ID copy button gains a non-secure-context fallback (execCommand path for plain-HTTP LAN access, where navigator.clipboard is absent), visible copied/failed feedback, and an accessible name. Search debounce widens 300→800ms so regex queries don't fire per keystroke. --- .../controllers/conversation_controller.py | 3 +- .../routers/modules/conversation_routes.py | 8 +- .../tests/test_conversation_search.py | 25 ++++- .../webui/src/pages/ConversationDetail.tsx | 92 ++++++++++++++++++- .../webui/src/pages/Conversations.tsx | 15 ++- backends/advanced/webui/src/services/api.ts | 2 +- 6 files changed, 127 insertions(+), 18 deletions(-) diff --git a/backends/advanced/src/advanced_omi_backend/controllers/conversation_controller.py b/backends/advanced/src/advanced_omi_backend/controllers/conversation_controller.py index 9afc7b9f8..aa7563155 100644 --- a/backends/advanced/src/advanced_omi_backend/controllers/conversation_controller.py +++ b/backends/advanced/src/advanced_omi_backend/controllers/conversation_controller.py @@ -483,6 +483,7 @@ async def get_conversations( # MongoDB fields covered by each independently selectable search category. _SEARCH_CATEGORY_FIELDS: dict[str, list[str]] = { + "id": ["conversation_id"], "title": ["title"], "summary": ["summary", "detailed_summary"], "speakers": ["_search_active_version.segments.speaker"], @@ -591,7 +592,7 @@ async def search_conversations( categories: list[str] | None = None, ): """Search conversations by literal pattern across selected field categories.""" - categories = categories or ["title", "summary", "speakers"] + categories = categories or ["id", "title", "summary", "speakers"] fields = _search_fields(categories) try: result = await _regex_search_conversations(query, user, fields, limit, offset) diff --git a/backends/advanced/src/advanced_omi_backend/routers/modules/conversation_routes.py b/backends/advanced/src/advanced_omi_backend/routers/modules/conversation_routes.py index 08dfde10d..50e9cae0f 100644 --- a/backends/advanced/src/advanced_omi_backend/routers/modules/conversation_routes.py +++ b/backends/advanced/src/advanced_omi_backend/routers/modules/conversation_routes.py @@ -88,13 +88,13 @@ async def search_conversations( q: str = Query("", description="Optional text search query"), limit: int = Query(50, ge=1, le=200, description="Max results to return"), offset: int = Query(0, ge=0, description="Number of results to skip"), - fields: list[Literal["title", "summary", "speakers"]] = Query( - default=["title", "summary", "speakers"], - description="Search categories: title, summary, and/or speakers", + fields: list[Literal["id", "title", "summary", "speakers"]] = Query( + default=["id", "title", "summary", "speakers"], + description="Search categories: conversation ID, title, summary, and/or speakers", ), current_user: User = Depends(current_active_user), ): - """Search conversations and identified people by literal case-insensitive pattern.""" + """Search conversation metadata by literal case-insensitive pattern.""" return await conversation_controller.search_conversations( q.strip(), current_user, limit, offset, fields ) diff --git a/backends/advanced/tests/test_conversation_search.py b/backends/advanced/tests/test_conversation_search.py index 6f1e7ccbd..41f9af387 100644 --- a/backends/advanced/tests/test_conversation_search.py +++ b/backends/advanced/tests/test_conversation_search.py @@ -6,9 +6,10 @@ ) -def test_everything_search_includes_all_three_categories(): - fields = _search_fields(["title", "summary", "speakers"]) +def test_everything_search_includes_all_categories(): + fields = _search_fields(["id", "title", "summary", "speakers"]) + assert "conversation_id" in fields assert "title" in fields assert "summary" in fields assert "detailed_summary" in fields @@ -17,11 +18,31 @@ def test_everything_search_includes_all_three_categories(): def test_search_categories_are_independent(): + assert _search_fields(["id"]) == ["conversation_id"] assert _search_fields(["title"]) == ["title"] assert _search_fields(["summary"]) == ["summary", "detailed_summary"] assert _search_fields(["speakers"]) == ["_search_active_version.segments.speaker"] +def test_conversation_id_search_matches_literal_fragments(): + stages = _search_query_stages("abc.123", _search_fields(["id"])) + + assert stages == [ + { + "$match": { + "$or": [ + { + "conversation_id": { + "$regex": r"abc\.123", + "$options": "i", + } + } + ] + } + } + ] + + def test_speaker_search_resolves_only_the_active_transcript_version(): fields = _search_fields(["speakers"]) stages = _search_query_stages("unshull", fields) diff --git a/backends/advanced/webui/src/pages/ConversationDetail.tsx b/backends/advanced/webui/src/pages/ConversationDetail.tsx index 0f2431215..13ebe6ac5 100644 --- a/backends/advanced/webui/src/pages/ConversationDetail.tsx +++ b/backends/advanced/webui/src/pages/ConversationDetail.tsx @@ -4,7 +4,8 @@ import { useQueryClient } from '@tanstack/react-query' import { ArrowLeft, Calendar, User, Trash2, RefreshCw, MoreVertical, RotateCcw, Zap, Download, Scissors, - Save, X, Pencil, Clock, Database, Layers, Star, BarChart3, Hash, AudioLines, ChevronRight + Save, X, Pencil, Clock, Database, Layers, Star, BarChart3, Hash, AudioLines, ChevronRight, + Check, Copy } from 'lucide-react' import { annotationsApi, speakerApi, systemApi, BACKEND_URL } from '../services/api' import { @@ -109,8 +110,18 @@ export default function ConversationDetail() { const [reprocessingMemory, setReprocessingMemory] = useState(false) const [reprocessingSpeakers, setReprocessingSpeakers] = useState(false) const [actionError, setActionError] = useState(null) + const [idCopyStatus, setIdCopyStatus] = useState<'idle' | 'copied' | 'error'>('idle') + const idCopyResetTimer = useRef(null) const toggleStarMutation = useToggleStar() + useEffect(() => { + return () => { + if (idCopyResetTimer.current !== null) { + window.clearTimeout(idCopyResetTimer.current) + } + } + }, []) + const handleToggleStar = async () => { if (!id) return try { @@ -173,6 +184,45 @@ export default function ConversationDetail() { return `${mins}:${secs.toString().padStart(2, '0')}` } + const handleCopyConversationId = async () => { + const conversationId = conversation?.conversation_id + if (!conversationId) return + + let copied = false + if (window.isSecureContext && navigator.clipboard) { + try { + await navigator.clipboard.writeText(conversationId) + copied = true + } catch { + // Fall through to the selection-based copy path below. + } + } + + if (!copied) { + const textArea = document.createElement('textarea') + textArea.value = conversationId + textArea.setAttribute('readonly', '') + textArea.style.position = 'fixed' + textArea.style.left = '-9999px' + document.body.appendChild(textArea) + textArea.select() + + try { + copied = document.execCommand('copy') + } catch { + copied = false + } finally { + textArea.remove() + } + } + + setIdCopyStatus(copied ? 'copied' : 'error') + if (idCopyResetTimer.current !== null) { + window.clearTimeout(idCopyResetTimer.current) + } + idCopyResetTimer.current = window.setTimeout(() => setIdCopyStatus('idle'), 2000) + } + // Action handlers const handleDownloadAudio = async () => { if (!id) return @@ -646,11 +696,43 @@ export default function ConversationDetail() {
diff --git a/backends/advanced/webui/src/pages/Conversations.tsx b/backends/advanced/webui/src/pages/Conversations.tsx index 24dfd788a..e10bcd3cc 100644 --- a/backends/advanced/webui/src/pages/Conversations.tsx +++ b/backends/advanced/webui/src/pages/Conversations.tsx @@ -54,6 +54,7 @@ const isUnknownLabel = (name?: string): boolean => { } const PAGE_SIZE = 20 +const SEARCH_DEBOUNCE_MS = 800 const SORT_OPTIONS = [ { label: 'Date (newest)', sortBy: 'created_at', sortOrder: 'desc' }, @@ -130,8 +131,8 @@ export default function Conversations() { // Search state (regex-only; semantic search was removed for performance reasons) const [searchQuery, setSearchQuery] = useState('') - type SearchField = 'title' | 'summary' | 'speakers' - const allSearchFields: SearchField[] = ['title', 'summary', 'speakers'] + type SearchField = 'id' | 'title' | 'summary' | 'speakers' + const allSearchFields: SearchField[] = ['id', 'title', 'summary', 'speakers'] const [searchFields, setSearchFields] = useState(allSearchFields) const [searchResults, setSearchResults] = useState(null) const [isSearching, setIsSearching] = useState(false) @@ -216,7 +217,10 @@ export default function Conversations() { } setIsSearching(true) - searchTimeoutRef.current = setTimeout(() => runSearch(trimmed, searchFields), 300) + searchTimeoutRef.current = setTimeout( + () => runSearch(trimmed, searchFields), + SEARCH_DEBOUNCE_MS, + ) return () => { if (searchTimeoutRef.current) clearTimeout(searchTimeoutRef.current) @@ -652,7 +656,7 @@ export default function Conversations() { type="text" value={searchQuery} onChange={(e) => setSearchQuery(e.target.value)} - placeholder="Search conversations or people..." + placeholder="Search conversations, IDs, or people..." className="w-full pl-9 pr-9 py-2 rounded-lg border border-gray-300 dark:border-gray-600 bg-white dark:bg-gray-800 text-gray-900 dark:text-gray-100 placeholder-gray-400 focus:outline-none focus:ring-2 focus:ring-blue-500 focus:border-transparent text-sm" /> {searchQuery && ( @@ -684,7 +688,7 @@ export default function Conversations() { - {/* Search fields: Everything mirrors the three individual checkboxes. */} + {/* Search fields: Everything mirrors the individual checkboxes. */}
+ Cleanup Old Sessions + {streamingStatus?.stream_health && Object.keys(streamingStatus.stream_health).length > 0 && ( )} - + })`} +
{/* Stream Workers Section - Shows audio streams + listen jobs */}
-

Stream Workers (Client Sessions)

+ Stream Workers (Client Sessions)
{streamingStatus?.stream_health && Object.entries(streamingStatus.stream_health).map(([streamKey, health]) => { // Extract client_id from stream key (format: audio:stream:{client_id}) @@ -843,34 +837,34 @@ const Queue: React.FC = () => { : []; return ( -
+
- {streamKey} - Active + {streamKey} + Active
- Stream Length: - {health.stream_length} + Stream Length: + {health.stream_length}
- Age: - {(health.stream_age_seconds || 0).toFixed(0)}s + Age: + {(health.stream_age_seconds || 0).toFixed(0)}s
- Pending: - 0 ? 'text-yellow-600' : 'text-green-600'}`}> + Pending: + 0 ? 'text-yellow-600 dark:text-yellow-400' : 'text-green-600 dark:text-green-400'}`}> {health.total_pending}
{health.consumer_groups && health.consumer_groups.map((group) => ( -
-
{group.name}:
+
+
{group.name}:
{(group.consumers || []).map((consumer) => (
- {consumer.name} - 0 ? 'text-yellow-600' : 'text-green-600'}> + {consumer.name} + 0 ? 'text-yellow-600 dark:text-yellow-400' : 'text-green-600 dark:text-green-400'}> {consumer.pending} pending
@@ -880,8 +874,8 @@ const Queue: React.FC = () => { {/* Current Speech Detection Job */} {listenJobs.length > 0 && ( -
-
Current Speech Detection:
+
+
Current Speech Detection:
{listenJobs.map((job) => { const runtime = job.started_at ? Math.floor((Date.now() - new Date(job.started_at).getTime()) / 1000) @@ -890,51 +884,50 @@ const Queue: React.FC = () => { const seconds = runtime % 60; return ( -
+
-
+
{getStatusIcon(job.status)} - {job.job_type} - - {job.status} - + {job.job_type} + {job.status}
- +
{/* Job metadata */} -
+
Job ID: - {job.job_id.substring(0, 12)}... + {job.job_id.substring(0, 12)}...
{job.started_at && (
Runtime: - {minutes}m {seconds}s + {minutes}m {seconds}s
)} {job.created_at && (
Created: - {new Date(job.created_at).toLocaleTimeString()} + {new Date(job.created_at).toLocaleTimeString()}
)} {job.meta?.speech_detected_at && (
Speech Detected: - {new Date(job.meta.speech_detected_at).toLocaleString()} + {new Date(job.meta.speech_detected_at).toLocaleString()}
)} {job.meta?.status && (
Status: - {job.meta.status.replace(/_/g, ' ')} + {job.meta.status.replace(/_/g, ' ')}
)}
@@ -945,30 +938,30 @@ const Queue: React.FC = () => { if (!session) return null; return ( -
-
Speech Detection Events:
+
+
Speech Detection Events:
{session.last_event && (
- Last Event: - {session.last_event.split(':')[0]} + Last Event: + {session.last_event.split(':')[0]}
)} {session.speaker_check_status && (
- Speaker Check: + Speaker Check: {session.speaker_check_status}
)} {session.identified_speakers && (
- Speakers: - {session.identified_speakers} + Speakers: + {session.identified_speakers}
)}
@@ -990,7 +983,7 @@ const Queue: React.FC = () => {
{/* Active Conversations - Grouped by conversation_id */}
-

Active Conversations

+ Active Conversations {(() => { // Group all jobs by conversation_id with deduplication const allJobsRaw = Object.values(conversationJobs).flat().filter(job => job != null); @@ -1037,11 +1030,7 @@ const Queue: React.FC = () => { }); if (conversationMap.size === 0) { - return ( -
- No active conversations -
- ); + return No active conversations; } return ( @@ -1067,45 +1056,45 @@ const Queue: React.FC = () => { const failedJobCount = jobs.filter(j => j.status === 'failed').length; return ( -
+
toggleConversationExpansion(conversationId)} >
{isExpanded ? ( - + ) : ( - + )} {hasFailedJob ? ( - + ) : ( - + )} - {clientId} + {clientId} {hasFailedJob ? ( - + {failedJobCount} Error{failedJobCount > 1 ? 's' : ''} - + ) : ( - Active + Active )} {speakers.length > 0 && ( - + {speakers.length} speaker{speakers.length > 1 ? 's' : ''} - + )}
-
+
Conversation: {conversationId.substring(0, 8)}... • {createdAt && `Started: ${new Date(createdAt).toLocaleTimeString()} • `} Words: {wordCount} {lastUpdate && ` • Updated: ${new Date(lastUpdate).toLocaleTimeString()}`}
{transcript && ( -
+
"{transcript.substring(0, 100)}{transcript.length > 100 ? '...' : ''}"
)} @@ -1136,10 +1125,10 @@ const Queue: React.FC = () => { {/* Expanded Jobs Section */} {isExpanded && ( -
+
{/* Pipeline Timeline */}
-
Pipeline Timeline:
+
Pipeline Timeline:
{(() => { // Helper function to get display name from job type const getJobDisplayName = (jobType: string) => { @@ -1190,7 +1179,7 @@ const Queue: React.FC = () => { const validTimes = jobTimes.filter(t => t !== null); if (validTimes.length === 0) { return ( -
No job timing data available
+
No job timing data available
); } @@ -1216,7 +1205,7 @@ const Queue: React.FC = () => { return (
{/* Time axis */} -
+
{timeMarkers.map((marker, idx) => (
{ style={{ left: `${marker.percent}%`, transform: 'translateX(-50%)' }} >
-
+
{marker.time}
@@ -1253,10 +1242,10 @@ const Queue: React.FC = () => {
{/* Stage Name */} - {name} + {name} {/* Timeline Container */} -
+
{/* Job Bar */}
{
{/* Total Duration */} -
+
Total: {formatDuration(totalDuration)}
@@ -1285,33 +1274,29 @@ const Queue: React.FC = () => { })()}
-
Conversation Jobs:
+
Conversation Jobs:
{jobs.filter(j => j != null && j.job_id).length > 0 ? (
{jobs .filter(j => j != null && j.job_id) .sort((a, b) => new Date(a.created_at).getTime() - new Date(b.created_at).getTime()) .map((job, index) => ( -
+
toggleJobExpansion(job.job_id)} >
- #{index + 1} + #{index + 1} {getJobTypeIcon(job.job_type)} {getStatusIcon(job.status)} - {job.job_type} - - {job.status} - - {job.queue} + {job.job_type} + {job.status} + {job.queue} {/* Show memory count badge on collapsed card */} {!expandedJobs.has(job.job_id) && job.job_type === 'process_memory_job' && job.result?.memories_created !== undefined && ( - - {job.result.memories_created} memories - + {job.result.memories_created} memories )}
@@ -1319,7 +1304,7 @@ const Queue: React.FC = () => { {/* Collapsible metadata section */} {expandedJobs.has(job.job_id) && ( -
+
{job.started_at && ( Started: {new Date(job.started_at).toLocaleTimeString()} @@ -1331,7 +1316,7 @@ const Queue: React.FC = () => { {/* Show job-specific metadata */} {job.meta && ( -
+
{/* open_conversation_job metadata */} {job.job_type === 'open_conversation_job' && ( <> @@ -1345,7 +1330,7 @@ const Queue: React.FC = () => {
Idle: {Math.floor(job.meta.inactivity_seconds)}s
)} {job.meta.transcript && ( -
+
"{job.meta.transcript.substring(0, 80)}..."
)} @@ -1356,10 +1341,10 @@ const Queue: React.FC = () => { {job.job_type === 'transcribe_full_audio_job' && job.status === 'started' && job.meta?.batch_progress && (
- {job.meta.batch_progress.message} - {job.meta.batch_progress.percent}% + {job.meta.batch_progress.message} + {job.meta.batch_progress.percent}%
-
+
@@ -1400,9 +1385,9 @@ const Queue: React.FC = () => { )} {job.meta.memory_details && job.meta.memory_details.length > 0 && (
-
Memories Created:
+
Memories Created:
{job.meta.memory_details.map((memory: any, idx: number) => ( -
+
"{memory.text}"
))} @@ -1413,7 +1398,7 @@ const Queue: React.FC = () => { {/* Show conversation_id if present */} {job.meta.conversation_id && ( -
+
Conv: {job.meta.conversation_id.substring(0, 8)}...
)} @@ -1421,20 +1406,21 @@ const Queue: React.FC = () => { )}
)} - +
))}
) : ( -
No jobs found for this conversation
+
No jobs found for this conversation
)}
)} @@ -1448,23 +1434,24 @@ const Queue: React.FC = () => { {/* Completed Conversations - Grouped by conversation_id */}
-
-

Completed Conversations

+
+ Completed Conversations
- - { setCompletedConvTimeRange(Number(e.target.value)); setCompletedConvPage(1); // Reset to first page }} - className="text-xs border border-gray-300 rounded px-2 py-1" > - +
{(() => { @@ -1513,11 +1500,7 @@ const Queue: React.FC = () => { }); if (conversationMap.size === 0) { - return ( -
- No completed conversations -
- ); + return No completed conversations; } // Convert to array and filter by time range @@ -1545,11 +1528,7 @@ const Queue: React.FC = () => { const paginatedConversations = conversationsArray.slice(startIndex, endIndex); if (conversationsArray.length === 0) { - return ( -
- No completed conversations in the selected time range -
- ); + return No completed conversations in the selected time range; } return ( @@ -1582,25 +1561,25 @@ const Queue: React.FC = () => { const failedJobCount = jobs.filter(j => j.status === 'failed').length; // Determine status styling - let bgColor = 'bg-yellow-50 border-yellow-200'; - let hoverColor = 'hover:bg-yellow-100'; - let iconColor = 'text-yellow-600'; - let statusBadge = 'bg-yellow-100 text-yellow-700'; + let bgColor = 'bg-yellow-50 border-yellow-200 dark:bg-yellow-900/20 dark:border-yellow-800'; + let hoverColor = 'hover:bg-yellow-100 dark:hover:bg-yellow-900/30'; + let iconColor = 'text-yellow-600 dark:text-yellow-400'; + let statusTone: StateTone = 'warning'; let statusText = 'Processing'; let StatusIcon = Clock; if (hasFailedJob) { - bgColor = 'bg-red-50 border-red-300'; - hoverColor = 'hover:bg-red-100'; - iconColor = 'text-red-600'; - statusBadge = 'bg-red-200 text-red-800'; + bgColor = 'bg-red-50 border-red-300 dark:bg-red-900/20 dark:border-red-800'; + hoverColor = 'hover:bg-red-100 dark:hover:bg-red-900/30'; + iconColor = 'text-red-600 dark:text-red-400'; + statusTone = 'danger'; statusText = `${failedJobCount} Error${failedJobCount > 1 ? 's' : ''}`; StatusIcon = AlertTriangle; } else if (allComplete) { - bgColor = 'bg-green-50 border-green-200'; - hoverColor = 'hover:bg-green-100'; - iconColor = 'text-green-600'; - statusBadge = 'bg-green-100 text-green-700'; + bgColor = 'bg-green-50 border-green-200 dark:bg-green-900/20 dark:border-green-800'; + hoverColor = 'hover:bg-green-100 dark:hover:bg-green-900/30'; + iconColor = 'text-green-600 dark:text-green-400'; + statusTone = 'success'; statusText = 'Complete'; StatusIcon = CheckCircle; } @@ -1611,7 +1590,7 @@ const Queue: React.FC = () => { className={`flex items-center justify-between p-3 cursor-pointer transition-colors ${hoverColor}`} onClick={() => toggleConversationExpansion(conversationId)} > -
+
{isExpanded ? ( @@ -1619,17 +1598,15 @@ const Queue: React.FC = () => { )} - {clientId} - - {statusText} - + {clientId} + {statusText} {speakers.length > 0 && ( - + {speakers.length} speaker{speakers.length > 1 ? 's' : ''} - + )}
-
+
Conversation: {conversationId.substring(0, 8)}... • Words: {wordCount} {createdAt && ( @@ -1640,23 +1617,23 @@ const Queue: React.FC = () => { {allComplete ? ( <> {title ? ( -
+
{title}
) : transcript ? ( -
+
"{transcript.substring(0, 100)}{transcript.length > 100 ? '...' : ''}"
) : null} {summary && ( -
+
{summary}
)} ) : ( transcript && ( -
+
"{transcript.substring(0, 100)}{transcript.length > 100 ? '...' : ''}"
) @@ -1666,12 +1643,12 @@ const Queue: React.FC = () => { {/* Expanded Jobs Section */} {isExpanded && ( -
{/* Pipeline Timeline */}
-
Pipeline Timeline:
+
Pipeline Timeline:
{(() => { // Helper function to get display name from job type const getJobDisplayName = (jobType: string) => { @@ -1722,7 +1699,7 @@ const Queue: React.FC = () => { const validTimes = jobTimes.filter(t => t !== null); if (validTimes.length === 0) { return ( -
No job timing data available
+
No job timing data available
); } @@ -1748,7 +1725,7 @@ const Queue: React.FC = () => { return (
{/* Time axis */} -
+
{timeMarkers.map((marker, idx) => (
{ style={{ left: `${marker.percent}%`, transform: 'translateX(-50%)' }} >
-
+
{marker.time}
@@ -1785,10 +1762,10 @@ const Queue: React.FC = () => {
{/* Stage Name */} - {name} + {name} {/* Timeline Container */} -
+
{/* Job Bar */}
{
{/* Total Duration */} -
+
Total: {formatDuration(totalDuration)}
@@ -1819,48 +1796,45 @@ const Queue: React.FC = () => { })()}
-
Conversation Jobs:
+
Conversation Jobs:
{jobs.filter(j => j != null && j.job_id).length > 0 ? (
{jobs .filter(j => j != null && j.job_id) .sort((a, b) => new Date(a.created_at).getTime() - new Date(b.created_at).getTime()) .map((job, index) => ( -
+
toggleJobExpansion(job.job_id)} > - #{index + 1} + #{index + 1} {getJobTypeIcon(job.job_type)} {getStatusIcon(job.status)} - {job.job_type} - - {job.status} - - {job.queue || job.data?.queue || 'unknown'} + {job.job_type} + {job.status} + {job.queue || job.data?.queue || 'unknown'} {/* Show memory count badge on collapsed card */} {!expandedJobs.has(job.job_id) && job.job_type === 'process_memory_job' && job.result?.memories_created !== undefined && ( - - {job.result.memories_created} memories - + {job.result.memories_created} memories )}
- +
{/* Collapsible metadata section */} {expandedJobs.has(job.job_id) && ( -
+
{job.started_at && ( Started: {new Date(job.started_at).toLocaleTimeString()} @@ -1872,7 +1846,7 @@ const Queue: React.FC = () => { {/* Show job-specific metadata */} {job.meta && ( -
+
{/* open_conversation_job metadata */} {job.job_type === 'open_conversation_job' && ( <> @@ -1886,7 +1860,7 @@ const Queue: React.FC = () => {
Idle: {Math.floor(job.meta.inactivity_seconds)}s
)} {job.meta.transcript && ( -
+
"{job.meta.transcript.substring(0, 80)}..."
)} @@ -1931,7 +1905,7 @@ const Queue: React.FC = () => { {/* Show conversation_id if present */} {job.meta.conversation_id && ( -
+
Conv: {job.meta.conversation_id.substring(0, 8)}...
)} @@ -1943,7 +1917,7 @@ const Queue: React.FC = () => { ))}
) : ( -
No jobs found for this conversation
+
No jobs found for this conversation
)}
)} @@ -1954,23 +1928,21 @@ const Queue: React.FC = () => { {/* Pagination Controls */} {totalPages > 1 && ( -
-
+
+
Showing {startIndex + 1}-{Math.min(endIndex, totalConversations)} of {totalConversations} conversations
- + Page {completedConvPage} of {totalPages}
-
+ )} {/* Events */} -
+
{ const next = !eventsExpanded; setEventsExpanded(next); @@ -1999,9 +1971,9 @@ const Queue: React.FC = () => { }} >
- -

Events

- + + Events + {(() => { const includes = Object.entries(eventFilters).filter(([, v]) => v === 'include').map(([k]) => k); const excludes = Object.entries(eventFilters).filter(([, v]) => v === 'exclude').map(([k]) => k); @@ -2016,7 +1988,11 @@ const Queue: React.FC = () => {
{eventsExpanded && events.length > 0 && ( - + )} - {eventsExpanded ? : } + {eventsExpanded + ? + : }
{eventsExpanded && [...new Set(events.map(e => e.event))].sort().length > 0 && ( -
+
{[...new Set(events.map(e => e.event))].sort().map(eventType => { const state = eventFilters[eventType]; return ( @@ -2045,10 +2020,10 @@ const Queue: React.FC = () => { onClick={() => cycleEventFilter(eventType)} className={`inline-flex items-center px-2.5 py-0.5 rounded-full text-xs font-medium border cursor-pointer transition-colors ${ state === 'include' - ? 'bg-blue-100 text-blue-700 border-blue-400' + ? 'bg-blue-100 text-blue-700 border-blue-400 dark:bg-blue-900/40 dark:text-blue-300 dark:border-blue-600' : state === 'exclude' - ? 'bg-red-100 text-red-700 border-red-400 line-through' - : 'bg-gray-100 text-gray-500 border-gray-300' + ? 'bg-red-100 text-red-700 border-red-400 line-through dark:bg-red-900/40 dark:text-red-300 dark:border-red-600' + : 'bg-gray-100 text-gray-500 border-gray-300 dark:bg-gray-700/60 dark:text-gray-400 dark:border-gray-600' }`} > {eventType} @@ -2058,7 +2033,7 @@ const Queue: React.FC = () => { {Object.keys(eventFilters).length > 0 && ( @@ -2079,24 +2054,24 @@ const Queue: React.FC = () => { if (filtered.length === 0) { return ( -
+
No events recorded yet. Events are logged when system actions like conversation.complete, memory.processed, or button presses occur.
); } return ( - - +
+ - - - - - + + + + + - + {filtered.map((evt, idx) => { const pluginsExecuted = evt.plugins_executed || []; // A plugin can intentionally no-op (e.g. wake word armed on a @@ -2108,8 +2083,8 @@ const Queue: React.FC = () => { const allSkipped = pluginsExecuted.length > 0 && ranPlugins.length === 0; return ( - - + - - @@ -2165,15 +2136,18 @@ const Queue: React.FC = () => { })()} )} - + {/* Filters */} -

Filters

-
+
+ Filters +
+
- + setFilters({ ...filters, job_type: e.target.value })} > - - - - - -
- -
- -
@@ -2218,123 +2179,108 @@ const Queue: React.FC = () => { - +
{/* Jobs Table */} -
-
-

Jobs

+ +
+ Jobs {jobs.length > 0 && ( - + )}
-
TimeEventUserPlugins TriggeredStatusTimeEventUserPlugins TriggeredStatus
+
{new Date(evt.timestamp * 1000).toLocaleTimeString()} @@ -2117,42 +2092,38 @@ const Queue: React.FC = () => { {evt.event} + {(evt.user_id || '').length > 12 ? `${evt.user_id.slice(-8)}` : evt.user_id} + {pluginsExecuted.length > 0 ? pluginsExecuted.map(p => p.plugin_id).join(', ') - : none + : none }
{pluginsExecuted.length === 0 ? ( - no plugins ran + no plugins ran ) : allSkipped ? ( - Skipped + Skipped ) : allSuccess ? ( - + OK ) : anyFailure ? ( - + Error ) : ( - partial + partial )} {pluginsExecuted.length > 0 && ( - + )}
- +
+ - - - - - - - + + + + + + + - + {jobs .filter((job) => { if (filters.status && job.status !== filters.status) return false; if (filters.job_type && job.job_type !== filters.job_type) return false; - if (filters.priority && job.meta?.priority !== filters.priority) return false; return true; }) .sort((a, b) => new Date(b.created_at).getTime() - new Date(a.created_at).getTime()).map((job) => ( - - + - - @@ -2346,29 +2292,21 @@ const Queue: React.FC = () => { {/* Pagination */} {pagination.total > pagination.limit && ( -
-
+
+
Showing {pagination.offset + 1} to {Math.min(pagination.offset + pagination.limit, pagination.total)} of {pagination.total} results
- - + +
)} -
+ {/* Old Jobs Table and Pagination - Removed in favor of session-based view above */} {/* Job Details Modal */} @@ -2393,46 +2331,46 @@ const Queue: React.FC = () => {
- -

{selectedJob.job_id}

+ +

{selectedJob.job_id}

- - + + {getStatusIcon(selectedJob.status)} {selectedJob.status.charAt(0).toUpperCase() + selectedJob.status.slice(1)} - +
{selectedJob.description && (
- -

{selectedJob.description}

+ +

{selectedJob.description}

)} {selectedJob.func_name && (
- -

{selectedJob.func_name}

+ +

{selectedJob.func_name}

)}
- -

{selectedJob.created_at ? formatDate(selectedJob.created_at) : '-'}

+ +

{selectedJob.created_at ? formatDate(selectedJob.created_at) : '-'}

- -

{selectedJob.started_at ? formatDate(selectedJob.started_at) : '-'}

+ +

{selectedJob.started_at ? formatDate(selectedJob.started_at) : '-'}

- -

{selectedJob.ended_at ? formatDate(selectedJob.ended_at) : '-'}

+ +

{selectedJob.ended_at ? formatDate(selectedJob.ended_at) : '-'}

{selectedJob.args && selectedJob.args.length > 0 && (
- - 
+                    
+                    
                       {JSON.stringify(selectedJob.args, null, 2)}
@@ -2440,8 +2378,8 @@ const Queue: React.FC = () => { {selectedJob.kwargs && Object.keys(selectedJob.kwargs).length > 0 && (
- - 
+                    
+                    
                       {JSON.stringify(selectedJob.kwargs, null, 2)}
@@ -2449,8 +2387,8 @@ const Queue: React.FC = () => { {selectedJob.error_message && (
- - 
+                    
+                    
                       {selectedJob.error_message}
@@ -2458,8 +2396,8 @@ const Queue: React.FC = () => { {selectedJob.result && (
- - 
+                    
+                    
                       {JSON.stringify(selectedJob.result, null, 2)}
@@ -2468,11 +2406,11 @@ const Queue: React.FC = () => { {/* Formatted Job Metadata - Job-specific displays */} {selectedJob.meta && Object.keys(selectedJob.meta).length > 0 && (
- + {/* open_conversation_job formatted metadata */} {selectedJob.func_name?.includes('open_conversation_job') && ( -
+
{selectedJob.meta.word_count !== undefined && (
Word Count: {selectedJob.meta.word_count} @@ -2506,7 +2444,7 @@ const Queue: React.FC = () => { {selectedJob.meta.transcript && (
Transcript:
-
+
"{selectedJob.meta.transcript}"
@@ -2516,7 +2454,7 @@ const Queue: React.FC = () => { {/* process_memory_job formatted metadata */} {selectedJob.func_name?.includes('process_memory_job') && selectedJob.meta.memory_details && selectedJob.meta.memory_details.length > 0 && ( -
+
Memories Created: {selectedJob.meta.memories_created || selectedJob.meta.memory_details.length}
@@ -2529,7 +2467,7 @@ const Queue: React.FC = () => {
Memory Details:
{selectedJob.meta.memory_details.map((mem: any, idx: number) => ( -
+
{mem.text}
))} @@ -2540,7 +2478,7 @@ const Queue: React.FC = () => { {/* stream_speech_detection_job formatted metadata */} {selectedJob.func_name?.includes('stream_speech_detection_job') && ( -
+
{selectedJob.meta.speech_detected_at && (
Speech Detected At: {new Date(selectedJob.meta.speech_detected_at).toLocaleString()} @@ -2561,7 +2499,7 @@ const Queue: React.FC = () => { {/* transcribe_full_audio_job formatted metadata */} {selectedJob.func_name?.includes('transcribe_full_audio_job') && (selectedJob.meta.title || selectedJob.meta.summary) && ( -
+
{selectedJob.meta.title && (
Title: {selectedJob.meta.title} @@ -2592,10 +2530,10 @@ const Queue: React.FC = () => { {/* Raw JSON metadata (collapsible) */}
- + Raw Metadata JSON - 
+                      
                         {JSON.stringify(selectedJob.meta, null, 2)}
@@ -2623,37 +2561,37 @@ const Queue: React.FC = () => {
- -

{new Date(selectedEvent.timestamp * 1000).toLocaleString()}

+ +

{new Date(selectedEvent.timestamp * 1000).toLocaleString()}

- + {selectedEvent.event}
- -

{selectedEvent.user_id}

+ +

{selectedEvent.user_id}

{selectedEvent.metadata?.client_id && (
- -

{selectedEvent.metadata.client_id}

+ +

{selectedEvent.metadata.client_id}

)}
- +
{(selectedEvent.plugins_executed || []).map((p, i) => { const skipped = !!p.data?.skipped; - const tone = skipped - ? { card: 'bg-gray-50 border-gray-200', badge: 'bg-gray-100 text-gray-600', text: 'text-gray-700', label: 'Skipped' } + const tone: { card: string; badge: StateTone; text: string; label: string } = skipped + ? { card: 'bg-gray-50 border-gray-200 dark:bg-gray-900/40 dark:border-gray-700', badge: 'neutral', text: 'text-gray-700 dark:text-gray-300', label: 'Skipped' } : p.success - ? { card: 'bg-green-50 border-green-200', badge: 'bg-green-100 text-green-700', text: 'text-green-800', label: 'OK' } - : { card: 'bg-red-50 border-red-200', badge: 'bg-red-100 text-red-700', text: 'text-red-800', label: 'Error' }; + ? { card: 'bg-green-50 border-green-200 dark:bg-green-900/20 dark:border-green-800', badge: 'success', text: 'text-green-800 dark:text-green-300', label: 'OK' } + : { card: 'bg-red-50 border-red-200 dark:bg-red-900/20 dark:border-red-800', badge: 'danger', text: 'text-red-800 dark:text-red-300', label: 'Error' }; // Show the plugin's structured output minus the skip flags we // already render via the badge/detail. const { skipped: _s, skip_reason: _r, detail, ...restData } = p.data || {}; @@ -2661,15 +2599,13 @@ const Queue: React.FC = () => {
{skipped - ? + ? : p.success - ? - : + ? + : } - {p.plugin_id} - - {tone.label} - + {p.plugin_id} + {tone.label}
{(p.message || detail) && (

@@ -2677,7 +2613,7 @@ const Queue: React.FC = () => {

)} {Object.keys(restData).length > 0 && ( - 
+                          
                             {JSON.stringify(restData, null, 2)}
                           

                         )}
@@ -2689,10 +2625,10 @@ const Queue: React.FC = () => {
 
               {selectedEvent.metadata && Object.keys(selectedEvent.metadata).length > 0 && (
                 

-                  
+                  
                     Raw Metadata
                   
-                  
+                  
                     {JSON.stringify(selectedEvent.metadata, null, 2)}
                   

                 

@@ -2744,12 +2680,9 @@ const Queue: React.FC = () => {
           }
         >
             

-              

-                

-                  
-                  This will permanently remove jobs from the database
-                

-              

+              }>
+                This will permanently remove jobs from the database
+              
 
               

                 

@@ -2761,14 +2694,15 @@ const Queue: React.FC = () => {
                       onChange={() => setFlushSettings(prev => ({ ...prev, flush_all: false }))}
                       className="text-blue-600"
                     />
-                    Flush old inactive jobs (recommended)
+                    Flush old inactive jobs (recommended)
                   
 
                   {!flushSettings.flush_all && (
                     

                       

-                        
+                        
                          onToggle(key)}
+                      title={isDropped ? 'Include this clip' : 'Drop this clip from the export'}
+                      className="mt-1"
+                    />
+                    
+                    
+                      {formatDuration(clip.duration_seconds)}
+                    
+                    
+                      {clip.text ? (
+                        
+                          {clip.text}
+                        
+                      ) : (
+                        
+                          no transcript
+                        
+                      )}
+                    
+                  

+                )
+              })}
+            

+          ))}
+      

+    

+  )
+}
+
 /** Review panel: flagged segments grouped by conversation, each a withhold toggle. */
 function ScreenReview({
   report,
diff --git a/backends/advanced/webui/src/components/dataAudit/filters.tsx b/backends/advanced/webui/src/components/dataAudit/filters.tsx
index 2bd84ed7b..f472cf01c 100644
--- a/backends/advanced/webui/src/components/dataAudit/filters.tsx
+++ b/backends/advanced/webui/src/components/dataAudit/filters.tsx
@@ -15,6 +15,7 @@ import {
   FileArchive,
   LucideIcon,
   Mic,
+  PackageOpen,
   Search,
   Users,
 } from 'lucide-react'
@@ -408,6 +409,49 @@ const datasetFilter: FilterDef = {
   ),
 }
 
+// ---------------------------------------------------------------------------
+// Export history (from the on-disk annotation-export metadata)
+// ---------------------------------------------------------------------------
+
+type ExportedValue = '' | 'never' | 'exported'
+
+const exportedFilter: FilterDef = {
+  key: 'exported',
+  label: 'Export history',
+  icon: PackageOpen,
+  defaultValue: '',
+  isActive: (v) => v !== '',
+  chipLabel: (v) => (v === 'never' ? 'Not yet exported' : 'Previously exported'),
+  toParams: (v) => ({ exported: v || undefined }),
+  Editor: ({ value, onChange }) => (
+    

+      {(
+        [
+          { key: '', label: 'All conversations' },
+          { key: 'never', label: 'Not yet exported' },
+          { key: 'exported', label: 'Previously exported' },
+        ] as const
+      ).map((opt) => (
+        
+      ))}
+      

+        Whether a previous annotation export shipped the conversation.
+      

+    

+  ),
+}
+
 // ---------------------------------------------------------------------------
 // Hide failed (processing_status == 'failed')
 // ---------------------------------------------------------------------------
@@ -444,6 +488,7 @@ export const AUDIT_FILTERS: FilterDef[] = [
   speakersFilter,
   dateFilter,
   datasetFilter,
+  exportedFilter,
   hideFailedFilter,
   hideReviewedFilter,
 ]
diff --git a/backends/advanced/webui/src/services/api.ts b/backends/advanced/webui/src/services/api.ts
index feaf21081..048125327 100644
--- a/backends/advanced/webui/src/services/api.ts
+++ b/backends/advanced/webui/src/services/api.ts
@@ -843,6 +843,9 @@ export interface AuditConversation {
   analyzed: boolean
   speech_fraction: number | null
   derived_operation: 'split' | 'merge' | null
+  // Most recent annotation export that shipped this conversation (null = never
+  // exported) — lets curation sessions skip audio already sent to annotators.
+  last_export: { export_id: string; created_at: string } | null
   audio_archived: boolean
   audio_archived_at: string | null
   archive_reason: string | null
@@ -1051,6 +1054,42 @@ export interface ExportRecord {
   zip_ready: boolean
 }
 
+// One clip the export would produce, as returned by the preview dry-run.
+export interface ExportPreviewClip {
+  clip_index: number
+  clip_id: string
+  start: number
+  end: number
+  duration_seconds: number
+  // The sliced transcript this clip's manifest record would carry ('' = no
+  // transcript covers the clip).
+  text: string
+  segment_count: number
+}
+
+export interface ExportPreviewConversation {
+  conversation_id: string
+  title: string | null
+  client_id: string | null
+  created_at: string | null
+  skipped_reason?: string
+  clips?: ExportPreviewClip[]
+  sample_rate?: number
+  clip_seconds?: number
+  excluded_seconds?: number
+}
+
+export interface ExportPreviewResult {
+  conversations: ExportPreviewConversation[]
+  totals: {
+    conversation_count: number
+    exported_conversations: number
+    clip_count: number
+    total_clip_seconds: number
+    excluded_seconds: number
+  }
+}
+
 // One transcript segment flagged by the privacy screen as too sensitive to share.
 export interface ScreenFlaggedSegment {
   index: number
@@ -1776,9 +1815,28 @@ export const dataAuditApi = {
       policy: policy ?? null,
     }),
 
+  // Dry-run of the export: the exact clips (boundaries + sliced transcripts)
+  // the current settings would produce, without writing any audio. Runs the
+  // same plan computation as the export job, so what it shows is what ships.
+  previewExport: (
+    conversationIds: string[],
+    params?: {
+      mode?: 'clips' | 'full'
+      pad_seconds?: number
+      speech_threshold?: number
+      merge_gap_seconds?: number
+      excluded_ranges?: Record
+    }
+  ) =>
+    api.post('/api/data-audit/export/preview', {
+      conversation_ids: conversationIds,
+      ...params,
+    }),
+
   // Enqueue an annotation-dataset export (speech-cropped clips + manifest).
   // `excluded_ranges` (conversation_id → withheld [start,end] ranges from the
-  // privacy screen) are carved out of the exported audio + transcript.
+  // privacy screen) and `dropped_ranges` (clips unticked in the preview) are
+  // carved out of the exported audio + transcript.
   // Returns { job_id, export_id, status }.
   startExport: (
     conversationIds: string[],
@@ -1788,6 +1846,7 @@ export const dataAuditApi = {
       speech_threshold?: number
       merge_gap_seconds?: number
       excluded_ranges?: Record
+      dropped_ranges?: Record
       sensitivity_policy?: string | null
     }
   ) =>
DateConversation IDJob IDTypeStatusDurationActionsDateConversation IDJob IDTypeStatusDurationActions
+
{new Date(job.created_at).toLocaleString('en-US', { month: 'short', day: 'numeric', hour: '2-digit', minute: '2-digit' })} -
+
{job.meta?.conversation_id ? job.meta.conversation_id.substring(0, 8) : '—'}
-
+
+
{job.job_id}
-
{getJobTypeShort(job.job_type)}
+ 		+
+ {getJobTypeShort(job.job_type)} +
- + {getStatusIcon(job.status)} {job.status.charAt(0).toUpperCase() + job.status.slice(1)} - + -
+
{formatDuration(job)}
-
+
{job.status === 'failed' && ( - + )} - + {(job.status === 'queued' || job.status === 'started') && ( - + )} {job.status === 'finished' && ( - + )}