diff --git a/AGENTS.md b/AGENTS.md index 1f4db7897..26629feca 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -54,6 +54,7 @@ Chronicle includes an **interactive setup wizard** for easy configuration. The w - Memory configuration (agentic Markdown vault — Chronicle's single memory provider) - Network configuration and HTTPS setup - Optional services (speaker recognition, Parakeet ASR) +- Immich photo library integration (photo discovery + person photos in the vault) ### Quick Start ```bash @@ -128,7 +129,11 @@ All test operations are managed through a simple Makefile interface: cd tests # Full test workflow (recommended) -make test # Start containers + run all tests +make test # Start containers + run all tests (profile: mock, no credentials) + +# Same suite against real backing services (see tests/profiles.yml) +make test PROFILE=deepgram-openai # real Deepgram STT + real OpenAI LLM +make test PROFILE=deepgram-openai-speaker # ...plus the real speaker service # Or step by step make start # Start test containers (with health checks) @@ -588,6 +593,7 @@ tailscale ip -4 ### Testing Strategy - **Makefile-Based**: All test operations through simple `make` commands (`make test`, `make start`, `make stop`) +- **One suite, N service profiles**: tests are never selected by whether an API key is present. `tests/profiles.yml` declares which backing services are real for a run; stubs replay recorded real responses from `tests/cassettes/`, so the same assertions hold with or without credentials. Do not add a tag or a skip to work around a missing key — record a cassette (`make record-cassettes`) or fix the stub. - **Log Preservation**: Container logs always saved before cleanup (never lose debugging info) - **End-to-End Integration**: Robot Framework validates complete audio processing pipeline - **Environment Flexibility**: Tests work with both local .env files and CI environment variables @@ -634,6 +640,7 @@ For detailed technical documentation, see: - **[@docs/podman.md](docs/podman.md)**: Running with Podman instead of Docker (engine selection, rootless/GPU setup) - **[@docs/screenpipe.md](docs/screenpipe.md)**: ScreenPipe capture-node architecture, services, desktop controls, and troubleshooting - **[@docs/audio-pipeline-architecture.md](docs/audio-pipeline-architecture.md)**: Audio pipeline design +- **[@docs/backend/compose-stack.md](docs/backend/compose-stack.md)**: Backend compose services, shared mounts, and profiles - **[@docs/backend/auth.md](docs/backend/auth.md)**: Authentication architecture - **[@docs/backend/memories.md](docs/backend/memories.md)**: Memory system documentation - **[@docs/backend/plugin-development-guide.md](docs/backend/plugin-development-guide.md)**: Plugin development guide diff --git a/backends/advanced/Dockerfile b/backends/advanced/Dockerfile index 4841b6d46..e1f8380de 100644 --- a/backends/advanced/Dockerfile +++ b/backends/advanced/Dockerfile @@ -22,7 +22,7 @@ COPY pyproject.toml uv.lock ./ # Build a project .venv from the lockfile (includes git deps like Graphiti) RUN --mount=type=cache,target=/root/.cache/uv \ - uv sync --frozen --no-dev --extra deepgram --extra galileo --extra benchmark --no-install-project + uv sync --frozen --no-dev --extra galileo --extra benchmark --no-install-project # ============================================ @@ -138,7 +138,7 @@ WORKDIR /app COPY --from=ghcr.io/astral-sh/uv:latest /uv /uvx /bin/ COPY pyproject.toml uv.lock ./ RUN --mount=type=cache,target=/root/.cache/uv \ - uv sync --frozen --extra deepgram --extra galileo --extra benchmark --group test --no-install-project && \ + uv sync --frozen --extra galileo --extra benchmark --group test --no-install-project && \ rm /bin/uv /bin/uvx ENV VIRTUAL_ENV=/app/.venv diff --git a/backends/advanced/Dockerfile.k8s b/backends/advanced/Dockerfile.k8s index 6500ccf53..6f55affb5 100644 --- a/backends/advanced/Dockerfile.k8s +++ b/backends/advanced/Dockerfile.k8s @@ -23,12 +23,12 @@ COPY uv.lock . RUN mkdir -p src/advanced_omi_backend COPY src/advanced_omi_backend/__init__.py src/advanced_omi_backend/ -# Install dependencies using uv with deepgram extra +# Install dependencies using uv # Use cache mount for BuildKit, fallback for legacy builds # RUN --mount=type=cache,target=/root/.cache/uv \ -# uv sync --extra deepgram +# uv sync # Fallback for legacy Docker builds (CI compatibility) -RUN uv sync --extra deepgram +RUN uv sync # Copy all application code COPY . . diff --git a/backends/advanced/Docs/data-audit.md b/backends/advanced/Docs/data-audit.md index 6997d2432..611f9f77c 100644 --- a/backends/advanced/Docs/data-audit.md +++ b/backends/advanced/Docs/data-audit.md @@ -135,6 +135,36 @@ records by `clip_id` + `conversation_id` + source times. Helpers live in the Data Audit toolbar (exports selected rows; modal also lists/downloads/ deletes server-side exports). +### Contents preview (verify before shipping) + +`POST /export/preview` (same body minus `sensitivity_policy`) is a synchronous +dry-run: it returns the exact clips the settings would produce — boundaries, +durations, and the sliced transcript each manifest record would carry — +without writing any audio. Preview and job share one code path +(`utils/export_planning.plan_conversation_clips`), so the preview cannot drift +from the export. Unanalyzed conversations are reported skipped +(`not analyzed`) rather than analyzed inline; the export job still runs VAD +for them. + +The modal's **Dataset contents** panel renders this plan live (auto-refreshing +as params or screen withholdings change): per-clip play (gapless player over +the clip's range), the transcript slice (a `no transcript` badge marks clips +an annotator would receive silent), and an include checkbox per clip. Unticked +clips are passed to `/export` as `dropped_ranges` +(`{conversation_id: [[start, end], …]}`) and carved out exactly like privacy +ranges but accounted separately (`params.curated`, per-conversation and total +`dropped_seconds`): one bucket means "too sensitive to share", the other "not +worth annotating". + +### Export history + +The listing joins the on-disk `export.json` metadata to mark conversations a +previous export actually shipped (skipped ones don't count): each row carries +`last_export` (chipped `exported` in the table), and the `exported=never|exported` +filter scopes a curation session to un-shipped audio. Reading history from the +export directories means deleting an export naturally un-marks its +conversations — no second source of truth. + ## Privacy screen (shareability gate) Before sharing audio + transcripts with an outside annotator, the export can diff --git a/backends/advanced/docker-compose-test.yml b/backends/advanced/docker-compose-test.yml index 87ba52768..9a59a1715 100644 --- a/backends/advanced/docker-compose-test.yml +++ b/backends/advanced/docker-compose-test.yml @@ -166,10 +166,12 @@ services: build: context: ../.. dockerfile: tests/Dockerfile.mock-streaming-stt + # 8879 on purpose: the old 9999 is a popular default port for other + # tools and collided with unrelated services on a dev host. ports: - - "9999:9999" + - "8879:8879" healthcheck: - test: ["CMD", "python", "-c", "import socket; s=socket.socket(); s.connect(('localhost',9999)); s.close()"] + test: ["CMD", "python", "-c", "import socket; s=socket.socket(); s.connect(('localhost',8879)); s.close()"] interval: 10s timeout: 5s retries: 3 diff --git a/backends/advanced/docker-compose.yml b/backends/advanced/docker-compose.yml index c886402cc..f22e10b63 100644 --- a/backends/advanced/docker-compose.yml +++ b/backends/advanced/docker-compose.yml @@ -1,10 +1,12 @@ -# Explicit DNS upstreams. On a network with DNS enabled these are handed to the -# engine's embedded resolver (aardvark under Podman) as *upstream* servers — the -# container's resolv.conf still points at the resolver, so container-name lookups -# are unaffected. Without them, aardvark falls back to the host's resolver, and -# aardvark 1.4.0 stops forwarding external queries for good after one transient -# upstream failure while still answering container names. See docs/podman.md. +# The advanced backend stack. What each service is for, and why the non-obvious +# settings below are the way they are: docs/backend/compose-stack.md +# Drive it with ./start.sh / ./stop.sh / ./restart.sh, not by hand. + +# Explicit DNS upstreams; Tailscale's resolver first. Without these, aardvark +# (Podman) can silently stop forwarding external queries; with only public ones, +# *.ts.net stops resolving. See docs/backend/compose-stack.md#dns-pinning-x-public-dns x-public-dns: &public-dns + - 100.100.100.100 - 1.1.1.1 - 8.8.8.8 @@ -16,33 +18,29 @@ services: dockerfile: Dockerfile target: prod # Use prod stage without test dependencies args: - # git describe of the checkout (exported by services.py before builds; - # CI sets it to the release tag) — reported by the /version endpoint. + # git describe, exported by services.py; must match the other services + # sharing this image tag. See compose-stack.md#the-shared-backend-image CHRONICLE_BUILD_VERSION: ${CHRONICLE_BUILD_VERSION:-dev} ports: - "8000:8000" dns: *public-dns env_file: - .env + # Shared mounts (src/config/plugins are bind-mounted, so code changes need a + # restart, not a rebuild): compose-stack.md#shared-mounts volumes: - - ./src:/app/src # Mount source code for development + - ./src:/app/src - ./benchmark:/app/benchmark # LongMemEval benchmark harness (Phase A+) - ./data/audio_chunks:/app/audio_chunks - ./data/debug_dir:/app/debug_dir - ./data:/app/data - - ../../config:/app/config # Mount entire config directory (includes config.yml, defaults.yml, plugins.yml) - - ../../plugins:/app/plugins # External plugins directory - - ../../discovery.py:/app/discovery.py:ro # Service discovery module - # Mount the DIRECTORY, not the socket file. Bind-mounting a unix socket pins - # an inode, and systemd's RuntimeDirectory=tailscale deletes and recreates - # /run/tailscale on every tailscaled restart — leaving the container holding a - # deleted socket that refuses every connection until it is restarted. Pair with - # RuntimeDirectoryPreserve=yes (installed by services.py) so the directory - # itself also survives. See docs/ssl-certificates.md. - - /var/run/tailscale:/var/run/tailscale:ro # Tailscale socket dir for minidisc - # Codex CLI auth for the optional codex memory-agent executor. The wizard points - # CODEX_HOME_DIR at the host's ~/.codex; rw because codex rotates its tokens. - - ${CODEX_HOME_DIR:-./data/codex-home}:/codex-home + - ../../config:/app/config + - ../../plugins:/app/plugins + - ../../discovery.py:/app/discovery.py:ro + # The DIRECTORY, not the socket file — a mounted socket goes stale on every + # tailscaled restart. compose-stack.md#the-tailscale-socket-directory + - /var/run/tailscale:/var/run/tailscale:ro + - ${CODEX_HOME_DIR:-./data/codex-home}:/codex-home # Codex CLI auth (rw: it rotates tokens) environment: - CODEX_HOME=/codex-home - PARAKEET_ASR_URL=${PARAKEET_ASR_URL} @@ -57,18 +55,12 @@ services: - CORS_ORIGINS=http://localhost:5173,http://localhost:8000,http://192.168.1.153:5173,http://192.168.1.153:8000,https://localhost:5173,https://localhost:8000,https://100.105.225.45,https://localhost - REDIS_URL=redis://redis:6379/0 - MONGODB_URI=mongodb://mongo:27017 - # Vault sync broker -> server Syncthing REST API (internal docker network) + # Service endpoints — what each is for: compose-stack.md#chronicle-backend - VAULT_SYNC_SYNCTHING_URL=http://vault-syncthing:8384 - VAULT_SYNC_API_KEY=${VAULT_SYNC_API_KEY:-} - VAULT_SYNC_ADDRESS=${VAULT_SYNC_ADDRESS:-} - # Wake-word data-collection proxy -> standalone wakeword-service (chronicle-network) - WAKEWORD_SERVICE_URL=${WAKEWORD_SERVICE_URL:-http://chronicle-wakeword-service:8770} - # TTS service (kitten/etc) for spoken replies on the device. Empty → the backend - # discovers chronicle-tts on the Tailnet (set explicitly by the wizard for a - # local/own/pinned endpoint). - - CHRONICLE_TTS_URL=${CHRONICLE_TTS_URL:-} - # Host service-manager agent (start/stop services from the WebUI). - # Token comes from .env via env_file (auto-generated by services.py). + - CHRONICLE_TTS_URL=${CHRONICLE_TTS_URL:-} # empty → discover chronicle-tts on the Tailnet - SERVICE_MANAGER_URL=${SERVICE_MANAGER_URL:-http://host.docker.internal:8775} depends_on: mongo: @@ -85,13 +77,9 @@ services: start_period: 5s restart: unless-stopped - # Unified Worker Container - # No CUDA needed for chronicle-backend and workers, workers only orchestrate jobs and call external services - # Runs all workers in a single container for efficiency: - # - 6 RQ workers (transcription, memory, default queues) - # - 1 Audio persistence worker (audio queue) - # - 1+ Stream workers (conditional based on config.yml - Deepgram/Parakeet) - # Uses Python orchestrator for process management, health monitoring, and self-healing + # The whole worker fleet in one container (RQ workers + audio persistence + + # stream consumers), supervised by worker_orchestrator.py. No CUDA — workers + # orchestrate jobs and call external services. compose-stack.md#workers workers: image: ${CHRONICLE_REGISTRY:-}chronicle-backend:${CHRONICLE_TAG:-latest} build: @@ -105,24 +93,19 @@ services: dns: *public-dns env_file: - .env + # Same shared mounts as chronicle-backend: compose-stack.md#shared-mounts volumes: - ./src:/app/src - ./worker_orchestrator.py:/app/worker_orchestrator.py - - ./worker_healthcheck.py:/app/worker_healthcheck.py # Container healthcheck probe + - ./worker_healthcheck.py:/app/worker_healthcheck.py - ./data/audio_chunks:/app/audio_chunks - ./data:/app/data - - ../../config:/app/config # Mount entire config directory (includes config.yml, defaults.yml, plugins.yml) - - ../../plugins:/app/plugins # External plugins directory - - ../../discovery.py:/app/discovery.py:ro # Service discovery module - # Mount the DIRECTORY, not the socket file. Bind-mounting a unix socket pins - # an inode, and systemd's RuntimeDirectory=tailscale deletes and recreates - # /run/tailscale on every tailscaled restart — leaving the container holding a - # deleted socket that refuses every connection until it is restarted. Pair with - # RuntimeDirectoryPreserve=yes (installed by services.py) so the directory - # itself also survives. See docs/ssl-certificates.md. - - /var/run/tailscale:/var/run/tailscale:ro # Tailscale socket dir for minidisc - # Codex CLI auth for the optional codex memory-agent executor (memory jobs run here). - - ${CODEX_HOME_DIR:-./data/codex-home}:/codex-home + - ../../config:/app/config + - ../../plugins:/app/plugins + - ../../discovery.py:/app/discovery.py:ro + # The DIRECTORY, not the socket file. compose-stack.md#the-tailscale-socket-directory + - /var/run/tailscale:/var/run/tailscale:ro + - ${CODEX_HOME_DIR:-./data/codex-home}:/codex-home # Codex CLI auth (memory jobs run here) environment: - CODEX_HOME=/codex-home - PARAKEET_ASR_URL=${PARAKEET_ASR_URL} @@ -131,14 +114,12 @@ services: - HA_TOKEN=${HA_TOKEN} - REDIS_URL=redis://redis:6379/0 - MONGODB_URI=mongodb://mongo:27017 - # Worker orchestrator configuration (optional - defaults shown) + # Orchestrator tunables (optional - defaults shown): compose-stack.md#workers - WORKER_CHECK_INTERVAL=${WORKER_CHECK_INTERVAL:-10} - MIN_RQ_WORKERS=${MIN_RQ_WORKERS:-6} - WORKER_STARTUP_GRACE_PERIOD=${WORKER_STARTUP_GRACE_PERIOD:-30} - WORKER_SHUTDOWN_TIMEOUT=${WORKER_SHUTDOWN_TIMEOUT:-30} - # TTS service (kitten/etc) for spoken replies on the device — dispatcher runs here. - # Empty → discover chronicle-tts on the Tailnet (wizard sets it for local/pinned). - - CHRONICLE_TTS_URL=${CHRONICLE_TTS_URL:-} + - CHRONICLE_TTS_URL=${CHRONICLE_TTS_URL:-} # TTS dispatcher runs here too extra_hosts: - "host.docker.internal:host-gateway" # Access host services depends_on: @@ -147,9 +128,8 @@ services: mongo: condition: service_healthy restart: unless-stopped - # Probe the actual work, not just the process: fails if the RQ worker fleet - # shrank below MIN_RQ_WORKERS or a stream-consumer heartbeat went stale - # (wedged-but-alive). start_period covers orchestrator startup + worker boot. + # Probes the actual work, not the process — catches wedged-but-alive workers. + # compose-stack.md#workers healthcheck: test: ["CMD", "python", "worker_healthcheck.py"] interval: 30s @@ -157,11 +137,8 @@ services: start_period: 90s retries: 3 - # Annotation Cron Scheduler - # Runs periodic jobs for AI-powered annotation suggestions: - # - Daily: Surface potential errors in transcripts/memories - # - Weekly: Fine-tune error detection models using user feedback - # Set DEV_MODE=true in .env for 1-minute intervals (testing) + # Periodic AI-annotation jobs (daily error surfacing, weekly fine-tuning). + # DEV_MODE=true gives 1-minute intervals. compose-stack.md#annotation-cron annotation-cron: image: ${CHRONICLE_REGISTRY:-}chronicle-backend:${CHRONICLE_TAG:-latest} dns: *public-dns @@ -189,10 +166,8 @@ services: profiles: - annotation # Optional profile - enable with: docker compose --profile annotation up - # Lightweight intent-router microservice: classifies a voice command as a - # home-automation request vs a general agent/chat query (sub-ms Model2Vec + - # logreg). Kept out of the backend image so the ML deps don't bloat it. The - # Home Assistant plugin calls it at http://intent-router:8791/classify. + # Classifies a voice command as home-automation vs general agent/chat query. + # Own image so its ML deps stay out of the backend. compose-stack.md#intent-router intent-router: build: context: ../../extras/intent-router @@ -217,9 +192,8 @@ services: retries: 3 start_period: 25s - # Caddy reverse proxy - provides HTTPS for microphone access - # Access at: https://localhost (accepts self-signed cert warning) - # Only starts when HTTPS is configured (Caddyfile exists) + # HTTPS for the dashboard + Langfuse (browsers require it for mic access). + # Starts only under the https profile. See docs/ssl-certificates.md caddy: image: caddy:2-alpine dns: *public-dns @@ -239,9 +213,8 @@ services: profiles: - https - # WebUI with hot reload — source is volume-mounted, changes appear instantly - # without rebuilds. This is the only webui (home use); served on :5173 and - # fronted by Caddy for HTTPS. + # The only WebUI: Vite dev server on :5173 with hot reload, fronted by Caddy + # for HTTPS. compose-stack.md#webui-dev webui-dev: build: context: ./webui @@ -282,8 +255,8 @@ services: - "6379:6379" # Avoid conflict with dev on 6379 volumes: - ./data/redis_data:/data - # Redis is the raw-audio WAL. ACK XADD only after the append is fsynced so a - # host crash cannot erase the last second of already-accepted audio. + # Redis is the raw-audio WAL — fsync every append, or a host crash erases + # audio the system already accepted. docs/backend/audio-durability.md command: redis-server --appendonly yes --appendfsync always restart: unless-stopped healthcheck: @@ -292,19 +265,16 @@ services: timeout: 3s retries: 5 - # Vault sync - Syncthing instance that shares each user's Obsidian vault - # (data/conversation_docs/{user_id}) with their Mac so it can be opened in Obsidian. - # Configured exclusively by the backend's /api/vault-sync broker - not by hand. - # REST API stays on the internal docker network (reachable as vault-syncthing:8384); - # only the sync protocol port 22000 is published (reach it over Tailscale). - # Enable with: docker compose --profile vault-sync up -d + # Shares each user's vault (data/conversation_docs/{user_id}) to Obsidian. + # Configured exclusively by the backend's /api/vault-sync broker, never by + # hand. compose-stack.md#vault-syncthing vault-syncthing: image: syncthing/syncthing:latest dns: *public-dns container_name: chronicle-vault-syncthing hostname: chronicle-vault-syncthing environment: - - STGUIADDRESS=0.0.0.0:8384 # REST API reachable from the backend container + - STGUIADDRESS=0.0.0.0:8384 # REST API — internal network only - STGUIAPIKEY=${VAULT_SYNC_API_KEY} # backend authenticates to Syncthing with this - PUID=${VAULT_SYNC_PUID:-0} # match conversation_docs file ownership - PGID=${VAULT_SYNC_PGID:-0} @@ -319,6 +289,8 @@ services: profiles: - vault-sync + # Optional in-container tailnet membership; most deployments run Tailscale on + # the host instead. compose-stack.md#tailscale tailscale: image: tailscale/tailscale:latest container_name: advanced-tailscale diff --git a/backends/advanced/pyproject.toml b/backends/advanced/pyproject.toml index 66f519b86..86a585713 100644 --- a/backends/advanced/pyproject.toml +++ b/backends/advanced/pyproject.toml @@ -40,10 +40,6 @@ dependencies = [ ] [project.optional-dependencies] -deepgram = [ - "deepgram-sdk>=4.0.0", -] - local-audio = [ "easy-audio-interfaces[local-audio]>=0.7.1", ] diff --git a/backends/advanced/src/advanced_omi_backend/controllers/conversation_controller.py b/backends/advanced/src/advanced_omi_backend/controllers/conversation_controller.py index 9afc7b9f8..aa7563155 100644 --- a/backends/advanced/src/advanced_omi_backend/controllers/conversation_controller.py +++ b/backends/advanced/src/advanced_omi_backend/controllers/conversation_controller.py @@ -483,6 +483,7 @@ async def get_conversations( # MongoDB fields covered by each independently selectable search category. _SEARCH_CATEGORY_FIELDS: dict[str, list[str]] = { + "id": ["conversation_id"], "title": ["title"], "summary": ["summary", "detailed_summary"], "speakers": ["_search_active_version.segments.speaker"], @@ -591,7 +592,7 @@ async def search_conversations( categories: list[str] | None = None, ): """Search conversations by literal pattern across selected field categories.""" - categories = categories or ["title", "summary", "speakers"] + categories = categories or ["id", "title", "summary", "speakers"] fields = _search_fields(categories) try: result = await _regex_search_conversations(query, user, fields, limit, offset) diff --git a/backends/advanced/src/advanced_omi_backend/controllers/data_audit_controller.py b/backends/advanced/src/advanced_omi_backend/controllers/data_audit_controller.py index 9aa01bf42..6afb0c2ef 100644 --- a/backends/advanced/src/advanced_omi_backend/controllers/data_audit_controller.py +++ b/backends/advanced/src/advanced_omi_backend/controllers/data_audit_controller.py @@ -14,7 +14,7 @@ import statistics import uuid from datetime import datetime, timezone -from typing import Dict, List, Optional, Tuple +from typing import Any, Dict, List, Optional, Tuple from fastapi.responses import FileResponse, JSONResponse @@ -38,6 +38,7 @@ EXPORTS_DIR, META_NAME, ZIP_NAME, + active_segments, export_dir, new_export_id, validate_export_id, @@ -55,6 +56,10 @@ AudioValidationError, validate_and_prepare_audio, ) +from advanced_omi_backend.utils.export_planning import ( + export_eligibility, + plan_conversation_clips, +) from advanced_omi_backend.utils.transcript_slicing import ( build_transcript_text, shift_segments, @@ -198,6 +203,38 @@ def _vad_stale(va: Optional[dict], duration: float) -> bool: return not audio_cache_duration_matches(cached, duration) +def _latest_exports_by_conversation(user: User) -> Dict[str, dict]: + """conversation_id → the most recent export that actually shipped it. + + Read from the on-disk export metadata (the audit trail the export job + already writes) rather than a new Mongo field, so deleting an export + directory naturally un-marks its conversations. Conversations that were + selected but skipped don't count as exported. Scoped by the same + ownership rule as ``list_exports``. + """ + latest: Dict[str, dict] = {} + if not EXPORTS_DIR.is_dir(): + return latest + for meta_path in EXPORTS_DIR.glob(f"*/{META_NAME}"): + try: + meta = json.loads(meta_path.read_text()) + except Exception: + continue + if not user.is_superuser and meta.get("created_by") != str(user.user_id): + continue + created_at = meta.get("created_at") or "" + for conv in meta.get("conversations", []): + if conv.get("skipped_reason"): + continue + cid = conv.get("conversation_id") + if cid and created_at > (latest.get(cid, {}).get("created_at") or ""): + latest[cid] = { + "export_id": meta.get("export_id"), + "created_at": created_at, + } + return latest + + async def list_for_audit( user: User, speech_threshold: float = 0.5, @@ -210,6 +247,7 @@ async def list_for_audit( include_speakers: Optional[List[str]] = None, exclude_speakers: Optional[List[str]] = None, dataset_id: Optional[str] = None, + exported: Optional[str] = None, archived_only: bool = False, hide_failed: bool = False, hide_reviewed: bool = False, @@ -225,6 +263,12 @@ async def list_for_audit( the ``exclude_speakers``. Speech bounds exclude unanalyzed conversations; ``max_speech_fraction=1`` / ``min_speech_fraction=0`` / ``max_duration=0`` disable the respective bound. + + ``exported`` filters on annotation-export history (from the on-disk export + metadata): ``never`` keeps only conversations no export has shipped, + ``exported`` only those a previous export contains. Each row carries + ``last_export`` either way, so curation sessions can skip audio already + sent to annotators. """ try: base: dict = {} if user.is_superuser else {"user_id": str(user.user_id)} @@ -303,6 +347,7 @@ async def list_for_audit( include_set = set(include_speakers or []) exclude_set = set(exclude_speakers or []) + export_history = _latest_exports_by_conversation(user) matched: List[dict] = [] # Speakers present anywhere in the scanned working set (before the # compound predicate), so the filter UI offers exactly the labels that @@ -347,6 +392,11 @@ async def list_for_audit( # speech segment already has an identified_as). if hide_reviewed and unknown_count == 0: continue + in_export = doc.get("conversation_id") in export_history + if exported == "never" and in_export: + continue + if exported == "exported" and not in_export: + continue created_at = doc.get("created_at") archived_at = doc.get("audio_archived_at") @@ -374,6 +424,7 @@ async def list_for_audit( "derived_operation": ( derived_from.get("operation") if derived_from else None ), + "last_export": export_history.get(doc.get("conversation_id")), "audio_archived": doc.get("audio_archived", False), "audio_archived_at": ( archived_at.isoformat() if archived_at else None @@ -1705,6 +1756,96 @@ async def get_default_sensitivity_policy(): return {"policy": get_sensitivity_policy()} +async def preview_export( + user: User, + conversation_ids: List[str], + mode: str = "clips", + pad_seconds: float = 1.0, + speech_threshold: float = 0.5, + merge_gap_seconds: float = 3.0, + excluded_ranges: Optional[Dict[str, List[List[float]]]] = None, +): + """Dry-run of the export: the exact clips it would produce, without + writing any audio. + + Runs the same plan computation as the export job + (``utils/export_planning.plan_conversation_clips``), so the boundaries, + durations, and sliced transcripts returned here are byte-for-byte what + the manifest would contain. Unanalyzed conversations are reported as + skipped (``not analyzed``) rather than analyzed inline — VAD over a long + recording is too slow for a synchronous endpoint; the UI points at the + Analyze button. + """ + excluded_ranges = excluded_ranges or {} + user_id = str(user.user_id) + conversations: List[dict] = [] + totals = {"clip_count": 0, "total_clip_seconds": 0.0, "excluded_seconds": 0.0} + + try: + for cid in dict.fromkeys(conversation_ids): + conv = await Conversation.find_one(Conversation.conversation_id == cid) + entry: Dict[str, Any] = { + "conversation_id": cid, + "title": conv.title if conv else None, + "client_id": conv.client_id if conv else None, + "created_at": ( + conv.created_at.isoformat() if conv and conv.created_at else None + ), + } + skipped = export_eligibility(conv, user_id, user.is_superuser) + if not skipped: + plan = await plan_conversation_clips( + conv, + mode, + pad_seconds, + speech_threshold, + merge_gap_seconds, + excluded_ranges.get(cid), + ) + skipped = plan.skipped_reason + if skipped: + entry["skipped_reason"] = skipped + conversations.append(entry) + continue + + segments = active_segments(conv) + clips = [] + for clip in plan.clips: + sliced = slice_segments(segments, clip.start, clip.end) + clips.append( + { + "clip_index": clip.clip_index, + "clip_id": f"{cid}_{clip.clip_index:03d}", + "start": round(clip.start, 2), + "end": round(clip.end, 2), + "duration_seconds": round(clip.duration, 2), + "text": build_transcript_text(sliced), + "segment_count": len(sliced), + } + ) + entry["clips"] = clips + entry["sample_rate"] = plan.sample_rate + entry["clip_seconds"] = round(plan.clip_seconds, 2) + entry["excluded_seconds"] = plan.excluded_seconds + totals["clip_count"] += len(clips) + totals["total_clip_seconds"] += plan.clip_seconds + totals["excluded_seconds"] += plan.excluded_seconds + conversations.append(entry) + + totals["total_clip_seconds"] = round(totals["total_clip_seconds"], 2) + totals["excluded_seconds"] = round(totals["excluded_seconds"], 2) + totals["conversation_count"] = len(conversations) + totals["exported_conversations"] = sum( + 1 for c in conversations if "skipped_reason" not in c + ) + return {"conversations": conversations, "totals": totals} + except Exception as e: + logger.exception(f"Error previewing annotation export: {e}") + return JSONResponse( + status_code=500, content={"error": "Error previewing export"} + ) + + async def start_export( user: User, conversation_ids: List[str], @@ -1713,12 +1854,14 @@ async def start_export( speech_threshold: float = 0.5, merge_gap_seconds: float = 3.0, excluded_ranges: Optional[Dict[str, List[List[float]]]] = None, + dropped_ranges: Optional[Dict[str, List[List[float]]]] = None, sensitivity_policy: Optional[str] = None, ): """Enqueue the annotation-dataset export job for selected conversations. ``excluded_ranges`` maps conversation_id → withheld ``[start, end]`` ranges - confirmed from the privacy screen; those are carved out of the export. + confirmed from the privacy screen; ``dropped_ranges`` → clips the user + unticked in the export preview. Both are carved out of the export. """ try: export_id = new_export_id() @@ -1732,6 +1875,7 @@ async def start_export( speech_threshold=speech_threshold, merge_gap_seconds=merge_gap_seconds, excluded_ranges=excluded_ranges, + dropped_ranges=dropped_ranges, sensitivity_policy=sensitivity_policy, job_timeout=3600, result_ttl=JOB_RESULT_TTL, diff --git a/backends/advanced/src/advanced_omi_backend/models/memory_audit.py b/backends/advanced/src/advanced_omi_backend/models/memory_audit.py index a259f0b22..f3e6b6ee3 100644 --- a/backends/advanced/src/advanced_omi_backend/models/memory_audit.py +++ b/backends/advanced/src/advanced_omi_backend/models/memory_audit.py @@ -39,7 +39,8 @@ class MemoryAuditEntry(Document): None, description="Why the memory changed (provenance), one of MemoryCause: " "auto_extraction, memory_replay, memory_rebuild, transcript_reprocess, " - "speaker_reprocess, annotation_apply, obsidian_sync, delete_all. " + "speaker_reprocess, annotation_apply, obsidian_sync, obsidian_action, " + "delete_all. " "See services/memory/audit.py.", ) strategy: Optional[str] = Field( diff --git a/backends/advanced/src/advanced_omi_backend/routers/modules/conversation_routes.py b/backends/advanced/src/advanced_omi_backend/routers/modules/conversation_routes.py index 08dfde10d..50e9cae0f 100644 --- a/backends/advanced/src/advanced_omi_backend/routers/modules/conversation_routes.py +++ b/backends/advanced/src/advanced_omi_backend/routers/modules/conversation_routes.py @@ -88,13 +88,13 @@ async def search_conversations( q: str = Query("", description="Optional text search query"), limit: int = Query(50, ge=1, le=200, description="Max results to return"), offset: int = Query(0, ge=0, description="Number of results to skip"), - fields: list[Literal["title", "summary", "speakers"]] = Query( - default=["title", "summary", "speakers"], - description="Search categories: title, summary, and/or speakers", + fields: list[Literal["id", "title", "summary", "speakers"]] = Query( + default=["id", "title", "summary", "speakers"], + description="Search categories: conversation ID, title, summary, and/or speakers", ), current_user: User = Depends(current_active_user), ): - """Search conversations and identified people by literal case-insensitive pattern.""" + """Search conversation metadata by literal case-insensitive pattern.""" return await conversation_controller.search_conversations( q.strip(), current_user, limit, offset, fields ) diff --git a/backends/advanced/src/advanced_omi_backend/routers/modules/data_audit_routes.py b/backends/advanced/src/advanced_omi_backend/routers/modules/data_audit_routes.py index 3acdfd2e1..d81ddfd83 100644 --- a/backends/advanced/src/advanced_omi_backend/routers/modules/data_audit_routes.py +++ b/backends/advanced/src/advanced_omi_backend/routers/modules/data_audit_routes.py @@ -101,11 +101,30 @@ class ExportRequest(BaseModel): description="conversation_id → withheld [start, end] time ranges (seconds) " "from the privacy screen; carved out of the exported audio + transcript", ) + dropped_ranges: Dict[str, List[List[float]]] = Field( + default_factory=dict, + description="conversation_id → [start, end] ranges of clips the user " + "unticked in the export preview; removed from the export (accounted " + "separately from privacy withholdings)", + ) sensitivity_policy: Optional[str] = Field( None, description="Policy used for the screen (recorded in export metadata)" ) +class ExportPreviewRequest(BaseModel): + conversation_ids: List[str] = Field(..., min_length=1, max_length=200) + mode: str = Field("clips", pattern="^(clips|full)$") + pad_seconds: float = Field(1.0, ge=0.0, le=10.0) + speech_threshold: float = Field(0.5, ge=0.0, le=1.0) + merge_gap_seconds: float = Field(3.0, ge=0.0, le=60.0) + excluded_ranges: Dict[str, List[List[float]]] = Field( + default_factory=dict, + description="Privacy-screen withholdings to apply to the preview, so the " + "clips shown match what the export would produce", + ) + + @router.post("/analyze") async def analyze( body: AnalyzeRequest, @@ -159,6 +178,12 @@ async def list_conversations( max_length=200, description="Only conversations imported from this annotation dataset", ), + exported: Optional[str] = Query( + None, + pattern="^(never|exported)$", + description="Filter on annotation-export history: never = not in any " + "export, exported = shipped by a previous export", + ), archived_only: bool = Query( False, description="List archived metadata stubs instead of active conversations", @@ -192,6 +217,7 @@ def _csv(v: Optional[str]) -> Optional[list]: include_speakers=_csv(include_speakers), exclude_speakers=_csv(exclude_speakers), dataset_id=dataset_id, + exported=exported, archived_only=archived_only, hide_failed=hide_failed, hide_reviewed=hide_reviewed, @@ -814,6 +840,28 @@ async def screen_export( ) +@router.post("/export/preview") +async def preview_export( + body: ExportPreviewRequest, + current_user: User = Depends(current_active_user), +): + """Dry-run of the export: the exact clips (boundaries, durations, sliced + transcripts) the current settings would produce, computed synchronously + without writing any audio. Unanalyzed conversations are reported skipped + (``not analyzed``) — run /analyze first. Untick clips here and pass their + ranges to /export as ``dropped_ranges``. + """ + return await data_audit_controller.preview_export( + current_user, + body.conversation_ids, + mode=body.mode, + pad_seconds=body.pad_seconds, + speech_threshold=body.speech_threshold, + merge_gap_seconds=body.merge_gap_seconds, + excluded_ranges=body.excluded_ranges, + ) + + @router.post("/export") async def start_export( body: ExportRequest, @@ -822,8 +870,9 @@ async def start_export( """Enqueue an annotation-dataset export: WAV audio + transcript manifest, zipped for download. Mode ``clips`` cuts one padded WAV per VAD speech region (silence cropped); mode ``full`` exports each conversation as a - single untouched WAV. ``excluded_ranges`` from the privacy screen are - carved out of the exported audio + transcript. + single untouched WAV. ``excluded_ranges`` from the privacy screen and + ``dropped_ranges`` from the export preview are carved out of the exported + audio + transcript. Poll job status via /api/queue/jobs/{id}/status, then download from /api/data-audit/exports/{export_id}/download. @@ -836,6 +885,7 @@ async def start_export( speech_threshold=body.speech_threshold, merge_gap_seconds=body.merge_gap_seconds, excluded_ranges=body.excluded_ranges, + dropped_ranges=body.dropped_ranges, sensitivity_policy=body.sensitivity_policy, ) diff --git a/backends/advanced/src/advanced_omi_backend/routers/modules/device_input_routes.py b/backends/advanced/src/advanced_omi_backend/routers/modules/device_input_routes.py index e903568ab..e3f1ae313 100644 --- a/backends/advanced/src/advanced_omi_backend/routers/modules/device_input_routes.py +++ b/backends/advanced/src/advanced_omi_backend/routers/modules/device_input_routes.py @@ -83,6 +83,11 @@ def _effective_source_status( ) -> str: if source.status != "online": return source.status + # Immich is polled by Chronicle on a schedule; it does not send the frequent + # heartbeats expected from live ScreenPipe capture agents. Its last_seen_at + # therefore means "last successful sync", not "last heartbeat". + if source.provider == "immich": + return "online" checked_at = _as_utc(now or utcnow()) if source.last_seen_at is None: return "offline" diff --git a/backends/advanced/src/advanced_omi_backend/routers/modules/memory_routes.py b/backends/advanced/src/advanced_omi_backend/routers/modules/memory_routes.py index ebd99e8eb..f31d4983c 100644 --- a/backends/advanced/src/advanced_omi_backend/routers/modules/memory_routes.py +++ b/backends/advanced/src/advanced_omi_backend/routers/modules/memory_routes.py @@ -5,13 +5,23 @@ """ import logging -from typing import Optional +from typing import Literal, Optional -from fastapi import APIRouter, Body, Depends, Query +from fastapi import APIRouter, Body, Depends, HTTPException, Query from pydantic import BaseModel from advanced_omi_backend.auth import current_active_user, current_superuser from advanced_omi_backend.controllers import memory_controller +from advanced_omi_backend.services.memory.person_merge import ( + PersonMergeError, + PersonMergeStale, +) +from advanced_omi_backend.services.memory.person_merge_actions import ( + apply_person_merge, + get_person_suggestions, + preview_person_merge, + set_people_distinct, +) from advanced_omi_backend.users import User logger = logging.getLogger(__name__) @@ -26,6 +36,37 @@ class AddMemoryRequest(BaseModel): source_id: Optional[str] = None +class PersonMergePreviewRequest(BaseModel): + """Local state supplied by an Obsidian or automation client.""" + + source_name: str + target_name: str + source_hash: Optional[str] = None + target_hash: Optional[str] = None + + +class PersonMergeApplyRequest(BaseModel): + """Apply the exact server-side plan previously shown to the user.""" + + source_name: str + target_name: str + plan_token: str + + +class PersonIdentityDecisionRequest(BaseModel): + """A durable user decision about whether two person notes are distinct.""" + + person_a: str + person_b: str + decision: Literal["distinct", "clear_distinct"] + revision: Optional[str] = None + + +def _person_merge_http_error(error: PersonMergeError) -> HTTPException: + status = 409 if isinstance(error, PersonMergeStale) else 422 + return HTTPException(status_code=status, detail=str(error)) + + @router.get("") async def get_memories( current_user: User = Depends(current_active_user), @@ -109,6 +150,70 @@ async def add_memory( ) +@router.post("/people/merge/preview") +async def preview_people_merge( + request: PersonMergePreviewRequest, + current_user: User = Depends(current_active_user), +): + """Preview a deterministic person merge without changing the vault.""" + try: + return await preview_person_merge( + current_user.user_id, + request.source_name, + request.target_name, + request.source_hash, + request.target_hash, + ) + except PersonMergeError as error: + raise _person_merge_http_error(error) from error + + +@router.post("/people/merge") +async def merge_people( + request: PersonMergeApplyRequest, + current_user: User = Depends(current_active_user), +): + """Apply a previously previewed deterministic person merge.""" + try: + return await apply_person_merge( + current_user.user_id, + request.source_name, + request.target_name, + request.plan_token, + ) + except PersonMergeError as error: + raise _person_merge_http_error(error) from error + + +@router.get("/people/suggestions") +async def get_people_suggestions( + limit: int = Query(default=20, ge=1, le=100), + current_user: User = Depends(current_active_user), +): + """Return deterministic duplicate-person candidates for user review.""" + return { + "suggestions": await get_person_suggestions(current_user.user_id, limit), + } + + +@router.post("/people/identity") +async def set_people_identity( + request: PersonIdentityDecisionRequest, + current_user: User = Depends(current_active_user), +): + """Persist or clear a symmetric distinct-person annotation.""" + try: + return await set_people_distinct( + current_user.user_id, + request.person_a, + request.person_b, + distinct=request.decision == "distinct", + revision=request.revision, + ) + except PersonMergeError as error: + raise _person_merge_http_error(error) from error + + @router.delete("/{memory_id}") async def delete_memory( memory_id: str, current_user: User = Depends(current_active_user) diff --git a/backends/advanced/src/advanced_omi_backend/services/immich_discovery.py b/backends/advanced/src/advanced_omi_backend/services/immich_discovery.py index fd2f69c78..14ad74a76 100644 --- a/backends/advanced/src/advanced_omi_backend/services/immich_discovery.py +++ b/backends/advanced/src/advanced_omi_backend/services/immich_discovery.py @@ -47,6 +47,13 @@ def _asset_time(asset: dict[str, Any]) -> datetime | None: return value if value.tzinfo else value.replace(tzinfo=timezone.utc) +def _as_utc(value: datetime) -> datetime: + """Restore UTC stripped by MongoDB before serializing times for Immich.""" + if value.tzinfo is None: + return value.replace(tzinfo=timezone.utc) + return value.astimezone(timezone.utc) + + def select_candidates( assets: list[dict[str, Any]], limit: int = _DAILY_LIMIT ) -> list[dict[str, Any]]: @@ -105,9 +112,8 @@ async def scan_immich_memories() -> dict[str, Any]: DeviceInputItem.source_id == source_id, sort=[("captured_at", -1)], ) - since = ( - newest.captured_at if newest else utcnow() - timedelta(days=2) - ) - timedelta(hours=48) + newest_at = _as_utc(newest.captured_at) if newest else utcnow() - timedelta(days=2) + since = newest_at - timedelta(hours=48) page: int | None = 1 assets: list[dict[str, Any]] = [] async with httpx.AsyncClient( diff --git a/backends/advanced/src/advanced_omi_backend/services/memory/agent/__init__.py b/backends/advanced/src/advanced_omi_backend/services/memory/agent/__init__.py index 806522efa..f710cb442 100644 --- a/backends/advanced/src/advanced_omi_backend/services/memory/agent/__init__.py +++ b/backends/advanced/src/advanced_omi_backend/services/memory/agent/__init__.py @@ -1,22 +1,31 @@ -"""Chronicle memory agent: a tool-calling agent that maintains the markdown vault.""" +"""Chronicle memory agent package. -from .codex_agent import CodexMemoryAgent, codex_executor_available -from .memory_agent import MemoryAgent, MemoryAgentResult, search_vault -from .vault_tools import ( - VAULT_SEARCH_TOOL_SCHEMAS, - VAULT_TOOL_SCHEMAS, - VaultToolError, - VaultTools, -) +Public symbols are loaded lazily so low-level deterministic vault helpers can import +``agent.section_edit`` without initializing the LLM agents (or creating cycles with +``vault_tools``). +""" -__all__ = [ - "CodexMemoryAgent", - "codex_executor_available", - "MemoryAgent", - "MemoryAgentResult", - "search_vault", - "VaultTools", - "VaultToolError", - "VAULT_TOOL_SCHEMAS", - "VAULT_SEARCH_TOOL_SCHEMAS", -] +from importlib import import_module + +_EXPORTS = { + "CodexMemoryAgent": (".codex_agent", "CodexMemoryAgent"), + "codex_executor_available": (".codex_agent", "codex_executor_available"), + "MemoryAgent": (".memory_agent", "MemoryAgent"), + "MemoryAgentResult": (".memory_agent", "MemoryAgentResult"), + "search_vault": (".memory_agent", "search_vault"), + "VaultTools": (".vault_tools", "VaultTools"), + "VaultToolError": (".vault_tools", "VaultToolError"), + "VAULT_TOOL_SCHEMAS": (".vault_tools", "VAULT_TOOL_SCHEMAS"), + "VAULT_SEARCH_TOOL_SCHEMAS": (".vault_tools", "VAULT_SEARCH_TOOL_SCHEMAS"), +} + +__all__ = list(_EXPORTS) + + +def __getattr__(name: str): + if name not in _EXPORTS: + raise AttributeError(name) + module_name, attribute = _EXPORTS[name] + value = getattr(import_module(module_name, __name__), attribute) + globals()[name] = value + return value diff --git a/backends/advanced/src/advanced_omi_backend/services/memory/agent/codex_agent.py b/backends/advanced/src/advanced_omi_backend/services/memory/agent/codex_agent.py index 0d23fa53c..3de224378 100644 --- a/backends/advanced/src/advanced_omi_backend/services/memory/agent/codex_agent.py +++ b/backends/advanced/src/advanced_omi_backend/services/memory/agent/codex_agent.py @@ -29,11 +29,13 @@ import os import shutil import tempfile +import time from datetime import datetime, timezone from pathlib import Path from typing import Dict, List, Optional from ..vault_templates import CONVERSATION_TEMPLATE, PERSON_TEMPLATE, TOPIC_TEMPLATE +from . import codex_quota from .memory_agent import MemoryAgentResult, _for_prompt, _get_prompt logger = logging.getLogger("memory_service.agent.codex") @@ -228,6 +230,22 @@ async def run( ) binary = detail + quota_payload, quota_block = await asyncio.to_thread( + self._check_quota, conversation_id + ) + if quota_block: + from .memory_agent import MemoryAgent + + return await MemoryAgent(self.root).run( + transcript, + conversation_id, + date=date, + duration_minutes=duration_minutes, + title=title, + vault_summary=vault_summary, + guidance=guidance, + ) + date = date or datetime.now(timezone.utc).isoformat() system_prompt = await _get_prompt( CODEX_AGENT_SYSTEM_PROMPT_ID, @@ -279,6 +297,9 @@ async def run( "chronicle.memory.executor": "codex", "chronicle.memory.sandbox_mode": sandbox_mode, "chronicle.memory.transcript_chars": len(transcript), + **codex_quota.quota_span_attributes( + quota_payload, str(settings.get("limit_id") or "") + ), "langfuse.observation.input": json.dumps( { "conversation_id": conversation_id, @@ -316,6 +337,11 @@ async def run( ) span.set_attribute("chronicle.memory.error_count", len(result.errors)) span.set_attribute("chronicle.memory.truncated", result.truncated) + # Mirrored onto the agent span for at-a-glance filtering; the + # ingestable copy lives on the child codex_turn span (see + # _record_usage_span for why it cannot live here). + for key, value in result.usage.items(): + span.set_attribute(f"chronicle.memory.usage.{key}", value) span.set_attribute( "langfuse.observation.output", json.dumps( @@ -401,6 +427,7 @@ def _run_locked( model or "default", timeout, ) + started_ns = time.time_ns() try: proc = subprocess.run( cmd, @@ -428,8 +455,11 @@ def _run_locked( except OSError as e: errors.append(f"codex exec failed to start: {e}") - command_count, turn_count, event_errors = self._parse_events(stdout) + ended_ns = time.time_ns() + + command_count, turn_count, event_errors, usage = self._parse_events(stdout) errors.extend(event_errors) + self._record_usage_span(usage, model, started_ns, ended_ns) summary = "" try: @@ -458,22 +488,109 @@ def _run_locked( tool_calls=command_count, removed=removed, errors=errors, + usage=usage, truncated=failed, ) logger.info( "codex agent done: conv=%s turns=%d commands=%d touched=%d removed=%d " - "errors=%d%s — %s", + "errors=%d tokens=in:%d/cached:%d/out:%d%s — %s", conversation_id, result.rounds, command_count, len(touched), len(removed), len(errors), + usage.get("input_tokens", 0), + usage.get("input_cached_tokens", 0), + usage.get("output_tokens", 0), " (FAILED)" if failed else "", summary[:160], ) return result + @staticmethod + def _check_quota(conversation_id: str) -> tuple[Optional[dict], bool]: + """Return the quota snapshot and whether this run should yield the budget. + + Chronicle's background recording shares one account-wide weekly budget with + the user's interactive Codex sessions, and is the cheaper consumer to give + up: a yielded run still records the conversation via the direct (metered + API) executor, while a blocked interactive session is stuck for days. + + Fails OPEN — an unreadable quota yields ``False`` and the run proceeds. The + probe is an optimisation over Codex's own limit error, not a correctness + gate, so a broken probe must not stop memory extraction entirely. + """ + settings = _codex_settings() + threshold = settings.get("max_used_percent") + if threshold is None: + return None, False + try: + threshold = int(threshold) + except (TypeError, ValueError): + logger.warning("ignoring non-numeric memory.codex.max_used_percent") + return None, False + + limit_id = str(settings.get("limit_id") or "") + payload = codex_quota.read_rate_limits() + used = codex_quota.bucket_used_percent(payload, limit_id) + if used is None: + logger.debug("codex quota unknown for conv=%s; proceeding", conversation_id) + return payload, False + if used < threshold: + return payload, False + + logger.warning( + "codex quota %d%% used (>= %d%% budget for Chronicle) — recording conv=%s " + "via the direct memory agent instead, leaving the remainder for " + "interactive use", + used, + threshold, + conversation_id, + ) + return payload, True + + @staticmethod + def _record_usage_span( + usage: Dict[str, int], model: str, started_ns: int, ended_ns: int + ) -> None: + """Emit the model call as a child LLM span carrying the run's token usage. + + Deliberately NOT on the parent ``codex_memory_agent`` span: current Langfuse + drops usage from spans whose ``gen_ai.operation.name`` is ``invoke_agent`` or + ``agent_step`` (it assumes the agent span duplicates usage from child + model-call spans) and would ingest the tokens as zero without erroring. Older + Langfuse — including 3.x — has no such guard, so putting usage on the agent + span works today and silently breaks on upgrade. A child model-call span is + correct under both. + + Created after the subprocess returns, since usage is only known then, with + explicit timestamps so it still spans the real call window. + """ + if not usage: + return + try: + from advanced_omi_backend.observability.otel_setup import get_tracer + + tracer = get_tracer("chronicle.memory.codex") + if tracer is None: + return + attributes = { + "openinference.span.kind": "LLM", + "gen_ai.operation.name": "chat", + "gen_ai.system": "openai", + "gen_ai.provider.name": "openai_codex_cli", + "gen_ai.request.model": model or "codex-default", + "gen_ai.response.model": model or "codex-default", + **{f"gen_ai.usage.{k}": v for k, v in usage.items()}, + } + span = tracer.start_span( + "codex_turn", attributes=attributes, start_time=started_ns + ) + span.end(end_time=ended_ns) + except Exception: # noqa: BLE001 - telemetry must never fail the run + logger.debug("failed to record codex usage span", exc_info=True) + def _snapshot(self) -> Dict[str, str]: """Vault-relative ``*.md`` contents (same shape the provider's audit diff uses).""" snapshot: Dict[str, str] = {} @@ -489,11 +606,17 @@ def _snapshot(self) -> Dict[str, str]: return snapshot @staticmethod - def _parse_events(stdout: str) -> tuple[int, int, List[str]]: - """Tolerantly scan the ``--json`` JSONL stream for counts and errors.""" + def _parse_events(stdout: str) -> tuple[int, int, List[str], Dict[str, int]]: + """Tolerantly scan the ``--json`` JSONL stream for counts, errors, and usage. + + ``turn.completed`` carries the turn's token ``usage``; it is the only place + the CLI reports what a run actually cost, so it is summed across turns and + translated into Langfuse's usage-detail key names. + """ commands = 0 turns = 0 errors: List[str] = [] + usage: Dict[str, int] = {} for line in stdout.splitlines(): line = line.strip() if not line.startswith("{"): @@ -509,10 +632,42 @@ def _parse_events(stdout: str) -> tuple[int, int, List[str]]: commands += 1 elif etype == "turn.completed": turns += 1 + for key, value in CodexMemoryAgent._turn_usage(event).items(): + usage[key] = usage.get(key, 0) + value elif etype == "turn.failed": turns += 1 failure = event.get("error") or {} errors.append(f"codex turn failed: {failure.get('message', failure)}") elif etype == "error": errors.append(f"codex error: {event.get('message', event)}") - return commands, turns, errors + return commands, turns, errors, usage + + @staticmethod + def _turn_usage(event: dict) -> Dict[str, int]: + """Map one ``turn.completed`` event's ``usage`` to Langfuse usage details. + + Tolerant by design: the CLI's field names are not a stable contract, so an + absent or oddly-shaped block yields ``{}`` rather than failing the run. + """ + raw = event.get("usage") + if not isinstance(raw, dict): + return {} + # Codex reports cached input tokens *inside* input_tokens, which is what + # Langfuse's normaliser assumes (it derives uncached input as + # input_tokens - input_cached_tokens), so both pass through unchanged. + # Caveat on Langfuse 3.x: it instead adds the two, so the rollup `usage.input` + # and `total` over-count cached tokens there. `usageDetails.input` is right + # on both. + field_map = { + "input_tokens": "input_tokens", + "cached_input_tokens": "input_cached_tokens", + "output_tokens": "output_tokens", + "reasoning_output_tokens": "output_reasoning_tokens", + } + usage: Dict[str, int] = {} + for source, target in field_map.items(): + value = raw.get(source) + if isinstance(value, bool) or not isinstance(value, (int, float)): + continue + usage[target] = int(value) + return usage diff --git a/backends/advanced/src/advanced_omi_backend/services/memory/agent/codex_quota.py b/backends/advanced/src/advanced_omi_backend/services/memory/agent/codex_quota.py new file mode 100644 index 000000000..3d3e34cd9 --- /dev/null +++ b/backends/advanced/src/advanced_omi_backend/services/memory/agent/codex_quota.py @@ -0,0 +1,189 @@ +"""Read the ChatGPT subscription's Codex quota, so background runs can yield to it. + +Codex's rate limit is an account-wide weekly budget shared with the user's own +interactive sessions. Chronicle's background vault recording is the lower-priority +consumer of it: a memory extraction that is skipped still lands via the direct +(metered API) executor, whereas an interactive session that hits the wall is simply +blocked for days. So the agent checks headroom before spawning ``codex exec``. + +The numbers come from the CLI itself rather than from parsing its error text: the +``codex app-server`` JSON-RPC surface exposes ``account/rateLimits/read``, which +returns per-bucket ``usedPercent`` / ``resetsAt`` / ``windowDurationMins``. That is +the same source the TUI's own usage display reads. +""" + +import contextlib +import json +import logging +import os +import shutil +import subprocess +import threading +import time +from typing import Dict, Optional + +logger = logging.getLogger("memory_service.agent.codex.quota") + +# The app-server is spawned per read and killed as soon as the response arrives; the +# cache keeps that off the hot path when conversations close in quick succession. +_CACHE_TTL_SECONDS = 120 +_READ_TIMEOUT_SECONDS = 20 + +_cache_lock = threading.Lock() +_cached: Optional[tuple[float, Optional[dict]]] = None + + +def read_rate_limits( + *, timeout: int = _READ_TIMEOUT_SECONDS, use_cache: bool = True +) -> Optional[dict]: + """Return the ``account/rateLimits/read`` payload, or ``None`` if unavailable. + + ``None`` means "could not determine" (no binary, no auth, timeout, protocol + change) and is deliberately distinct from a snapshot reporting 100% used. + Callers must not treat it as exhausted — see :func:`bucket_used_percent`. + """ + global _cached + if use_cache: + with _cache_lock: + if _cached and (time.monotonic() - _cached[0]) < _CACHE_TTL_SECONDS: + return _cached[1] + + payload = _read_uncached(timeout) + with _cache_lock: + _cached = (time.monotonic(), payload) + return payload + + +def _read_uncached(timeout: int) -> Optional[dict]: + binary = shutil.which(os.environ.get("CODEX_BINARY", "codex")) + if not binary: + return None + + proc = None + try: + proc = subprocess.Popen( + [binary, "app-server"], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.DEVNULL, + text=True, + bufsize=1, + ) + requests = ( + json.dumps( + { + "jsonrpc": "2.0", + "id": 1, + "method": "initialize", + "params": { + "clientInfo": { + "name": "chronicle-memory", + "title": "Chronicle memory agent", + "version": "1", + } + }, + } + ) + + "\n" + + json.dumps( + { + "jsonrpc": "2.0", + "id": 2, + "method": "account/rateLimits/read", + "params": {}, + } + ) + + "\n" + ) + assert proc.stdin is not None and proc.stdout is not None + proc.stdin.write(requests) + proc.stdin.flush() + + # The stream interleaves unsolicited notifications with responses, so read + # until the id=2 reply appears or the deadline passes. + deadline = time.monotonic() + timeout + result: Optional[dict] = None + while time.monotonic() < deadline: + line = proc.stdout.readline() + if not line: + break + try: + message = json.loads(line) + except json.JSONDecodeError: + continue + if message.get("id") == 2: + result = message.get("result") + break + if result is None: + logger.debug("codex app-server returned no rate-limit response in time") + return result + except Exception as e: # noqa: BLE001 — a quota probe must never break recording + logger.debug("could not read codex rate limits (%s)", e) + return None + finally: + if proc is not None: + proc.kill() + # Best-effort reap; the kill above is what actually ends the server. + with contextlib.suppress(Exception): + proc.wait(timeout=5) + + +def bucket_used_percent(payload: Optional[dict], limit_id: str = "") -> Optional[int]: + """Percent of the weekly Codex budget already spent, or ``None`` if unknown. + + ``limit_id`` selects a specific bucket from ``rateLimitsByLimitId`` (models are + metered against different buckets — e.g. ``codex`` vs ``codex_bengalfox`` for + Spark — and the account may have one exhausted while another is untouched). + Empty selects the payload's own backward-compatible single-bucket view. + """ + if not isinstance(payload, dict): + return None + snapshot = None + if limit_id: + by_id = payload.get("rateLimitsByLimitId") + if isinstance(by_id, dict): + snapshot = by_id.get(limit_id) + if snapshot is None: + # An unknown limit_id must not silently fall back to a different + # bucket's headroom — that would gate on the wrong budget. + logger.warning("codex rate-limit bucket %r not in payload", limit_id) + return None + else: + snapshot = payload.get("rateLimits") + if not isinstance(snapshot, dict): + return None + primary = snapshot.get("primary") + if not isinstance(primary, dict): + return None + used = primary.get("usedPercent") + if isinstance(used, bool) or not isinstance(used, (int, float)): + return None + return int(used) + + +def quota_span_attributes( + payload: Optional[dict], limit_id: str = "" +) -> Dict[str, object]: + """Flatten a snapshot into span attributes (empty when nothing is known).""" + used = bucket_used_percent(payload, limit_id) + if used is None: + return {} + attributes: Dict[str, object] = {"chronicle.memory.quota.used_percent": used} + snapshot = ( + (payload or {}).get("rateLimitsByLimitId", {}).get(limit_id) + if limit_id + else (payload or {}).get("rateLimits") + ) + if isinstance(snapshot, dict): + primary = snapshot.get("primary") + if isinstance(primary, dict): + for source, target in ( + ("resetsAt", "chronicle.memory.quota.resets_at"), + ("windowDurationMins", "chronicle.memory.quota.window_minutes"), + ): + value = primary.get(source) + if isinstance(value, int) and not isinstance(value, bool): + attributes[target] = value + if snapshot.get("limitId"): + attributes["chronicle.memory.quota.limit_id"] = snapshot["limitId"] + return attributes diff --git a/backends/advanced/src/advanced_omi_backend/services/memory/agent/memory_agent.py b/backends/advanced/src/advanced_omi_backend/services/memory/agent/memory_agent.py index 4365af5d7..d3aeb4fd7 100644 --- a/backends/advanced/src/advanced_omi_backend/services/memory/agent/memory_agent.py +++ b/backends/advanced/src/advanced_omi_backend/services/memory/agent/memory_agent.py @@ -185,6 +185,10 @@ class MemoryAgentResult: # so a note disappearing is never invisible in the ledger. removed: List[dict] = field(default_factory=list) errors: List[str] = field(default_factory=list) + # Token counts for the run, keyed as Langfuse usage details (``input_tokens``, + # ``output_tokens``, ``input_cached_tokens``, ``output_reasoning_tokens``). Empty + # when the executor reports none. + usage: Dict[str, int] = field(default_factory=dict) truncated: bool = ( False # loop ended on a truncated/empty LLM response, not a deliberate finish ) diff --git a/backends/advanced/src/advanced_omi_backend/services/memory/agent/vault_tools.py b/backends/advanced/src/advanced_omi_backend/services/memory/agent/vault_tools.py index c54488e01..acbfe029b 100644 --- a/backends/advanced/src/advanced_omi_backend/services/memory/agent/vault_tools.py +++ b/backends/advanced/src/advanced_omi_backend/services/memory/agent/vault_tools.py @@ -28,6 +28,7 @@ from pathlib import Path from typing import Any, Dict, Iterator, List +from ..person_merge import PersonMergeService from ..vault_lock import VaultLockTimeout, vault_note_lock from ..vault_scaffold import write_category from .edit_engine import Edit, EditError, apply_edits @@ -406,20 +407,22 @@ def rename_person(self, old_name: str, new_name: str) -> str: # keeps its final content and the merge never loses facts unrecorded. old_content = old_fp.read_text(encoding="utf-8") if new_fp.exists(): - # Merge case — a plain move would clobber the target. Migrate the old - # note's facts into the target *before* deleting it (non-lossy by - # construction — never rely on a follow-up edit_note that may not come), - # rewrite backlinks, then remove the old note. - migrated = self._migrate_person_facts(old_content, new_fp, old_rel) - n = self._rewrite_backlinks_python(old_name, new_name) - old_fp.unlink() - self.touched.add(new_rel) + # Merge case — delegate to the same deterministic operation exposed to + # Obsidian and automation clients. The caller already owns the vault + # lock, so use its locked implementation directly. + service = PersonMergeService(self.root) + preview = service.preview(old_name, new_name) + result = service.apply_preview_locked(preview) + for rel, after in result.after.items(): + if after is not None: + self.touched.add(rel) self._record_removal(old_rel, new_rel, old_content) return ( f"'{new_name}' already existed — merged into People/{new_name}.md: " - f"migrated {migrated} fact bullet(s), rewrote {n} backlink(s), and " - f"deleted People/{old_name}.md. Review People/{new_name}.md and use " - f"edit_note to de-duplicate any overlapping facts." + f"migrated {preview.facts_to_add} fact bullet(s), skipped " + f"{preview.duplicate_facts_skipped} duplicate(s), rewrote " + f"{preview.backlink_occurrences} backlink(s), added '{old_name}' as " + f"an alias, and deleted People/{old_name}.md." ) self.touched.add(new_rel) if self._notesmd: diff --git a/backends/advanced/src/advanced_omi_backend/services/memory/audit.py b/backends/advanced/src/advanced_omi_backend/services/memory/audit.py index 9961e9a07..d4d6ac2d6 100644 --- a/backends/advanced/src/advanced_omi_backend/services/memory/audit.py +++ b/backends/advanced/src/advanced_omi_backend/services/memory/audit.py @@ -53,6 +53,7 @@ class MemoryCause(str, Enum): SPEAKER_REPROCESS = "speaker_reprocess" # re-ran diarization ANNOTATION_APPLY = "annotation_apply" # user applied annotation corrections OBSIDIAN_SYNC = "obsidian_sync" # inbound human edit via Syncthing + OBSIDIAN_ACTION = "obsidian_action" # explicit semantic action from Obsidian DELETE_ALL = "delete_all" # bulk vault wipe @@ -73,6 +74,7 @@ class UpdateStrategy(str, Enum): MemoryCause.SPEAKER_REPROCESS: "reprocess", MemoryCause.ANNOTATION_APPLY: "reprocess", MemoryCause.OBSIDIAN_SYNC: "human", + MemoryCause.OBSIDIAN_ACTION: "human", MemoryCause.DELETE_ALL: "bulk", } @@ -84,6 +86,7 @@ class UpdateStrategy(str, Enum): MemoryCause.SPEAKER_REPROCESS: "Speaker reprocess", MemoryCause.ANNOTATION_APPLY: "Annotation applied", MemoryCause.OBSIDIAN_SYNC: "Human · Obsidian", + MemoryCause.OBSIDIAN_ACTION: "Obsidian action", MemoryCause.DELETE_ALL: "Bulk delete", } @@ -132,7 +135,7 @@ def actor_for(cause: Optional[str], agent_mode: bool, operation: Optional[str]) if agent_mode: return "agent" c = _as_cause(cause) - if c == MemoryCause.OBSIDIAN_SYNC: + if c in (MemoryCause.OBSIDIAN_SYNC, MemoryCause.OBSIDIAN_ACTION): return "human_external" if c == MemoryCause.AUTO_EXTRACTION: return "system" diff --git a/backends/advanced/src/advanced_omi_backend/services/memory/person_identity.py b/backends/advanced/src/advanced_omi_backend/services/memory/person_identity.py new file mode 100644 index 000000000..06fbd37d1 --- /dev/null +++ b/backends/advanced/src/advanced_omi_backend/services/memory/person_identity.py @@ -0,0 +1,354 @@ +"""Deterministic duplicate-person suggestions and durable identity annotations.""" + +import json +import re +import uuid +from dataclasses import dataclass +from datetime import date +from difflib import SequenceMatcher +from itertools import combinations +from pathlib import Path +from typing import Any, Optional + +from .person_merge import ( + PersonMergeError, + PersonMergeStale, + _as_list, + _atomic_write, + _join_frontmatter, + _linked_person_names, + _resolve_person, + _section_bullets, + _sha256, + _split_frontmatter, +) +from .vault_lock import VaultLockTimeout, vault_note_lock + +_TOKEN_RE = re.compile(r"[a-z0-9]+") +_LINK_RE = re.compile(r"\[\[([^\]|#]+)") +_CONVERSATION_RE = re.compile(r"Conversations/([0-9a-f-]{36})", re.IGNORECASE) +_PHOTO_RE = re.compile(r"_media/([^\]|]+)", re.IGNORECASE) +_IGNORED_CONTEXT_LINKS = { + "people", + "conversations.base", + "conversations", +} + + +@dataclass +class PersonRecord: + name: str + path: str + text: str + content_hash: str + aliases: set[str] + distinct_from: set[str] + org: str + role: str + links: set[str] + conversations: set[str] + photos: set[str] + snippets: list[str] + + +@dataclass +class IdentityChangeResult: + action_id: str + person_a: str + person_b: str + decision: str + changed_paths: list[str] + before: dict[str, str] + after: dict[str, str] + + def to_dict(self) -> dict[str, Any]: + return { + "action_id": self.action_id, + "person_a": self.person_a, + "person_b": self.person_b, + "decision": self.decision, + "changed_paths": self.changed_paths, + } + + +def _normalise_name(name: str) -> str: + return "".join(_TOKEN_RE.findall(name.casefold())) + + +def _tokens(name: str) -> set[str]: + return set(_TOKEN_RE.findall(name.casefold())) + + +def _edit_distance(left: str, right: str) -> int: + left = _normalise_name(left) + right = _normalise_name(right) + row = list(range(len(right) + 1)) + for index, left_char in enumerate(left, 1): + next_row = [index] + for right_index, right_char in enumerate(right, 1): + next_row.append( + min( + next_row[-1] + 1, + row[right_index] + 1, + row[right_index - 1] + (left_char != right_char), + ) + ) + row = next_row + return row[-1] + + +def _plain_value(value: Any) -> str: + return str(value).strip().casefold() if value else "" + + +def _aliases(value: Any) -> set[str]: + result: set[str] = set() + for item in _as_list(value): + result.update(_linked_person_names(item)) + return result + + +def _record(path: Path, root: Path) -> PersonRecord: + text = path.read_text(encoding="utf-8") + frontmatter, _ = _split_frontmatter(text) + links = { + link.strip().casefold() + for link in _LINK_RE.findall(text) + if link.strip().casefold() not in _IGNORED_CONTEXT_LINKS + and not link.startswith(("../", "Conversations/")) + } + snippets = [ + bullet.strip().lstrip("-").strip() + for bullet in _section_bullets(text, "About")[:2] + ] + return PersonRecord( + name=path.stem, + path=path.relative_to(root).as_posix(), + text=text, + content_hash=_sha256(text), + aliases=_aliases(frontmatter.get("aliases")), + distinct_from=_linked_person_names(frontmatter.get("distinct_from")), + org=_plain_value(frontmatter.get("org")), + role=_plain_value(frontmatter.get("role")), + links=links, + conversations=set(_CONVERSATION_RE.findall(text)), + photos={photo.casefold() for photo in _PHOTO_RE.findall(text)}, + snippets=snippets, + ) + + +def _pair_revision(left: PersonRecord, right: PersonRecord) -> str: + payload = { + "people": sorted( + [(left.path, left.content_hash), (right.path, right.content_hash)] + ) + } + return _sha256(json.dumps(payload, sort_keys=True, separators=(",", ":"))) + + +def _score_pair(left: PersonRecord, right: PersonRecord) -> tuple[int, list[str]]: + left_name = _normalise_name(left.name) + right_name = _normalise_name(right.name) + shorter = min(len(left_name), len(right_name)) + distance = _edit_distance(left.name, right.name) + similarity = SequenceMatcher(None, left_name, right_name).ratio() + score = 0 + identity_signal = False + reasons: list[str] = [] + + if right.name.casefold() in left.aliases or left.name.casefold() in right.aliases: + score += 100 + identity_signal = True + reasons.append("one name is already an alias of the other") + elif left_name == right_name: + score += 90 + identity_signal = True + reasons.append("names match after normalization") + + shared_photos = left.photos & right.photos + if shared_photos: + score += 90 + identity_signal = True + reasons.append("same person photo") + elif left.photos and right.photos: + score -= 25 + + if shorter >= 4 and distance == 1: + score += 45 + identity_signal = True + reasons.append("names differ by one character") + elif shorter >= 6 and distance == 2: + score += 25 + identity_signal = True + reasons.append("names differ by two characters") + + if similarity >= 0.88: + score += 25 + identity_signal = True + reasons.append("very similar spelling") + elif similarity >= 0.80: + score += 15 + identity_signal = True + reasons.append("similar spelling") + + left_tokens = _tokens(left.name) + right_tokens = _tokens(right.name) + if ( + left_tokens + and right_tokens + and left_tokens != right_tokens + and (left_tokens < right_tokens or right_tokens < left_tokens) + and shorter >= 4 + ): + score += 25 + identity_signal = True + reasons.append("one name appears to be a fuller form") + + shared_links = left.links & right.links + if shared_links: + score += min(18, len(shared_links) * 6) + reasons.append(f"shared context in {len(shared_links)} linked note(s)") + + shared_conversations = left.conversations & right.conversations + if shared_conversations: + score += min(24, len(shared_conversations) * 12) + reasons.append(f"same source conversation ({len(shared_conversations)})") + + if left.org and left.org == right.org: + score += 15 + reasons.append("same organization") + if left.role and left.role == right.role: + score += 8 + reasons.append("same role") + if not identity_signal: + return 0, [] + return score, reasons + + +class PersonIdentityService: + """Read identity candidates and write symmetric distinct-person decisions.""" + + def __init__(self, root: Path): + self.root = Path(root) + + def suggestions(self, limit: int = 20, min_score: int = 40) -> list[dict[str, Any]]: + people_dir = self.root / "People" + if not people_dir.is_dir(): + return [] + records = [_record(path, self.root) for path in sorted(people_dir.glob("*.md"))] + suggestions = [] + for left, right in combinations(records, 2): + if ( + right.name.casefold() in left.distinct_from + or left.name.casefold() in right.distinct_from + ): + continue + score, reasons = _score_pair(left, right) + if score < min_score: + continue + suggestions.append( + { + "pair_id": _sha256( + "\0".join(sorted([left.name.casefold(), right.name.casefold()])) + )[:16], + "revision": _pair_revision(left, right), + "score": score, + "reasons": reasons, + "person_a": { + "name": left.name, + "path": left.path, + "hash": left.content_hash, + "snippets": left.snippets, + }, + "person_b": { + "name": right.name, + "path": right.path, + "hash": right.content_hash, + "snippets": right.snippets, + }, + } + ) + suggestions.sort( + key=lambda item: ( + -item["score"], + item["person_a"]["name"].casefold(), + item["person_b"]["name"].casefold(), + ) + ) + return suggestions[:limit] + + def set_distinct( + self, + person_a: str, + person_b: str, + *, + distinct: bool, + revision: Optional[str] = None, + ) -> IdentityChangeResult: + try: + with vault_note_lock(self.root.name): + return self._set_distinct_locked( + person_a, person_b, distinct=distinct, revision=revision + ) + except VaultLockTimeout as exc: + raise PersonMergeError("The vault is busy. Retry shortly.") from exc + + def _set_distinct_locked( + self, + person_a: str, + person_b: str, + *, + distinct: bool, + revision: Optional[str], + ) -> IdentityChangeResult: + path_a = _resolve_person(self.root, person_a) + path_b = _resolve_person(self.root, person_b) + if path_a == path_b: + raise PersonMergeError( + "A person cannot be marked distinct from themselves." + ) + record_a = _record(path_a, self.root) + record_b = _record(path_b, self.root) + if revision and revision != _pair_revision(record_a, record_b): + raise PersonMergeStale( + "One of these people changed after the suggestion was shown. Review again." + ) + + new_a = self._update_distinct(record_a, record_b.name, distinct) + new_b = self._update_distinct(record_b, record_a.name, distinct) + before = {record_a.path: record_a.text, record_b.path: record_b.text} + after = {record_a.path: new_a, record_b.path: new_b} + changed = [path for path in after if after[path] != before[path]] + try: + for path in changed: + _atomic_write(self.root / path, after[path]) + except Exception: + for path, content in before.items(): + _atomic_write(self.root / path, content) + raise + return IdentityChangeResult( + action_id=str(uuid.uuid4()), + person_a=record_a.name, + person_b=record_b.name, + decision="distinct" if distinct else "clear_distinct", + changed_paths=sorted(changed), + before=before, + after=after, + ) + + def _update_distinct( + self, record: PersonRecord, other_name: str, distinct: bool + ) -> str: + frontmatter, body = _split_frontmatter(record.text) + values = _as_list(frontmatter.get("distinct_from")) + filtered = [ + value + for value in values + if other_name.casefold() not in _linked_person_names(value) + ] + if distinct: + filtered.append(f"[[{other_name}]]") + frontmatter["distinct_from"] = filtered + if "updated" in frontmatter: + frontmatter["updated"] = date.today().isoformat() + return _join_frontmatter(frontmatter, body) diff --git a/backends/advanced/src/advanced_omi_backend/services/memory/person_merge.py b/backends/advanced/src/advanced_omi_backend/services/memory/person_merge.py new file mode 100644 index 000000000..07681ce49 --- /dev/null +++ b/backends/advanced/src/advanced_omi_backend/services/memory/person_merge.py @@ -0,0 +1,464 @@ +"""Deterministic, transactional-ish person-note merges for the Chronicle vault. + +Identity resolution is intentionally outside this module: a human or agent decides +that two notes describe the same person. This module only executes the mechanical +operation with fixed rules, a preview token, the per-user vault lock, and rollback on +ordinary failures. +""" + +import hashlib +import io +import json +import os +import re +import tempfile +import uuid +from dataclasses import dataclass, field +from datetime import date +from pathlib import Path +from typing import Any, Optional + +from ruamel.yaml import YAML + +from .agent.section_edit import SectionEditError, apply_section_edit +from .vault_lock import VaultLockTimeout, vault_note_lock + +_H2_RE = re.compile(r"^##\s+(.+?)\s*$", re.MULTILINE) +_MERGE_SECTIONS = ("About", "Mentions") +_SCALAR_IDENTITY_FIELDS = ("org", "role", "relationship", "location") + + +class PersonMergeError(Exception): + """A person merge cannot be previewed or applied safely.""" + + +class PersonMergeStale(PersonMergeError): + """The vault changed after the caller read or previewed it.""" + + +@dataclass(frozen=True) +class MetadataConflict: + field: str + source_value: Any + target_value: Any + + def to_dict(self) -> dict[str, Any]: + return { + "field": self.field, + "source_value": self.source_value, + "target_value": self.target_value, + } + + +@dataclass +class PersonMergePreview: + source_name: str + target_name: str + source_path: str + target_path: str + source_hash: str + target_hash: str + plan_token: str + facts_to_add: int + duplicate_facts_skipped: int + backlink_files: list[str] + backlink_occurrences: int + metadata_conflicts: list[MetadataConflict] + _source_text: str = field(repr=False) + _target_text: str = field(repr=False) + _before: dict[str, str] = field(repr=False) + + def to_dict(self) -> dict[str, Any]: + return { + "source_name": self.source_name, + "target_name": self.target_name, + "source_path": self.source_path, + "target_path": self.target_path, + "source_hash": self.source_hash, + "target_hash": self.target_hash, + "plan_token": self.plan_token, + "facts_to_add": self.facts_to_add, + "duplicate_facts_skipped": self.duplicate_facts_skipped, + "backlink_files": self.backlink_files, + "backlink_occurrences": self.backlink_occurrences, + "metadata_conflicts": [item.to_dict() for item in self.metadata_conflicts], + } + + +@dataclass +class PersonMergeResult: + action_id: str + preview: PersonMergePreview + changed_paths: list[str] + before: dict[str, str] + after: dict[str, Optional[str]] + + def to_dict(self) -> dict[str, Any]: + return { + "action_id": self.action_id, + **self.preview.to_dict(), + "changed_paths": self.changed_paths, + } + + +def _sha256(text: str) -> str: + return hashlib.sha256(text.encode("utf-8")).hexdigest() + + +def _validate_name(name: str) -> str: + cleaned = name.strip() + if ( + not cleaned + or cleaned in (".", "..") + or Path(cleaned).name != cleaned + or "/" in cleaned + or "\\" in cleaned + ): + raise PersonMergeError( + "Person names must be plain note titles without slashes." + ) + return cleaned + + +def _resolve_person(root: Path, name: str) -> Path: + people = root / "People" + wanted = f"{_validate_name(name)}.md".casefold() + if not people.is_dir(): + raise PersonMergeError("The vault has no People folder.") + matches = [path for path in people.glob("*.md") if path.name.casefold() == wanted] + if not matches: + raise PersonMergeError(f"People/{name}.md does not exist.") + if len(matches) > 1: + raise PersonMergeError(f"Multiple case-variant notes match People/{name}.md.") + return matches[0] + + +def _split_frontmatter(text: str) -> tuple[dict[str, Any], str]: + if not text.startswith("---\n"): + raise PersonMergeError("Person note is missing YAML frontmatter.") + end = text.find("\n---\n", 4) + if end < 0: + raise PersonMergeError("Person note has malformed YAML frontmatter.") + yaml = YAML(typ="rt") + data = yaml.load(text[4:end]) or {} + if not isinstance(data, dict): + raise PersonMergeError("Person note frontmatter must be a mapping.") + return data, text[end + 5 :] + + +def _join_frontmatter(data: dict[str, Any], body: str) -> str: + yaml = YAML(typ="rt") + yaml.preserve_quotes = True + yaml.indent(mapping=2, sequence=4, offset=2) + stream = io.StringIO() + yaml.dump(data, stream) + return f"---\n{stream.getvalue()}---\n{body.lstrip()}" + + +def _as_list(value: Any) -> list[Any]: + if value is None or value == "": + return [] + return list(value) if isinstance(value, list) else [value] + + +def _union_values(*collections: list[Any]) -> list[Any]: + result: list[Any] = [] + seen: set[str] = set() + for collection in collections: + for value in collection: + marker = str(value).strip().casefold() + if marker and marker not in seen: + result.append(value) + seen.add(marker) + return result + + +def _linked_person_names(value: Any) -> set[str]: + """Return case-folded person titles from a frontmatter link/list value.""" + names: set[str] = set() + for item in _as_list(value): + text = str(item).strip() + match = re.fullmatch(r"\[\[(?:People/)?([^\]|#]+)(?:[|#][^\]]*)?\]\]", text) + title = match.group(1) if match else text + if title: + names.add(title.strip().casefold()) + return names + + +def _section_bullets(content: str, heading: str) -> list[str]: + wanted = heading.casefold() + found = False + result: list[str] = [] + for line in content.splitlines(): + match = _H2_RE.match(line.rstrip()) + if match: + found = match.group(1).casefold() == wanted + continue + stripped = line.strip() + if found and stripped.startswith("-") and stripped.lstrip("-").strip(): + result.append(line.rstrip()) + return result + + +def _normalise_bullet(line: str) -> str: + return " ".join(line.lstrip().lstrip("-").split()).casefold() + + +def _media_embeds(body: str) -> list[str]: + prologue = body.split("\n## ", 1)[0] + return [ + line.strip() for line in prologue.splitlines() if line.strip().startswith("![[") + ] + + +def _add_media_embeds(body: str, embeds: list[str]) -> str: + missing = [embed for embed in embeds if embed not in body] + if not missing: + return body + block = "\n".join(missing) + "\n" + return block + body.lstrip() + + +def _link_pattern(source_name: str) -> re.Pattern[str]: + return re.compile( + rf"(?P\[\[(?:People/)?)({re.escape(source_name)})" + rf"(?P(?:[#|][^\]]*)?\]\])", + re.IGNORECASE, + ) + + +def _rewrite_links(text: str, source_name: str, target_name: str) -> tuple[str, int]: + pattern = _link_pattern(source_name) + return pattern.subn( + lambda match: f"{match.group('prefix')}{target_name}{match.group('suffix')}", + text, + ) + + +def _atomic_write(path: Path, content: str) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + fd, temporary = tempfile.mkstemp(prefix=f".{path.name}.", dir=path.parent) + try: + with os.fdopen(fd, "w", encoding="utf-8") as handle: + handle.write(content) + handle.flush() + os.fsync(handle.fileno()) + os.replace(temporary, path) + except Exception: + try: + os.unlink(temporary) + except FileNotFoundError: + pass + raise + + +class PersonMergeService: + """Preview and apply one deterministic person merge inside a vault root.""" + + def __init__(self, root: Path): + self.root = Path(root) + + def preview( + self, + source_name: str, + target_name: str, + *, + expected_source_hash: Optional[str] = None, + expected_target_hash: Optional[str] = None, + ) -> PersonMergePreview: + source = _resolve_person(self.root, source_name) + target = _resolve_person(self.root, target_name) + if source == target: + raise PersonMergeError("Source and target resolve to the same person note.") + + source_text = source.read_text(encoding="utf-8") + target_text = target.read_text(encoding="utf-8") + source_hash = _sha256(source_text) + target_hash = _sha256(target_text) + if expected_source_hash and expected_source_hash != source_hash: + raise PersonMergeStale("The source note differs from the server copy.") + if expected_target_hash and expected_target_hash != target_hash: + raise PersonMergeStale("The target note differs from the server copy.") + + source_frontmatter, _ = _split_frontmatter(source_text) + target_frontmatter, _ = _split_frontmatter(target_text) + if target.stem.casefold() in _linked_person_names( + source_frontmatter.get("distinct_from") + ) or source.stem.casefold() in _linked_person_names( + target_frontmatter.get("distinct_from") + ): + raise PersonMergeError( + f"{source.stem} and {target.stem} are marked as separate people. " + "Clear that identity annotation before merging them." + ) + conflicts = [] + for key in _SCALAR_IDENTITY_FIELDS: + source_value = source_frontmatter.get(key) + target_value = target_frontmatter.get(key) + if source_value and target_value and source_value != target_value: + conflicts.append(MetadataConflict(key, source_value, target_value)) + + facts_to_add = 0 + duplicate_facts = 0 + for heading in _MERGE_SECTIONS: + existing = { + _normalise_bullet(line) + for line in _section_bullets(target_text, heading) + } + for bullet in _section_bullets(source_text, heading): + if _normalise_bullet(bullet) in existing: + duplicate_facts += 1 + else: + facts_to_add += 1 + existing.add(_normalise_bullet(bullet)) + + backlink_files: list[str] = [] + backlink_occurrences = 0 + before: dict[str, str] = { + source.relative_to(self.root).as_posix(): source_text, + target.relative_to(self.root).as_posix(): target_text, + } + pattern = _link_pattern(source.stem) + for path in sorted(self.root.rglob("*.md")): + if path == source: + continue + text = path.read_text(encoding="utf-8") + count = len(pattern.findall(text)) + if count: + rel = path.relative_to(self.root).as_posix() + backlink_files.append(rel) + backlink_occurrences += count + before[rel] = text + + token_payload = { + "source": source.relative_to(self.root).as_posix(), + "target": target.relative_to(self.root).as_posix(), + "files": {path: _sha256(text) for path, text in sorted(before.items())}, + } + plan_token = _sha256( + json.dumps(token_payload, sort_keys=True, separators=(",", ":")) + ) + return PersonMergePreview( + source_name=source.stem, + target_name=target.stem, + source_path=source.relative_to(self.root).as_posix(), + target_path=target.relative_to(self.root).as_posix(), + source_hash=source_hash, + target_hash=target_hash, + plan_token=plan_token, + facts_to_add=facts_to_add, + duplicate_facts_skipped=duplicate_facts, + backlink_files=backlink_files, + backlink_occurrences=backlink_occurrences, + metadata_conflicts=conflicts, + _source_text=source_text, + _target_text=target_text, + _before=before, + ) + + def apply( + self, source_name: str, target_name: str, plan_token: str + ) -> PersonMergeResult: + try: + with vault_note_lock(self.root.name): + preview = self.preview(source_name, target_name) + if preview.plan_token != plan_token: + raise PersonMergeStale( + "The vault changed after this merge was previewed. Preview it again." + ) + return self.apply_preview_locked(preview) + except VaultLockTimeout as exc: + raise PersonMergeError( + "The vault is busy. Retry the merge shortly." + ) from exc + + def apply_preview_locked(self, preview: PersonMergePreview) -> PersonMergeResult: + """Apply a preview while the caller already holds the per-user vault lock.""" + source = self.root / preview.source_path + target = self.root / preview.target_path + before = dict(preview._before) + after: dict[str, Optional[str]] = {} + changed: list[str] = [] + try: + merged = self._merge_person_notes(preview) + merged, _ = _rewrite_links(merged, preview.source_name, preview.target_name) + _atomic_write(target, merged) + after[preview.target_path] = merged + changed.append(preview.target_path) + + for rel in preview.backlink_files: + if rel == preview.target_path: + continue + rewritten, count = _rewrite_links( + before[rel], preview.source_name, preview.target_name + ) + if count: + _atomic_write(self.root / rel, rewritten) + after[rel] = rewritten + changed.append(rel) + + source.unlink() + after[preview.source_path] = None + changed.append(preview.source_path) + except Exception: + for rel, content in before.items(): + _atomic_write(self.root / rel, content) + raise + + return PersonMergeResult( + action_id=str(uuid.uuid4()), + preview=preview, + changed_paths=sorted(set(changed)), + before=before, + after=after, + ) + + def _merge_person_notes(self, preview: PersonMergePreview) -> str: + source_frontmatter, source_body = _split_frontmatter(preview._source_text) + target_frontmatter, target_body = _split_frontmatter(preview._target_text) + + target_frontmatter["categories"] = _union_values( + _as_list(target_frontmatter.get("categories")), + _as_list(source_frontmatter.get("categories")), + ) + target_frontmatter["aliases"] = _union_values( + _as_list(target_frontmatter.get("aliases")), + _as_list(source_frontmatter.get("aliases")), + [preview.source_name], + ) + source_distinct = [ + value + for value in _as_list(source_frontmatter.get("distinct_from")) + if preview.target_name.casefold() not in _linked_person_names(value) + ] + target_frontmatter["distinct_from"] = _union_values( + _as_list(target_frontmatter.get("distinct_from")), source_distinct + ) + for key in _SCALAR_IDENTITY_FIELDS: + if not target_frontmatter.get(key) and source_frontmatter.get(key): + target_frontmatter[key] = source_frontmatter[key] + if "updated" in target_frontmatter: + target_frontmatter["updated"] = date.today().isoformat() + + target_body = _add_media_embeds(target_body, _media_embeds(source_body)) + for heading in _MERGE_SECTIONS: + existing = { + _normalise_bullet(line) + for line in _section_bullets(target_body, heading) + } + additions = [] + for bullet in _section_bullets(source_body, heading): + marker = _normalise_bullet(bullet) + if marker not in existing: + additions.append(bullet) + existing.add(marker) + if additions: + try: + target_body = apply_section_edit( + target_body, heading, "\n".join(additions), "append" + ) + except SectionEditError as exc: + raise PersonMergeError( + f"Target person note is missing its {heading} section." + ) from exc + return _join_frontmatter(target_frontmatter, target_body) diff --git a/backends/advanced/src/advanced_omi_backend/services/memory/person_merge_actions.py b/backends/advanced/src/advanced_omi_backend/services/memory/person_merge_actions.py new file mode 100644 index 000000000..4bc5c312c --- /dev/null +++ b/backends/advanced/src/advanced_omi_backend/services/memory/person_merge_actions.py @@ -0,0 +1,122 @@ +"""Authenticated-service orchestration for deterministic person merges.""" + +import asyncio + +from .audit import MemoryCause, memory_provenance, record_vault_change +from .person_identity import IdentityChangeResult, PersonIdentityService +from .person_merge import PersonMergeResult, PersonMergeService +from .vault_manager import ConvDocVaultManager + + +def _service(user_id: str) -> PersonMergeService: + return PersonMergeService(ConvDocVaultManager().user_root(user_id)) + + +def _identity_service(user_id: str) -> PersonIdentityService: + return PersonIdentityService(ConvDocVaultManager().user_root(user_id)) + + +async def get_person_suggestions(user_id: str, limit: int = 20) -> list[dict]: + return await asyncio.to_thread(_identity_service(user_id).suggestions, limit) + + +async def set_people_distinct( + user_id: str, + person_a: str, + person_b: str, + *, + distinct: bool, + revision: str | None = None, +) -> dict: + result = await asyncio.to_thread( + _identity_service(user_id).set_distinct, + person_a, + person_b, + distinct=distinct, + revision=revision, + ) + await _record_identity_audit(user_id, result) + return result.to_dict() + + +async def preview_person_merge( + user_id: str, + source_name: str, + target_name: str, + source_hash: str | None = None, + target_hash: str | None = None, +) -> dict: + preview = await asyncio.to_thread( + _service(user_id).preview, + source_name, + target_name, + expected_source_hash=source_hash, + expected_target_hash=target_hash, + ) + return preview.to_dict() + + +async def apply_person_merge( + user_id: str, source_name: str, target_name: str, plan_token: str +) -> dict: + result = await asyncio.to_thread( + _service(user_id).apply, source_name, target_name, plan_token + ) + await _record_merge_audit(user_id, result) + return result.to_dict() + + +async def _record_merge_audit(user_id: str, result: PersonMergeResult) -> None: + action_id = result.action_id + source_path = result.preview.source_path + target_path = result.preview.target_path + with memory_provenance(MemoryCause.OBSIDIAN_ACTION): + for path in result.changed_paths: + before = result.before.get(path) + after = result.after.get(path) + if path == source_path: + await record_vault_change( + user_id=user_id, + operation="rename", + note_path=path, + before=before, + after=None, + summary=f"merged into {target_path}", + action_id=action_id, + new_path=target_path, + ) + continue + await record_vault_change( + user_id=user_id, + operation="update", + note_path=path, + before=before, + after=after, + summary=( + f"person merge {result.preview.source_name} → " + f"{result.preview.target_name}" + ), + action_id=action_id, + source_path=source_path, + target_path=target_path, + ) + + +async def _record_identity_audit(user_id: str, result: IdentityChangeResult) -> None: + with memory_provenance(MemoryCause.OBSIDIAN_ACTION): + for path in result.changed_paths: + await record_vault_change( + user_id=user_id, + operation="update", + note_path=path, + before=result.before[path], + after=result.after[path], + summary=( + f"identity decision: {result.person_a} and {result.person_b} are " + f"{'separate people' if result.decision == 'distinct' else 'no longer marked separate'}" + ), + action_id=result.action_id, + identity_decision=result.decision, + person_a=result.person_a, + person_b=result.person_b, + ) diff --git a/backends/advanced/src/advanced_omi_backend/services/memory/vault_templates.py b/backends/advanced/src/advanced_omi_backend/services/memory/vault_templates.py index 763d9d7e1..b876d1c82 100644 --- a/backends/advanced/src/advanced_omi_backend/services/memory/vault_templates.py +++ b/backends/advanced/src/advanced_omi_backend/services/memory/vault_templates.py @@ -51,6 +51,7 @@ categories: - "[[People]]" aliases: [] +distinct_from: [] org: role: relationship: diff --git a/backends/advanced/src/advanced_omi_backend/services/observability/health_poller.py b/backends/advanced/src/advanced_omi_backend/services/observability/health_poller.py index f8578e0f7..6f2dba314 100644 --- a/backends/advanced/src/advanced_omi_backend/services/observability/health_poller.py +++ b/backends/advanced/src/advanced_omi_backend/services/observability/health_poller.py @@ -45,6 +45,9 @@ # Redis keys for last-known state. _HEALTH_KEY = "system:health:last" # hash: "{node}/{service}" -> health +# Reserved field in that hash; "local/..." and "{node}/..." keys can't collide with it. +_AGENT_KEY = "node-agent" +_AGENT_INCIDENT = "node-agent-unreachable" _SEEN_FAILED_KEY = "system:health:seen_failed_jobs" # set of job ids _CONFIG_SEEN_KEY = "system:health:config_issues" # set of issue keys _WORKER_HEALTH_FIELD = "internal/workers-fleet" @@ -121,8 +124,44 @@ def _bad_severity(health: str | None, detail: str) -> str | None: async def _poll_external_services(redis) -> None: data = await get_external_services() + + # An unreachable agent is itself a reportable fault, not merely an absence of + # data. The agent runs natively on the host and is the only thing that sees + # host-level faults — dead container DNS, a logged-out Tailscale, a stale + # socket mount. Returning silently here meant the failure mode this poller + # exists to catch produced no signal at all. Reported as a transition, like + # every other state below, so a persistent outage does not spam the ledger. + agent_state = "reachable" if data.get("available") else "unreachable" + prev_agent = await redis.hget(_HEALTH_KEY, _AGENT_KEY) + if prev_agent != agent_state: + await redis.hset(_HEALTH_KEY, _AGENT_KEY, agent_state) + if agent_state == "unreachable": + await record_event( + severity="warning", + category="service", + source="node-agent", + title="Node agent unreachable", + detail=( + "The service manager could not be reached " + f"({data.get('reason') or 'unknown'}). Host-level checks are not " + "running, so DNS, Tailscale and certificate faults will go " + "unreported until it returns." + ), + incident_key=_AGENT_INCIDENT, + ) + else: + await record_event( + severity="info", + category="service", + source="node-agent", + title="Node agent reachable", + detail="The service manager is responding again.", + incident_key=_AGENT_INCIDENT, + resolves_incident=True, + ) + if not data.get("available"): - return # agent unreachable/unconfigured → unknown, don't fabricate transitions + return # no per-service data to reconcile; don't fabricate transitions for svc in data.get("services", []) or []: if not svc.get("enabled", True): diff --git a/backends/advanced/src/advanced_omi_backend/services/observation_curation.py b/backends/advanced/src/advanced_omi_backend/services/observation_curation.py index b6b657905..f86c2a34f 100644 --- a/backends/advanced/src/advanced_omi_backend/services/observation_curation.py +++ b/backends/advanced/src/advanced_omi_backend/services/observation_curation.py @@ -24,6 +24,7 @@ codex_executor_available, ) from advanced_omi_backend.services.memory.vault_manager import ConvDocVaultManager +from advanced_omi_backend.services.memory.vault_media import promote_image_bytes logger = logging.getLogger(__name__) @@ -392,30 +393,6 @@ async def _immich_image( return response.content, content_type -def _promote_image_bytes(data: bytes, content_type: str, root: Path) -> tuple[str, str]: - if not data or not content_type: - raise ValueError("cannot promote empty image data") - suffixes = { - "image/jpeg": ".jpg", - "image/png": ".png", - "image/webp": ".webp", - "image/heic": ".heic", - "image/avif": ".avif", - } - suffix = suffixes.get(content_type) - if suffix is None: - raise ValueError("unsupported vault image type") - digest = hashlib.sha256(data).hexdigest() - media_dir = root / "_media" - media_dir.mkdir(parents=True, exist_ok=True) - target = media_dir / f"{digest}{suffix}" - if not target.exists(): - temporary = target.with_suffix(target.suffix + ".part") - temporary.write_bytes(data) - os.replace(temporary, target) - return target.relative_to(root).as_posix(), digest - - def _write_media_provenance( root: Path, digest: str, @@ -566,7 +543,7 @@ async def apply_curation_decision( raise ValueError("agent selected an invalid Immich candidate") asset_id = immich_item.metadata.get("asset_id") data, content_type = await _immich_image(str(asset_id), "original") - promoted, digest = _promote_image_bytes(data, content_type, root) + promoted, digest = promote_image_bytes(data, content_type, root) immich_item.promoted_path = promoted immich_item.content_hash = digest immich_item.state = "promoted" @@ -582,7 +559,7 @@ async def apply_curation_decision( elif retain_image: if not item.media_data or not item.media_content_type: raise ValueError("selected ScreenPipe image is unavailable") - promoted, digest = _promote_image_bytes( + promoted, digest = promote_image_bytes( item.media_data, item.media_content_type, root ) item.content_hash = digest diff --git a/backends/advanced/src/advanced_omi_backend/services/transcription/streaming_consumer.py b/backends/advanced/src/advanced_omi_backend/services/transcription/streaming_consumer.py index b80c7dc05..c1e33ef17 100644 --- a/backends/advanced/src/advanced_omi_backend/services/transcription/streaming_consumer.py +++ b/backends/advanced/src/advanced_omi_backend/services/transcription/streaming_consumer.py @@ -61,6 +61,15 @@ # ride out brief network blips (producer emits chunks every 0.25s when healthy). STREAM_IDLE_TIMEOUT_SECONDS = 300 +# How recently a still-ACTIVE session must have appended for its stream to count as +# resumed rather than merely quiet. A healthy producer emits a chunk every 0.25s. +STREAM_RESUME_MAX_AGE_SECONDS = 10.0 + +# Entries read from the tail when probing for the producer's end marker. The marker is +# the last thing finalize_session appends, so 1 would normally do; a small window keeps +# the probe correct if a chunk raced in behind it. +STREAM_TAIL_PROBE_ENTRIES = 5 + def _is_connection_error(e: Exception) -> bool: """Check if exception indicates WebSocket connection death.""" @@ -285,28 +294,61 @@ async def discover_streams(self) -> list[str]: return streams - async def _stream_has_fresh_entries( - self, stream_name: str, max_age_seconds: float = 10.0 - ) -> bool: - """True if the stream's newest entry is younger than ``max_age_seconds``. - - Used to distinguish a genuinely-finished stream from one a reconnecting - device has resumed writing to. A live producer emits a chunk every ~0.25s, - so a last entry within 10s means audio is actively flowing. Redis stream IDs - are ``-``, so the timestamp comes free from the entry id — no need - to decode the payload. Errors return False (treat as not-fresh → safe skip). + async def _session_resumed(self, stream_name: str, session_id: str) -> bool: + """True if a stream that carries a completion flag is still being written to. + + Answering this wrong in the permissive direction is expensive: clearing the + flag revokes the handshake ``open_conversation_job`` is blocked on, and no + replacement signal ever arrives, so the conversation stalls for that job's + full 30s wait before finishing without it. + + Recency alone cannot answer it. ``finalize_session`` flushes the residual + audio and appends the end marker as its *last* act, so at the exact moment + the flag is set the newest entry is milliseconds old — a closing session is + indistinguishable from a resuming one by age. Two causal facts decide it + instead: + + - **Session status.** ``producer._append_owned_message`` appends inside a + WATCH/MULTI whose precondition is ``status == "active"``, so a session that + has left ACTIVE can never receive another entry. Its stream is frozen, and + whatever sits at the tail is its own closing flush. + - **The end marker.** It is appended (while still ACTIVE) strictly before the + consumer can read it and set the flag, so its presence proves the producer + finished even if the FINALIZING status write has not landed yet. + + Only when neither says "finished" does recency get to speak, and there it + answers the question it is actually good at: whether audio is flowing now, or + the consumer gave up on a stream that has been silent for a long time. + + Errors return False — declining to re-attach costs a resumed session its + streaming transcription, but wrongly re-attaching corrupts the handshake for + every conversation on the session. """ try: - entries = await self.redis_client.xrevrange(stream_name, count=1) + if await self.store.get_status(session_id) != SessionStatus.ACTIVE: + return False + + entries = await self.redis_client.xrevrange( + stream_name, count=STREAM_TAIL_PROBE_ENTRIES + ) if not entries: return False + if any( + fields.get(b"end_marker") or fields.get("end_marker") + for _, fields in entries + ): + return False + + # Redis stream ids are ``-``, so age comes free from the id. entry_id = entries[0][0] if isinstance(entry_id, bytes): entry_id = entry_id.decode() entry_ms = int(entry_id.split("-")[0]) - return (time.time() * 1000 - entry_ms) < (max_age_seconds * 1000) + return (time.time() * 1000 - entry_ms) < ( + STREAM_RESUME_MAX_AGE_SECONDS * 1000 + ) except Exception as e: # noqa: BLE001 — best-effort liveness probe - logger.debug(f"Freshness check failed for {stream_name}: {e}") + logger.debug(f"Resume probe failed for {stream_name}: {e}") return False async def setup_consumer_group(self, stream_name: str): @@ -1200,19 +1242,21 @@ async def start_consuming(self, heartbeat_name: str | None = None): session_id = stream_name.replace("audio:stream:", "") completion_key = f"transcription:complete:{session_id}" if await self.redis_client.exists(completion_key): - # session_id is stable across reconnects, so the flag may be - # stale: a device reconnected onto the same stream after the - # prior connection's provider stream closed. Producer.init_session - # clears the flag on (re)connect, but a backend-only restart - # leaves THIS worker's old process_stream task alive, and it can - # set the flag (idle-timeout exit) AFTER init_session cleared it - # — re-poisoning the resumed stream until the 5-min TTL. - # Self-heal: if fresh audio is flowing into a "completed" stream, - # the session resumed — drop the flag and re-attach. - if await self._stream_has_fresh_entries(stream_name): + # The flag can outlive the provider stream it describes: a + # process_stream task that exits on its idle heartbeat sets it + # while the session is still ACTIVE, and the device may resume + # sending afterwards. Discovery would then skip that live stream + # until the 5-min TTL, starving it of transcription. + # + # Self-heal, but only for a session that can still produce. The + # flag is also the handshake open_conversation_job waits on, so + # clearing it for a session that has finished stalls that job for + # its full 30s wait (see _session_resumed). + if await self._session_resumed(stream_name, session_id): logger.info( - f"Stream {stream_name} marked complete but has fresh " - f"audio — session resumed, clearing flag and re-attaching" + f"Stream {stream_name} marked complete but its session " + f"is still active and producing — clearing flag and " + f"re-attaching" ) await self.redis_client.delete(completion_key) else: diff --git a/backends/advanced/src/advanced_omi_backend/utils/export_planning.py b/backends/advanced/src/advanced_omi_backend/utils/export_planning.py new file mode 100644 index 000000000..a4f2bfff6 --- /dev/null +++ b/backends/advanced/src/advanced_omi_backend/utils/export_planning.py @@ -0,0 +1,189 @@ +"""Clip-plan computation for annotation exports. + +One code path decides what an annotation export will contain — which speech +regions become clips, at which boundaries, with which transcript slices — +shared by the export RQ job (``workers/data_audit_jobs.py``, which renders the +plan to WAVs in a zip) and the synchronous preview endpoint (which returns the +plan for the user to verify and curate before anything is written). Keeping +both on the same functions means the preview can never drift from the export. + +Two kinds of range carving, deliberately accounted separately: + +- ``excluded_ranges`` — privacy-screen withholdings; reported as + ``excluded_seconds`` ("withheld"). +- ``dropped_ranges`` — clips the user unticked in the export preview; reported + as ``dropped_seconds``. Same subtraction mechanics, different meaning: one is + "too sensitive to share", the other is "not worth annotating". +""" + +import logging +from dataclasses import dataclass, field +from typing import List, Optional, Tuple + +from advanced_omi_backend.models.audio_chunk import AudioChunkDocument +from advanced_omi_backend.models.conversation import Conversation +from advanced_omi_backend.utils.vad_analysis import ( + frame_speech_intervals, + merge_speech_regions, + subtract_intervals, +) + +logger = logging.getLogger(__name__) + + +@dataclass +class ClipPlan: + """One planned clip: a speech region and its position in the source.""" + + clip_index: int + start: float + end: float + + @property + def duration(self) -> float: + return self.end - self.start + + +@dataclass +class ConversationPlan: + """The export plan for one conversation (or the reason it has none).""" + + conversation: Optional[Conversation] + clips: List[ClipPlan] = field(default_factory=list) + sample_rate: int = 16000 + excluded_seconds: float = 0.0 # privacy-screen withholdings + dropped_seconds: float = 0.0 # preview-dropped clips + skipped_reason: Optional[str] = None + + @property + def clip_seconds(self) -> float: + return sum(c.duration for c in self.clips) + + +def export_eligibility( + conv: Optional[Conversation], user_id: str, is_superuser: bool +) -> Optional[str]: + """Why this conversation cannot be exported, or None if it can.""" + if not conv: + return "not found" + if not is_superuser and conv.user_id != user_id: + return "access forbidden" + if conv.deleted: + return "deleted" + if conv.audio_archived: + return "audio archived" + if not conv.audio_chunks_count: + return "no audio" + return None + + +async def collect_raw_intervals( + conversation_id: str, threshold: float +) -> Tuple[Optional[List[List[float]]], float, int]: + """Raw speech intervals from cached chunk frame scores (streaming cursor). + + Returns (intervals, last_chunk_end_seconds, sample_rate); intervals is + None when any chunk lacks VAD scores (caller should analyze first). + """ + collection = AudioChunkDocument.get_pymongo_collection() + cursor = collection.find( + {"conversation_id": conversation_id}, + { + "start_time": 1, + "end_time": 1, + "sample_rate": 1, + "vad.scores": 1, + "vad.frame_hop_ms": 1, + }, + ).sort("chunk_index", 1) + + intervals: List[List[float]] = [] + last_end = 0.0 + sample_rate = 16000 + first = True + async for chunk in cursor: + if first: + sample_rate = int(chunk.get("sample_rate") or 16000) + first = False + vad = chunk.get("vad") + if not vad or vad.get("scores") is None: + return None, 0.0, sample_rate + intervals.extend( + frame_speech_intervals( + vad["scores"], + float(vad["frame_hop_ms"]) / 1000.0, + float(chunk["start_time"]), + threshold=threshold, + ) + ) + last_end = float(chunk["end_time"]) + return intervals, last_end, sample_rate + + +async def plan_conversation_clips( + conv: Conversation, + mode: str, + pad_seconds: float, + speech_threshold: float, + merge_gap_seconds: float, + excluded_ranges: Optional[List[List[float]]] = None, + dropped_ranges: Optional[List[List[float]]] = None, +) -> ConversationPlan: + """Compute the clip plan for one eligible conversation. + + Mode ``clips``: one region per VAD speech run, padded and gap-merged at + the requested settings (the cached ``speech_regions`` use the default + 0.3s pad, so regions are always re-merged here). Mode ``full``: a single + region spanning the whole recording. + + ``excluded_ranges`` (privacy screen) and ``dropped_ranges`` (preview + curation) are subtracted **after** padding/merge so padding cannot + re-expose a cut. A dropped clip's exact [start, end] therefore removes + precisely that region. + + Unanalyzed audio yields ``skipped_reason='not analyzed'`` — the caller + decides whether to run VAD (the export job does; the synchronous preview + endpoint does not, pointing the user at the Analyze button instead). + """ + cid = conv.conversation_id + + if mode == "full": + duration = conv.audio_total_duration or 0.0 + if duration <= 0: + return ConversationPlan(conv, skipped_reason="no audio duration") + regions: List[List[float]] = [[0.0, duration]] + first = await AudioChunkDocument.find_one( + AudioChunkDocument.conversation_id == cid + ) + sample_rate = first.sample_rate if first else 16000 + else: + intervals, last_end, sample_rate = await collect_raw_intervals( + cid, speech_threshold + ) + if intervals is None: + return ConversationPlan( + conv, sample_rate=sample_rate, skipped_reason="not analyzed" + ) + duration = conv.audio_total_duration or last_end + regions = merge_speech_regions( + intervals, + duration, + pad_seconds=pad_seconds, + merge_gap_seconds=merge_gap_seconds, + ) + + kept = sum(t1 - t0 for t0, t1 in regions) + if excluded_ranges: + regions = subtract_intervals(regions, excluded_ranges) + after_privacy = sum(t1 - t0 for t0, t1 in regions) + if dropped_ranges: + regions = subtract_intervals(regions, dropped_ranges) + after_drop = sum(t1 - t0 for t0, t1 in regions) + + return ConversationPlan( + conv, + clips=[ClipPlan(i, t0, t1) for i, (t0, t1) in enumerate(regions)], + sample_rate=sample_rate, + excluded_seconds=round(max(0.0, kept - after_privacy), 2), + dropped_seconds=round(max(0.0, after_privacy - after_drop), 2), + ) diff --git a/backends/advanced/src/advanced_omi_backend/workers/data_audit_jobs.py b/backends/advanced/src/advanced_omi_backend/workers/data_audit_jobs.py index bed4640f2..bca4832d8 100644 --- a/backends/advanced/src/advanced_omi_backend/workers/data_audit_jobs.py +++ b/backends/advanced/src/advanced_omi_backend/workers/data_audit_jobs.py @@ -28,7 +28,6 @@ archive_conversation_audio_doc, ) from advanced_omi_backend.llm_client import async_generate -from advanced_omi_backend.models.audio_chunk import AudioChunkDocument from advanced_omi_backend.models.conversation import Conversation from advanced_omi_backend.models.job import async_job from advanced_omi_backend.services.observability.system_events import record_event_sync @@ -45,6 +44,10 @@ audio_cache_duration_matches, reconstruct_audio_segment, ) +from advanced_omi_backend.utils.export_planning import ( + export_eligibility, + plan_conversation_clips, +) from advanced_omi_backend.utils.sensitivity_screening import ( DEFAULT_SENSITIVITY_POLICY, build_screening_prompt, @@ -52,12 +55,7 @@ screenable_segments, ) from advanced_omi_backend.utils.transcript_slicing import slice_segments -from advanced_omi_backend.utils.vad_analysis import ( - analyze_conversation_audio, - frame_speech_intervals, - merge_speech_regions, - subtract_intervals, -) +from advanced_omi_backend.utils.vad_analysis import analyze_conversation_audio logger = logging.getLogger(__name__) @@ -445,49 +443,6 @@ def _progress(done: int, label: str) -> None: return summary -async def _collect_raw_intervals( - conversation_id: str, threshold: float -) -> Tuple[Optional[List[List[float]]], float, int]: - """Raw speech intervals from cached chunk frame scores (streaming cursor). - - Returns (intervals, last_chunk_end_seconds, sample_rate); intervals is - None when any chunk lacks VAD scores (caller should analyze first). - """ - collection = AudioChunkDocument.get_pymongo_collection() - cursor = collection.find( - {"conversation_id": conversation_id}, - { - "start_time": 1, - "end_time": 1, - "sample_rate": 1, - "vad.scores": 1, - "vad.frame_hop_ms": 1, - }, - ).sort("chunk_index", 1) - - intervals: List[List[float]] = [] - last_end = 0.0 - sample_rate = 16000 - first = True - async for chunk in cursor: - if first: - sample_rate = int(chunk.get("sample_rate") or 16000) - first = False - vad = chunk.get("vad") - if not vad or vad.get("scores") is None: - return None, 0.0, sample_rate - intervals.extend( - frame_speech_intervals( - vad["scores"], - float(vad["frame_hop_ms"]) / 1000.0, - float(chunk["start_time"]), - threshold=threshold, - ) - ) - last_end = float(chunk["end_time"]) - return intervals, last_end, sample_rate - - async def _export_conversation_clips( zf: zipfile.ZipFile, conv: Conversation, @@ -496,82 +451,65 @@ async def _export_conversation_clips( speech_threshold: float, merge_gap_seconds: float, excluded_ranges: Optional[List[List[float]]] = None, -) -> Tuple[List[dict], float, float]: + dropped_ranges: Optional[List[List[float]]] = None, +) -> Tuple[List[dict], float, float, float]: """Write the conversation's WAV clip(s) into the zip; return its manifest - records, total clipped seconds, and excluded (withheld) seconds. + records, total clipped seconds, excluded (privacy-withheld) seconds, and + dropped (preview-unticked) seconds. - Mode ``clips``: one padded WAV per VAD speech region (silence cropped). - Mode ``full``: a single untouched WAV spanning the whole conversation — - no VAD needed. - - ``excluded_ranges`` (absolute conversation seconds, from the privacy - screen) are carved out of the regions so the withheld audio + its - transcript never enter a clip. + The clip boundaries come from ``plan_conversation_clips`` — the same + computation the preview endpoint serves — so what the user approved is + exactly what gets written. Unanalyzed audio gets VAD run inline + (idempotent) and the plan retried. """ - cid = conv.conversation_id - - if mode == "full": - duration = conv.audio_total_duration or 0.0 - if duration <= 0: - raise ValueError("Conversation has no audio duration") - regions = [[0.0, duration]] - first = await AudioChunkDocument.find_one( - AudioChunkDocument.conversation_id == cid - ) - sample_rate = first.sample_rate if first else 16000 - else: - intervals, last_end, sample_rate = await _collect_raw_intervals( - cid, speech_threshold - ) - if intervals is None: - # Unanalyzed audio — run VAD inline (idempotent), then retry. - if await _analyze_and_store(conv) is None: - raise ValueError("VAD analysis failed") - intervals, last_end, sample_rate = await _collect_raw_intervals( - cid, speech_threshold - ) - if intervals is None: - raise ValueError("VAD scores missing after analysis") - - duration = conv.audio_total_duration or last_end - # Cached speech_regions are built with the default 0.3s pad — always - # re-merge here so the export honors the requested padding. - regions = merge_speech_regions( - intervals, - duration, - pad_seconds=pad_seconds, - merge_gap_seconds=merge_gap_seconds, + plan = await plan_conversation_clips( + conv, + mode, + pad_seconds, + speech_threshold, + merge_gap_seconds, + excluded_ranges, + dropped_ranges, + ) + if plan.skipped_reason == "not analyzed": + if await _analyze_and_store(conv) is None: + raise ValueError("VAD analysis failed") + plan = await plan_conversation_clips( + conv, + mode, + pad_seconds, + speech_threshold, + merge_gap_seconds, + excluded_ranges, + dropped_ranges, ) - - # Carve out privacy-screened ranges so withheld audio/transcript is - # never written. Done after padding/merge so padding can't re-expose a cut. - kept_seconds = sum(t1 - t0 for t0, t1 in regions) - if excluded_ranges: - regions = subtract_intervals(regions, excluded_ranges) - excluded_seconds = kept_seconds - sum(t1 - t0 for t0, t1 in regions) + if plan.skipped_reason == "not analyzed": + raise ValueError("VAD scores missing after analysis") + if plan.skipped_reason: + raise ValueError(plan.skipped_reason.capitalize()) segments = active_segments(conv) created_at = conv.created_at.isoformat() if conv.created_at else None records: List[dict] = [] - clip_seconds = 0.0 - for i, (t0, t1) in enumerate(regions): - wav = await reconstruct_audio_segment(cid, t0, t1) + for clip in plan.clips: + wav = await reconstruct_audio_segment( + conv.conversation_id, clip.start, clip.end + ) record = build_clip_record( - conversation_id=cid, + conversation_id=conv.conversation_id, conversation_title=conv.title, client_id=conv.client_id, conversation_created_at=created_at, - clip_index=i, - region_start=t0, - region_end=t1, - sample_rate=sample_rate, - segments=slice_segments(segments, t0, t1), + clip_index=clip.clip_index, + region_start=clip.start, + region_end=clip.end, + sample_rate=plan.sample_rate, + segments=slice_segments(segments, clip.start, clip.end), ) zf.writestr(record["audio_path"], wav) records.append(record) - clip_seconds += t1 - t0 - return records, clip_seconds, round(max(0.0, excluded_seconds), 2) + return records, plan.clip_seconds, plan.excluded_seconds, plan.dropped_seconds @async_job(redis=False, beanie=True, timeout=3600) @@ -584,6 +522,7 @@ async def export_annotation_dataset_job( speech_threshold: float = 0.5, merge_gap_seconds: float = 3.0, excluded_ranges: Optional[Dict[str, List[List[float]]]] = None, + dropped_ranges: Optional[Dict[str, List[List[float]]]] = None, sensitivity_policy: Optional[str] = None, ) -> Dict[str, Any]: """Build an annotation dataset zip for the selected conversations. @@ -597,15 +536,17 @@ async def export_annotation_dataset_job( the download endpoint. ``excluded_ranges`` maps ``conversation_id`` → withheld time ranges (from - the privacy screen); those ranges are carved out of each conversation's - audio and transcript. ``sensitivity_policy`` is recorded in the metadata - for auditability. + the privacy screen) and ``dropped_ranges`` → clips the user unticked in + the export preview; both are carved out of each conversation's audio and + transcript, accounted separately. ``sensitivity_policy`` is recorded in + the metadata for auditability. Per-conversation failures are recorded as ``skipped_reason``; the job only raises on export-level failures (e.g. disk errors). """ start = time.time() excluded_ranges = excluded_ranges or {} + dropped_ranges = dropped_ranges or {} user = await User.get(PydanticObjectId(user_id)) is_super = bool(user and user.is_superuser) @@ -617,6 +558,7 @@ async def export_annotation_dataset_job( manifest_records: List[dict] = [] total_clip_seconds = 0.0 total_excluded_seconds = 0.0 + total_dropped_seconds = 0.0 with zipfile.ZipFile(zip_path, "w", zipfile.ZIP_DEFLATED) as zf: for cid in dict.fromkeys(conversation_ids): @@ -630,19 +572,12 @@ async def export_annotation_dataset_job( summary["title"] = conv.title summary["client_id"] = conv.client_id - if not conv: - summary["skipped_reason"] = "not found" - elif not is_super and conv.user_id != user_id: - summary["skipped_reason"] = "access forbidden" - elif conv.deleted: - summary["skipped_reason"] = "deleted" - elif conv.audio_archived: - summary["skipped_reason"] = "audio archived" - elif not conv.audio_chunks_count: - summary["skipped_reason"] = "no audio" + skipped = export_eligibility(conv, user_id, is_super) + if skipped: + summary["skipped_reason"] = skipped else: try: - records, clip_seconds, excluded_seconds = ( + records, clip_seconds, excluded_seconds, dropped_seconds = ( await _export_conversation_clips( zf, conv, @@ -651,15 +586,19 @@ async def export_annotation_dataset_job( speech_threshold, merge_gap_seconds, excluded_ranges.get(cid), + dropped_ranges.get(cid), ) ) manifest_records.extend(records) total_clip_seconds += clip_seconds total_excluded_seconds += excluded_seconds + total_dropped_seconds += dropped_seconds summary["clip_count"] = len(records) summary["clip_seconds"] = round(clip_seconds, 2) if excluded_seconds > 0: summary["excluded_seconds"] = excluded_seconds + if dropped_seconds > 0: + summary["dropped_seconds"] = dropped_seconds except Exception as e: logger.exception(f"Export failed for conversation {cid[:12]}") summary["skipped_reason"] = f"error: {e}" @@ -678,6 +617,7 @@ async def export_annotation_dataset_job( "merge_gap_seconds": merge_gap_seconds, "screened": bool(excluded_ranges), "sensitivity_policy": sensitivity_policy if excluded_ranges else None, + "curated": bool(dropped_ranges), }, "conversations": conv_summaries, "totals": { @@ -686,6 +626,7 @@ async def export_annotation_dataset_job( "clip_count": len(manifest_records), "total_clip_seconds": round(total_clip_seconds, 2), "excluded_seconds": round(total_excluded_seconds, 2), + "dropped_seconds": round(total_dropped_seconds, 2), }, } zf.writestr( diff --git a/backends/advanced/tests/test_codex_executor.py b/backends/advanced/tests/test_codex_executor.py index 26eb8e9d0..3b7b9826e 100644 --- a/backends/advanced/tests/test_codex_executor.py +++ b/backends/advanced/tests/test_codex_executor.py @@ -1,12 +1,17 @@ """Codex CLI memory-agent executor: selection, filesystem-diff auditing, failure paths.""" import contextlib +import json import subprocess from types import SimpleNamespace import pytest -from advanced_omi_backend.services.memory.agent import codex_agent, memory_agent +from advanced_omi_backend.services.memory.agent import ( + codex_agent, + codex_quota, + memory_agent, +) from advanced_omi_backend.services.memory.agent.codex_agent import CodexMemoryAgent from advanced_omi_backend.services.memory.agent.memory_agent import ( MemoryAgent, @@ -59,7 +64,9 @@ def test_agent_class_falls_back_when_codex_unavailable(monkeypatch): # --------------------------------------------------------------------------- -def _fake_codex_run(vault_root, *, summary="Recorded the conversation.", returncode=0): +def _fake_codex_run( + vault_root, *, summary="Recorded the conversation.", returncode=0, usage=None +): """A subprocess.run stand-in that mimics one codex exec editing the vault.""" def fake_run(cmd, **kwargs): @@ -72,9 +79,13 @@ def fake_run(cmd, **kwargs): last_msg = cmd[cmd.index("--output-last-message") + 1] with open(last_msg, "w") as f: f.write(summary) + turn = {"type": "turn.completed"} + if usage is not None: + turn["usage"] = usage stdout = ( '{"type":"item.completed","item":{"item_type":"command_execution"}}\n' - '{"type":"turn.completed"}\n' + + json.dumps(turn) + + "\n" ) return SimpleNamespace(returncode=returncode, stdout=stdout, stderr="") @@ -132,6 +143,244 @@ def failing_run(cmd, **kwargs): assert result.touched == [] # nothing was written +# --------------------------------------------------------------------------- +# Token usage +# --------------------------------------------------------------------------- + + +def test_parse_events_sums_turn_usage(): + stdout = ( + '{"type":"turn.completed","usage":{"input_tokens":1200,' + '"cached_input_tokens":900,"output_tokens":40}}\n' + '{"type":"turn.completed","usage":{"input_tokens":300,' + '"cached_input_tokens":100,"output_tokens":10,' + '"reasoning_output_tokens":7}}\n' + ) + + _, turns, _, usage = CodexMemoryAgent._parse_events(stdout) + + assert turns == 2 + assert usage == { + "input_tokens": 1500, + "input_cached_tokens": 1000, + "output_tokens": 50, + "output_reasoning_tokens": 7, + } + + +@pytest.mark.parametrize( + "event", + [ + {"type": "turn.completed"}, # older CLI: no usage block at all + {"type": "turn.completed", "usage": None}, + {"type": "turn.completed", "usage": "unexpected"}, + {"type": "turn.completed", "usage": {"input_tokens": "many"}}, + ], +) +def test_turn_usage_tolerates_missing_or_odd_shapes(event): + """The CLI's field names are not a stable contract; usage must never break a run.""" + assert CodexMemoryAgent._turn_usage(event) == {} + + +@pytest.mark.asyncio +async def test_run_reports_usage_from_the_json_stream(tmp_path, monkeypatch, unlocked): + root = _seed_vault(tmp_path) + monkeypatch.setattr( + codex_agent, "codex_executor_available", lambda: (True, "/usr/bin/codex") + ) + monkeypatch.setattr( + subprocess, + "run", + _fake_codex_run(root, usage={"input_tokens": 8000, "output_tokens": 120}), + ) + + result = await CodexMemoryAgent(root).run("a real transcript", "conv1") + + assert result.usage == {"input_tokens": 8000, "output_tokens": 120} + + +def test_usage_span_is_a_child_not_the_agent_span(monkeypatch): + """Langfuse drops usage on ``invoke_agent`` spans, so it must ride a child LLM span. + + Pins why usage lives on ``codex_turn``: moved onto ``codex_memory_agent`` + (``gen_ai.operation.name: invoke_agent``), current Langfuse's OTEL processor + ingests the tokens as zero without erroring anywhere. Langfuse 3.x has no such + guard, so that regression would not show up on an older deployment. + """ + recorded = {} + + class _Span: + def end(self, end_time=None): + recorded["end_time"] = end_time + + class _Tracer: + def start_span(self, name, attributes=None, start_time=None): + recorded.update(name=name, attributes=attributes, start_time=start_time) + return _Span() + + monkeypatch.setattr( + "advanced_omi_backend.observability.otel_setup.get_tracer", + lambda _name: _Tracer(), + ) + + CodexMemoryAgent._record_usage_span( + {"input_tokens": 10, "input_cached_tokens": 4}, "gpt-5.6-terra", 111, 222 + ) + + assert recorded["name"] == "codex_turn" + attrs = recorded["attributes"] + assert attrs["gen_ai.operation.name"] == "chat" # NOT invoke_agent + assert attrs["gen_ai.usage.input_tokens"] == 10 + assert attrs["gen_ai.usage.input_cached_tokens"] == 4 + # Explicit timestamps: the span is created only after the subprocess returns. + assert (recorded["start_time"], recorded["end_time"]) == (111, 222) + + +def test_no_usage_emits_no_span(monkeypatch): + def _boom(_name): + raise AssertionError("tracer must not be built when there is no usage") + + monkeypatch.setattr( + "advanced_omi_backend.observability.otel_setup.get_tracer", _boom + ) + + CodexMemoryAgent._record_usage_span({}, "gpt-5.6-terra", 1, 2) + + +# --------------------------------------------------------------------------- +# Quota guard +# --------------------------------------------------------------------------- + +# Verbatim shape of a real `account/rateLimits/read` reply (codex-cli 0.144.4), +# trimmed to the fields the guard reads. Two buckets, one exhausted, one untouched. +REAL_RATE_LIMITS = { + "rateLimits": { + "limitId": "codex", + "primary": { + "usedPercent": 100, + "windowDurationMins": 10080, + "resetsAt": 1785612921, + }, + "secondary": None, + "planType": "prolite", + "rateLimitReachedType": "rate_limit_reached", + }, + "rateLimitsByLimitId": { + "codex": { + "limitId": "codex", + "primary": { + "usedPercent": 100, + "windowDurationMins": 10080, + "resetsAt": 1785612921, + }, + }, + "codex_bengalfox": { + "limitId": "codex_bengalfox", + "limitName": "GPT-5.3-Codex-Spark", + "primary": { + "usedPercent": 0, + "windowDurationMins": 10080, + "resetsAt": 1785798988, + }, + }, + }, +} + + +def test_bucket_used_percent_reads_the_default_and_named_buckets(): + assert codex_quota.bucket_used_percent(REAL_RATE_LIMITS) == 100 + assert codex_quota.bucket_used_percent(REAL_RATE_LIMITS, "codex") == 100 + # A different model's bucket can be untouched while the default is exhausted. + assert codex_quota.bucket_used_percent(REAL_RATE_LIMITS, "codex_bengalfox") == 0 + + +def test_unknown_bucket_is_unknown_not_the_default_bucket(): + """Must not silently gate on some other budget's headroom.""" + assert codex_quota.bucket_used_percent(REAL_RATE_LIMITS, "codex_nope") is None + + +@pytest.mark.parametrize("payload", [None, {}, {"rateLimits": {"primary": {}}}]) +def test_bucket_used_percent_unknown_shapes(payload): + assert codex_quota.bucket_used_percent(payload) is None + + +def test_quota_span_attributes_carry_window_and_reset(): + attrs = codex_quota.quota_span_attributes(REAL_RATE_LIMITS) + assert attrs["chronicle.memory.quota.used_percent"] == 100 + assert attrs["chronicle.memory.quota.window_minutes"] == 10080 + assert attrs["chronicle.memory.quota.resets_at"] == 1785612921 + assert codex_quota.quota_span_attributes(None) == {} + + +@pytest.mark.parametrize( + "settings,used,expect_block", + [ + ({"max_used_percent": 80}, 100, True), # over budget -> yield + ({"max_used_percent": 80}, 80, True), # at budget -> yield + ({"max_used_percent": 80}, 79, False), + ({}, 100, False), # unconfigured -> guard off + ({"max_used_percent": None}, 100, False), + ({"max_used_percent": "abc"}, 100, False), # unparseable -> guard off + ({"max_used_percent": 80}, None, False), # unreadable -> fail OPEN + ], +) +def test_quota_guard_decision(monkeypatch, settings, used, expect_block): + monkeypatch.setattr(codex_agent, "_codex_settings", lambda: settings) + monkeypatch.setattr(codex_quota, "read_rate_limits", lambda **_: {"stub": True}) + monkeypatch.setattr(codex_quota, "bucket_used_percent", lambda *_a, **_k: used) + + _, blocked = CodexMemoryAgent._check_quota("conv1") + + assert blocked is expect_block + + +@pytest.mark.asyncio +async def test_exhausted_quota_records_via_direct_agent_instead( + tmp_path, monkeypatch, unlocked +): + """Yielding must still record the conversation, not drop it.""" + root = _seed_vault(tmp_path) + monkeypatch.setattr( + codex_agent, "codex_executor_available", lambda: (True, "/usr/bin/codex") + ) + monkeypatch.setattr( + codex_agent, "_codex_settings", lambda: {"max_used_percent": 80} + ) + monkeypatch.setattr(codex_quota, "read_rate_limits", lambda **_: REAL_RATE_LIMITS) + + def _no_subprocess(*_a, **_k): + raise AssertionError("codex must not be spawned when over budget") + + monkeypatch.setattr(subprocess, "run", _no_subprocess) + + delegated = {} + + class _Direct: + def __init__(self, root, *a, **kw): + delegated["constructed"] = True + # The yield is a budget decision, not a failed run: it must NOT use the + # note-guarantee recovery path's forced fallback LLM. + delegated["force_fallback"] = kw.get("force_fallback", False) + + async def run(self, transcript, conversation_id, **kwargs): + delegated["conversation_id"] = conversation_id + return MemoryAgentResult( + conversation_id=conversation_id, + rounds=1, + touched=["Conversations/conv1.md"], + summary="recorded by the direct agent", + ) + + monkeypatch.setattr(memory_agent, "MemoryAgent", _Direct) + + result = await CodexMemoryAgent(root).run("a real transcript", "conv1") + + assert delegated["constructed"] is True + assert delegated["force_fallback"] is False + assert result.touched == ["Conversations/conv1.md"] + assert not result.truncated + + @pytest.mark.asyncio async def test_run_unavailable_executor_returns_truncated(tmp_path, monkeypatch): root = _seed_vault(tmp_path) diff --git a/backends/advanced/tests/test_conversation_search.py b/backends/advanced/tests/test_conversation_search.py index 6f1e7ccbd..41f9af387 100644 --- a/backends/advanced/tests/test_conversation_search.py +++ b/backends/advanced/tests/test_conversation_search.py @@ -6,9 +6,10 @@ ) -def test_everything_search_includes_all_three_categories(): - fields = _search_fields(["title", "summary", "speakers"]) +def test_everything_search_includes_all_categories(): + fields = _search_fields(["id", "title", "summary", "speakers"]) + assert "conversation_id" in fields assert "title" in fields assert "summary" in fields assert "detailed_summary" in fields @@ -17,11 +18,31 @@ def test_everything_search_includes_all_three_categories(): def test_search_categories_are_independent(): + assert _search_fields(["id"]) == ["conversation_id"] assert _search_fields(["title"]) == ["title"] assert _search_fields(["summary"]) == ["summary", "detailed_summary"] assert _search_fields(["speakers"]) == ["_search_active_version.segments.speaker"] +def test_conversation_id_search_matches_literal_fragments(): + stages = _search_query_stages("abc.123", _search_fields(["id"])) + + assert stages == [ + { + "$match": { + "$or": [ + { + "conversation_id": { + "$regex": r"abc\.123", + "$options": "i", + } + } + ] + } + } + ] + + def test_speaker_search_resolves_only_the_active_transcript_version(): fields = _search_fields(["speakers"]) stages = _search_query_stages("unshull", fields) diff --git a/backends/advanced/tests/test_device_input_routes_helpers.py b/backends/advanced/tests/test_device_input_routes_helpers.py new file mode 100644 index 000000000..feb279b86 --- /dev/null +++ b/backends/advanced/tests/test_device_input_routes_helpers.py @@ -0,0 +1,63 @@ +from datetime import datetime, timedelta, timezone +from types import SimpleNamespace + +from advanced_omi_backend.routers.modules.device_input_routes import ( + _effective_source_status, + _utc_iso, +) + + +def test_utc_iso_marks_naive_mongo_datetimes_as_utc(): + assert _utc_iso(datetime(2026, 7, 24, 16, 0, 23, 834000)) == ( + "2026-07-24T16:00:23.834000Z" + ) + + +def test_utc_iso_converts_aware_datetimes_to_utc(): + india = timezone(timedelta(hours=5, minutes=30)) + assert _utc_iso(datetime(2026, 7, 24, 21, 30, tzinfo=india)) == ( + "2026-07-24T16:00:00Z" + ) + + +def test_online_source_becomes_offline_when_heartbeat_is_stale(): + now = datetime(2026, 7, 24, 16, 5, tzinfo=timezone.utc) + source = SimpleNamespace( + provider="screenpipe", + status="online", + last_seen_at=datetime(2026, 7, 24, 16, 2, tzinfo=timezone.utc), + ) + + assert _effective_source_status(source, now) == "offline" + + +def test_recent_source_remains_online(): + now = datetime(2026, 7, 24, 16, 5, tzinfo=timezone.utc) + source = SimpleNamespace( + provider="screenpipe", + status="online", + last_seen_at=datetime(2026, 7, 24, 16, 4, 30, tzinfo=timezone.utc), + ) + + assert _effective_source_status(source, now) == "online" + + +def test_immich_source_uses_last_seen_as_sync_time_not_heartbeat(): + now = datetime(2026, 7, 31, 16, 5, tzinfo=timezone.utc) + source = SimpleNamespace( + provider="immich", + status="online", + last_seen_at=datetime(2026, 7, 24, 16, 2, tzinfo=timezone.utc), + ) + + assert _effective_source_status(source, now) == "online" + + +def test_immich_source_preserves_explicit_error_status(): + source = SimpleNamespace( + provider="immich", + status="error", + last_seen_at=datetime(2026, 7, 24, 16, 2, tzinfo=timezone.utc), + ) + + assert _effective_source_status(source) == "error" diff --git a/backends/advanced/tests/test_export_planning.py b/backends/advanced/tests/test_export_planning.py new file mode 100644 index 000000000..a69df7c87 --- /dev/null +++ b/backends/advanced/tests/test_export_planning.py @@ -0,0 +1,342 @@ +"""Tests for the shared export clip planner and the export-preview flow. + +The planner (``utils/export_planning.py``) is the single source of clip +boundaries for both the export job and the preview endpoint, so these tests +pin the properties the preview → curate → export loop relies on: preview and +export agree, dropped clips disappear exactly, and privacy vs curation +carve-outs are accounted separately. +""" + +import json +from datetime import datetime, timezone +from types import SimpleNamespace + +import pytest + +from advanced_omi_backend.controllers import data_audit_controller +from advanced_omi_backend.models.audio_chunk import AudioChunkDocument +from advanced_omi_backend.models.conversation import Conversation +from advanced_omi_backend.utils.export_planning import ( + export_eligibility, + plan_conversation_clips, +) + + +def _conv(**overrides): + base = dict( + conversation_id="conv-1", + title="Test conversation", + client_id="user01-phone", + created_at=datetime(2026, 8, 1, tzinfo=timezone.utc), + user_id="user-1", + deleted=False, + audio_archived=False, + audio_chunks_count=3, + audio_total_duration=60.0, + transcript_versions=[], + active_transcript_version=None, + ) + base.update(overrides) + return SimpleNamespace(**base) + + +class _ChunkCursor: + def __init__(self, docs): + self.docs = docs + + def sort(self, *_args): + return self + + def __aiter__(self): + self._it = iter(self.docs) + return self + + async def __anext__(self): + try: + return next(self._it) + except StopIteration: + raise StopAsyncIteration + + +def _chunk(start: float, end: float, scores, hop_ms: float = 100.0): + return { + "start_time": start, + "end_time": end, + "sample_rate": 16000, + "vad": {"scores": scores, "frame_hop_ms": hop_ms}, + } + + +def _mock_chunks(monkeypatch, docs): + collection = SimpleNamespace(find=lambda *_a, **_k: _ChunkCursor(docs)) + monkeypatch.setattr( + AudioChunkDocument, "get_pymongo_collection", lambda: collection + ) + + +# Two speech runs: 2.0–5.0s and 20.0–24.0s (frames at 100ms hop). +def _two_region_chunks(): + scores = [0.0] * 600 + for i in range(20, 50): + scores[i] = 0.9 + for i in range(200, 240): + scores[i] = 0.9 + return [_chunk(0.0, 60.0, scores)] + + +class TestPlanConversationClips: + @pytest.mark.asyncio + async def test_clips_mode_pads_and_keeps_separate_regions(self, monkeypatch): + _mock_chunks(monkeypatch, _two_region_chunks()) + plan = await plan_conversation_clips( + _conv(), + "clips", + pad_seconds=1.0, + speech_threshold=0.5, + merge_gap_seconds=3.0, + ) + assert plan.skipped_reason is None + assert [(c.start, c.end) for c in plan.clips] == [(1.0, 6.0), (19.0, 25.0)] + assert plan.excluded_seconds == 0.0 + assert plan.dropped_seconds == 0.0 + assert plan.sample_rate == 16000 + + @pytest.mark.asyncio + async def test_wide_merge_gap_joins_regions(self, monkeypatch): + _mock_chunks(monkeypatch, _two_region_chunks()) + plan = await plan_conversation_clips( + _conv(), + "clips", + pad_seconds=1.0, + speech_threshold=0.5, + merge_gap_seconds=30.0, + ) + assert [(c.start, c.end) for c in plan.clips] == [(1.0, 25.0)] + + @pytest.mark.asyncio + async def test_unanalyzed_audio_is_reported_not_analyzed(self, monkeypatch): + chunk = _chunk(0.0, 60.0, [0.9] * 600) + chunk["vad"] = None + _mock_chunks(monkeypatch, [chunk]) + plan = await plan_conversation_clips( + _conv(), + "clips", + pad_seconds=1.0, + speech_threshold=0.5, + merge_gap_seconds=3.0, + ) + assert plan.skipped_reason == "not analyzed" + assert plan.clips == [] + + @pytest.mark.asyncio + async def test_dropping_a_previewed_clip_removes_exactly_that_clip( + self, monkeypatch + ): + """The curation contract: unticking a clip in the preview and passing + its exact [start, end] as a dropped range removes that clip and only + that clip from a recomputed plan.""" + _mock_chunks(monkeypatch, _two_region_chunks()) + preview = await plan_conversation_clips( + _conv(), + "clips", + 1.0, + 0.5, + 3.0, + ) + dropped = preview.clips[0] + plan = await plan_conversation_clips( + _conv(), + "clips", + 1.0, + 0.5, + 3.0, + dropped_ranges=[[dropped.start, dropped.end]], + ) + assert [(c.start, c.end) for c in plan.clips] == [(19.0, 25.0)] + assert plan.dropped_seconds == 5.0 + assert plan.excluded_seconds == 0.0 + + @pytest.mark.asyncio + async def test_privacy_and_curation_carves_are_accounted_separately( + self, monkeypatch + ): + _mock_chunks(monkeypatch, _two_region_chunks()) + plan = await plan_conversation_clips( + _conv(), + "clips", + 1.0, + 0.5, + 3.0, + excluded_ranges=[[2.0, 4.0]], # privacy: carve inside clip 1 + dropped_ranges=[[19.0, 25.0]], # curation: drop clip 2 whole + ) + assert plan.excluded_seconds == 2.0 + assert plan.dropped_seconds == 6.0 + # Clip 1 splits around the privacy cut; clip 2 is gone. + assert [(c.start, c.end) for c in plan.clips] == [(1.0, 2.0), (4.0, 6.0)] + + @pytest.mark.asyncio + async def test_full_mode_is_one_untouched_region(self, monkeypatch): + async def _no_chunk(*_a, **_k): + return None + + # Uninitialized Beanie models raise on field access — give the class a + # plain attribute so the planner's find_one filter expression evaluates. + monkeypatch.setattr( + AudioChunkDocument, "conversation_id", "field", raising=False + ) + monkeypatch.setattr(AudioChunkDocument, "find_one", _no_chunk) + plan = await plan_conversation_clips( + _conv(audio_total_duration=42.5), + "full", + 1.0, + 0.5, + 3.0, + ) + assert [(c.start, c.end) for c in plan.clips] == [(0.0, 42.5)] + + +class TestExportEligibility: + def test_owner_with_audio_is_eligible(self): + assert export_eligibility(_conv(), "user-1", False) is None + + def test_reasons(self): + assert export_eligibility(None, "user-1", False) == "not found" + assert ( + export_eligibility(_conv(user_id="other"), "user-1", False) + == "access forbidden" + ) + assert export_eligibility(_conv(user_id="other"), "user-1", True) is None + assert export_eligibility(_conv(deleted=True), "user-1", False) == "deleted" + assert ( + export_eligibility(_conv(audio_archived=True), "user-1", False) + == "audio archived" + ) + assert ( + export_eligibility(_conv(audio_chunks_count=0), "user-1", False) + == "no audio" + ) + + +class _FakeConversationCls: + """Stands in for the Beanie model in controller tests: the field's + ``==`` returns the queried id so ``find_one`` can look it up.""" + + docs: dict = {} + + class _Field: + def __eq__(self, other): + return other + + conversation_id = _Field() + + @classmethod + async def find_one(cls, cid): + return cls.docs.get(cid) + + +def _segment(start, end, text, speaker="speaker_0"): + return Conversation.SpeakerSegment(start=start, end=end, text=text, speaker=speaker) + + +class TestPreviewExport: + @pytest.mark.asyncio + async def test_preview_returns_clips_with_sliced_transcripts(self, monkeypatch): + segments = [ + _segment(2.0, 4.0, "hello there"), + _segment(21.0, 23.0, "second clip words"), + ] + version = SimpleNamespace(version_id="v1", segments=segments) + conv = _conv(transcript_versions=[version], active_transcript_version="v1") + _FakeConversationCls.docs = {"conv-1": conv} + monkeypatch.setattr(data_audit_controller, "Conversation", _FakeConversationCls) + _mock_chunks(monkeypatch, _two_region_chunks()) + user = SimpleNamespace(is_superuser=False, user_id="user-1") + + result = await data_audit_controller.preview_export( + user, + ["conv-1", "missing"], + mode="clips", + ) + + assert result["totals"]["conversation_count"] == 2 + assert result["totals"]["exported_conversations"] == 1 + assert result["totals"]["clip_count"] == 2 + previewed, missing = result["conversations"] + assert missing["skipped_reason"] == "not found" + clips = previewed["clips"] + assert [c["clip_id"] for c in clips] == ["conv-1_000", "conv-1_001"] + assert clips[0]["text"] == "hello there" + assert clips[1]["text"] == "second clip words" + assert clips[0]["segment_count"] == 1 + assert previewed["clip_seconds"] == 11.0 + + @pytest.mark.asyncio + async def test_preview_reports_unanalyzed_instead_of_running_vad(self, monkeypatch): + conv = _conv() + _FakeConversationCls.docs = {"conv-1": conv} + monkeypatch.setattr(data_audit_controller, "Conversation", _FakeConversationCls) + chunk = _chunk(0.0, 60.0, []) + chunk["vad"] = None + _mock_chunks(monkeypatch, [chunk]) + user = SimpleNamespace(is_superuser=False, user_id="user-1") + + result = await data_audit_controller.preview_export(user, ["conv-1"]) + + assert result["conversations"][0]["skipped_reason"] == "not analyzed" + assert result["totals"]["exported_conversations"] == 0 + + +class TestLatestExportsByConversation: + def _write_export(self, exports_dir, export_id, created_at, conversations): + d = exports_dir / export_id + d.mkdir(parents=True) + (d / "export.json").write_text( + json.dumps( + { + "export_id": export_id, + "created_at": created_at, + "created_by": "user-1", + "conversations": conversations, + } + ) + ) + + def test_latest_export_wins_and_skipped_do_not_count(self, tmp_path, monkeypatch): + monkeypatch.setattr(data_audit_controller, "EXPORTS_DIR", tmp_path) + self._write_export( + tmp_path, + "annotation_20260601_000000_aaaa", + "2026-06-01T00:00:00+00:00", + [ + {"conversation_id": "c1"}, + {"conversation_id": "c2", "skipped_reason": "no audio"}, + ], + ) + self._write_export( + tmp_path, + "annotation_20260701_000000_bbbb", + "2026-07-01T00:00:00+00:00", + [{"conversation_id": "c1"}, {"conversation_id": "c3"}], + ) + user = SimpleNamespace(is_superuser=False, user_id="user-1") + + latest = data_audit_controller._latest_exports_by_conversation(user) + + assert latest["c1"]["export_id"] == "annotation_20260701_000000_bbbb" + assert latest["c3"]["export_id"] == "annotation_20260701_000000_bbbb" + assert "c2" not in latest # skipped conversations were never shipped + + def test_other_users_exports_are_invisible(self, tmp_path, monkeypatch): + monkeypatch.setattr(data_audit_controller, "EXPORTS_DIR", tmp_path) + self._write_export( + tmp_path, + "annotation_20260601_000000_aaaa", + "2026-06-01T00:00:00+00:00", + [{"conversation_id": "c1"}], + ) + stranger = SimpleNamespace(is_superuser=False, user_id="user-2") + superuser = SimpleNamespace(is_superuser=True, user_id="admin") + + assert data_audit_controller._latest_exports_by_conversation(stranger) == {} + assert "c1" in data_audit_controller._latest_exports_by_conversation(superuser) diff --git a/backends/advanced/tests/test_immich_discovery.py b/backends/advanced/tests/test_immich_discovery.py index c9e7c5800..bdd70ab83 100644 --- a/backends/advanced/tests/test_immich_discovery.py +++ b/backends/advanced/tests/test_immich_discovery.py @@ -1,6 +1,6 @@ from datetime import datetime, timedelta, timezone -from advanced_omi_backend.services.immich_discovery import select_candidates +from advanced_omi_backend.services.immich_discovery import _as_utc, select_candidates def asset(identifier: str, when: datetime, name: str = "photo.jpg"): @@ -29,3 +29,9 @@ def test_candidate_selection_honors_analysis_budget(): start = datetime(2026, 7, 22, tzinfo=timezone.utc) rows = [asset(str(i), start + timedelta(hours=i)) for i in range(20)] assert len(select_candidates(rows)) == 12 + + +def test_mongo_datetime_is_restored_to_utc_for_immich_search(): + stored = datetime(2026, 7, 27, 16, 46, 7) + + assert _as_utc(stored).isoformat() == "2026-07-27T16:46:07+00:00" diff --git a/backends/advanced/tests/test_person_identity.py b/backends/advanced/tests/test_person_identity.py new file mode 100644 index 000000000..8544b038e --- /dev/null +++ b/backends/advanced/tests/test_person_identity.py @@ -0,0 +1,226 @@ +"""Duplicate suggestions and durable distinct-person identity decisions.""" + +import contextlib + +import pytest +from ruamel.yaml import YAML + +from advanced_omi_backend.services.memory import person_identity, person_merge_actions +from advanced_omi_backend.services.memory.person_identity import PersonIdentityService +from advanced_omi_backend.services.memory.person_merge import ( + PersonMergeError, + PersonMergeService, + PersonMergeStale, +) + + +def _person( + name: str, + *, + aliases: list[str] | None = None, + distinct_from: list[str] | None = None, + org: str = "", + topic: str = "", + conversation: str = "", + photo: str = "", +) -> str: + aliases_yaml = ( + "\n" + "\n".join(f" - {value}" for value in aliases) if aliases else " []" + ) + distinct_yaml = ( + "\n" + "\n".join(f' - "[[{value}]]"' for value in distinct_from) + if distinct_from + else " []" + ) + image = f"![[../_media/{photo}|200]]\n" if photo else "" + context = f" Discussed [[{topic}]]." if topic else "" + mention = ( + f"- Met in [[Conversations/{conversation}|Conversation]].\n" + if conversation + else "- Mentioned once.\n" + ) + return ( + "---\n" + 'categories:\n - "[[People]]"\n' + f"aliases:{aliases_yaml}\n" + f"distinct_from:{distinct_yaml}\n" + f"org: {org}\n" + "role:\nrelationship:\nlocation:\n" + "created: 2026-08-01\nupdated: 2026-08-01\n" + "---\n" + f"{image}" + "## About\n" + f"- Information about {name}.{context}\n\n" + "## Conversations\n![[Conversations.base#Person]]\n\n" + "## Mentions\n" + f"{mention}" + ) + + +def _metadata(path) -> dict: + text = path.read_text(encoding="utf-8") + end = text.index("\n---\n", 4) + return YAML(typ="safe").load(text[4:end]) + + +@pytest.fixture +def vault(tmp_path): + people = tmp_path / "People" + people.mkdir() + (people / "Sabi.md").write_text( + _person( + "Sabi", + org="Acme", + topic="Model Training", + conversation="11111111-1111-1111-1111-111111111111", + ), + encoding="utf-8", + ) + (people / "Sabri.md").write_text( + _person( + "Sabri", + org="Acme", + topic="Model Training", + conversation="11111111-1111-1111-1111-111111111111", + ), + encoding="utf-8", + ) + (people / "Robert.md").write_text( + _person("Robert", aliases=["Bob"]), encoding="utf-8" + ) + (people / "Bob.md").write_text(_person("Bob"), encoding="utf-8") + (people / "Alice.md").write_text(_person("Alice"), encoding="utf-8") + (people / "Carlos.md").write_text( + _person( + "Carlos", + topic="Shared Project", + conversation="22222222-2222-2222-2222-222222222222", + ), + encoding="utf-8", + ) + (people / "Diana.md").write_text( + _person( + "Diana", + topic="Shared Project", + conversation="22222222-2222-2222-2222-222222222222", + ), + encoding="utf-8", + ) + return tmp_path + + +def test_suggestions_combine_name_alias_and_context_evidence(vault): + suggestions = PersonIdentityService(vault).suggestions() + by_pair = { + frozenset((item["person_a"]["name"], item["person_b"]["name"])): item + for item in suggestions + } + + sabi = by_pair[frozenset(("Sabi", "Sabri"))] + assert sabi["score"] >= 100 + assert "names differ by one character" in sabi["reasons"] + assert "same organization" in sabi["reasons"] + assert any( + reason.startswith("same source conversation") for reason in sabi["reasons"] + ) + assert sabi["revision"] + + robert = by_pair[frozenset(("Robert", "Bob"))] + assert "one name is already an alias of the other" in robert["reasons"] + assert frozenset(("Alice", "Bob")) not in by_pair + assert frozenset(("Carlos", "Diana")) not in by_pair + + +def test_distinct_decision_is_symmetric_and_removes_suggestion(vault, monkeypatch): + monkeypatch.setattr( + person_identity, "vault_note_lock", lambda _user: contextlib.nullcontext() + ) + service = PersonIdentityService(vault) + suggestion = next( + item + for item in service.suggestions() + if {item["person_a"]["name"], item["person_b"]["name"]} == {"Sabi", "Sabri"} + ) + + result = service.set_distinct( + "Sabi", "Sabri", distinct=True, revision=suggestion["revision"] + ) + + assert result.decision == "distinct" + assert set(result.changed_paths) == {"People/Sabi.md", "People/Sabri.md"} + assert _metadata(vault / "People/Sabi.md")["distinct_from"] == ["[[Sabri]]"] + assert _metadata(vault / "People/Sabri.md")["distinct_from"] == ["[[Sabi]]"] + assert not any( + {item["person_a"]["name"], item["person_b"]["name"]} == {"Sabi", "Sabri"} + for item in service.suggestions() + ) + + +def test_distinct_decision_blocks_merge_until_cleared(vault, monkeypatch): + monkeypatch.setattr( + person_identity, "vault_note_lock", lambda _user: contextlib.nullcontext() + ) + identity = PersonIdentityService(vault) + identity.set_distinct("Sabi", "Sabri", distinct=True) + + with pytest.raises(PersonMergeError, match="marked as separate people"): + PersonMergeService(vault).preview("Sabi", "Sabri") + + identity.set_distinct("Sabi", "Sabri", distinct=False) + preview = PersonMergeService(vault).preview("Sabi", "Sabri") + assert preview.source_name == "Sabi" + + +def test_distinct_decision_rejects_stale_suggestion(vault, monkeypatch): + monkeypatch.setattr( + person_identity, "vault_note_lock", lambda _user: contextlib.nullcontext() + ) + service = PersonIdentityService(vault) + suggestion = next( + item + for item in service.suggestions() + if {item["person_a"]["name"], item["person_b"]["name"]} == {"Sabi", "Sabri"} + ) + path = vault / "People/Sabi.md" + path.write_text(path.read_text(encoding="utf-8") + "\nChanged.\n", encoding="utf-8") + + with pytest.raises(PersonMergeStale, match="changed after the suggestion"): + service.set_distinct( + "Sabi", "Sabri", distinct=True, revision=suggestion["revision"] + ) + + +def test_existing_one_sided_annotation_is_respected(vault): + path = vault / "People/Sabi.md" + path.write_text( + _person("Sabi", distinct_from=["Sabri"]), + encoding="utf-8", + ) + suggestions = PersonIdentityService(vault).suggestions() + assert not any( + {item["person_a"]["name"], item["person_b"]["name"]} == {"Sabi", "Sabri"} + for item in suggestions + ) + with pytest.raises(PersonMergeError, match="marked as separate people"): + PersonMergeService(vault).preview("Sabi", "Sabri") + + +async def test_identity_decision_audits_both_notes(vault, monkeypatch): + monkeypatch.setattr( + person_identity, "vault_note_lock", lambda _user: contextlib.nullcontext() + ) + result = PersonIdentityService(vault).set_distinct("Sabi", "Sabri", distinct=True) + entries = [] + + async def capture(**kwargs): + entries.append(kwargs) + + monkeypatch.setattr(person_merge_actions, "record_vault_change", capture) + await person_merge_actions._record_identity_audit("user-1", result) + + assert {entry["note_path"] for entry in entries} == { + "People/Sabi.md", + "People/Sabri.md", + } + assert {entry["identity_decision"] for entry in entries} == {"distinct"} + assert {entry["action_id"] for entry in entries} == {result.action_id} diff --git a/backends/advanced/tests/test_person_merge.py b/backends/advanced/tests/test_person_merge.py new file mode 100644 index 000000000..729875738 --- /dev/null +++ b/backends/advanced/tests/test_person_merge.py @@ -0,0 +1,236 @@ +"""Deterministic person-merge behavior shared by API, Obsidian, and agents.""" + +import contextlib +import hashlib + +import pytest +from ruamel.yaml import YAML + +from advanced_omi_backend.services.memory import person_merge, person_merge_actions +from advanced_omi_backend.services.memory.audit import ( + MemoryCause, + actor_for, + source_label_for, +) +from advanced_omi_backend.services.memory.person_merge import ( + PersonMergeService, + PersonMergeStale, +) + + +def _person( + name: str, + about: list[str], + mentions: list[str], + *, + aliases: list[str] | None = None, + distinct_from: list[str] | None = None, + org: str = "", + role: str = "", + photo: str = "", +) -> str: + alias_lines = "\n".join(f" - {alias}" for alias in aliases or []) or "[]" + if aliases: + alias_value = f"\n{alias_lines}" + else: + alias_value = " []" + distinct_lines = ( + "\n" + "\n".join(f' - "[[{name}]]"' for name in distinct_from) + if distinct_from + else " []" + ) + image = f"![[../_media/{photo}|200]]\n" if photo else "" + return ( + "---\n" + 'categories:\n - "[[People]]"\n' + f"aliases:{alias_value}\n" + f"distinct_from:{distinct_lines}\n" + f"org: {org}\n" + f"role: {role}\n" + "relationship:\nlocation:\ncreated: 2026-07-28\nupdated: 2026-07-28\n" + "---\n" + f"{image}" + "## About\n" + + "\n".join(f"- {fact}" for fact in about) + + "\n\n## Conversations\n![[Conversations.base#Person]]\n\n## Mentions\n" + + "\n".join(f"- {mention}" for mention in mentions) + + "\n" + ) + + +def _frontmatter(text: str) -> dict: + end = text.index("\n---\n", 4) + return YAML(typ="safe").load(text[4:end]) + + +@pytest.fixture +def vault(tmp_path): + people = tmp_path / "People" + conversations = tmp_path / "Conversations" + people.mkdir() + conversations.mkdir() + (people / "Amay.md").write_text( + _person( + "Amay", + ["Owns the radar pipeline.", "Shared fact."], + ["2026-07-28 — Planned work."], + aliases=["A. May"], + distinct_from=["Carol"], + org="Acme", + role="Engineer", + photo="amay.jpg", + ), + encoding="utf-8", + ) + (people / "Amey.md").write_text( + _person( + "Amey", + ["Discussed model parsing.", "Shared fact."], + ["2026-07-28 — Discussed metrics."], + aliases=["A Mehta"], + org="Acme", + role="Lead", + photo="amey.jpg", + ), + encoding="utf-8", + ) + (conversations / "one.md").write_text( + 'people:\n - "[[Amay]]"\n- [[Amay]] owns this.\n', encoding="utf-8" + ) + (conversations / "two.md").write_text( + "See [[People/Amay|Amay from work]].\n", encoding="utf-8" + ) + return tmp_path + + +def test_preview_is_read_only_and_reports_complete_plan(vault): + service = PersonMergeService(vault) + source_before = (vault / "People/Amay.md").read_text(encoding="utf-8") + + preview = service.preview("amay", "AMEY") + + assert preview.source_name == "Amay" + assert preview.target_name == "Amey" + assert preview.facts_to_add == 2 + assert preview.duplicate_facts_skipped == 1 + assert preview.backlink_files == ["Conversations/one.md", "Conversations/two.md"] + assert preview.backlink_occurrences == 3 + assert [conflict.field for conflict in preview.metadata_conflicts] == ["role"] + assert (vault / "People/Amay.md").read_text(encoding="utf-8") == source_before + + +def test_apply_merges_metadata_facts_media_and_backlinks(vault, monkeypatch): + monkeypatch.setattr( + person_merge, "vault_note_lock", lambda _user: contextlib.nullcontext() + ) + service = PersonMergeService(vault) + preview = service.preview("Amay", "Amey") + + result = service.apply("Amay", "Amey", preview.plan_token) + + assert not (vault / "People/Amay.md").exists() + merged = (vault / "People/Amey.md").read_text(encoding="utf-8") + metadata = _frontmatter(merged) + assert metadata["aliases"] == ["A Mehta", "A. May", "Amay"] + assert metadata["distinct_from"] == ["[[Carol]]"] + assert metadata["org"] == "Acme" + assert metadata["role"] == "Lead" + assert "Owns the radar pipeline" in merged + assert merged.count("Shared fact") == 1 + assert "amey.jpg" in merged and "amay.jpg" in merged + assert "[[Amay]]" not in (vault / "Conversations/one.md").read_text( + encoding="utf-8" + ) + assert "[[People/Amey|Amay from work]]" in ( + vault / "Conversations/two.md" + ).read_text(encoding="utf-8") + assert set(result.changed_paths) == { + "People/Amay.md", + "People/Amey.md", + "Conversations/one.md", + "Conversations/two.md", + } + + +def test_apply_rejects_a_stale_preview(vault, monkeypatch): + monkeypatch.setattr( + person_merge, "vault_note_lock", lambda _user: contextlib.nullcontext() + ) + service = PersonMergeService(vault) + preview = service.preview("Amay", "Amey") + target = vault / "People/Amey.md" + target.write_text( + target.read_text(encoding="utf-8") + "\nNew edit.\n", encoding="utf-8" + ) + + with pytest.raises(PersonMergeStale, match="Preview it again"): + service.apply("Amay", "Amey", preview.plan_token) + + +def test_preview_rejects_a_local_copy_that_is_not_synced(vault): + local_hash = hashlib.sha256(b"older local note").hexdigest() + with pytest.raises(PersonMergeStale, match="source note differs"): + PersonMergeService(vault).preview( + "Amay", "Amey", expected_source_hash=local_hash + ) + + +def test_apply_rolls_back_files_when_a_write_fails(vault, monkeypatch): + monkeypatch.setattr( + person_merge, "vault_note_lock", lambda _user: contextlib.nullcontext() + ) + service = PersonMergeService(vault) + preview = service.preview("Amay", "Amey") + before = { + path.relative_to(vault).as_posix(): path.read_text(encoding="utf-8") + for path in vault.rglob("*.md") + } + real_write = person_merge._atomic_write + failed = False + + def fail_once(path, content): + nonlocal failed + if not failed and path.name == "one.md": + failed = True + raise OSError("simulated write failure") + real_write(path, content) + + monkeypatch.setattr(person_merge, "_atomic_write", fail_once) + with pytest.raises(OSError, match="simulated"): + service.apply("Amay", "Amey", preview.plan_token) + + after = { + path.relative_to(vault).as_posix(): path.read_text(encoding="utf-8") + for path in vault.rglob("*.md") + } + assert after == before + + +async def test_merge_audit_covers_every_changed_note(vault, monkeypatch): + monkeypatch.setattr( + person_merge, "vault_note_lock", lambda _user: contextlib.nullcontext() + ) + service = PersonMergeService(vault) + preview = service.preview("Amay", "Amey") + result = service.apply("Amay", "Amey", preview.plan_token) + entries = [] + + async def capture(**kwargs): + entries.append(kwargs) + + monkeypatch.setattr(person_merge_actions, "record_vault_change", capture) + await person_merge_actions._record_merge_audit("user-1", result) + + assert {entry["note_path"] for entry in entries} == set(result.changed_paths) + assert {entry["action_id"] for entry in entries} == {result.action_id} + source = next(entry for entry in entries if entry["note_path"] == "People/Amay.md") + assert source["operation"] == "rename" + assert source["after"] is None + assert source["new_path"] == "People/Amey.md" + + +def test_obsidian_action_provenance_is_human(): + assert source_label_for(MemoryCause.OBSIDIAN_ACTION, False, "update") == ( + "Obsidian action" + ) + assert actor_for(MemoryCause.OBSIDIAN_ACTION, False, "update") == "human_external" diff --git a/backends/advanced/tests/test_streaming_resume_probe.py b/backends/advanced/tests/test_streaming_resume_probe.py new file mode 100644 index 000000000..2f75c55ff --- /dev/null +++ b/backends/advanced/tests/test_streaming_resume_probe.py @@ -0,0 +1,160 @@ +"""Regression tests for the streaming consumer's "did this session resume?" probe. + +``transcription:complete:{session_id}`` does double duty: it stops the discovery +loop from re-attaching a second provider connection to a stream it already +finished, *and* it is the handshake ``open_conversation_job`` blocks on before +reading the final transcript. Clearing it therefore has a cost the discovery loop +cannot see — no replacement signal is ever produced, so the conversation job waits +out its full 30s timeout and finishes without the streaming result. + +That is what CI run 30884816710 hit. The probe used to ask only whether the +stream's newest entry was recent, but ``finalize_session`` flushes the residual +audio and appends the end marker as its last act, so at the moment the flag is set +the newest entry is milliseconds old. A closing session looked exactly like a +resuming one, and the discovery loop cleared the flag 122ms after it was written: + + 06:52:29,006 end_reason determined: websocket_disconnect + 06:52:29,128 marked complete but has fresh audio — clearing flag + 06:52:59,062 Timed out waiting for streaming completion signal (waited 30s) + +The probe now decides on causal state instead: a session that has left ACTIVE can +never append again (``producer._append_owned_message`` appends inside a WATCH/MULTI +whose precondition is ``status == "active"``), and an end marker in the stream +proves the producer finished even if the FINALIZING write has not landed yet. +""" + +import time + +import pytest +from fakeredis import aioredis as fake_aioredis + +import advanced_omi_backend.services.transcription.streaming_consumer as sc_module +from advanced_omi_backend.services.audio_stream.session_store import SessionStore +from advanced_omi_backend.services.transcription.streaming_consumer import ( + StreamingTranscriptionConsumer, +) + +pytestmark = pytest.mark.unit + +SESSION_ID = "989f33-plugin-tes-b43abe11e4a640f58c7f2ca8eee2aa20" +STREAM = f"audio:stream:{SESSION_ID}" + + +class _StubProvider: + """The consumer resolves a provider in __init__; nothing here calls it.""" + + capabilities: list[str] = [] + + +@pytest.fixture +def consumer(monkeypatch): + redis = fake_aioredis.FakeRedis() + monkeypatch.setattr( + sc_module, "get_transcription_provider", lambda mode: _StubProvider() + ) + return StreamingTranscriptionConsumer(redis_client=redis), redis + + +async def _append_chunk(redis, *, age_seconds: float = 0.0, end_marker: bool = False): + """Append one WAL entry, stamped ``age_seconds`` in the past. + + Redis stream ids are ``-`` and the probe reads the age straight off + the id, so an explicit id makes staleness deterministic without sleeping. + """ + entry_id = f"{int((time.time() - age_seconds) * 1000)}-*" + fields = {b"audio_data": b"\x00" * 8000, b"session_id": SESSION_ID.encode()} + if end_marker: + fields = {b"audio_data": b"", b"end_marker": b"true", b"chunk_id": b"END"} + await redis.xadd(STREAM, fields, id=entry_id) + + +async def _finalize_like_producer(redis, *, write_end_marker: bool = True): + """Replay ``finalize_session``: flush residual audio, end marker, then status.""" + await _append_chunk(redis) + if write_end_marker: + await _append_chunk(redis, end_marker=True) + await SessionStore(redis).mark_finalizing(SESSION_ID, "websocket_disconnect") + + +async def test_finalized_session_is_not_resumed_despite_a_fresh_tail(consumer): + """The CI failure: finalize's own closing writes must not read as a resume.""" + c, redis = consumer + await SessionStore(redis).set_status_active(SESSION_ID) + await _finalize_like_producer(redis) + + assert await c._session_resumed(STREAM, SESSION_ID) is False + + +async def test_end_marker_blocks_reattach_before_the_status_write_lands(consumer): + """The marker is appended strictly before the flag can exist, so it decides. + + ``finalize_session`` appends the marker while the session is still ACTIVE and + only then calls ``mark_finalizing``. A consumer that reads the marker and sets + the completion flag inside that window would otherwise see status=active. + """ + c, redis = consumer + await SessionStore(redis).set_status_active(SESSION_ID) + await _append_chunk(redis) + await _append_chunk(redis, end_marker=True) + + assert await c._session_resumed(STREAM, SESSION_ID) is False + + +async def test_end_marker_is_found_behind_a_late_chunk(consumer): + """A chunk racing in behind the marker must not hide it from the tail probe.""" + c, redis = consumer + await SessionStore(redis).set_status_active(SESSION_ID) + await _append_chunk(redis, end_marker=True) + await _append_chunk(redis) + + assert await c._session_resumed(STREAM, SESSION_ID) is False + + +async def test_finalized_session_without_an_end_marker_is_not_resumed(consumer): + """A backend restart loses the producer buffer, so finalize writes no marker. + + Status is then the only evidence the session is over — and it is enough. + """ + c, redis = consumer + await SessionStore(redis).set_status_active(SESSION_ID) + await _finalize_like_producer(redis, write_end_marker=False) + + assert await c._session_resumed(STREAM, SESSION_ID) is False + + +async def test_active_session_still_producing_is_resumed(consumer): + """The self-heal this probe exists for: an idle-exited task must re-attach. + + ``process_stream`` sets the completion flag when its idle heartbeat fires, but + the device may resume afterwards. Without re-attaching, that live stream gets + no transcription until the flag's 5-minute TTL expires. + """ + c, redis = consumer + await SessionStore(redis).set_status_active(SESSION_ID) + await _append_chunk(redis) + + assert await c._session_resumed(STREAM, SESSION_ID) is True + + +async def test_active_session_gone_quiet_is_not_resumed(consumer): + """No audio for a long while — re-attaching would just churn provider sockets.""" + c, redis = consumer + await SessionStore(redis).set_status_active(SESSION_ID) + await _append_chunk(redis, age_seconds=60.0) + + assert await c._session_resumed(STREAM, SESSION_ID) is False + + +async def test_session_without_a_hash_is_not_resumed(consumer): + """No session hash means no producer, whatever the stream still holds.""" + c, redis = consumer + await _append_chunk(redis) + + assert await c._session_resumed(STREAM, SESSION_ID) is False + + +async def test_empty_stream_is_not_resumed(consumer): + c, redis = consumer + await SessionStore(redis).set_status_active(SESSION_ID) + + assert await c._session_resumed(STREAM, SESSION_ID) is False diff --git a/backends/advanced/uv.lock b/backends/advanced/uv.lock index cdb3bae7d..9b2959762 100644 --- a/backends/advanced/uv.lock +++ b/backends/advanced/uv.lock @@ -57,9 +57,6 @@ benchmark = [ { name = "huggingface-hub" }, { name = "ijson" }, ] -deepgram = [ - { name = "deepgram-sdk" }, -] galileo = [ { name = "galileo" }, { name = "opentelemetry-exporter-otlp" }, @@ -92,7 +89,6 @@ test = [ requires-dist = [ { name = "aiohttp", specifier = ">=3.8.0" }, { name = "croniter", specifier = ">=1.3.0" }, - { name = "deepgram-sdk", marker = "extra == 'deepgram'", specifier = ">=4.0.0" }, { name = "easy-audio-interfaces", specifier = ">=0.7.1" }, { name = "easy-audio-interfaces", extras = ["local-audio"], marker = "extra == 'local-audio'", specifier = ">=0.7.1" }, { name = "en-core-web-sm", url = "https://github.com/explosion/spacy-models/releases/download/en_core_web_sm-3.8.0/en_core_web_sm-3.8.0-py3-none-any.whl" }, @@ -129,7 +125,7 @@ requires-dist = [ { name = "websockets", specifier = ">=12.0" }, { name = "wyoming", specifier = ">=1.6.1" }, ] -provides-extras = ["deepgram", "local-audio", "galileo", "benchmark"] +provides-extras = ["local-audio", "galileo", "benchmark"] [package.metadata.requires-dev] dev = [ @@ -989,22 +985,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/89/94/b7ff6279e642b014cd4aef4d914b9fca3917c2c9c35df49db062023cbdfc/dbus_fast-3.1.2-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:1d7cc1315586e4c50875c9a2d56b9ad2e056ec75e2f27c43cd80392f72d0f6e3", size = 1623709, upload_time = "2025-11-17T03:49:59.571Z" }, ] -[[package]] -name = "deepgram-sdk" -version = "5.3.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "httpx" }, - { name = "pydantic" }, - { name = "pydantic-core" }, - { name = "typing-extensions" }, - { name = "websockets" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/2d/9c/4529cc5818e9305ac9be3c24545249ad57418cbc3736c3f1c0a8397b59f5/deepgram_sdk-5.3.0.tar.gz", hash = "sha256:4e682a53f64c26dc49d8fd70865eae1e98236d313870d1bcf5f107f125e53793", size = 148179, upload_time = "2025-11-03T15:24:02.436Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/58/e2/cda09edad156199cc9e330533f6b72cb5276c0d476cab7f1744be7ffa16e/deepgram_sdk-5.3.0-py3-none-any.whl", hash = "sha256:431418fdffbd93cdf6a78a168984e3df3cb696818ced1cfc52ce336e0bc6a7fe", size = 390669, upload_time = "2025-11-03T15:24:01.078Z" }, -] - [[package]] name = "diskcache" version = "5.6.3" diff --git a/backends/advanced/webui/src/components/dataAudit/AuditTable.tsx b/backends/advanced/webui/src/components/dataAudit/AuditTable.tsx index d09ae705d..b96ba39e5 100644 --- a/backends/advanced/webui/src/components/dataAudit/AuditTable.tsx +++ b/backends/advanced/webui/src/components/dataAudit/AuditTable.tsx @@ -257,6 +257,13 @@ export default function AuditTable({ {r.derived_operation && ( {r.derived_operation} )} + {r.last_export && ( + + exported + + )} {(() => { const chip = processingStatusChip(r.processing_status, r.failure_stage) return chip ? ( diff --git a/backends/advanced/webui/src/components/dataAudit/ExportModal.tsx b/backends/advanced/webui/src/components/dataAudit/ExportModal.tsx index 9c3dd973b..b9430810f 100644 --- a/backends/advanced/webui/src/components/dataAudit/ExportModal.tsx +++ b/backends/advanced/webui/src/components/dataAudit/ExportModal.tsx @@ -5,17 +5,22 @@ import { HelpCircle, Loader2, PackageOpen, + Pause, + Play, ShieldCheck, Trash2, } from 'lucide-react' import { AuditConversation, + ExportPreviewClip, + ExportPreviewResult, ExportRecord, ScreenConversationReport, ScreenResult, dataAuditApi, } from '../../services/api' -import { Alert, Button, Modal, Textarea } from '../../components/ui' +import { Alert, Button, Modal, StateBadge, Textarea } from '../../components/ui' +import { useGaplessPlayer } from '../../hooks/useGaplessPlayer' import { useJobPolling } from '../../hooks/useJobPolling' import { formatDate, formatDuration } from './format' @@ -46,6 +51,12 @@ function Hint({ text }: { text: string }) { // A flagged segment is keyed by conversation + its segment index. const segKey = (cid: string, index: number) => `${cid}:${index}` +// A previewed clip is keyed by conversation + its exact boundaries, so a +// boundary change after re-preview (different clip) naturally resets the +// include/drop decision instead of applying it to the wrong audio. +const clipKey = (cid: string, clip: ExportPreviewClip) => + `${cid}@${clip.start}-${clip.end}` + // 16 kHz mono 16-bit PCM — what exported WAV clips contain (pre-zip). const WAV_BYTES_PER_SECOND = 32000 @@ -80,6 +91,14 @@ export default function ExportModal({ selected, onClose }: Props) { // Flagged segments the user has chosen to withhold (default: all flagged). const [excluded, setExcluded] = useState>(new Set()) + // Contents preview (dry-run): the exact clips the current settings would + // ship, from the same plan computation the export job runs. + const [preview, setPreview] = useState(null) + const [previewing, setPreviewing] = useState(false) + const [previewError, setPreviewError] = useState(null) + // Clips unticked in the preview (keyed by exact boundaries — see clipKey). + const [dropped, setDropped] = useState>(new Set()) + // Run state const [exporting, setExporting] = useState(false) const [status, setStatus] = useState(null) @@ -246,6 +265,21 @@ export default function ExportModal({ selected, onClose }: Props) { return ranges } + // dropped_ranges for the export request: each unticked clip's exact + // [start, end], so the job carves out precisely what the user reviewed away. + const buildDroppedRanges = (): Record => { + const ranges: Record = {} + for (const conv of preview?.conversations ?? []) { + const picked = (conv.clips ?? []).filter((c) => + dropped.has(clipKey(conv.conversation_id, c)) + ) + if (picked.length) { + ranges[conv.conversation_id] = picked.map((c) => [c.start, c.end]) + } + } + return ranges + } + const runExport = async () => { setExporting(true) setError(null) @@ -260,6 +294,7 @@ export default function ExportModal({ selected, onClose }: Props) { speech_threshold: speechThreshold, merge_gap_seconds: mergeGap, excluded_ranges: excludedRanges, + dropped_ranges: buildDroppedRanges(), sensitivity_policy: screenEnabled && Object.keys(excludedRanges).length ? policy : null, } @@ -310,6 +345,78 @@ export default function ExportModal({ selected, onClose }: Props) { const totalFlagged = screenResult?.totals.flagged_segments ?? 0 const totalExcluded = excluded.size + // ── Contents preview ────────────────────────────────────────────────────── + // Auto-refresh (debounced) whenever anything that changes the plan changes: + // selection, mode, clip params, or the privacy-screen withholdings. The + // response is the export job's own plan computation, so what's listed here + // is exactly what the zip would contain. + const screenRanges = screenEnabled && resultValid ? buildExcludedRanges() : {} + const previewSig = JSON.stringify({ + idsSig, + mode, + padSeconds, + speechThreshold, + mergeGap, + screenRanges, + }) + useEffect(() => { + if (selected.length === 0) return + let cancelled = false + setPreviewing(true) + setPreviewError(null) + const timer = setTimeout(() => { + dataAuditApi + .previewExport( + selected.map((c) => c.conversation_id), + { + mode, + pad_seconds: padSeconds, + speech_threshold: speechThreshold, + merge_gap_seconds: mergeGap, + excluded_ranges: screenRanges, + } + ) + .then((res) => { + if (cancelled) return + setPreview(res.data) + setPreviewing(false) + }) + .catch((e) => { + if (cancelled) return + setPreviewError(e?.response?.data?.error || 'Failed to preview export contents') + setPreviewing(false) + }) + }, 400) + return () => { + cancelled = true + clearTimeout(timer) + } + // previewSig captures every input the request uses + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [previewSig]) + + // Live totals for what will actually ship (preview minus unticked clips). + const included = (() => { + let clips = 0 + let seconds = 0 + let droppedClips = 0 + let droppedSeconds = 0 + let unanalyzed = 0 + for (const conv of preview?.conversations ?? []) { + if (conv.skipped_reason === 'not analyzed') unanalyzed += 1 + for (const c of conv.clips ?? []) { + if (dropped.has(clipKey(conv.conversation_id, c))) { + droppedClips += 1 + droppedSeconds += c.duration_seconds + } else { + clips += 1 + seconds += c.duration_seconds + } + } + } + return { clips, seconds, droppedClips, droppedSeconds, unanalyzed } + })() + // Dataset-impact estimate, live-updated as withhold toggles change. // Baseline = what the export would contain without the screen: full mode is // the exact summed duration; clips mode estimates speech via the cached @@ -356,7 +463,7 @@ export default function ExportModal({ selected, onClose }: Props) { onClose={onClose} title="Export for annotation" icon={} - maxWidthClassName="max-w-2xl" + maxWidthClassName="max-w-3xl" className="max-h-[85vh] overflow-y-auto" footer={ + + {formatDuration(clip.duration_seconds)} + + + {clip.text ? ( + + {clip.text} + + ) : ( + + no transcript + + )} + + + ) + })} + + ))} + + + ) +} + /** Review panel: flagged segments grouped by conversation, each a withhold toggle. */ function ScreenReview({ report, diff --git a/backends/advanced/webui/src/components/dataAudit/filters.tsx b/backends/advanced/webui/src/components/dataAudit/filters.tsx index 2bd84ed7b..f472cf01c 100644 --- a/backends/advanced/webui/src/components/dataAudit/filters.tsx +++ b/backends/advanced/webui/src/components/dataAudit/filters.tsx @@ -15,6 +15,7 @@ import { FileArchive, LucideIcon, Mic, + PackageOpen, Search, Users, } from 'lucide-react' @@ -408,6 +409,49 @@ const datasetFilter: FilterDef = { ), } +// --------------------------------------------------------------------------- +// Export history (from the on-disk annotation-export metadata) +// --------------------------------------------------------------------------- + +type ExportedValue = '' | 'never' | 'exported' + +const exportedFilter: FilterDef = { + key: 'exported', + label: 'Export history', + icon: PackageOpen, + defaultValue: '', + isActive: (v) => v !== '', + chipLabel: (v) => (v === 'never' ? 'Not yet exported' : 'Previously exported'), + toParams: (v) => ({ exported: v || undefined }), + Editor: ({ value, onChange }) => ( +
+ {( + [ + { key: '', label: 'All conversations' }, + { key: 'never', label: 'Not yet exported' }, + { key: 'exported', label: 'Previously exported' }, + ] as const + ).map((opt) => ( + + ))} +

+ Whether a previous annotation export shipped the conversation. +

+
+ ), +} + // --------------------------------------------------------------------------- // Hide failed (processing_status == 'failed') // --------------------------------------------------------------------------- @@ -444,6 +488,7 @@ export const AUDIT_FILTERS: FilterDef[] = [ speakersFilter, dateFilter, datasetFilter, + exportedFilter, hideFailedFilter, hideReviewedFilter, ] diff --git a/backends/advanced/webui/src/hooks/useQueue.ts b/backends/advanced/webui/src/hooks/useQueue.ts index 2d8d9c58e..1d2a488bf 100644 --- a/backends/advanced/webui/src/hooks/useQueue.ts +++ b/backends/advanced/webui/src/hooks/useQueue.ts @@ -1,10 +1,15 @@ -import { useQuery, useMutation, useQueryClient } from '@tanstack/react-query' +import { useQuery, useMutation, useQueryClient, keepPreviousData } from '@tanstack/react-query' import { queueApi } from '../services/api' export function useQueueDashboard(expandedSessions: string[]) { return useQuery({ queryKey: ['queue', 'dashboard', expandedSessions], queryFn: () => queueApi.getDashboard(expandedSessions).then(r => r.data), + // Expanding a conversation adds to the key, which would otherwise be a cold + // query and blank the whole page to its loading spinner. Keep rendering the + // previous result while the wider payload loads; `isFetching` still drives + // the Refresh spinner. + placeholderData: keepPreviousData, }) } diff --git a/backends/advanced/webui/src/pages/ConversationDetail.tsx b/backends/advanced/webui/src/pages/ConversationDetail.tsx index 0f2431215..13ebe6ac5 100644 --- a/backends/advanced/webui/src/pages/ConversationDetail.tsx +++ b/backends/advanced/webui/src/pages/ConversationDetail.tsx @@ -4,7 +4,8 @@ import { useQueryClient } from '@tanstack/react-query' import { ArrowLeft, Calendar, User, Trash2, RefreshCw, MoreVertical, RotateCcw, Zap, Download, Scissors, - Save, X, Pencil, Clock, Database, Layers, Star, BarChart3, Hash, AudioLines, ChevronRight + Save, X, Pencil, Clock, Database, Layers, Star, BarChart3, Hash, AudioLines, ChevronRight, + Check, Copy } from 'lucide-react' import { annotationsApi, speakerApi, systemApi, BACKEND_URL } from '../services/api' import { @@ -109,8 +110,18 @@ export default function ConversationDetail() { const [reprocessingMemory, setReprocessingMemory] = useState(false) const [reprocessingSpeakers, setReprocessingSpeakers] = useState(false) const [actionError, setActionError] = useState(null) + const [idCopyStatus, setIdCopyStatus] = useState<'idle' | 'copied' | 'error'>('idle') + const idCopyResetTimer = useRef(null) const toggleStarMutation = useToggleStar() + useEffect(() => { + return () => { + if (idCopyResetTimer.current !== null) { + window.clearTimeout(idCopyResetTimer.current) + } + } + }, []) + const handleToggleStar = async () => { if (!id) return try { @@ -173,6 +184,45 @@ export default function ConversationDetail() { return `${mins}:${secs.toString().padStart(2, '0')}` } + const handleCopyConversationId = async () => { + const conversationId = conversation?.conversation_id + if (!conversationId) return + + let copied = false + if (window.isSecureContext && navigator.clipboard) { + try { + await navigator.clipboard.writeText(conversationId) + copied = true + } catch { + // Fall through to the selection-based copy path below. + } + } + + if (!copied) { + const textArea = document.createElement('textarea') + textArea.value = conversationId + textArea.setAttribute('readonly', '') + textArea.style.position = 'fixed' + textArea.style.left = '-9999px' + document.body.appendChild(textArea) + textArea.select() + + try { + copied = document.execCommand('copy') + } catch { + copied = false + } finally { + textArea.remove() + } + } + + setIdCopyStatus(copied ? 'copied' : 'error') + if (idCopyResetTimer.current !== null) { + window.clearTimeout(idCopyResetTimer.current) + } + idCopyResetTimer.current = window.setTimeout(() => setIdCopyStatus('idle'), 2000) + } + // Action handlers const handleDownloadAudio = async () => { if (!id) return @@ -646,11 +696,43 @@ export default function ConversationDetail() {
diff --git a/backends/advanced/webui/src/pages/Conversations.tsx b/backends/advanced/webui/src/pages/Conversations.tsx index 24dfd788a..e10bcd3cc 100644 --- a/backends/advanced/webui/src/pages/Conversations.tsx +++ b/backends/advanced/webui/src/pages/Conversations.tsx @@ -54,6 +54,7 @@ const isUnknownLabel = (name?: string): boolean => { } const PAGE_SIZE = 20 +const SEARCH_DEBOUNCE_MS = 800 const SORT_OPTIONS = [ { label: 'Date (newest)', sortBy: 'created_at', sortOrder: 'desc' }, @@ -130,8 +131,8 @@ export default function Conversations() { // Search state (regex-only; semantic search was removed for performance reasons) const [searchQuery, setSearchQuery] = useState('') - type SearchField = 'title' | 'summary' | 'speakers' - const allSearchFields: SearchField[] = ['title', 'summary', 'speakers'] + type SearchField = 'id' | 'title' | 'summary' | 'speakers' + const allSearchFields: SearchField[] = ['id', 'title', 'summary', 'speakers'] const [searchFields, setSearchFields] = useState(allSearchFields) const [searchResults, setSearchResults] = useState(null) const [isSearching, setIsSearching] = useState(false) @@ -216,7 +217,10 @@ export default function Conversations() { } setIsSearching(true) - searchTimeoutRef.current = setTimeout(() => runSearch(trimmed, searchFields), 300) + searchTimeoutRef.current = setTimeout( + () => runSearch(trimmed, searchFields), + SEARCH_DEBOUNCE_MS, + ) return () => { if (searchTimeoutRef.current) clearTimeout(searchTimeoutRef.current) @@ -652,7 +656,7 @@ export default function Conversations() { type="text" value={searchQuery} onChange={(e) => setSearchQuery(e.target.value)} - placeholder="Search conversations or people..." + placeholder="Search conversations, IDs, or people..." className="w-full pl-9 pr-9 py-2 rounded-lg border border-gray-300 dark:border-gray-600 bg-white dark:bg-gray-800 text-gray-900 dark:text-gray-100 placeholder-gray-400 focus:outline-none focus:ring-2 focus:ring-blue-500 focus:border-transparent text-sm" /> {searchQuery && ( @@ -684,7 +688,7 @@ export default function Conversations() { - {/* Search fields: Everything mirrors the three individual checkboxes. */} + {/* Search fields: Everything mirrors the individual checkboxes. */}
+ Cleanup Old Sessions + {streamingStatus?.stream_health && Object.keys(streamingStatus.stream_health).length > 0 && ( )} - + })`} +
{/* Stream Workers Section - Shows audio streams + listen jobs */}
-

Stream Workers (Client Sessions)

+ Stream Workers (Client Sessions)
{streamingStatus?.stream_health && Object.entries(streamingStatus.stream_health).map(([streamKey, health]) => { // Extract client_id from stream key (format: audio:stream:{client_id}) @@ -843,34 +837,34 @@ const Queue: React.FC = () => { : []; return ( -
+
- {streamKey} - Active + {streamKey} + Active
- Stream Length: - {health.stream_length} + Stream Length: + {health.stream_length}
- Age: - {(health.stream_age_seconds || 0).toFixed(0)}s + Age: + {(health.stream_age_seconds || 0).toFixed(0)}s
- Pending: - 0 ? 'text-yellow-600' : 'text-green-600'}`}> + Pending: + 0 ? 'text-yellow-600 dark:text-yellow-400' : 'text-green-600 dark:text-green-400'}`}> {health.total_pending}
{health.consumer_groups && health.consumer_groups.map((group) => ( -
-
{group.name}:
+
+
{group.name}:
{(group.consumers || []).map((consumer) => (
- {consumer.name} - 0 ? 'text-yellow-600' : 'text-green-600'}> + {consumer.name} + 0 ? 'text-yellow-600 dark:text-yellow-400' : 'text-green-600 dark:text-green-400'}> {consumer.pending} pending
@@ -880,8 +874,8 @@ const Queue: React.FC = () => { {/* Current Speech Detection Job */} {listenJobs.length > 0 && ( -
-
Current Speech Detection:
+
+
Current Speech Detection:
{listenJobs.map((job) => { const runtime = job.started_at ? Math.floor((Date.now() - new Date(job.started_at).getTime()) / 1000) @@ -890,51 +884,50 @@ const Queue: React.FC = () => { const seconds = runtime % 60; return ( -
+
-
+
{getStatusIcon(job.status)} - {job.job_type} - - {job.status} - + {job.job_type} + {job.status}
- +
{/* Job metadata */} -
+
Job ID: - {job.job_id.substring(0, 12)}... + {job.job_id.substring(0, 12)}...
{job.started_at && (
Runtime: - {minutes}m {seconds}s + {minutes}m {seconds}s
)} {job.created_at && (
Created: - {new Date(job.created_at).toLocaleTimeString()} + {new Date(job.created_at).toLocaleTimeString()}
)} {job.meta?.speech_detected_at && (
Speech Detected: - {new Date(job.meta.speech_detected_at).toLocaleString()} + {new Date(job.meta.speech_detected_at).toLocaleString()}
)} {job.meta?.status && (
Status: - {job.meta.status.replace(/_/g, ' ')} + {job.meta.status.replace(/_/g, ' ')}
)}
@@ -945,30 +938,30 @@ const Queue: React.FC = () => { if (!session) return null; return ( -
-
Speech Detection Events:
+
+
Speech Detection Events:
{session.last_event && (
- Last Event: - {session.last_event.split(':')[0]} + Last Event: + {session.last_event.split(':')[0]}
)} {session.speaker_check_status && (
- Speaker Check: + Speaker Check: {session.speaker_check_status}
)} {session.identified_speakers && (
- Speakers: - {session.identified_speakers} + Speakers: + {session.identified_speakers}
)}
@@ -990,7 +983,7 @@ const Queue: React.FC = () => {
{/* Active Conversations - Grouped by conversation_id */}
-

Active Conversations

+ Active Conversations {(() => { // Group all jobs by conversation_id with deduplication const allJobsRaw = Object.values(conversationJobs).flat().filter(job => job != null); @@ -1037,11 +1030,7 @@ const Queue: React.FC = () => { }); if (conversationMap.size === 0) { - return ( -
- No active conversations -
- ); + return No active conversations; } return ( @@ -1067,45 +1056,45 @@ const Queue: React.FC = () => { const failedJobCount = jobs.filter(j => j.status === 'failed').length; return ( -
+
toggleConversationExpansion(conversationId)} >
{isExpanded ? ( - + ) : ( - + )} {hasFailedJob ? ( - + ) : ( - + )} - {clientId} + {clientId} {hasFailedJob ? ( - + {failedJobCount} Error{failedJobCount > 1 ? 's' : ''} - + ) : ( - Active + Active )} {speakers.length > 0 && ( - + {speakers.length} speaker{speakers.length > 1 ? 's' : ''} - + )}
-
+
Conversation: {conversationId.substring(0, 8)}... • {createdAt && `Started: ${new Date(createdAt).toLocaleTimeString()} • `} Words: {wordCount} {lastUpdate && ` • Updated: ${new Date(lastUpdate).toLocaleTimeString()}`}
{transcript && ( -
+
"{transcript.substring(0, 100)}{transcript.length > 100 ? '...' : ''}"
)} @@ -1136,10 +1125,10 @@ const Queue: React.FC = () => { {/* Expanded Jobs Section */} {isExpanded && ( -
+
{/* Pipeline Timeline */}
-
Pipeline Timeline:
+
Pipeline Timeline:
{(() => { // Helper function to get display name from job type const getJobDisplayName = (jobType: string) => { @@ -1190,7 +1179,7 @@ const Queue: React.FC = () => { const validTimes = jobTimes.filter(t => t !== null); if (validTimes.length === 0) { return ( -
No job timing data available
+
No job timing data available
); } @@ -1216,7 +1205,7 @@ const Queue: React.FC = () => { return (
{/* Time axis */} -
+
{timeMarkers.map((marker, idx) => (
{ style={{ left: `${marker.percent}%`, transform: 'translateX(-50%)' }} >
-
+
{marker.time}
@@ -1253,10 +1242,10 @@ const Queue: React.FC = () => {
{/* Stage Name */} - {name} + {name} {/* Timeline Container */} -
+
{/* Job Bar */}
{
{/* Total Duration */} -
+
Total: {formatDuration(totalDuration)}
@@ -1285,33 +1274,29 @@ const Queue: React.FC = () => { })()}
-
Conversation Jobs:
+
Conversation Jobs:
{jobs.filter(j => j != null && j.job_id).length > 0 ? (
{jobs .filter(j => j != null && j.job_id) .sort((a, b) => new Date(a.created_at).getTime() - new Date(b.created_at).getTime()) .map((job, index) => ( -
+
toggleJobExpansion(job.job_id)} >
- #{index + 1} + #{index + 1} {getJobTypeIcon(job.job_type)} {getStatusIcon(job.status)} - {job.job_type} - - {job.status} - - {job.queue} + {job.job_type} + {job.status} + {job.queue} {/* Show memory count badge on collapsed card */} {!expandedJobs.has(job.job_id) && job.job_type === 'process_memory_job' && job.result?.memories_created !== undefined && ( - - {job.result.memories_created} memories - + {job.result.memories_created} memories )}
@@ -1319,7 +1304,7 @@ const Queue: React.FC = () => { {/* Collapsible metadata section */} {expandedJobs.has(job.job_id) && ( -
+
{job.started_at && ( Started: {new Date(job.started_at).toLocaleTimeString()} @@ -1331,7 +1316,7 @@ const Queue: React.FC = () => { {/* Show job-specific metadata */} {job.meta && ( -
+
{/* open_conversation_job metadata */} {job.job_type === 'open_conversation_job' && ( <> @@ -1345,7 +1330,7 @@ const Queue: React.FC = () => {
Idle: {Math.floor(job.meta.inactivity_seconds)}s
)} {job.meta.transcript && ( -
+
"{job.meta.transcript.substring(0, 80)}..."
)} @@ -1356,10 +1341,10 @@ const Queue: React.FC = () => { {job.job_type === 'transcribe_full_audio_job' && job.status === 'started' && job.meta?.batch_progress && (
- {job.meta.batch_progress.message} - {job.meta.batch_progress.percent}% + {job.meta.batch_progress.message} + {job.meta.batch_progress.percent}%
-
+
@@ -1400,9 +1385,9 @@ const Queue: React.FC = () => { )} {job.meta.memory_details && job.meta.memory_details.length > 0 && (
-
Memories Created:
+
Memories Created:
{job.meta.memory_details.map((memory: any, idx: number) => ( -
+
"{memory.text}"
))} @@ -1413,7 +1398,7 @@ const Queue: React.FC = () => { {/* Show conversation_id if present */} {job.meta.conversation_id && ( -
+
Conv: {job.meta.conversation_id.substring(0, 8)}...
)} @@ -1421,20 +1406,21 @@ const Queue: React.FC = () => { )}
)} - +
))}
) : ( -
No jobs found for this conversation
+
No jobs found for this conversation
)}
)} @@ -1448,23 +1434,24 @@ const Queue: React.FC = () => { {/* Completed Conversations - Grouped by conversation_id */}
-
-

Completed Conversations

+
+ Completed Conversations
- - { setCompletedConvTimeRange(Number(e.target.value)); setCompletedConvPage(1); // Reset to first page }} - className="text-xs border border-gray-300 rounded px-2 py-1" > - +
{(() => { @@ -1513,11 +1500,7 @@ const Queue: React.FC = () => { }); if (conversationMap.size === 0) { - return ( -
- No completed conversations -
- ); + return No completed conversations; } // Convert to array and filter by time range @@ -1545,11 +1528,7 @@ const Queue: React.FC = () => { const paginatedConversations = conversationsArray.slice(startIndex, endIndex); if (conversationsArray.length === 0) { - return ( -
- No completed conversations in the selected time range -
- ); + return No completed conversations in the selected time range; } return ( @@ -1582,25 +1561,25 @@ const Queue: React.FC = () => { const failedJobCount = jobs.filter(j => j.status === 'failed').length; // Determine status styling - let bgColor = 'bg-yellow-50 border-yellow-200'; - let hoverColor = 'hover:bg-yellow-100'; - let iconColor = 'text-yellow-600'; - let statusBadge = 'bg-yellow-100 text-yellow-700'; + let bgColor = 'bg-yellow-50 border-yellow-200 dark:bg-yellow-900/20 dark:border-yellow-800'; + let hoverColor = 'hover:bg-yellow-100 dark:hover:bg-yellow-900/30'; + let iconColor = 'text-yellow-600 dark:text-yellow-400'; + let statusTone: StateTone = 'warning'; let statusText = 'Processing'; let StatusIcon = Clock; if (hasFailedJob) { - bgColor = 'bg-red-50 border-red-300'; - hoverColor = 'hover:bg-red-100'; - iconColor = 'text-red-600'; - statusBadge = 'bg-red-200 text-red-800'; + bgColor = 'bg-red-50 border-red-300 dark:bg-red-900/20 dark:border-red-800'; + hoverColor = 'hover:bg-red-100 dark:hover:bg-red-900/30'; + iconColor = 'text-red-600 dark:text-red-400'; + statusTone = 'danger'; statusText = `${failedJobCount} Error${failedJobCount > 1 ? 's' : ''}`; StatusIcon = AlertTriangle; } else if (allComplete) { - bgColor = 'bg-green-50 border-green-200'; - hoverColor = 'hover:bg-green-100'; - iconColor = 'text-green-600'; - statusBadge = 'bg-green-100 text-green-700'; + bgColor = 'bg-green-50 border-green-200 dark:bg-green-900/20 dark:border-green-800'; + hoverColor = 'hover:bg-green-100 dark:hover:bg-green-900/30'; + iconColor = 'text-green-600 dark:text-green-400'; + statusTone = 'success'; statusText = 'Complete'; StatusIcon = CheckCircle; } @@ -1611,7 +1590,7 @@ const Queue: React.FC = () => { className={`flex items-center justify-between p-3 cursor-pointer transition-colors ${hoverColor}`} onClick={() => toggleConversationExpansion(conversationId)} > -
+
{isExpanded ? ( @@ -1619,17 +1598,15 @@ const Queue: React.FC = () => { )} - {clientId} - - {statusText} - + {clientId} + {statusText} {speakers.length > 0 && ( - + {speakers.length} speaker{speakers.length > 1 ? 's' : ''} - + )}
-
+
Conversation: {conversationId.substring(0, 8)}... • Words: {wordCount} {createdAt && ( @@ -1640,23 +1617,23 @@ const Queue: React.FC = () => { {allComplete ? ( <> {title ? ( -
+
{title}
) : transcript ? ( -
+
"{transcript.substring(0, 100)}{transcript.length > 100 ? '...' : ''}"
) : null} {summary && ( -
+
{summary}
)} ) : ( transcript && ( -
+
"{transcript.substring(0, 100)}{transcript.length > 100 ? '...' : ''}"
) @@ -1666,12 +1643,12 @@ const Queue: React.FC = () => { {/* Expanded Jobs Section */} {isExpanded && ( -
{/* Pipeline Timeline */}
-
Pipeline Timeline:
+
Pipeline Timeline:
{(() => { // Helper function to get display name from job type const getJobDisplayName = (jobType: string) => { @@ -1722,7 +1699,7 @@ const Queue: React.FC = () => { const validTimes = jobTimes.filter(t => t !== null); if (validTimes.length === 0) { return ( -
No job timing data available
+
No job timing data available
); } @@ -1748,7 +1725,7 @@ const Queue: React.FC = () => { return (
{/* Time axis */} -
+
{timeMarkers.map((marker, idx) => (
{ style={{ left: `${marker.percent}%`, transform: 'translateX(-50%)' }} >
-
+
{marker.time}
@@ -1785,10 +1762,10 @@ const Queue: React.FC = () => {
{/* Stage Name */} - {name} + {name} {/* Timeline Container */} -
+
{/* Job Bar */}
{
{/* Total Duration */} -
+
Total: {formatDuration(totalDuration)}
@@ -1819,48 +1796,45 @@ const Queue: React.FC = () => { })()}
-
Conversation Jobs:
+
Conversation Jobs:
{jobs.filter(j => j != null && j.job_id).length > 0 ? (
{jobs .filter(j => j != null && j.job_id) .sort((a, b) => new Date(a.created_at).getTime() - new Date(b.created_at).getTime()) .map((job, index) => ( -
+
toggleJobExpansion(job.job_id)} > - #{index + 1} + #{index + 1} {getJobTypeIcon(job.job_type)} {getStatusIcon(job.status)} - {job.job_type} - - {job.status} - - {job.queue || job.data?.queue || 'unknown'} + {job.job_type} + {job.status} + {job.queue || job.data?.queue || 'unknown'} {/* Show memory count badge on collapsed card */} {!expandedJobs.has(job.job_id) && job.job_type === 'process_memory_job' && job.result?.memories_created !== undefined && ( - - {job.result.memories_created} memories - + {job.result.memories_created} memories )}
- +
{/* Collapsible metadata section */} {expandedJobs.has(job.job_id) && ( -
+
{job.started_at && ( Started: {new Date(job.started_at).toLocaleTimeString()} @@ -1872,7 +1846,7 @@ const Queue: React.FC = () => { {/* Show job-specific metadata */} {job.meta && ( -
+
{/* open_conversation_job metadata */} {job.job_type === 'open_conversation_job' && ( <> @@ -1886,7 +1860,7 @@ const Queue: React.FC = () => {
Idle: {Math.floor(job.meta.inactivity_seconds)}s
)} {job.meta.transcript && ( -
+
"{job.meta.transcript.substring(0, 80)}..."
)} @@ -1931,7 +1905,7 @@ const Queue: React.FC = () => { {/* Show conversation_id if present */} {job.meta.conversation_id && ( -
+
Conv: {job.meta.conversation_id.substring(0, 8)}...
)} @@ -1943,7 +1917,7 @@ const Queue: React.FC = () => { ))}
) : ( -
No jobs found for this conversation
+
No jobs found for this conversation
)}
)} @@ -1954,23 +1928,21 @@ const Queue: React.FC = () => { {/* Pagination Controls */} {totalPages > 1 && ( -
-
+
+
Showing {startIndex + 1}-{Math.min(endIndex, totalConversations)} of {totalConversations} conversations
- + Page {completedConvPage} of {totalPages}
-
+ )} {/* Events */} -
+
{ const next = !eventsExpanded; setEventsExpanded(next); @@ -1999,9 +1971,9 @@ const Queue: React.FC = () => { }} >
- -

Events

- + + Events + {(() => { const includes = Object.entries(eventFilters).filter(([, v]) => v === 'include').map(([k]) => k); const excludes = Object.entries(eventFilters).filter(([, v]) => v === 'exclude').map(([k]) => k); @@ -2016,7 +1988,11 @@ const Queue: React.FC = () => {
{eventsExpanded && events.length > 0 && ( - + )} - {eventsExpanded ? : } + {eventsExpanded + ? + : }
{eventsExpanded && [...new Set(events.map(e => e.event))].sort().length > 0 && ( -
+
{[...new Set(events.map(e => e.event))].sort().map(eventType => { const state = eventFilters[eventType]; return ( @@ -2045,10 +2020,10 @@ const Queue: React.FC = () => { onClick={() => cycleEventFilter(eventType)} className={`inline-flex items-center px-2.5 py-0.5 rounded-full text-xs font-medium border cursor-pointer transition-colors ${ state === 'include' - ? 'bg-blue-100 text-blue-700 border-blue-400' + ? 'bg-blue-100 text-blue-700 border-blue-400 dark:bg-blue-900/40 dark:text-blue-300 dark:border-blue-600' : state === 'exclude' - ? 'bg-red-100 text-red-700 border-red-400 line-through' - : 'bg-gray-100 text-gray-500 border-gray-300' + ? 'bg-red-100 text-red-700 border-red-400 line-through dark:bg-red-900/40 dark:text-red-300 dark:border-red-600' + : 'bg-gray-100 text-gray-500 border-gray-300 dark:bg-gray-700/60 dark:text-gray-400 dark:border-gray-600' }`} > {eventType} @@ -2058,7 +2033,7 @@ const Queue: React.FC = () => { {Object.keys(eventFilters).length > 0 && ( @@ -2079,24 +2054,24 @@ const Queue: React.FC = () => { if (filtered.length === 0) { return ( -
+
No events recorded yet. Events are logged when system actions like conversation.complete, memory.processed, or button presses occur.
); } return ( - - +
+ - - - - - + + + + + - + {filtered.map((evt, idx) => { const pluginsExecuted = evt.plugins_executed || []; // A plugin can intentionally no-op (e.g. wake word armed on a @@ -2108,8 +2083,8 @@ const Queue: React.FC = () => { const allSkipped = pluginsExecuted.length > 0 && ranPlugins.length === 0; return ( - - + - - @@ -2165,15 +2136,18 @@ const Queue: React.FC = () => { })()} )} - + {/* Filters */} -

Filters

-
+
+ Filters +
+
- + setFilters({ ...filters, job_type: e.target.value })} > - - - - - -
- -
- -
@@ -2218,123 +2179,108 @@ const Queue: React.FC = () => { - +
{/* Jobs Table */} -
-
-

Jobs

+ +
+ Jobs {jobs.length > 0 && ( - + )}
-
TimeEventUserPlugins TriggeredStatusTimeEventUserPlugins TriggeredStatus
+
{new Date(evt.timestamp * 1000).toLocaleTimeString()} @@ -2117,42 +2092,38 @@ const Queue: React.FC = () => { {evt.event} + {(evt.user_id || '').length > 12 ? `${evt.user_id.slice(-8)}` : evt.user_id} + {pluginsExecuted.length > 0 ? pluginsExecuted.map(p => p.plugin_id).join(', ') - : none + : none }
{pluginsExecuted.length === 0 ? ( - no plugins ran + no plugins ran ) : allSkipped ? ( - Skipped + Skipped ) : allSuccess ? ( - + OK ) : anyFailure ? ( - + Error ) : ( - partial + partial )} {pluginsExecuted.length > 0 && ( - + )}
- +
+ - - - - - - - + + + + + + + - + {jobs .filter((job) => { if (filters.status && job.status !== filters.status) return false; if (filters.job_type && job.job_type !== filters.job_type) return false; - if (filters.priority && job.meta?.priority !== filters.priority) return false; return true; }) .sort((a, b) => new Date(b.created_at).getTime() - new Date(a.created_at).getTime()).map((job) => ( - - + - - @@ -2346,29 +2292,21 @@ const Queue: React.FC = () => { {/* Pagination */} {pagination.total > pagination.limit && ( -
-
+
+
Showing {pagination.offset + 1} to {Math.min(pagination.offset + pagination.limit, pagination.total)} of {pagination.total} results
- - + +
)} -
+ {/* Old Jobs Table and Pagination - Removed in favor of session-based view above */} {/* Job Details Modal */} @@ -2393,46 +2331,46 @@ const Queue: React.FC = () => {
- -

{selectedJob.job_id}

+ +

{selectedJob.job_id}

- - + + {getStatusIcon(selectedJob.status)} {selectedJob.status.charAt(0).toUpperCase() + selectedJob.status.slice(1)} - +
{selectedJob.description && (
- -

{selectedJob.description}

+ +

{selectedJob.description}

)} {selectedJob.func_name && (
- -

{selectedJob.func_name}

+ +

{selectedJob.func_name}

)}
- -

{selectedJob.created_at ? formatDate(selectedJob.created_at) : '-'}

+ +

{selectedJob.created_at ? formatDate(selectedJob.created_at) : '-'}

- -

{selectedJob.started_at ? formatDate(selectedJob.started_at) : '-'}

+ +

{selectedJob.started_at ? formatDate(selectedJob.started_at) : '-'}

- -

{selectedJob.ended_at ? formatDate(selectedJob.ended_at) : '-'}

+ +

{selectedJob.ended_at ? formatDate(selectedJob.ended_at) : '-'}

{selectedJob.args && selectedJob.args.length > 0 && (
- - 
+                    
+                    
                       {JSON.stringify(selectedJob.args, null, 2)}
@@ -2440,8 +2378,8 @@ const Queue: React.FC = () => { {selectedJob.kwargs && Object.keys(selectedJob.kwargs).length > 0 && (
- - 
+                    
+                    
                       {JSON.stringify(selectedJob.kwargs, null, 2)}
@@ -2449,8 +2387,8 @@ const Queue: React.FC = () => { {selectedJob.error_message && (
- - 
+                    
+                    
                       {selectedJob.error_message}
@@ -2458,8 +2396,8 @@ const Queue: React.FC = () => { {selectedJob.result && (
- - 
+                    
+                    
                       {JSON.stringify(selectedJob.result, null, 2)}
@@ -2468,11 +2406,11 @@ const Queue: React.FC = () => { {/* Formatted Job Metadata - Job-specific displays */} {selectedJob.meta && Object.keys(selectedJob.meta).length > 0 && (
- + {/* open_conversation_job formatted metadata */} {selectedJob.func_name?.includes('open_conversation_job') && ( -
+
{selectedJob.meta.word_count !== undefined && (
Word Count: {selectedJob.meta.word_count} @@ -2506,7 +2444,7 @@ const Queue: React.FC = () => { {selectedJob.meta.transcript && (
Transcript:
-
+
"{selectedJob.meta.transcript}"
@@ -2516,7 +2454,7 @@ const Queue: React.FC = () => { {/* process_memory_job formatted metadata */} {selectedJob.func_name?.includes('process_memory_job') && selectedJob.meta.memory_details && selectedJob.meta.memory_details.length > 0 && ( -
+
Memories Created: {selectedJob.meta.memories_created || selectedJob.meta.memory_details.length}
@@ -2529,7 +2467,7 @@ const Queue: React.FC = () => {
Memory Details:
{selectedJob.meta.memory_details.map((mem: any, idx: number) => ( -
+
{mem.text}
))} @@ -2540,7 +2478,7 @@ const Queue: React.FC = () => { {/* stream_speech_detection_job formatted metadata */} {selectedJob.func_name?.includes('stream_speech_detection_job') && ( -
+
{selectedJob.meta.speech_detected_at && (
Speech Detected At: {new Date(selectedJob.meta.speech_detected_at).toLocaleString()} @@ -2561,7 +2499,7 @@ const Queue: React.FC = () => { {/* transcribe_full_audio_job formatted metadata */} {selectedJob.func_name?.includes('transcribe_full_audio_job') && (selectedJob.meta.title || selectedJob.meta.summary) && ( -
+
{selectedJob.meta.title && (
Title: {selectedJob.meta.title} @@ -2592,10 +2530,10 @@ const Queue: React.FC = () => { {/* Raw JSON metadata (collapsible) */}
- + Raw Metadata JSON - 
+                      
                         {JSON.stringify(selectedJob.meta, null, 2)}
@@ -2623,37 +2561,37 @@ const Queue: React.FC = () => {
- -

{new Date(selectedEvent.timestamp * 1000).toLocaleString()}

+ +

{new Date(selectedEvent.timestamp * 1000).toLocaleString()}

- + {selectedEvent.event}
- -

{selectedEvent.user_id}

+ +

{selectedEvent.user_id}

{selectedEvent.metadata?.client_id && (
- -

{selectedEvent.metadata.client_id}

+ +

{selectedEvent.metadata.client_id}

)}
- +
{(selectedEvent.plugins_executed || []).map((p, i) => { const skipped = !!p.data?.skipped; - const tone = skipped - ? { card: 'bg-gray-50 border-gray-200', badge: 'bg-gray-100 text-gray-600', text: 'text-gray-700', label: 'Skipped' } + const tone: { card: string; badge: StateTone; text: string; label: string } = skipped + ? { card: 'bg-gray-50 border-gray-200 dark:bg-gray-900/40 dark:border-gray-700', badge: 'neutral', text: 'text-gray-700 dark:text-gray-300', label: 'Skipped' } : p.success - ? { card: 'bg-green-50 border-green-200', badge: 'bg-green-100 text-green-700', text: 'text-green-800', label: 'OK' } - : { card: 'bg-red-50 border-red-200', badge: 'bg-red-100 text-red-700', text: 'text-red-800', label: 'Error' }; + ? { card: 'bg-green-50 border-green-200 dark:bg-green-900/20 dark:border-green-800', badge: 'success', text: 'text-green-800 dark:text-green-300', label: 'OK' } + : { card: 'bg-red-50 border-red-200 dark:bg-red-900/20 dark:border-red-800', badge: 'danger', text: 'text-red-800 dark:text-red-300', label: 'Error' }; // Show the plugin's structured output minus the skip flags we // already render via the badge/detail. const { skipped: _s, skip_reason: _r, detail, ...restData } = p.data || {}; @@ -2661,15 +2599,13 @@ const Queue: React.FC = () => {
{skipped - ? + ? : p.success - ? - : + ? + : } - {p.plugin_id} - - {tone.label} - + {p.plugin_id} + {tone.label}
{(p.message || detail) && (

@@ -2677,7 +2613,7 @@ const Queue: React.FC = () => {

)} {Object.keys(restData).length > 0 && ( - 
+                          
                             {JSON.stringify(restData, null, 2)}
                           

                         )}
@@ -2689,10 +2625,10 @@ const Queue: React.FC = () => {
 
               {selectedEvent.metadata && Object.keys(selectedEvent.metadata).length > 0 && (
                 

-                  
+                  
                     Raw Metadata
                   
-                  
+                  
                     {JSON.stringify(selectedEvent.metadata, null, 2)}
                   

                 

@@ -2744,12 +2680,9 @@ const Queue: React.FC = () => {
           }
         >
             

-              

-                

-                  
-                  This will permanently remove jobs from the database
-                

-              

+              }>
+                This will permanently remove jobs from the database
+              
 
               

                 

@@ -2761,14 +2694,15 @@ const Queue: React.FC = () => {
                       onChange={() => setFlushSettings(prev => ({ ...prev, flush_all: false }))}
                       className="text-blue-600"
                     />
-                    Flush old inactive jobs (recommended)
+                    Flush old inactive jobs (recommended)
                   
 
                   {!flushSettings.flush_all && (
                     

                       

-                        
+
DateConversation IDJob IDTypeStatusDurationActionsDateConversation IDJob IDTypeStatusDurationActions
+
{new Date(job.created_at).toLocaleString('en-US', { month: 'short', day: 'numeric', hour: '2-digit', minute: '2-digit' })} -
+
{job.meta?.conversation_id ? job.meta.conversation_id.substring(0, 8) : '—'}
-
+
+
{job.job_id}
-
{getJobTypeShort(job.job_type)}
+ 		+
+ {getJobTypeShort(job.job_type)} +
- + {getStatusIcon(job.status)} {job.status.charAt(0).toUpperCase() + job.status.slice(1)} - + -
+
{formatDuration(job)}
-
+
{job.status === 'failed' && ( - + )} - + {(job.status === 'queued' || job.status === 'started') && ( - + )} {job.status === 'finished' && ( - + )}