From f442ac1c10a81c62b3673ebc8630d5280529c25c Mon Sep 17 00:00:00 2001 From: itisar-345 <143781880+itisar-345@users.noreply.github.com> Date: Sat, 17 Oct 2026 18:23:25 +0530 Subject: [PATCH 1/2] Bound list endpoints, multi-process app, async summaries, a11y fixes --- CHANGELOG.md | 18 +++ backend/app/arcadedb.py | 71 ++++++++- backend/app/auth.py | 38 +++++ backend/app/coordination.py | 24 +++ backend/app/main.py | 3 + backend/app/ratelimit.py | 49 +++++-- backend/app/repositories/_common.py | 22 ++- backend/app/repositories/artifacts.py | 17 ++- backend/app/repositories/graph.py | 29 +++- backend/app/repositories/knowledge_items.py | 56 +++++-- backend/app/repositories/relationships.py | 48 +++++- backend/app/routers/admin.py | 10 +- backend/app/routers/auth.py | 24 ++- backend/app/routers/knowledge.py | 119 +++++++++++++-- backend/app/routers/lineage.py | 6 +- backend/app/routers/models.py | 3 + backend/app/serializers.py | 8 +- backend/app/services/artifact_pipeline.py | 116 +++++++++------ backend/app/services/graphrag.py | 69 +++++++-- backend/app/services/lineage.py | 19 ++- backend/app/services/llm_catalog.py | 25 +++- backend/tests/conftest.py | 2 + backend/tests/test_connectors.py | 2 +- backend/tests/test_frontend_contract.py | 5 +- backend/tests/test_graphrag.py | 38 ++++- backend/tests/test_models.py | 24 +++ backend/tests/test_paging.py | 137 ++++++++++++++++++ backend/tests/test_scaling.py | 132 +++++++++++++++++ backend/tests/test_security.py | 62 ++++++++ deploy/.env.example | 5 + deploy/docker-compose.yml | 5 + docs/api.md | 10 +- docs/configuration.md | 2 + docs/deployment.md | 9 +- docs/known-limitations.md | 3 + docs/security.md | 4 +- frontend/src/app/app.routes.ts | 6 +- frontend/src/app/models/api.ts | 36 ++++- .../admin-users/admin-users.component.html | 16 +- .../connectors/connectors.component.html | 10 +- .../pages/decisions/decisions.component.html | 14 +- .../pages/decisions/decisions.component.ts | 15 +- .../pages/graphrag/graphrag.component.html | 6 +- .../app/pages/groups/groups.component.html | 18 +-- .../knowledge-detail.component.html | 6 +- .../knowledge-detail.component.ts | 47 +++--- .../pages/knowledge/knowledge.component.html | 48 +++--- .../pages/knowledge/knowledge.component.ts | 59 ++++++-- .../src/app/pages/login/login.component.css | 2 +- .../src/app/pages/login/login.component.html | 7 +- .../src/app/pages/login/login.component.ts | 2 + .../model-manager/model-manager.component.css | 2 +- .../model-manager.component.html | 4 +- .../onboarding/onboarding.component.html | 18 ++- .../onboarding/onboarding.component.spec.ts | 69 +++++++++ .../pages/onboarding/onboarding.component.ts | 16 +- .../pages/playbooks/playbooks.component.html | 8 +- .../pages/playbooks/playbooks.component.ts | 7 +- .../app/pages/review/review.component.html | 8 +- .../src/app/pages/review/review.component.ts | 8 +- .../app/pages/search/search.component.html | 30 ++-- frontend/src/app/services/auth.guard.ts | 9 ++ frontend/src/app/services/auth.service.ts | 4 +- .../src/app/services/space.service.spec.ts | 49 +++++++ frontend/src/app/services/space.service.ts | 4 +- frontend/src/app/services/usernames.ts | 4 + frontend/src/styles.css | 41 +++--- 67 files changed, 1519 insertions(+), 268 deletions(-) create mode 100644 backend/tests/test_paging.py create mode 100644 frontend/src/app/pages/onboarding/onboarding.component.spec.ts create mode 100644 frontend/src/app/services/space.service.spec.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index 07a4034..a305c53 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,7 +18,16 @@ Notable changes to Knowledge Hubs. The format follows [Keep a Changelog](https:/ - `API_DOCS`: `false` hides `/docs`, `/redoc` and `/openapi.json`; the production stack sets it. - [`deploy/backup.sh`](deploy/backup.sh): a consistent ArcadeDB backup copied to `deploy/backups/`, with old copies pruned. The deployment guide describes restoring. +- `GET /knowledge/items` (filtered, paged items) and `GET /knowledge/items/{id}/context` (what an item's page shows). +- GraphRAG answers whose model names its sources without writing their ids get those sources inferred from the wording, marked "matched". + ### Changed +- **Lists are bounded.** `GET /knowledge` returns a page (default 100) with `totals`, filters items on the server, and leaves out document text unless `include_content=true`; the review queue, decision register, graph and search have limits too. On 3,000 items the Hub's response was 4 MB and, under load, ran ArcadeDB out of memory. The pages show **Load more**. +- **Ingestion no longer waits for the summary.** The LLM summary of a new source is built after the response (about 30 s on a CPU). +- **App instances run several server processes** (`APP_WORKERS`, default 2), and bcrypt runs on a worker thread. +- The Ollama status behind the public model catalog and sign-in is cached for 5 seconds (`OLLAMA_STATUS_CACHE_SECONDS`). +- **Passwords:** at least 8 characters (was 6), not a common password, not the username. Existing passwords keep working. +- Accessibility: one `

` per page, labelled filters, a group role for the graph, underlined links in text, and text colours with at least 4.5:1 contrast in both themes. - **License: GPL-3.0 → AGPL-3.0.** - Angular 17 → 21. Building the frontend needs Node 20.19+ or 22.12+; the Docker images use Node 22. - Backend dependencies upgraded to current releases, including FastAPI 0.142 (Starlette 1.7), Pydantic 2.13, sentence-transformers 6, NumPy 2 and Celery 5.6. @@ -27,6 +36,15 @@ Notable changes to Knowledge Hubs. The format follows [Keep a Changelog](https:/ - Docker base images: Caddy 2.10, Node 22. ### Fixed +- A write that collided with another on the same ArcadeDB storage page failed outright ("Concurrent modification… Please retry"): a 3,000-item import was lost when a summary was saved meanwhile. Ingest, embedding and summary writes now run again on such a conflict (only the writes; extraction isn't repeated). +- Finding the relationships for a page scanned every edge in the space: about a second for the Hub on 6,000 items. They're now found by walking out from the page's nodes (Hub 2.4 s → 0.2 s, graph 1.3 s → 0.16 s). +- Signing in with an unknown username answered faster than with a known one, which told them apart. +- Source results in search couldn't be opened. +- First run: the onboarding page had no setup-token field, so with `SETUP_TOKEN` set the first account couldn't be created there. It now asks for the token, as the sign-in page does. +- Switching spaces while on the Hub kept showing the previous space's knowledge until a reload (the switch navigated through `/`, which redirects to the Hub itself). +- A group, playbook or other record created a moment ago could be missing from the next list in the clustered stack: reads went to a follower that hadn't applied the write yet. The app now sends its work to the cluster's leader. +- With Redis down, sign-in took up to 15 seconds and sometimes answered 503 "Ollama is not running". The rate limiter and the health probe now give Redis 2 seconds and then leave it alone for 30. +- Members opening `/admin/users` saw the account form with an error; they're now sent to the Hub. - URL fetching could be redirected to a private address by DNS rebinding. The address is now checked again when connecting, and the connection goes to exactly that address. - The production frontend build no longer loads its stylesheet through an inline `onload` handler (`inlineCritical` off), which the Content Security Policy would block. - Reloading a page whose path is also an API path (such as `/knowledge`) made the app show "Failed to load knowledge": the browser reused the cached page for the app's data request. The web server now sends `Vary: Accept`. diff --git a/backend/app/arcadedb.py b/backend/app/arcadedb.py index 064626e..24a0fb0 100644 --- a/backend/app/arcadedb.py +++ b/backend/app/arcadedb.py @@ -9,7 +9,10 @@ import asyncio import logging -from typing import Any, Dict, List, Optional, Tuple +import random +import time +from typing import Any, Awaitable, Callable, Dict, List, Optional, Tuple, TypeVar +from urllib.parse import urlsplit import httpx @@ -49,12 +52,20 @@ class ArcadeClient: election). Timeouts and dropped connections are NOT retried elsewhere: the statement may have been applied, and running it twice could duplicate a write. A transaction stays on the node where it began. + + With several nodes the client prefers the current leader (looked up at + most every LEADER_REFRESH seconds). A follower can answer a read before it + has applied a write just made through it, so a group created a moment ago + could be missing from the next list; the leader has applied everything it + acknowledged. ArcadeDB's linearizable reads would fix that too, but cost + about three seconds per read that follows a write. """ # A dead node must cost seconds, not the whole request timeout: connecting is # quick or it isn't happening. Queries themselves may legitimately run longer. CONNECT_TIMEOUT = 3.0 PROBE_TIMEOUT = 2.0 + LEADER_REFRESH = 10.0 def __init__(self, url: str, database: str, user: str, password: str, timeout: float = 60) -> None: self.database = database @@ -65,6 +76,7 @@ def __init__(self, url: str, database: str, user: str, password: str, timeout: f timeout=httpx.Timeout(timeout, connect=self.CONNECT_TIMEOUT)) self._preferred = 0 self._session_nodes: Dict[str, str] = {} # transaction id -> node it lives on + self._leader_checked = float("-inf") @staticmethod def _error(response: httpx.Response) -> ArcadeDBError: @@ -87,9 +99,33 @@ def _order(self) -> List[int]: n = len(self.nodes) return [(self._preferred + i) % n for i in range(n)] + async def _leader_address(self) -> Optional[str]: + """The leader's HTTP "host:port", as the cluster reports it; None if unknown.""" + for node in (self.nodes[i] for i in self._order()): + try: + response = await self.http.get(f"{node}/server", params={"mode": "cluster"}, + timeout=self.PROBE_TIMEOUT) + except httpx.HTTPError: + continue + if response.status_code < 400: + return (response.json().get("ha") or {}).get("leaderAddress") + return None + + async def _follow_leader(self) -> None: + if len(self.nodes) < 2 or time.monotonic() - self._leader_checked < self.LEADER_REFRESH: + return + self._leader_checked = time.monotonic() + address = (await self._leader_address() or "").lower() + for index, node in enumerate(self.nodes): + if address and urlsplit(node).netloc.lower() == address: + self._preferred = index + return + async def _post(self, path: str, body: Optional[Record] = None, session_id: Optional[str] = None) -> Tuple[httpx.Response, str]: headers = {"arcadedb-session-id": session_id} if session_id else None + if not session_id: + await self._follow_leader() if session_id: node = self._session_nodes.get(session_id, self.nodes[self._preferred]) try: @@ -229,9 +265,19 @@ async def aclose(self) -> None: await self.http.aclose() +T = TypeVar("T") + + +def is_write_conflict(exc: Exception) -> bool: + """ArcadeDB's "someone else changed this page first; retry the operation".""" + return isinstance(exc, ArcadeDBError) and "ConcurrentModification" in f"{exc.exception} {exc.detail}" + + class ArcadeSession: """One per request. Use query() for reads and execute() for writes.""" + WRITE_ATTEMPTS = 4 + def __init__(self, client: ArcadeClient) -> None: self.client = client self._session_id: Optional[str] = None @@ -263,6 +309,29 @@ async def commit(self) -> None: session_id, self._session_id = self._session_id, None await self.client.commit(session_id) + async def transaction(self, work: Callable[[], Awaitable[T]]) -> T: + """ + Run `work` (writes, no commit of its own) and commit it, running it again + after a write conflict. ArcadeDB checks concurrency per storage page, so two + transactions writing different records on the same page conflict and the one + that commits second is told to retry: a long import failed that way when a + summary was saved meanwhile. Retried only when no transaction was already + open here, since rolling back would also undo the writes made before it. + """ + retryable = self._session_id is None + for attempt in range(1, self.WRITE_ATTEMPTS + 1): + try: + result = await work() + await self.commit() + return result + except ArcadeDBError as exc: + await self.rollback() + if not (retryable and is_write_conflict(exc)) or attempt == self.WRITE_ATTEMPTS: + raise + logger.warning("Write conflict, retrying (%d of %d)", attempt + 1, self.WRITE_ATTEMPTS) + await asyncio.sleep(0.05 * 2 ** attempt + random.uniform(0, 0.1)) + raise AssertionError("unreachable") + async def rollback(self) -> None: if self._session_id is not None: session_id, self._session_id = self._session_id, None diff --git a/backend/app/auth.py b/backend/app/auth.py index a1637eb..fea27a7 100644 --- a/backend/app/auth.py +++ b/backend/app/auth.py @@ -53,6 +53,44 @@ def hash_password(password: str) -> str: return bcrypt.hashpw(_bcrypt_input(password), bcrypt.gensalt()).decode("ascii") +# New passwords only; existing ones keep working. Short of a full breached-password +# check, refuse the handful that guessing tools try first. +PASSWORD_MIN_LENGTH = 8 +PASSWORD_RULES = f"at least {PASSWORD_MIN_LENGTH} characters, not a common password and not your username" +_COMMON_PASSWORDS = frozenset(""" +password password1 password12 password123 passw0rd p@ssw0rd p@ssword 12345678 123456789 1234567890 +12341234 11111111 00000000 87654321 qwertyui qwerty123 qwertyuiop 1q2w3e4r 1qaz2wsx asdfghjk +iloveyou letmein1 welcome1 welcome123 admin123 administrator changeme changeme1 football baseball +sunshine princess superman starwars trustno1 whatever monkey123 dragon123 abcd1234 abc12345 +""".split()) + + +def check_new_password(password: str, username: str = "") -> str: + """Validate a password for a new account. Raises ValueError (a 422 from Pydantic validators).""" + if len(password) < PASSWORD_MIN_LENGTH: + raise ValueError(f"Password must be {PASSWORD_RULES}") + lowered = password.lower() + if lowered in _COMMON_PASSWORDS or len(set(password)) == 1: + raise ValueError("That password is too common. Choose a less predictable one.") + if username and lowered == username.strip().lower(): + raise ValueError("The password can't be the same as the username.") + return password + + +# bcrypt of a random value, checked when the username doesn't exist, so that answering +# "Invalid credentials" takes as long either way: the time no longer says whether a +# name is registered (it was 15 ms for an unknown name against 250 ms for a real one). +_DUMMY_HASH: Optional[str] = None + + +def burn_password_check(plain: str) -> None: + global _DUMMY_HASH + if _DUMMY_HASH is None: + import secrets + _DUMMY_HASH = hash_password(secrets.token_urlsafe(16)) + verify_password(plain, _DUMMY_HASH) + + def verify_password(plain: str, hashed: str) -> bool: try: return bcrypt.checkpw(_bcrypt_input(plain), hashed.encode("ascii")) diff --git a/backend/app/coordination.py b/backend/app/coordination.py index 67033dc..05f8b1d 100644 --- a/backend/app/coordination.py +++ b/backend/app/coordination.py @@ -32,6 +32,13 @@ # Redis being down must fail fast, never hang: /health would otherwise exceed the # load balancer's check and take every app instance out of rotation. REDIS_TIMEOUT = 2.0 +# After Redis fails, nothing that can do without it (the health probe, the sign-in +# rate limiter) dials it again for this long. With the Redis container gone, even +# resolving its name hangs in the event loop's shared resolver threads; probing it +# every few seconds piles those lookups up until unrelated ones, such as Ollama's +# host name, time out too. +REDIS_BACKOFF = 30.0 +_redis_down_until: Dict[str, float] = {} # REDIS_URL -> monotonic time to try again # compare-and-delete / compare-and-extend, so a lock is only touched by its holder _RELEASE = "if redis.call('get', KEYS[1]) == ARGV[1] then return redis.call('del', KEYS[1]) else return 0 end" @@ -139,13 +146,30 @@ async def sync_running(connector_id: str) -> bool: return await is_held(sync_lock(connector_id)) +def redis_backing_off() -> bool: + """True while Redis recently failed and optional callers should not dial it.""" + return worker_mode() and time.monotonic() < _redis_down_until.get(redis_url() or "", 0.0) + + +def note_redis_failure() -> None: + if worker_mode(): + _redis_down_until[redis_url() or ""] = time.monotonic() + REDIS_BACKOFF + + +def reset_redis_backoff() -> None: + _redis_down_until.clear() + + async def ping() -> Optional[bool]: """Redis reachability for /health: None in in-process mode.""" if not worker_mode(): return None + if redis_backing_off(): + return False try: return bool(await asyncio.wait_for(_redis().ping(), timeout=REDIS_TIMEOUT)) except Exception: + note_redis_failure() return False diff --git a/backend/app/main.py b/backend/app/main.py index c10ba36..2e192ea 100644 --- a/backend/app/main.py +++ b/backend/app/main.py @@ -29,6 +29,9 @@ @asynccontextmanager async def lifespan(app: FastAPI) -> AsyncIterator[None]: check_secret_key() # refuse to run with a forgeable JWT signing key + if int(os.getenv("WEB_CONCURRENCY", "1") or 1) > 1 and not coordination.worker_mode(): + # without Redis, locks, sign-in throttling and the clean-up live in each process's memory + raise RuntimeError("WEB_CONCURRENCY > 1 needs REDIS_URL (worker mode); see docs/deployment.md") # vector indexes are sized to the active embedding model embedder = get_embedding_provider() await init_db(embedder.dimensions, embedder.name) diff --git a/backend/app/ratelimit.py b/backend/app/ratelimit.py index 5194a95..1db3559 100644 --- a/backend/app/ratelimit.py +++ b/backend/app/ratelimit.py @@ -13,17 +13,21 @@ If Redis is unreachable the limiter lets requests through (and logs it): sign-in must keep working while the queue is down, as everything else does. +Each operation gets REDIS_TIMEOUT in all, and after a failure Redis is left +alone for coordination.REDIS_BACKOFF seconds: with the Redis container gone, +even resolving its name can take seconds, and every sign-in would pay that. AUTH_RATE_LIMIT=false turns it off (the test suite does; its own tests turn it back on). """ from __future__ import annotations +import asyncio import logging import math import os import time -from typing import Dict, Tuple +from typing import Awaitable, Callable, Dict, Optional, Tuple, TypeVar from fastapi import HTTPException, Request @@ -40,6 +44,8 @@ _local: Dict[str, Tuple[int, float]] = {} # key -> (count, window ends at) _LOCAL_MAX_KEYS = 10_000 +T = TypeVar("T") + def enabled() -> bool: return os.getenv("AUTH_RATE_LIMIT", "true").strip().lower() not in ("false", "0", "no", "off") @@ -52,8 +58,27 @@ def client_address(request: Request) -> str: def reset() -> None: - """Forget every in-memory counter (tests).""" + """Forget every in-memory counter and any Redis back-off (tests).""" _local.clear() + coordination.reset_redis_backoff() + + +async def _guarded(op: Callable[[], Awaitable[T]], what: str) -> Optional[T]: + """ + Run op() within REDIS_TIMEOUT. If Redis fails (or is in back-off), fail open: + return None, so the request goes ahead unthrottled. A 429 passes through. + """ + if coordination.redis_backing_off(): + return None + try: + return await asyncio.wait_for(op(), coordination.REDIS_TIMEOUT) + except HTTPException: + raise + except Exception as exc: + coordination.note_redis_failure() + logger.warning("Rate limiter unavailable (%s), not enforcing it for %.0fs: %r", + what, coordination.REDIS_BACKOFF, exc) + return None def _prune(now: float) -> None: @@ -111,34 +136,29 @@ async def check(*limits: Tuple[str, int]) -> None: """Raise 429 if any (key, limit) has already been reached.""" if not enabled(): return - try: + + async def op() -> None: for key, limit in limits: count, retry_after = await _count(key) if count >= limit: raise _too_many(retry_after) - except HTTPException: - raise - except Exception as exc: # Redis down: fail open - logger.warning("Rate limiter unavailable, allowing the request: %s", exc) + await _guarded(op, "check") async def hit(*keys: str, window: int) -> None: if not enabled(): return - try: + + async def op() -> None: for key in keys: await _hit(key, window) - except Exception as exc: - logger.warning("Rate limiter unavailable, attempt not counted: %s", exc) + await _guarded(op, "count") async def clear(key: str) -> None: if not enabled(): return - try: - await _clear(key) - except Exception as exc: - logger.warning("Rate limiter unavailable, counter not cleared: %s", exc) + await _guarded(lambda: _clear(key), "clear") def login_keys(request: Request, username_key: str) -> Tuple[str, str]: @@ -148,4 +168,3 @@ def login_keys(request: Request, username_key: str) -> Tuple[str, str]: def register_key(request: Request) -> str: return f"register:{client_address(request)}" - diff --git a/backend/app/repositories/_common.py b/backend/app/repositories/_common.py index bf06117..ca9a334 100644 --- a/backend/app/repositories/_common.py +++ b/backend/app/repositories/_common.py @@ -1,9 +1,29 @@ from __future__ import annotations -from typing import Any, Dict, Iterable +from typing import Any, Dict, Iterable, Optional def tag_fields(tags: Iterable[str] | None) -> Dict[str, Any]: """tags plus the derived fields used for full-text search and case-insensitive tag filters.""" tags = [str(t) for t in (tags or [])] return {"tags": tags, "tags_text": " ".join(tags), "tags_lc": [t.lower() for t in tags]} + + +def page(sql: str, *, newest_first: bool, limit: Optional[int], offset: int = 0, newest_by: str = "created_at") -> str: + """ + Order and page. newest_first sorts by the `newest_by` timestamp, then id (so pages + never overlap); otherwise by @rid as before. Not by @rid for "newest": ArcadeDB + spreads a type's records over several buckets, so @rid order isn't insertion order. + limit/offset are ints, validated by the caller. + """ + sql += f" ORDER BY {newest_by} DESC, id DESC" if newest_first else " ORDER BY @rid" + if offset: + sql += f" SKIP {int(offset)}" + if limit is not None: + sql += f" LIMIT {int(limit)}" + return sql + + +def contains_pattern(q: str) -> str: + """A case-insensitive LIKE pattern for "contains q" (the value is bound, not inlined).""" + return f"%{q.strip().lower()}%" diff --git a/backend/app/repositories/artifacts.py b/backend/app/repositories/artifacts.py index 0695d39..5fb118d 100644 --- a/backend/app/repositories/artifacts.py +++ b/backend/app/repositories/artifacts.py @@ -3,13 +3,16 @@ from typing import Any, Dict, List, Optional from app.arcadedb import ArcadeSession -from app.repositories._common import tag_fields +from app.repositories._common import page, tag_fields Row = Dict[str, Any] # everything except the (large) vectors _FIELDS = ("id, user_id, title, content, source, source_type, author, tags, extraction_engine, " "created_at, metadata, summary") +# for lists: without the document text, which can be megabytes per space +_LIST_FIELDS = ("id, user_id, title, source, source_type, author, tags, extraction_engine, " + "created_at, metadata") class ArtifactRepository: @@ -21,13 +24,19 @@ async def get_owned(self, user_id: str, artifact_id: str) -> Optional[Row]: f"SELECT {_FIELDS} FROM Artifact WHERE id = :id AND user_id = :u", {"id": artifact_id, "u": user_id} ) - async def list(self, user_id: str, source_type: Optional[str] = None) -> List[Row]: - sql = f"SELECT {_FIELDS} FROM Artifact WHERE user_id = :u" + async def list( + self, user_id: str, source_type: Optional[str] = None, *, with_content: bool = True, + limit: Optional[int] = None, offset: int = 0, newest_first: bool = False, + ) -> List[Row]: + sql = f"SELECT {_FIELDS if with_content else _LIST_FIELDS} FROM Artifact WHERE user_id = :u" params: Row = {"u": user_id} if source_type: sql += " AND source_type = :st" params["st"] = source_type - return await self.session.query(sql + " ORDER BY @rid", params) + return await self.session.query(page(sql, newest_first=newest_first, limit=limit, offset=offset), params) + + async def count(self, user_id: str) -> int: + return await self.session.scalar("SELECT count(*) AS n FROM Artifact WHERE user_id = :u", {"u": user_id}, 0) async def ids_with_source_type(self, user_id: str, source_type: str) -> List[str]: rows = await self.session.query( diff --git a/backend/app/repositories/graph.py b/backend/app/repositories/graph.py index 3acb2f0..f76409a 100644 --- a/backend/app/repositories/graph.py +++ b/backend/app/repositories/graph.py @@ -12,7 +12,7 @@ """ from __future__ import annotations -from typing import Any, Dict, List +from typing import Any, Dict, List, Optional from app.arcadedb import ArcadeSession from app.repositories.search import ITEM_INDEX, lucene_query @@ -134,19 +134,32 @@ async def retrieve_for_rag(self, user_id: str, vector: List[float], top_k: int = combined.sort(key=lambda n: n.get("score") or 0.0, reverse=True) return combined - async def visualization(self, user_id: str) -> Row: + async def visualization(self, user_id: str, limit: Optional[int] = None) -> Row: + """ + The newest `limit` nodes (artifacts and items) and the edges between them. + A browser can't usefully draw thousands of nodes, and sending them all + cost seconds per page load on a large space. + """ + from app.repositories._common import page from app.repositories.relationships import RelationshipRepository + total = await self.session.scalar("SELECT count(*) AS n FROM Node WHERE user_id = :u", {"u": user_id}, 0) nodes = [ {"id": r["id"], "label": r.get("title") or r["id"], "type": "artifact" if r["node_type"] == "Artifact" else r.get("type")} for r in await self.session.query( - "SELECT id, title, type, @type AS node_type FROM Node WHERE user_id = :u ORDER BY @rid", + page("SELECT id, title, type, @type AS node_type, ifnull(created_at, date) AS sort_at " + "FROM Node WHERE user_id = :u", + newest_first=limit is not None, limit=limit, newest_by="sort_at"), {"u": user_id}, ) ] - edges = [ - {"source": e["from"], "target": e["to"], "label": e["type"]} - for e in await RelationshipRepository(self.session).list(user_id) - ] - return {"nodes": nodes, "edges": edges, "layout": "force-directed"} + rels = RelationshipRepository(self.session) + if limit is None: + found = await rels.list(user_id) + else: + found = await rels.among(user_id, [n["id"] for n in nodes if n["type"] != "artifact"], + [n["id"] for n in nodes if n["type"] == "artifact"]) + edges = [{"source": e["from"], "target": e["to"], "label": e["type"]} for e in found] + return {"nodes": nodes, "edges": edges, "layout": "force-directed", + "total_nodes": total, "truncated": len(nodes) < total} diff --git a/backend/app/repositories/knowledge_items.py b/backend/app/repositories/knowledge_items.py index 26612fd..4508ed1 100644 --- a/backend/app/repositories/knowledge_items.py +++ b/backend/app/repositories/knowledge_items.py @@ -3,7 +3,7 @@ from typing import Any, Dict, List, Optional from app.arcadedb import ArcadeSession -from app.repositories._common import tag_fields +from app.repositories._common import contains_pattern, page, tag_fields Row = Dict[str, Any] @@ -21,27 +21,59 @@ async def get_owned(self, user_id: str, item_id: str) -> Optional[Row]: f"SELECT {_FIELDS} FROM KnowledgeItem WHERE id = :id AND user_id = :u", {"id": item_id, "u": user_id} ) + @staticmethod + def _where( + user_id: str, *, type: Optional[str] = None, types: Optional[List[str]] = None, + review_status: Optional[str] = None, exclude_rejected: bool = False, q: Optional[str] = None, + ) -> tuple[str, Row]: + sql, params = "user_id = :u", {"u": user_id} + if type: + sql += " AND type = :type" + params["type"] = type + if types: + sql += " AND type IN :types" + params["types"] = list(types) + if review_status: + sql += " AND review_status = :rs" + params["rs"] = review_status + if exclude_rejected: + sql += " AND review_status <> 'rejected'" + if q and q.strip(): + sql += " AND (title.toLowerCase() LIKE :q OR tags_text.toLowerCase() LIKE :q)" + params["q"] = contains_pattern(q) + return sql, params + async def list( self, user_id: str, *, type: Optional[str] = None, + types: Optional[List[str]] = None, review_status: Optional[str] = None, exclude_rejected: bool = False, + q: Optional[str] = None, with_embedding: bool = False, + limit: Optional[int] = None, + offset: int = 0, + newest_first: bool = False, ) -> List[Row]: fields = _FIELDS + (", embedding" if with_embedding else "") - sql = f"SELECT {fields} FROM KnowledgeItem WHERE user_id = :u" - params: Row = {"u": user_id} - if type: - sql += " AND type = :type" - params["type"] = type - if review_status: - sql += " AND review_status = :rs" - params["rs"] = review_status - if exclude_rejected: - sql += " AND review_status <> 'rejected'" - return await self.session.query(sql + " ORDER BY @rid", params) + where, params = self._where(user_id, type=type, types=types, review_status=review_status, + exclude_rejected=exclude_rejected, q=q) + sql = page(f"SELECT {fields} FROM KnowledgeItem WHERE {where}", + newest_first=newest_first, limit=limit, offset=offset, newest_by="date") + return await self.session.query(sql, params) + + async def get_many(self, user_id: str, ids: List[str]) -> List[Row]: + if not ids: + return [] + return await self.session.query( + f"SELECT {_FIELDS} FROM KnowledgeItem WHERE user_id = :u AND id IN :ids", {"u": user_id, "ids": list(ids)} + ) + + async def count(self, user_id: str, **filters: Any) -> int: + where, params = self._where(user_id, **filters) + return await self.session.scalar(f"SELECT count(*) AS n FROM KnowledgeItem WHERE {where}", params, 0) async def count_embedded(self, user_id: str) -> int: return await self.session.scalar( diff --git a/backend/app/repositories/relationships.py b/backend/app/repositories/relationships.py index 0a1db00..580053d 100644 --- a/backend/app/repositories/relationships.py +++ b/backend/app/repositories/relationships.py @@ -7,18 +7,64 @@ """ from __future__ import annotations -from typing import Any, Dict, List +from typing import Any, Dict, Iterable, List, Optional from app.arcadedb import ArcadeSession Row = Dict[str, Any] +EDGE_TYPES = ("CONTAINS", "RELATED_TO", "LINK", "EVOLVES") + class RelationshipRepository: def __init__(self, session: ArcadeSession) -> None: self.session = session + async def touching( + self, user_id: str, ids: Iterable[str], edge_types: Iterable[str] = EDGE_TYPES, + ) -> List[Row]: + """ + The edges at the given nodes, found by walking out from them (the id index finds + the nodes). Filtering every edge of the space on its ends instead took a second + on 6,000 items. An artifact can hold thousands of items, so pass item ids, and + artifacts only with edge_types=("LINK",), unless all of its CONTAINS edges are wanted. + """ + ids = list(ids) + types = [t for t in edge_types if t in EDGE_TYPES] + if not ids or not types: + return [] + rows = await self.session.query( + "SELECT @type AS edge_type, @out.id AS src, @in.id AS dst, type AS link_type, kind " + f"FROM (SELECT expand(bothE({', '.join(repr(t) for t in types)})) FROM Node " + "WHERE id IN :ids AND user_id = :u)", + {"u": user_id, "ids": ids}, + ) + rels: List[Row] = [] + seen = set() + for r in rows: + edge_type = r["edge_type"] + label = {"LINK": r.get("link_type") or "", "EVOLVES": (r.get("kind") or "").upper()}.get(edge_type, edge_type) + key = (r["src"], r["dst"], edge_type, label) + if key not in seen: # an edge between two of the nodes is reached from both ends + seen.add(key) + rels.append({"from": r["src"], "to": r["dst"], "type": label}) + return rels + + async def among(self, user_id: str, item_ids: Iterable[str], artifact_ids: Iterable[str]) -> List[Row]: + """The edges between the given items and artifacts (both ends among them).""" + item_ids, artifact_ids = list(item_ids), list(artifact_ids) + shown = set(item_ids) | set(artifact_ids) + found = await self.touching(user_id, item_ids) + await self.touching(user_id, artifact_ids, ("LINK",)) + out, seen = [], set() + for r in found: + key = (r["from"], r["to"], r["type"]) + if r["from"] in shown and r["to"] in shown and key not in seen: + seen.add(key) + out.append(r) + return out + async def list(self, user_id: str) -> List[Row]: + """Every edge in the space.""" rels: List[Row] = [] for edge_type, label in (("CONTAINS", "'CONTAINS'"), ("RELATED_TO", "'RELATED_TO'"), ("LINK", "type"), ("EVOLVES", "kind")): diff --git a/backend/app/routers/admin.py b/backend/app/routers/admin.py index c8bf95b..75c4475 100644 --- a/backend/app/routers/admin.py +++ b/backend/app/routers/admin.py @@ -4,7 +4,7 @@ from typing import Any, Dict, List from fastapi import APIRouter, Depends, HTTPException -from pydantic import BaseModel, Field, field_validator +from pydantic import BaseModel, Field, field_validator, model_validator from app.arcadedb import ArcadeSession from app.db import User, get_session @@ -13,6 +13,7 @@ from app.routers.auth import create_user from app import activity from app.services import retention +from app.auth import check_new_password from app.usernames import check_new_username router = APIRouter(prefix="/admin", tags=["admin"]) @@ -22,7 +23,7 @@ class CreateUserRequest(BaseModel): username: str = Field(min_length=3, max_length=60) - password: str = Field(min_length=6) + password: str = Field(min_length=1, max_length=256) role: Role = Role.MEMBER @field_validator("username") @@ -30,6 +31,11 @@ class CreateUserRequest(BaseModel): def _valid_username(cls, value: str) -> str: return check_new_username(value) + @model_validator(mode="after") + def _valid_password(self) -> "CreateUserRequest": + check_new_password(self.password, self.username) + return self + class RoleUpdate(BaseModel): role: Role diff --git a/backend/app/routers/auth.py b/backend/app/routers/auth.py index f560a1e..9f45b18 100644 --- a/backend/app/routers/auth.py +++ b/backend/app/routers/auth.py @@ -8,10 +8,12 @@ from fastapi import APIRouter, Depends, HTTPException, Request from fastapi.security import OAuth2PasswordRequestForm -from pydantic import BaseModel, Field, field_validator +from starlette.concurrency import run_in_threadpool +from pydantic import BaseModel, Field, field_validator, model_validator from app.arcadedb import ArcadeSession, DuplicateKeyError -from app.auth import TokenResponse, create_token, get_current_user, hash_password, verify_password +from app.auth import (TokenResponse, burn_password_check, check_new_password, create_token, get_current_user, + hash_password, verify_password) from app.db import User, get_session from app.rbac import DEFAULT_ROLE, Permission, Role, has_permission from app.repositories import UserRepository @@ -24,7 +26,7 @@ class RegisterRequest(BaseModel): username: str = Field(min_length=3, max_length=60) - password: str = Field(min_length=6) + password: str = Field(min_length=1, max_length=256) # required for the first (admin) account when the server sets SETUP_TOKEN setup_token: str | None = Field(default=None, max_length=512) @@ -33,6 +35,11 @@ class RegisterRequest(BaseModel): def _valid_username(cls, value: str) -> str: return check_new_username(value) + @model_validator(mode="after") + def _valid_password(self) -> "RegisterRequest": + check_new_password(self.password, self.username) + return self + def registration_open() -> bool: """ALLOW_REGISTRATION=false closes self-sign-up; admins then create accounts (POST /admin/users).""" @@ -67,9 +74,13 @@ async def create_user(session: ArcadeSession, username: str, password: str, role if await users.get_by_username(username): raise HTTPException(status_code=409, detail="Username already taken") + # bcrypt takes about a quarter of a second of CPU: off the event loop, which + # every other request on this instance is waiting on + hashed = await run_in_threadpool(hash_password, password) + async def insert(as_role: Role, claim_admin: bool) -> Dict[str, Any]: user = {"id": str(uuid.uuid4()), "username": username, "username_key": username_key(username), - "hashed_password": hash_password(password), + "hashed_password": hashed, "role": as_role.value, "created_at": datetime.now(timezone.utc).isoformat()} if claim_admin: await session.execute("INSERT INTO Meta CONTENT :doc", @@ -156,7 +167,10 @@ async def login( (address_key, ratelimit.LOGIN_FAILURES_PER_ADDRESS)) users = UserRepository(session) user = await users.get_by_username(form.username) - if not user or not verify_password(form.password, user["hashed_password"]): + if not user: + # same cost as a wrong password: the timing reveals nothing + await run_in_threadpool(burn_password_check, form.password) + if not user or not await run_in_threadpool(verify_password, form.password, user["hashed_password"]): await ratelimit.hit(account_key, address_key, window=ratelimit.LOGIN_WINDOW) raise HTTPException(status_code=401, detail="Invalid credentials") await ratelimit.clear(account_key) diff --git a/backend/app/routers/knowledge.py b/backend/app/routers/knowledge.py index 04e2b9b..2c2b8b0 100644 --- a/backend/app/routers/knowledge.py +++ b/backend/app/routers/knowledge.py @@ -4,7 +4,7 @@ import logging from typing import Any, Dict, List, Optional -from fastapi import APIRouter, Depends, File, Form, HTTPException, Query, Response, UploadFile +from fastapi import APIRouter, BackgroundTasks, Depends, File, Form, HTTPException, Query, Response, UploadFile from pydantic import BaseModel, Field from app.arcadedb import ArcadeSession @@ -125,17 +125,48 @@ async def _update_item( # Read / export # --------------------------------------------------------------------------- +# A page of a space is bounded: a large space returned everything at once (4 MB for +# 3,000 items), which made each Hub load take seconds and, under load, ran the +# database out of memory. +PAGE_DEFAULT = 100 +PAGE_MAX = 1000 + + @router.get("") async def list_knowledge( + limit: int = Query(default=PAGE_DEFAULT, ge=1, le=PAGE_MAX), + offset: int = Query(default=0, ge=0), + q: Optional[str] = Query(default=None, max_length=200), + type: Optional[str] = None, + include_content: bool = False, space: Space = Depends(current_space), session: ArcadeSession = Depends(get_session), ) -> Dict[str, Any]: + """ + The space, a page at a time: the newest `limit` artifacts and items after + `offset` (items filtered by `q`, matching title or tags, and `type`), the + relationships between the returned nodes, every playbook, and `totals`. + Artifacts come without their text unless `include_content` is set. + """ user_id = space.id + items_repo, artifacts_repo = KnowledgeItemRepository(session), ArtifactRepository(session) + artifacts = await artifacts_repo.list(user_id, with_content=include_content, + limit=limit, offset=offset, newest_first=True) + items = await items_repo.list(user_id, q=q, type=type, limit=limit, offset=offset, newest_first=True) + types = await session.query("SELECT DISTINCT(type) AS type FROM KnowledgeItem WHERE user_id = :u", {"u": user_id}) return { - "artifacts": [artifact_dict(a) for a in await ArtifactRepository(session).list(user_id)], - "knowledge_items": [item_dict(i) for i in await KnowledgeItemRepository(session).list(user_id)], - "relationships": await RelationshipRepository(session).list(user_id), + "artifacts": [artifact_dict(a, content=include_content) for a in artifacts], + "knowledge_items": [item_dict(i) for i in items], + "relationships": await RelationshipRepository(session).among( + user_id, [i["id"] for i in items], [a["id"] for a in artifacts]), "playbooks": [playbook_dict(p) for p in await PlaybookRepository(session).list(user_id)], + "totals": { + "artifacts": await artifacts_repo.count(user_id), + "knowledge_items": await items_repo.count(user_id), + "matching_items": await items_repo.count(user_id, q=q, type=type), + "pending_review": await items_repo.count(user_id, review_status="pending"), + }, + "item_types": sorted({r["type"] for r in types if r.get("type")}), } @@ -169,17 +200,20 @@ async def export_okf( @router.post("/artifacts") async def ingest_artifact( request: ArtifactRequest, + background: BackgroundTasks, space: Space = Depends(current_space), session: ArcadeSession = Depends(get_session), ) -> Dict[str, Any]: return await pipeline.ingest_text( session, space.id, request.title, request.content, source=request.source, source_type="manual", author=request.author, tags=request.tags, + background=background, ) @router.post("/artifacts/upload") async def ingest_file( + background: BackgroundTasks, file: UploadFile = File(...), title: str = Form(...), author: str = Form("unknown"), @@ -191,13 +225,14 @@ async def ingest_file( tag_list = [t.strip() for t in tags.split(",") if t.strip()] return await pipeline.ingest_document( session, space.id, title, content, - source="file", source_type="file", author=author, tags=tag_list, + source="file", source_type="file", author=author, tags=tag_list, background=background, ) @router.post("/artifacts/url") async def ingest_url( request: UrlIngestRequest, + background: BackgroundTasks, space: Space = Depends(current_space), session: ArcadeSession = Depends(get_session), ) -> Dict[str, Any]: @@ -205,18 +240,21 @@ async def ingest_url( return await pipeline.ingest_document( session, space.id, request.title, content, source=request.url, source_type="url", author=request.author, tags=request.tags, + background=background, ) @router.post("/artifacts/transcript") async def ingest_transcript( request: TranscriptRequest, + background: BackgroundTasks, space: Space = Depends(current_space), session: ArcadeSession = Depends(get_session), ) -> Dict[str, Any]: return await pipeline.ingest_transcript( session, space.id, request.title, request.content, source_type=request.source_type, author=request.author, tags=request.tags, + background=background, ) @@ -226,10 +264,16 @@ async def ingest_transcript( @router.get("/review") async def list_review_queue( + response: Response, + limit: int = Query(default=50, ge=1, le=PAGE_MAX), + offset: int = Query(default=0, ge=0), space: Space = Depends(current_space), session: ArcadeSession = Depends(get_session), ) -> List[Dict[str, Any]]: - items = await KnowledgeItemRepository(session).list(space.id, review_status="pending") + """Pending items, oldest first, a page at a time; X-Total-Count holds the length of the queue.""" + repo = KnowledgeItemRepository(session) + items = await repo.list(space.id, review_status="pending", limit=limit, offset=offset) + response.headers["X-Total-Count"] = str(await repo.count(space.id, review_status="pending")) return [item_dict(i) for i in items] @@ -292,7 +336,7 @@ async def search_knowledge( type: Optional[str] = None, source_type: Optional[str] = None, tag: Optional[str] = None, - limit: int = Query(default=100, ge=1, le=500), + limit: int = Query(default=50, ge=1, le=500), space: Space = Depends(current_space), session: ArcadeSession = Depends(get_session), ) -> Dict[str, Any]: @@ -310,12 +354,24 @@ async def search_knowledge( artifacts, artifact_total, snippets = await search.search_artifacts( space.id, q, source_type=source_type, tag=tag, limit=limit ) + # sources have no page of their own: link each to an item extracted from it, whose + # page shows the source text (artifact results used to be dead ends) + first_items: Dict[str, str] = {} + if artifacts: + for row in await session.query( + "SELECT artifact_id, min(id) AS id FROM KnowledgeItem WHERE user_id = :u AND artifact_id IN :a " + "GROUP BY artifact_id", {"u": space.id, "a": [a["id"] for a in artifacts]}, + ): + first_items[row["artifact_id"]] = row["id"] return { "query": q, "filters": {"type": type, "source_type": source_type, "tag": tag}, "knowledge_items": [item_dict(i) for i in items], + # without their text: a matching passage comes back as `snippet` "artifacts": [ - {**artifact_dict(a), **({"snippet": snippets[a["id"]]} if a["id"] in snippets else {})} + {**artifact_dict(a, content=False), + **({"snippet": snippets[a["id"]]} if a["id"] in snippets else {}), + **({"first_item_id": first_items[a["id"]]} if a["id"] in first_items else {})} for a in artifacts ], "total": item_total + artifact_total, @@ -429,6 +485,49 @@ async def delete_artifact( # Knowledge items – read / update / delete # --------------------------------------------------------------------------- +@router.get("/items") +async def list_items( + q: Optional[str] = Query(default=None, max_length=200), + type: List[str] = Query(default=[]), + exclude_rejected: bool = False, + limit: int = Query(default=PAGE_DEFAULT, ge=1, le=PAGE_MAX), + offset: int = Query(default=0, ge=0), + space: Space = Depends(current_space), + session: ArcadeSession = Depends(get_session), +) -> Dict[str, Any]: + """Items newest first, filtered by `q` (title or tags) and any of `type`, with the total that match.""" + repo = KnowledgeItemRepository(session) + filters: Row = {"q": q, "types": type or None, "exclude_rejected": exclude_rejected} + items = await repo.list(space.id, **filters, limit=limit, offset=offset, newest_first=True) + return {"items": [item_dict(i) for i in items], "total": await repo.count(space.id, **filters)} + + +@router.get("/items/{item_id}/context") +async def item_context( + item_id: str, + space: Space = Depends(current_space), + session: ArcadeSession = Depends(get_session), +) -> Dict[str, Any]: + """What the page of an item shows around it: its source (with text), its relationships, their titles.""" + item = await _owned_item(session, space, item_id) + artifacts = ArtifactRepository(session) + artifact = await artifacts.get_owned(space.id, item["artifact_id"]) if item.get("artifact_id") else None + relationships = await RelationshipRepository(session).touching(space.id, [item_id]) + other_ids = sorted({r[end] for r in relationships for end in ("from", "to")} - {item_id}) + nodes = {i["id"]: {"title": i.get("title"), "kind": "item", "type": i.get("type")} + for i in await KnowledgeItemRepository(session).get_many(space.id, other_ids)} + for aid in [oid for oid in other_ids if oid not in nodes]: + a = await artifacts.get_owned(space.id, aid) + if a: + nodes[aid] = {"title": a.get("title"), "kind": "artifact", "type": "artifact"} + return { + "item": item_dict(item), + "artifact": artifact_dict(artifact) if artifact else None, + "relationships": relationships, + "nodes": nodes, + } + + @router.get("/items/{item_id}") async def get_item( item_id: str, @@ -479,10 +578,12 @@ async def delete_item( @router.get("/graph") async def knowledge_graph( + limit: int = Query(default=300, ge=1, le=PAGE_MAX), space: Space = Depends(current_space), session: ArcadeSession = Depends(get_session), ) -> Dict[str, Any]: - return await GraphStore(session).visualization(space.id) + """The newest `limit` nodes and the edges between them; `truncated` says whether there are more.""" + return await GraphStore(session).visualization(space.id, limit=limit) # Must stay last: the catch-all path would otherwise shadow the GET routes above. diff --git a/backend/app/routers/lineage.py b/backend/app/routers/lineage.py index ff375d0..e036c40 100644 --- a/backend/app/routers/lineage.py +++ b/backend/app/routers/lineage.py @@ -57,11 +57,13 @@ async def link_kinds(space: Space = Depends(current_space)) -> Dict[str, Any]: @router.get("/register") async def register( type: List[str] = Query(default=["decision"]), + limit: int = Query(default=300, ge=1, le=1000), + offset: int = Query(default=0, ge=0), space: Space = Depends(current_space), session: ArcadeSession = Depends(get_session), ) -> Dict[str, Any]: - """Decisions (and/or risks) with their status and lineage edges — the data behind the decision log.""" - return await lineage.register(session, space.id, type) + """Decisions (and/or risks) with their status and lineage edges, newest first: the data behind the decision log.""" + return await lineage.register(session, space.id, type, limit=limit, offset=offset) @router.get("/items/{item_id}/lineage") diff --git a/backend/app/routers/models.py b/backend/app/routers/models.py index d83eee3..4cb053c 100644 --- a/backend/app/routers/models.py +++ b/backend/app/routers/models.py @@ -69,6 +69,7 @@ async def events() -> AsyncIterator[str]: try: async for chunk in llm_catalog.pull_events(model_id): yield f"data: {json.dumps(chunk)}\n\n" + llm_catalog.forget_availability() installed = await llm_catalog.installed_models() if model_id not in installed: raise RuntimeError(f"{model_id} did not finish installing") @@ -166,6 +167,8 @@ async def remove_local_model( await llm_catalog.delete_model(body.model_id) except Exception as exc: raise HTTPException(status_code=503, detail=f"Could not remove the local model: {exc}") + finally: + llm_catalog.forget_availability() # Ollama is shared by every account on this machine: move anyone using it to another model replacement = llm_catalog.pick_default(remaining) diff --git a/backend/app/serializers.py b/backend/app/serializers.py index 15fbadf..6770d3a 100644 --- a/backend/app/serializers.py +++ b/backend/app/serializers.py @@ -6,8 +6,9 @@ Row = Dict[str, Any] -def artifact_dict(a: Row) -> Row: - return { +def artifact_dict(a: Row, content: bool = True) -> Row: + """content=False leaves out the document text (lists: it can be megabytes per space).""" + out = { "id": a["id"], "title": a.get("title"), "content": a.get("content"), @@ -19,6 +20,9 @@ def artifact_dict(a: Row) -> Row: "created_at": a.get("created_at"), "metadata": a.get("metadata") or {}, } + if not content: + del out["content"] + return out def item_dict(i: Row) -> Row: diff --git a/backend/app/services/artifact_pipeline.py b/backend/app/services/artifact_pipeline.py index c5671bd..e456122 100644 --- a/backend/app/services/artifact_pipeline.py +++ b/backend/app/services/artifact_pipeline.py @@ -8,6 +8,7 @@ from __future__ import annotations import hashlib +import logging from datetime import datetime, timezone from typing import Any, Dict, List, Optional, Tuple @@ -25,7 +26,9 @@ from app.services.llm_extraction import _summarise_text, extract_from_document, extract_from_transcript from app.services.okf import normalize_okf_payload from app.schema import record_embedding_model -from app.services.providers import get_embedding_provider +from app.services.providers import active_model, get_embedding_provider, use_model + +logger = logging.getLogger(__name__) Item = Dict[str, Any] @@ -138,30 +141,36 @@ async def persist_artifact( metadata: Dict[str, Any], extraction_engine: str = "regex", ) -> List[Dict[str, Any]]: - """Upsert the artifact, sync its items, ensure CONTAINS edges — one transaction. Returns the edges.""" - await ArtifactRepository(session).upsert( - user_id, - artifact["id"], - title=artifact["title"], - content=artifact["content"], - source=artifact["source"], - source_type=artifact["source_type"], - author=artifact["author"], - tags=artifact["tags"], - created_at=artifact["created_at"], - metadata=metadata, - extraction_engine=extraction_engine, - ) - await KnowledgeItemRepository(session).sync_for_artifact( - user_id, artifact["id"], items, extraction_engine - ) - relationships = await RelationshipRepository(session).ensure_contains( - user_id, artifact["id"], [i["id"] for i in items] - ) - # new decisions/risks get their first status; removed items may have freed others - await lineage.refresh_statuses(session, user_id) - await session.commit() - return relationships + """ + Upsert the artifact, sync its items, ensure CONTAINS edges — one transaction, + run again on a write conflict (only these writes: the extraction isn't repeated). + Returns the edges. + """ + async def write() -> List[Dict[str, Any]]: + await ArtifactRepository(session).upsert( + user_id, + artifact["id"], + title=artifact["title"], + content=artifact["content"], + source=artifact["source"], + source_type=artifact["source_type"], + author=artifact["author"], + tags=artifact["tags"], + created_at=artifact["created_at"], + metadata=metadata, + extraction_engine=extraction_engine, + ) + await KnowledgeItemRepository(session).sync_for_artifact( + user_id, artifact["id"], items, extraction_engine + ) + relationships = await RelationshipRepository(session).ensure_contains( + user_id, artifact["id"], [i["id"] for i in items] + ) + # new decisions/risks get their first status; removed items may have freed others + await lineage.refresh_statuses(session, user_id) + return relationships + + return await session.transaction(write) async def index_artifact( @@ -170,13 +179,35 @@ async def index_artifact( artifact: Dict[str, Any], items: List[Item], summarise: bool = True, + background: Optional[Any] = None, ) -> None: - """Embed items and (optionally) rebuild the artifact summary.""" + """ + Embed items and (optionally) rebuild the artifact summary. With `background` + (FastAPI's BackgroundTasks) the summary is built after the response has gone: + it's an LLM call, about 30 seconds on a CPU, and nothing in the response needs it. + """ await embed_and_store(session, user_id, items) - if summarise: + if not summarise: + return + if background is not None: + background.add_task(summarise_later, user_id, artifact["id"], artifact["content"], active_model()) + else: await build_artifact_summary(session, user_id, artifact["id"], artifact["content"]) +async def summarise_later(user_id: str, artifact_id: str, content: str, model: str) -> None: + """The deferred summary, in its own session: the request's has closed by now.""" + from app.db import get_client + use_model(model) # the model of the user who added it + session = ArcadeSession(get_client()) + try: + await build_artifact_summary(session, user_id, artifact_id, content) + except Exception: # the artifact may be gone, or the LLM down: GraphRAG copes + logger.exception("Couldn't summarise artifact %s", artifact_id) + finally: + await session.close() + + async def embed_and_store(session: ArcadeSession, user_id: str, items: List[Item]) -> int: """ Embed items and store the vectors (indexed by the KnowledgeItem[embedding] @@ -188,12 +219,15 @@ async def embed_and_store(session: ArcadeSession, user_id: str, items: List[Item return 0 provider = get_embedding_provider() repo = KnowledgeItemRepository(session) - updated = 0 - for item_id, vector in pairs: - if await repo.set_embedding(user_id, item_id, vector, provider.name, provider.dimensions): - updated += 1 - await session.commit() - return updated + + async def write() -> int: + updated = 0 + for item_id, vector in pairs: + if await repo.set_embedding(user_id, item_id, vector, provider.name, provider.dimensions): + updated += 1 + return updated + + return await session.transaction(write) async def build_artifact_summary( @@ -205,10 +239,9 @@ async def build_artifact_summary( return summary_vec = await embed(summary_text) provider = get_embedding_provider() - await ArtifactRepository(session).set_summary( + await session.transaction(lambda: ArtifactRepository(session).set_summary( user_id, artifact_id, summary_text, summary_vec, provider.name, provider.dimensions, - ) - await session.commit() + )) def _artifact( @@ -244,20 +277,20 @@ def _base_metadata(artifact: Dict[str, Any]) -> Dict[str, Any]: async def ingest_text( session: ArcadeSession, user_id: str, title: str, content: str, source: str, source_type: str, author: str, tags: List[str], - artifact_id: Optional[str] = None, created_at: Optional[str] = None, + artifact_id: Optional[str] = None, created_at: Optional[str] = None, background: Optional[Any] = None, ) -> Dict[str, Any]: """Regex extraction — used for manually entered text.""" artifact = _artifact(user_id, title, content, source, source_type, author, tags, artifact_id, created_at) items = build_regex_items(user_id, artifact["id"], content, author, tags, artifact["created_at"]) relationships = await persist_artifact(session, user_id, artifact, items, _base_metadata(artifact)) - await index_artifact(session, user_id, artifact, items) + await index_artifact(session, user_id, artifact, items, background=background) return {"artifact": artifact, "items": items, "relationships": relationships, "extracted_count": len(items)} async def ingest_document( session: ArcadeSession, user_id: str, title: str, content: str, source: str, source_type: str, author: str, tags: List[str], - artifact_id: Optional[str] = None, created_at: Optional[str] = None, + artifact_id: Optional[str] = None, created_at: Optional[str] = None, background: Optional[Any] = None, ) -> Dict[str, Any]: """ LLM extraction for uploaded files and fetched URLs. Regex is only a @@ -288,7 +321,7 @@ async def ingest_document( item.setdefault("extraction_engine", "regex" if item["type"] == "checklist" else engine) relationships = await persist_artifact(session, user_id, artifact, items, metadata, engine) - await index_artifact(session, user_id, artifact, items) + await index_artifact(session, user_id, artifact, items, background=background) return { "artifact": artifact, "items": items, @@ -302,6 +335,7 @@ async def ingest_transcript( session: ArcadeSession, user_id: str, title: str, content: str, source_type: str, author: str, tags: List[str], artifact_id: Optional[str] = None, created_at: Optional[str] = None, source: Optional[str] = None, + background: Optional[Any] = None, ) -> Dict[str, Any]: """LLM extraction for transcripts, emails and Slack exports.""" llm_result = await extract_from_transcript(content, source_type=source_type) @@ -314,7 +348,7 @@ async def ingest_transcript( items = build_llm_items(user_id, artifact["id"], llm_result, author, tags, artifact["created_at"]) relationships = await persist_artifact(session, user_id, artifact, items, metadata, engine) - await index_artifact(session, user_id, artifact, items) + await index_artifact(session, user_id, artifact, items, background=background) return { "artifact": artifact, "items": items, diff --git a/backend/app/services/graphrag.py b/backend/app/services/graphrag.py index f72af16..f380518 100644 --- a/backend/app/services/graphrag.py +++ b/backend/app/services/graphrag.py @@ -503,16 +503,10 @@ def _format_fallback_answer(question: str, context: str) -> str: # 11. Citation extraction # ───────────────────────────────────────────────────────────────────────────── -def _extract_citations( - answer: str, - nodes: List[Dict[str, Any]], - summaries: Optional[List[Dict[str, Any]]] = None, -) -> List[Dict[str, str]]: - """ - Citation format is the full id in brackets — `[item_id]` for knowledge - items, `[artifact_id]` for artifact summaries — exactly as written by - _build_context. Only exact matches against the supplied context count. - """ +def _citable( + nodes: List[Dict[str, Any]], summaries: Optional[List[Dict[str, Any]]] = None, +) -> Dict[str, Dict[str, str]]: + """Everything the answer could cite, by id: the context items and artifact summaries.""" by_id: Dict[str, Dict[str, str]] = {} for n in nodes: nid = n.get("id") @@ -528,7 +522,20 @@ def _extract_citations( aid = s.get("artifact_id") if aid and aid not in by_id: by_id[aid] = {"id": aid, "title": s.get("title") or aid, "type": "artifact"} + return by_id + +def _extract_citations( + answer: str, + nodes: List[Dict[str, Any]], + summaries: Optional[List[Dict[str, Any]]] = None, +) -> List[Dict[str, str]]: + """ + Citation format is the full id in brackets — `[item_id]` for knowledge + items, `[artifact_id]` for artifact summaries — exactly as written by + _build_context. Only exact matches against the supplied context count. + """ + by_id = _citable(nodes, summaries) cited: List[Dict[str, str]] = [] seen_ids = set() for token in re.findall(r"\[([^\]]+)\]", answer): @@ -541,6 +548,45 @@ def _extract_citations( return cited +_CITE_STOPWORDS = frozenset( + "the and for with that this from into our your their was were are has have had not but use used " + "using will would can could should may might its it's about which what when where who how all any " + "also than then them they there these those been being does did done per via".split() +) +_INFERRED_MAX = 4 + + +def _significant_words(text: str) -> set: + # "ADR-0001" stays one word: it names a record + return {w for w in re.findall(r"[a-z0-9]+(?:-[0-9]+)?", text.lower()) + if len(w) > 2 and w not in _CITE_STOPWORDS} + + +def _infer_citations( + answer: str, + nodes: List[Dict[str, Any]], + summaries: Optional[List[Dict[str, Any]]] = None, +) -> List[Dict[str, str]]: + """ + Sources for an answer that cites none. Small local models often name a + source ("(ADR-0001)") instead of writing its [id], which left a grounded + answer with no sources at all. A context entry counts when the answer names + an identifier from its title (a word with a digit, such as ADR-0003), or + when at least half of its title's words, and at least two, appear in the + answer. Marked `inferred`, so the UI can say these were matched, not cited. + """ + answer_words = _significant_words(answer) + scored = [] + for entry in _citable(nodes, summaries).values(): + title_words = _significant_words(entry["title"]) + shared = title_words & answer_words + names_it = any(any(ch.isdigit() for ch in w) for w in shared) + if title_words and (names_it or len(shared) >= max(2, math.ceil(len(title_words) / 2))): + scored.append((len(shared) / len(title_words), {**entry, "inferred": True})) + scored.sort(key=lambda pair: -pair[0]) + return [entry for _, entry in scored[:_INFERRED_MAX]] + + # ───────────────────────────────────────────────────────────────────────────── # 12. Main orchestrator # ───────────────────────────────────────────────────────────────────────────── @@ -659,7 +705,8 @@ async def embed_and_route() -> Tuple[Optional[List[float]], str]: context = _build_context(summaries_used, primary, graph_nbr) answer = await _generate(question, context, route, history) lap("generate") - citations = _extract_citations(answer, reranked, summaries_used) + citations = (_extract_citations(answer, reranked, summaries_used) + or _infer_citations(answer, reranked, summaries_used)) return { "answer": answer, diff --git a/backend/app/services/lineage.py b/backend/app/services/lineage.py index f36b51e..42e1a5d 100644 --- a/backend/app/services/lineage.py +++ b/backend/app/services/lineage.py @@ -297,10 +297,19 @@ async def suggest_links(session: ArcadeSession, user_id: str, item: Row, limit: return out -async def register(session: ArcadeSession, user_id: str, types: List[str]) -> Row: - """The decision / risk register: tracked items with their status and chain edges.""" - rows = [i for i in await KnowledgeItemRepository(session).list(user_id, exclude_rejected=True) - if i["type"] in types] +async def register(session: ArcadeSession, user_id: str, types: List[str], + limit: Optional[int] = None, offset: int = 0) -> Row: + """ + The decision / risk register: tracked items with their status and chain edges, + newest first. `total` counts them all; `items` holds at most `limit`. Items + named by the returned chain edges but outside the page come back as `related`. + """ + repo = KnowledgeItemRepository(session) + rows = await repo.list(user_id, types=types, exclude_rejected=True, + limit=limit, offset=offset, newest_first=True) + total = await repo.count(user_id, types=types, exclude_rejected=True) ids = {i["id"] for i in rows} edges = [e for e in await LineageRepository(session).list(user_id) if e["from"] in ids or e["to"] in ids] - return {"items": [item_dict(i) for i in rows], "edges": edges} + outside = {e[end] for e in edges for end in ("from", "to")} - ids + related = [item_dict(i) for i in await repo.get_many(user_id, sorted(outside))] + return {"items": [item_dict(i) for i in rows], "edges": edges, "total": total, "related": related} diff --git a/backend/app/services/llm_catalog.py b/backend/app/services/llm_catalog.py index 9c6fe8d..23ce8bf 100644 --- a/backend/app/services/llm_catalog.py +++ b/backend/app/services/llm_catalog.py @@ -9,6 +9,8 @@ import json import logging +import os +import time from dataclasses import asdict, dataclass from typing import Any, AsyncIterator, Dict, List, Optional, Set, Tuple @@ -60,12 +62,31 @@ async def installed_models() -> Set[str]: return {model_id for model_id in CATALOG_IDS if model_id in names} +# availability() is behind the unauthenticated catalog and every sign-in, so each call +# used to ask Ollama: anyone could put load on it, and it capped the catalog at about +# 10 requests a second. The answer is kept this long (0 = ask every time). +_STATUS_TTL = float(os.getenv("OLLAMA_STATUS_CACHE_SECONDS", "5")) +_status: Optional[Tuple[float, bool, Set[str]]] = None # (expires at, reachable, installed) + + async def availability() -> Tuple[bool, Set[str]]: """(ollama_reachable, installed catalog models) — never raises.""" + global _status + if _status and _status[0] > time.monotonic(): + return _status[1], set(_status[2]) try: - return True, await installed_models() + reachable, installed = True, await installed_models() except OllamaUnavailable: - return False, set() + reachable, installed = False, set() + if _STATUS_TTL > 0: + _status = (time.monotonic() + _STATUS_TTL, reachable, set(installed)) + return reachable, installed + + +def forget_availability() -> None: + """After a model is installed or removed: the next availability() asks Ollama again.""" + global _status + _status = None def pick_default(installed: Set[str], preferred: Optional[str] = None) -> Optional[str]: diff --git a/backend/tests/conftest.py b/backend/tests/conftest.py index ac2e39e..5c6341e 100644 --- a/backend/tests/conftest.py +++ b/backend/tests/conftest.py @@ -45,6 +45,8 @@ os.environ["ALLOW_REGISTRATION"] = "true" # every test signs in from the same address; test_security turns the limiter back on os.environ["AUTH_RATE_LIMIT"] = "false" +# the fake Ollama's installed models change mid-test: never serve a cached answer +os.environ["OLLAMA_STATUS_CACHE_SECONDS"] = "0" os.environ.pop("SETUP_TOKEN", None) # the inactivity job would race the tests on the shared database; tests run it explicitly os.environ["RETENTION_ENABLED"] = "false" diff --git a/backend/tests/test_connectors.py b/backend/tests/test_connectors.py index dd8fbec..78995fb 100644 --- a/backend/tests/test_connectors.py +++ b/backend/tests/test_connectors.py @@ -163,7 +163,7 @@ def sync(client, headers, connector_id: str) -> Dict[str, Any]: def artifacts(client, headers): - return client.get("/knowledge", headers=headers).json() + return client.get("/knowledge", headers=headers, params={"include_content": True}).json() # ── folder ────────────────────────────────────────────────────────────────── diff --git a/backend/tests/test_frontend_contract.py b/backend/tests/test_frontend_contract.py index ea05100..5376523 100644 --- a/backend/tests/test_frontend_contract.py +++ b/backend/tests/test_frontend_contract.py @@ -20,7 +20,10 @@ FRONTEND = Path(__file__).resolve().parents[2] / "frontend" / "src" / "app" # Endpoints that are deliberately not called from the UI, with the reason. -NOT_FOR_THE_UI: Set[Tuple[str, str]] = set() +NOT_FOR_THE_UI: Set[Tuple[str, str]] = { + # one item by id, for API clients; the item page loads it with its context (/context) + ("GET", "/knowledge/items/{item_id}"), +} _HTTP_CALL = re.compile( r"http\.(get|post|put|patch|delete|request)\s*(?:<(?:[^<>]|<[^<>]*>)*>)?\s*\(\s*(?:'([A-Z]+)'\s*,\s*)?" diff --git a/backend/tests/test_graphrag.py b/backend/tests/test_graphrag.py index 81052dc..68bf9c4 100644 --- a/backend/tests/test_graphrag.py +++ b/backend/tests/test_graphrag.py @@ -84,7 +84,8 @@ def test_works_with_no_llm_reachable(client, kb, llm): assert body["context_nodes"] assert "PostgreSQL" in body["answer"] # extractive fallback answer built from context assert body["sub_queries"] == ["What did we decide about PostgreSQL?"] - assert body["citations"] == [] + # that answer quotes the items it came from, so they're its (inferred) sources + assert body["citations"] and all(c["inferred"] for c in body["citations"]) def test_malformed_transform_output_is_tolerated(client, kb, llm): @@ -216,3 +217,38 @@ def test_query_is_logged(client, kb, llm, db_query): assert log["context_node_ids"] == [n["id"] for n in body["context_nodes"]] assert log["llm_provider"] == "fake:llm" assert log["citations"] == body["citations"] + + +# ── citations the model didn't write ─────────────────────────────────────── + +_ADRS = [ + {"id": "decisions_a3", "title": "ADR-0003: Use ArcadeDB for graph, documents and vectors", "type": "decision"}, + {"id": "decisions_a1", "title": "ADR-0001: Use PostgreSQL as the primary store", "type": "decision", + "status": "superseded"}, + {"id": "risks_r", "title": "Risk: on-call burnout if releases slip to Friday", "type": "risk"}, +] + + +def test_sources_are_inferred_when_the_model_names_them_without_ids(): + """Regression: llama3.1:8b wrote "(ADR-0001)" instead of [id], so a grounded answer showed no sources.""" + from app.services.graphrag import _infer_citations + answer = ("The chosen database is ArcadeDB. It replaced PostgreSQL, which was previously chosen as the " + "primary store (ADR-0001). Note that ADR-0003 supersedes ADR-0001.") + cited = _infer_citations(answer, _ADRS) + assert {c["id"] for c in cited} == {"decisions_a1", "decisions_a3"} + assert all(c["inferred"] for c in cited) + assert next(c for c in cited if c["id"] == "decisions_a1")["status"] == "superseded" + + +def test_no_sources_are_inferred_for_an_answer_that_uses_none(): + from app.services.graphrag import _infer_citations + refusal = ("I'm unable to answer the question as the provided context does not contain any " + "information about our parental leave policy.") + assert _infer_citations(refusal, _ADRS) == [] + assert _infer_citations("Use the release checklist.", _ADRS) == [] # one shared word is not enough + + +def test_written_citations_win_over_inferred_ones(client, kb, llm): + llm.handlers["generate"] = cite_first + body = ask(client, kb) + assert body["citations"] and not any(c.get("inferred") for c in body["citations"]) diff --git a/backend/tests/test_models.py b/backend/tests/test_models.py index 9b27827..d5984d9 100644 --- a/backend/tests/test_models.py +++ b/backend/tests/test_models.py @@ -231,3 +231,27 @@ def test_management_reports_ollama_down(client, ollama, make_user): ollama.reachable = False assert client.post("/models/install", headers=h, json={"model_id": "mistral:7b"}).status_code == 503 assert client.get("/models/status", headers=h).json()["llm"]["ollamaReachable"] is False + + +def test_ollama_status_is_cached_briefly_and_refreshed_after_changes(client, ollama, monkeypatch): + """Regression: every catalog request and sign-in asked Ollama, so anyone could load it from outside.""" + import asyncio + from app.services import llm_catalog + calls = [] + real = llm_catalog.installed_models + + async def counting(): + calls.append(1) + return await real() + monkeypatch.setattr(llm_catalog, "installed_models", counting) + monkeypatch.setattr(llm_catalog, "_STATUS_TTL", 60.0) + llm_catalog.forget_availability() + try: + for _ in range(5): + assert client.get("/models/catalog").json()["anyInstalled"] + assert len(calls) == 1 + llm_catalog.forget_availability() # what install and remove do + asyncio.run(llm_catalog.availability()) + assert len(calls) == 2 + finally: + llm_catalog.forget_availability() diff --git a/backend/tests/test_paging.py b/backend/tests/test_paging.py new file mode 100644 index 0000000..5241e45 --- /dev/null +++ b/backend/tests/test_paging.py @@ -0,0 +1,137 @@ +""" +Bounded responses: a large space used to come back whole (4 MB for 3,000 items), +which made pages slow and, under load, ran the database out of memory. Lists +are now pages with totals, and only the item page loads a document's text. +""" +from __future__ import annotations + +import pytest + +from tests.helpers import ARCH_NOTES, ingest_text + + +@pytest.fixture +def space(client, alice): + """Three sources in alice's space, newest last.""" + for n in range(3): + ingest_text(client, alice, title=f"Notes {n}", content=ARCH_NOTES.replace("PostgreSQL", f"PostgreSQL {n}")) + return alice + + +def test_the_hub_gets_a_page_with_totals_and_no_document_text(client, space): + full = client.get("/knowledge", headers=space).json() + totals = full["totals"] + assert totals["artifacts"] == 3 and totals["knowledge_items"] == len(full["knowledge_items"]) > 3 + assert totals["pending_review"] == totals["knowledge_items"] + assert all("content" not in a for a in full["artifacts"]) + assert "decision" in full["item_types"] + + page = client.get("/knowledge", headers=space, params={"limit": 2}).json() + assert [a["title"] for a in page["artifacts"]] == ["Notes 2", "Notes 1"] # newest first + assert len(page["knowledge_items"]) == 2 and page["totals"] == totals + shown = {a["id"] for a in page["artifacts"]} | {i["id"] for i in page["knowledge_items"]} + assert all(r["from"] in shown and r["to"] in shown for r in page["relationships"]) + + rest = client.get("/knowledge", headers=space, params={"limit": 2, "offset": 2}).json() + assert [a["title"] for a in rest["artifacts"]] == ["Notes 0"] + seen = {i["id"] for i in page["knowledge_items"]} | {i["id"] for i in rest["knowledge_items"]} + assert len(seen) == 4 # no overlap between pages + + with_text = client.get("/knowledge", headers=space, params={"include_content": True}).json() + assert all("PostgreSQL" in a["content"] for a in with_text["artifacts"]) + assert client.get("/knowledge", headers=space, params={"limit": 5000}).status_code == 422 + + +def test_the_hub_filters_on_the_server(client, space): + risks = client.get("/knowledge", headers=space, params={"type": "risk"}).json() + assert risks["knowledge_items"] and {i["type"] for i in risks["knowledge_items"]} == {"risk"} + assert risks["totals"]["matching_items"] == len(risks["knowledge_items"]) < risks["totals"]["knowledge_items"] + hits = client.get("/knowledge", headers=space, params={"q": "postgresql 1"}).json()["knowledge_items"] + assert [i["title"] for i in hits] and all("PostgreSQL 1" in i["title"] for i in hits) + tagged = client.get("/knowledge", headers=space, params={"q": "ARCH"}).json() # tags too, any case + assert len(tagged["knowledge_items"]) == tagged["totals"]["knowledge_items"] + + +def test_items_endpoint_filters_by_several_types(client, space): + body = client.get("/knowledge/items", headers=space, + params={"type": ["how-to", "checklist"], "exclude_rejected": True}).json() + assert body["items"] and {i["type"] for i in body["items"]} <= {"how-to", "checklist"} + assert body["total"] == len(body["items"]) + assert client.get("/knowledge/items", headers=space, params={"limit": 1}).json()["total"] > 1 + + +def test_the_item_page_gets_its_context(client, space): + item = client.get("/knowledge/items", headers=space, params={"type": "decision", "limit": 1}).json()["items"][0] + ctx = client.get(f"/knowledge/items/{item['id']}/context", headers=space).json() + assert ctx["item"]["id"] == item["id"] + assert ctx["artifact"]["id"] == item["artifact_id"] and "PostgreSQL" in ctx["artifact"]["content"] + assert {"from": item["artifact_id"], "to": item["id"], "type": "CONTAINS"} in ctx["relationships"] + assert ctx["nodes"][item["artifact_id"]]["kind"] == "artifact" + assert client.get("/knowledge/items/nope/context", headers=space).status_code == 404 + + +def test_another_space_gets_no_context(client, space, bob): + item = client.get("/knowledge/items", headers=space, params={"limit": 1}).json()["items"][0] + assert client.get(f"/knowledge/items/{item['id']}/context", headers=bob).status_code == 404 + + +def test_the_review_queue_is_paged(client, space): + r = client.get("/knowledge/review", headers=space, params={"limit": 3}) + assert r.status_code == 200 and len(r.json()) == 3 + assert int(r.headers["X-Total-Count"]) > 3 + + +def test_the_graph_is_capped_and_says_so(client, space): + full = client.get("/knowledge/graph", headers=space).json() + assert full["truncated"] is False and full["total_nodes"] == len(full["nodes"]) + capped = client.get("/knowledge/graph", headers=space, params={"limit": 4}).json() + assert len(capped["nodes"]) == 4 and capped["truncated"] and capped["total_nodes"] == full["total_nodes"] + ids = {n["id"] for n in capped["nodes"]} + assert all(e["source"] in ids and e["target"] in ids for e in capped["edges"]) + + +def test_the_register_is_paged_and_names_items_outside_the_page(client, space): + decisions = client.get("/knowledge/register", headers=space).json() + assert decisions["total"] == 3 == len(decisions["items"]) + newest, older = decisions["items"][0], decisions["items"][2] + r = client.post(f"/knowledge/items/{newest['id']}/lineage", headers=space, + json={"target_id": older["id"], "kind": "supersedes"}) + assert r.status_code == 201, r.text + page = client.get("/knowledge/register", headers=space, params={"limit": 1}).json() + assert [i["id"] for i in page["items"]] == [newest["id"]] and page["total"] == 3 + assert older["id"] in {i["id"] for i in page["related"]} # the chain still has a title + + +def test_search_results_carry_no_document_text_but_link_somewhere(client, space): + body = client.get("/knowledge/search", headers=space, params={"q": "migrations"}).json() + assert body["artifacts"] + for a in body["artifacts"]: + assert "content" not in a + assert a["first_item_id"].startswith(("decisions_", "risks_", "lessons_", "best", "how", "check")) + item = client.get(f"/knowledge/items/{body['artifacts'][0]['first_item_id']}", headers=space).json() + assert item["artifact_id"] == body["artifacts"][0]["id"] + + +def test_the_summary_is_built_after_the_response(client, alice, llm, db_query): + # through the route: the summary is a background task, which the test client runs before returning + llm.handlers["summary"] = lambda m: "A short summary of the billing notes." + body = ingest_text(client, alice, title="Deferred") + art = db_query("SELECT summary FROM Artifact WHERE id = :id", id=body["artifact"]["id"])[0] + assert art["summary"] == "A short summary of the billing notes." + + +def test_the_pipeline_schedules_the_summary_instead_of_waiting(llm): + import asyncio + import app.services.artifact_pipeline as pipeline + scheduled = [] + + class Recorder: # stands in for FastAPI's BackgroundTasks + def add_task(self, fn, *args): + scheduled.append((fn.__name__, args)) + + # no items: nothing to embed, so no database is needed + asyncio.run(pipeline.index_artifact(None, "space-1", {"id": "artifact_x", "content": "text"}, [], + background=Recorder())) + assert [name for name, _ in scheduled] == ["summarise_later"] + assert scheduled[0][1][:3] == ("space-1", "artifact_x", "text") + assert "summary" not in llm.kinds() # nothing waited on the LLM diff --git a/backend/tests/test_scaling.py b/backend/tests/test_scaling.py index a76398d..9c1ecdc 100644 --- a/backend/tests/test_scaling.py +++ b/backend/tests/test_scaling.py @@ -34,6 +34,7 @@ def wrapped(request: httpx.Request) -> httpx.Response: client = ArcadeClient("http://node-a:2480,http://node-b:2480", "db", "u", "p") client.http = httpx.AsyncClient(transport=httpx.MockTransport(wrapped)) + client._leader_checked = float("inf") # these tests are about failover; leader lookup has its own return client, seen @@ -273,6 +274,137 @@ def test_an_unreachable_redis_fails_fast(monkeypatch): assert time.monotonic() - started < coordination.REDIS_TIMEOUT + 1.5 +def test_a_failed_redis_is_left_alone_for_a_while(monkeypatch): + """ + Regression: with the Redis container stopped, every health probe started another + lookup of its (now unknown) host name; they piled up in the resolver threads until + the sign-in's Ollama check timed out and sign-in answered 503 "Ollama is not running". + """ + monkeypatch.setenv("REDIS_URL", "redis://10.255.255.2:6379/0") + coordination._redis_clients.clear() + coordination.reset_redis_backoff() + dials = [] + + class Hanging: + async def ping(self): + dials.append(1) + await asyncio.sleep(60) + monkeypatch.setattr(coordination, "_redis", lambda: Hanging()) + monkeypatch.setattr(coordination, "REDIS_TIMEOUT", 0.2) + try: + assert asyncio.run(coordination.ping()) is False and dials == [1] + assert coordination.redis_backing_off() + assert asyncio.run(coordination.ping()) is False and dials == [1] # not dialled again + monkeypatch.setattr(coordination, "_redis_down_until", {"redis://10.255.255.2:6379/0": 0.0}) + assert asyncio.run(coordination.ping()) is False and dials == [1, 1] # tried again after the back-off + finally: + coordination.reset_redis_backoff() + + +def test_the_client_sends_its_work_to_the_leader(monkeypatch): + """ + Regression: a follower answered reads before applying the write just made through + it, so a group created a moment ago was missing from the next list (4 in 30). + """ + seen: List[str] = [] + leader = {"address": "node-b:2480"} + down: set = set() + + def handler(request: httpx.Request) -> httpx.Response: + host = request.url.netloc.decode() + if host in down: + raise httpx.ConnectError("refused", request=request) + if request.url.path.endswith("/server"): + seen.append(f"lookup@{host}") + return httpx.Response(200, json={"ha": {"leader": "b", "leaderAddress": leader["address"]}}) + seen.append(host) + return httpx.Response(200, json={"result": []}) + + client = ArcadeClient("http://node-a:2480,http://node-b:2480,http://node-c:2480", "db", "u", "p") + client.http = httpx.AsyncClient(transport=httpx.MockTransport(handler)) + asyncio.run(client.command("SELECT 1")) + asyncio.run(client.command("SELECT 1")) + assert seen == ["lookup@node-a:2480", "node-b:2480", "node-b:2480"] # one lookup per LEADER_REFRESH + + # the leader dies: the request fails over to the next node, and the next lookup follows the new leader + seen.clear() + down.add("node-b:2480") + asyncio.run(client.command("SELECT 1")) + assert seen == ["node-c:2480"] + leader["address"] = "node-a:2480" + monkeypatch.setattr(client, "_leader_checked", float("-inf")) + seen.clear() + asyncio.run(client.command("SELECT 1")) + assert seen == ["lookup@node-c:2480", "node-a:2480"] + + # one node: nothing to look up + single = ArcadeClient("http://only:2480", "db", "u", "p") + single.http = client.http + seen.clear() + asyncio.run(single.command("SELECT 1")) + assert seen == ["only:2480"] + + +class _ConflictingClient: + """Fails the first `conflicts` commits with ArcadeDB's page-version conflict.""" + + def __init__(self, conflicts: int) -> None: + from app.arcadedb import ArcadeDBError + self.conflicts, self.error = conflicts, ArcadeDBError + self.log: List[str] = [] + + async def begin(self) -> str: + self.log.append("begin") + return f"tx{len(self.log)}" + + async def command(self, sql, params=None, *, language="sql", session_id=None): + self.log.append(sql) + return [] + + async def commit(self, session_id: str) -> None: + if self.conflicts: + self.conflicts -= 1 + raise self.error(503, "com.arcadedb.exception.ConcurrentModificationException", + "Concurrent modification on page PageId(db/9/8). Please retry the operation") + self.log.append("commit") + + async def rollback(self, session_id: str) -> None: + self.log.append("rollback") + + +def test_a_write_conflict_reruns_the_writes(monkeypatch): + """Regression: a long import failed at commit because a summary was saved on the same page meanwhile.""" + from app.arcadedb import ArcadeSession + monkeypatch.setattr(asyncio, "sleep", _no_sleep) + client = _ConflictingClient(conflicts=2) + session = ArcadeSession(client) + + async def work(): + await session.execute("UPDATE Artifact SET x = 1") + return "done" + assert asyncio.run(session.transaction(work)) == "done" + assert client.log.count("UPDATE Artifact SET x = 1") == 3 and client.log[-1] == "commit" + assert client.log.count("begin") == 3 # each attempt in a fresh transaction (a failed commit ends one) + + +def test_a_conflict_is_not_retried_over_earlier_writes(monkeypatch): + from app.arcadedb import ArcadeDBError, ArcadeSession + monkeypatch.setattr(asyncio, "sleep", _no_sleep) + client = _ConflictingClient(conflicts=1) + session = ArcadeSession(client) + + async def run(): + await session.execute("INSERT INTO Earlier SET x = 1") # already in this transaction + await session.transaction(lambda: session.execute("UPDATE Artifact SET x = 1")) + with pytest.raises(ArcadeDBError): + asyncio.run(run()) # rerunning would silently drop the earlier insert + assert client.log.count("UPDATE Artifact SET x = 1") == 1 + + +async def _no_sleep(_seconds): + return None + + def test_provisioning_waits_for_an_elected_leader_not_just_ready_nodes(): """ Regression: on a cold start nodes answer "ready" before the election ends, and diff --git a/backend/tests/test_security.py b/backend/tests/test_security.py index b640487..5ee4a17 100644 --- a/backend/tests/test_security.py +++ b/backend/tests/test_security.py @@ -238,6 +238,41 @@ def test_mass_registration_is_throttled(client, rate_limited): assert register(client)[0].status_code == 429 +def test_sign_in_stays_fast_when_redis_hangs(client, rate_limited, monkeypatch): + # with the Redis container gone, even resolving its name can take many seconds: + # the limiter must give up within its budget, then leave Redis alone for a while + import asyncio + import time + from app import coordination + calls = [] + + class Hanging: + def __getattr__(self, name): + async def never(*a, **kw): + calls.append(name) + await asyncio.sleep(60) + return never + + _, username = register(client) + t0 = time.perf_counter() + client.post("/auth/token", data={"username": username, "password": "secret123"}) + baseline = time.perf_counter() - t0 # a sign-in here, with the limiter in memory + monkeypatch.setenv("REDIS_URL", "redis://redis.invalid:6379/0") + monkeypatch.setattr(coordination, "_redis", lambda: Hanging()) + monkeypatch.setattr(coordination, "REDIS_TIMEOUT", 0.5) + t0 = time.perf_counter() + assert client.post("/auth/token", data={"username": username, "password": "secret123"}).status_code == 200 + # one Redis call hangs, then the limiter gives up after its 0.5 s budget + assert time.perf_counter() - t0 < baseline + 1.5, "the sign-in waited on Redis past the limiter's budget" + first = len(calls) + assert first == 1 + t0 = time.perf_counter() + assert client.post("/auth/token", data={"username": username, "password": "wrong"}).status_code == 401 + assert client.post("/auth/token", data={"username": username, "password": "secret123"}).status_code == 200 + assert len(calls) == first, "Redis was retried during its back-off" + assert time.perf_counter() - t0 < 2 * baseline + 1 + + def test_rate_limits_are_shared_through_redis(client, rate_limited, monkeypatch): # in worker mode every app instance counts in Redis, so guesses spread over instances add up import os @@ -419,3 +454,30 @@ async def go(): demoted, again, roles = client.portal.call(go) assert sorted(demoted) == ["later", "latest"] and again == [] assert roles == {"old_founder": "admin", "later": "member", "latest": "member"} + + +# ── passwords ─────────────────────────────────────────────────────────────── + +@pytest.mark.parametrize("password, reason", [ + ("short1", "at least 8"), ("password123", "too common"), ("aaaaaaaaaa", "too common"), +]) +def test_weak_passwords_are_refused(client, password, reason): + r = client.post("/auth/register", json={"username": f"user_{uuid.uuid4().hex[:10]}", "password": password}) + assert r.status_code == 422 and reason in r.text + + +def test_the_password_cannot_be_the_username(client, make_user): + name = f"Same_{uuid.uuid4().hex[:6]}" + r = client.post("/auth/register", json={"username": name, "password": name.lower()}) + assert r.status_code == 422 and "same as the username" in r.text + r = client.post("/admin/users", headers=make_user(role="admin"), json={"username": name, "password": "short"}) + assert r.status_code == 422 # admin-created accounts follow the same rules + + +def test_an_unknown_username_costs_a_password_check_too(client, monkeypatch): + """Regression: unknown names answered in 15 ms and real ones in 250 ms, which told them apart.""" + import app.routers.auth as auth_router + burned = [] + monkeypatch.setattr(auth_router, "burn_password_check", lambda plain: burned.append(plain)) + r = client.post("/auth/token", data={"username": f"ghost_{uuid.uuid4().hex[:8]}", "password": "guess-1234"}) + assert r.status_code == 401 and burned == ["guess-1234"] diff --git a/deploy/.env.example b/deploy/.env.example index 380c513..c861d2c 100644 --- a/deploy/.env.example +++ b/deploy/.env.example @@ -43,6 +43,11 @@ RETENTION_INACTIVE_DAYS=365 RETENTION_CHECK_HOURS=24 MAX_UPLOAD_MB=20 +# ── App instances ───────────────────────────────────────────────────────────── +# Server processes in each of app1 and app2. About 500 MB of memory each (every +# process loads the embedding model); a good value is the CPU cores per instance. +APP_WORKERS=2 + # ── Background worker ───────────────────────────────────────────────────────── # If a worker dies mid-sync (crash, kill -9), the sync is re-queued after this many # seconds. Keep it longer than your longest sync. diff --git a/deploy/docker-compose.yml b/deploy/docker-compose.yml index ad48da9..c382d64 100644 --- a/deploy/docker-compose.yml +++ b/deploy/docker-compose.yml @@ -128,6 +128,11 @@ services: # ── app instances (stateless; any can serve any request) ────────────────── app1: &app <<: *backend + environment: + <<: *backend-env + # server processes per instance (uvicorn reads WEB_CONCURRENCY). Each holds its own + # copy of the embedding model, about 500 MB. + WEB_CONCURRENCY: ${APP_WORKERS:-2} depends_on: provision: { condition: service_completed_successfully } redis: { condition: service_healthy } diff --git a/docs/api.md b/docs/api.md index f77f495..c5ac524 100644 --- a/docs/api.md +++ b/docs/api.md @@ -9,12 +9,12 @@ All endpoints except `/auth/register`, `/auth/setup`, `/auth/token`, `GET /healt | Spaces & groups | `GET /spaces`; `POST/GET /groups`, `PATCH/DELETE /groups/{id}`, `GET /groups/{id}/members`, `GET /groups/{id}/candidates?q=` (search users), `POST /groups/{id}/invitations`, `DELETE /groups/{id}/members/{user_id}`, `POST /groups/{id}/leave`; `GET /invitations`, `POST /invitations/{group_id}/accept`, `/decline` | | Ingest | `POST /knowledge/artifacts` (text), `/artifacts/upload` (multipart), `/artifacts/url`, `/artifacts/transcript` | | Artifacts | `PUT /knowledge/artifacts/{id}` (a content change re-extracts items), `DELETE /knowledge/artifacts/{id}`, `POST /knowledge/artifacts/{id}/share` (copy into another space) | -| Items | `GET/PUT/DELETE /knowledge/items/{id}`, `GET /knowledge/{ref}` (by id, id suffix or title) | -| Browse | `GET /knowledge` (everything in the space), `GET /knowledge/graph` | -| Review | `GET /knowledge/review`, `PATCH /knowledge/review/{id}` | -| Search | `GET /knowledge/search?q=&type=&source_type=&tag=&limit=` | +| Items | `GET /knowledge/items?q=&type=&type=&exclude_rejected=&limit=&offset=` (newest first, with `total`), `GET/PUT/DELETE /knowledge/items/{id}`, `GET /knowledge/items/{id}/context` (its source with text, its relationships and their titles), `GET /knowledge/{ref}` (by id, id suffix or title) | +| Browse | `GET /knowledge?limit=&offset=&q=&type=&include_content=` (a page of artifacts and items, newest first, default 100, at most 1,000; `totals`; artifacts without their text unless `include_content`), `GET /knowledge/graph?limit=` (the newest 300 nodes by default; `truncated`, `total_nodes`) | +| Review | `GET /knowledge/review?limit=&offset=` (oldest first, default 50; `X-Total-Count` header), `PATCH /knowledge/review/{id}` | +| Search | `GET /knowledge/search?q=&type=&source_type=&tag=&limit=` (default 50; artifacts come with a `snippet` and a `first_item_id`, not their text) | | Links | `POST /knowledge/link`, `GET /knowledge/links` | -| Lineage | `GET /knowledge/lineage/kinds`, `GET/POST /knowledge/items/{id}/lineage`, `DELETE /knowledge/items/{id}/lineage/{other}`, `GET …/lineage/suggestions`, `PUT /knowledge/items/{id}/status`, `GET /knowledge/register` | +| Lineage | `GET /knowledge/lineage/kinds`, `GET/POST /knowledge/items/{id}/lineage`, `DELETE /knowledge/items/{id}/lineage/{other}`, `GET …/lineage/suggestions`, `PUT /knowledge/items/{id}/status`, `GET /knowledge/register?type=&limit=&offset=` (newest first, default 300; `total`, and `related` for chain items outside the page) | | Playbooks | `GET /knowledge/playbooks`, `POST /knowledge/playbooks` (same title replaces), `DELETE /knowledge/playbooks/{id}` | | OKF | `GET /knowledge/okf/export`, `POST /knowledge/okf/import` | | GraphRAG | `POST /knowledge/graphrag/query`, `POST /knowledge/reembed` | diff --git a/docs/configuration.md b/docs/configuration.md index 21c6d41..cf9bd1e 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -30,6 +30,8 @@ Where settings go depends on how you run it: | `LOCAL_EMBED_MODEL` | `all-MiniLM-L6-v2` | sentence-transformers model | | `MAX_UPLOAD_MB` | `20` | Cap on request bodies, uploaded files and fetched pages | | `GRAPHRAG_RERANK` | `on` | `off` disables the LLM rerank step | +| `OLLAMA_STATUS_CACHE_SECONDS` | `5` | How long the answer to "is Ollama up, which models are installed" is reused (the public model catalog and every sign-in ask it). Installing or removing a model refreshes it. `0` asks every time | +| `APP_WORKERS` (deploy) / `WEB_CONCURRENCY` | `2` / `1` | Server processes per app instance. More than 1 needs `REDIS_URL` (worker mode): without it the API refuses to start. Each process loads the embedding model, about 500 MB | | `LLM_PROVIDER`, `EMBEDDING_PROVIDER` | `ollama`, `local` | Read but not used: those are the only implementations | **Changing the embedding model.** Vector indexes are sized to the model's dimension. diff --git a/docs/deployment.md b/docs/deployment.md index 4da3c44..0ef2d5a 100644 --- a/docs/deployment.md +++ b/docs/deployment.md @@ -65,15 +65,22 @@ Run against this stack, through the load balancer: | Sync started from the UI | queued by the API, run by the worker, never by an app instance | | Clean-up | scheduled by beat; locked so concurrent runs can't overlap | | Database access | the app connects as `kh_app`; the root password reaches only the ArcadeDB nodes and `provision` | -| Redis stopped | `/health` `degraded` in under 2 s; reads, ingest, search and the connectors list kept working; starting a sync answered 503 "background queue unavailable"; syncs queued again once Redis was back | +| Redis stopped | `/health` `degraded` in under 2 s; reads, ingest, search, sign-in and the connectors list kept working; starting a sync answered 503 "background queue unavailable"; syncs queued again once Redis was back | +| A group created, then listed at once (60 times, app on a follower node) | listed every time (before the fix: missing 4 times in 30) | | Rolling restart (`app1`, then `app2`) | one instance was serving at all times | Two scenarios found real bugs, both now fixed and covered by tests. They were the same mistake twice: a slow dependency made `/health` slower than the load balancer's 5-second check, so the balancer took every healthy instance out of rotation and turned a partial outage into a full one. The first run of the "leader stopped" scenario found the first one. `/health` probed the database nodes one by one, so a dead node made each check take 15 seconds. The load balancer then marked **both** healthy app instances as down. Probes now run in parallel with a 2-second limit, and the database client gives up connecting to a dead node after 3 seconds. The second: with Redis stopped, the health check's Redis ping had no timeout and hung. Redis calls now time out after 2 seconds. +A later walkthrough found two more: + +- **Stale reads after a write.** Every node accepts reads, but a follower can answer before it has applied a write the app just made through it (followers forward writes to the leader). A group created a moment ago was missing from the next list 4 times in 30. The app now sends its work to the current leader, which it looks up at most every 10 seconds; the other nodes take over if the leader is unreachable. ArcadeDB's `arcadedb.ha.readConsistency=linearizable` fixes this too, but measured at about 3 seconds per read that follows a write (against 15 ms). +- **Sign-in failing while Redis was down.** With the Redis container stopped, even looking up its host name hangs, in resolver threads the whole process shares. The health probe and the sign-in rate limiter kept starting new lookups until unrelated ones, such as Ollama's, timed out: 2 sign-ins in 8 answered 503 "Ollama is not running". After a Redis failure, both now leave Redis alone for 30 seconds, so `/health` can take up to 30 seconds to report `healthy` again once Redis is back. + ## Operating it +- **More processes per instance:** `APP_WORKERS` in `deploy/.env` (default 2) sets how many server processes each app instance runs. Each loads the embedding model (about 500 MB); one per CPU core is a good start. - **More app instances:** copy `app2` in `docker-compose.yml` and add it to `APP_UPSTREAMS`. More sync throughput: raise the worker's `--concurrency`, or run more `worker` replicas. Keep **one** `beat`. - **Logs:** `docker compose logs -f app1 worker`. Health: `curl -k https:///health` shows each node, the queue and the background mode. - **Backups:** run [`deploy/backup.sh`](../deploy/backup.sh) (`sh deploy/backup.sh`) while the stack is up, nightly from the host's crontab: `0 3 * * * sh /path/to/deploy/backup.sh`. It has ArcadeDB write a consistent snapshot, as the app's own user, copies the zip to `deploy/backups/` and deletes local copies older than `BACKUP_KEEP_DAYS` (14). Copy that folder off the machine as well. Queued syncs in `redis-data` don't need backing up: a lost sync runs again the next time someone clicks **Sync now**. diff --git a/docs/known-limitations.md b/docs/known-limitations.md index 084a546..97bf981 100644 --- a/docs/known-limitations.md +++ b/docs/known-limitations.md @@ -4,6 +4,8 @@ - **One LLM, one embedding model.** Only Ollama and local sentence-transformers are implemented. - **Models are per machine, choices are per user.** Everyone on one server shares the same Ollama, so removing a model affects every account. Anyone using it is moved to another installed model. - **Full-text search doesn't fold accents.** The default analyzer means `cafe` won't match "Café". +- **Lists come a page at a time.** The Hub, review queue, decision log and graph show the newest (or, for review, oldest) page with **Load more**; the Hub's filter runs on the server. The graph draws at most the 300 newest nodes. The status counts on the decision log cover the loaded entries. +- **Sources are found through their items.** A source in search results opens an item extracted from it, whose page shows the source; a source nothing was extracted from has no page. - **Search slows on very broad queries.** On 50,000 items a precise query takes about 75 ms, but a one-word query matching 30,000+ items takes about 300 ms, mostly spent counting the total. - **Extracted text isn't size-capped.** Uploads and URLs are capped by `MAX_UPLOAD_MB`, but the text extracted from them isn't (pasted text is capped at 100,000 characters). LLM extraction reads at most 8 chunks. @@ -26,6 +28,7 @@ **Production stack** - **The cluster needs a majority.** Three nodes survive one being down. With two down there's no leader, and writes stop until a node returns. +- **Reads don't spread over the nodes.** Every app instance sends its reads and writes to the current leader, so that a read always sees the write before it. The other nodes are standbys: they keep a copy and take over when the leader goes. More nodes add resilience, not read capacity. - **One Ollama for everything.** Every app instance and the worker call the same Ollama on the Docker host, so LLM throughput doesn't grow with more instances. - **Exactly one `beat`.** Two would schedule the clean-up twice. It's harmless, because runs are locked, but wasteful. - **No horizontal Redis.** Redis is a single container. If it's down, starting a sync returns "background queue unavailable" (503), and the Sources page can't show live sync state. Everything else (reading, ingesting, search, GraphRAG) keeps working, and `/health` reports `degraded`. This was tested against the stack. diff --git a/docs/security.md b/docs/security.md index 2e564c7..7a3972c 100644 --- a/docs/security.md +++ b/docs/security.md @@ -3,9 +3,9 @@ How Knowledge Hubs protects accounts and data. To report a vulnerability, see [SECURITY.md](../SECURITY.md). **Accounts** -- **Passwords and tokens.** Passwords are hashed with bcrypt (which reads only the first 72 bytes of a password). Tokens are HS256 JWTs (PyJWT) signed with `SECRET_KEY` and expire after 8 hours; a token without an expiry is refused. The frontend sends the token only to its own API. +- **Passwords and tokens.** Passwords are hashed with bcrypt (which reads only the first 72 bytes of a password), on a worker thread so a sign-in doesn't stall other requests. New passwords need at least 8 characters, can't be one of the passwords guessing tools try first, and can't be the username; existing passwords keep working. Signing in with an unknown username runs the same bcrypt check against a dummy hash, so the response time doesn't reveal which usernames exist (before: 15 ms against 250 ms). Tokens are HS256 JWTs (PyJWT) signed with `SECRET_KEY` and expire after 8 hours; a token without an expiry is refused. The frontend sends the token only to its own API. - **The first account.** It becomes the server admin. With `SETUP_TOKEN` set (required in the production stack), registering it needs that token, compared in constant time, so whoever finds a new server first can't claim it. -- **Throttling.** Failed sign-ins are limited to 5 per account and 30 in total per client address every 15 minutes, and sign-ups to 20 per address an hour. Past that the API answers 429 with `Retry-After`, before checking the password. A successful sign-in clears that account's count. In the production stack the counters are in Redis, so the limit holds across app instances. The per-account limit is keyed by address too, so nobody can lock a user out by failing on purpose from elsewhere. +- **Throttling.** Failed sign-ins are limited to 5 per account and 30 in total per client address every 15 minutes, and sign-ups to 20 per address an hour. Past that the API answers 429 with `Retry-After`, before checking the password. A successful sign-in clears that account's count. In the production stack the counters are in Redis, so the limit holds across app instances. If Redis is unreachable the limits aren't enforced, rather than blocking sign-in; after a failure Redis is left alone for 30 seconds, so sign-in doesn't wait on it each time. The per-account limit is keyed by address too, so nobody can lock a user out by failing on purpose from elsewhere. - **Signing out.** Each token carries the account's session version. **Sign out** (`POST /auth/logout`) increments it, so every token issued before stops working at once, a copied one included. - **Signing key.** The server refuses to start if `SECRET_KEY` is missing, shorter than 32 characters, or one of the placeholders this repo has ever shipped. Anyone who knows the key could sign in as any account. - **Roles.** [`backend/app/rbac.py`](../backend/app/rbac.py) maps roles to permissions, and each route declares the permission it needs. Roles are read from the database on every request, so a change applies to existing sessions at once. An unknown role grants nothing, and the last server admin can't be demoted. diff --git a/frontend/src/app/app.routes.ts b/frontend/src/app/app.routes.ts index 05d139d..0fda282 100644 --- a/frontend/src/app/app.routes.ts +++ b/frontend/src/app/app.routes.ts @@ -1,7 +1,7 @@ import { Routes } from '@angular/router' import { inject } from '@angular/core' import { Router } from '@angular/router' -import { authGuard } from './services/auth.guard' +import { adminGuard, authGuard } from './services/auth.guard' import { AuthService } from './services/auth.service' /** Decode the JWT payload without verifying the signature (client-side only). */ @@ -41,6 +41,8 @@ export const routes: Routes = [ { path: 'settings/models', loadComponent: () => import('./pages/model-manager/model-manager.component').then(m => m.ModelManagerComponent), canActivate: [authGuard] }, { path: 'playbooks', loadComponent: () => import('./pages/playbooks/playbooks.component').then(m => m.PlaybooksComponent), canActivate: [authGuard] }, { path: 'groups', loadComponent: () => import('./pages/groups/groups.component').then(m => m.GroupsComponent), canActivate: [authGuard] }, - { path: 'admin/users', loadComponent: () => import('./pages/admin-users/admin-users.component').then(m => m.AdminUsersComponent), canActivate: [authGuard] }, + { path: 'admin/users', loadComponent: () => import('./pages/admin-users/admin-users.component').then(m => m.AdminUsersComponent), canActivate: [adminGuard] }, + // an empty stop SpaceService.switchTo passes through, so the page you're on is rebuilt for the new space + { path: 'switching-space', children: [] }, { path: 'privacy-policy', loadComponent: () => import('./pages/privacy-policy/privacy-policy.component').then(m => m.PrivacyPolicyComponent) }, ] diff --git a/frontend/src/app/models/api.ts b/frontend/src/app/models/api.ts index 7b07161..9e276ec 100644 --- a/frontend/src/app/models/api.ts +++ b/frontend/src/app/models/api.ts @@ -44,7 +44,8 @@ export interface KnowledgeItem { export interface Artifact { id: string title: string - content: string + /** the document text: left out of lists (GET /knowledge, search) */ + content?: string source: string source_type: string author: string @@ -83,10 +84,32 @@ export interface HealthResponse { /** GET /knowledge */ export interface KnowledgeResponse { + /** a page: the newest `limit` after `offset` (see totals) */ artifacts: Artifact[] knowledge_items: KnowledgeItem[] + /** between the returned nodes only */ relationships: Relationship[] playbooks: Playbook[] + totals?: KnowledgeTotals + item_types?: string[] +} +export interface KnowledgeTotals { + artifacts: number + knowledge_items: number + /** items matching the q / type filters */ + matching_items: number + pending_review: number +} + +/** GET /knowledge/items */ +export interface ItemsResponse { items: KnowledgeItem[]; total: number } + +/** GET /knowledge/items/{id}/context */ +export interface ItemContext { + item: KnowledgeItem + artifact: Artifact | null + relationships: Relationship[] + nodes: Record } /** GET /knowledge/graph */ @@ -96,6 +119,9 @@ export interface GraphResponse { nodes: GraphNode[] edges: GraphEdge[] layout: string + /** the graph holds the newest nodes; total_nodes counts them all */ + total_nodes?: number + truncated?: boolean } /** GET /knowledge/links */ @@ -124,6 +150,8 @@ export interface IngestResult { /** GET /knowledge/search */ export interface SearchArtifact extends Artifact { snippet?: string // present when the artifact matched in its content + /** an item extracted from it, whose page shows the source; absent if it has none */ + first_item_id?: string } export interface SearchResult { query: string @@ -141,6 +169,8 @@ export interface Citation { title: string type: string status?: ItemStatus // e.g. "superseded" when the cited decision was replaced + /** matched to the answer's wording because the model wrote no [id] for it */ + inferred?: boolean } export interface ContextNode { id: string @@ -343,8 +373,12 @@ export interface LinkKindsResponse { /** GET /knowledge/register */ export interface RegisterResponse { + /** newest first, at most `limit`; total counts them all */ items: KnowledgeItem[] edges: LineageEdge[] + total?: number + /** items the edges name that are outside this page */ + related?: KnowledgeItem[] } /* ── Connectors ───────────────────────────────────────────────────────────── */ diff --git a/frontend/src/app/pages/admin-users/admin-users.component.html b/frontend/src/app/pages/admin-users/admin-users.component.html index 68a5a3d..82a2364 100644 --- a/frontend/src/app/pages/admin-users/admin-users.component.html +++ b/frontend/src/app/pages/admin-users/admin-users.component.html @@ -1,7 +1,7 @@
-

Users

+

Users

Admins manage accounts, the machine's shared models, and folder connectors. Members manage only their own knowledge.

@@ -10,16 +10,16 @@

Users

@if (notice) {

{{ notice }}

}
-

Create an account

+

Create an account

- - -

{{ usernameRules }}

@@ -28,7 +28,7 @@

Create an account

@if (retention) {
-

Inactive accounts and groups

+

Inactive accounts and groups

@if (retention.enabled) {

Accounts and groups unused for {{ retention.inactive_days }} days are deleted with all their data. Checked every {{ retention.check_hours }} h@if (retention.last_run) {; last run {{ retention.last_run.ran_at | date:'medium' }}}. @@ -37,7 +37,7 @@

Inactive accounts and groups

Automatic deletion is off on this server (RETENTION_ENABLED=false). The lists below show what it would delete.

} -

Due for deletion now ({{ dueCount }})

+

Due for deletion now ({{ dueCount }})

@if (dueCount === 0) {

Nothing is due.

}
@for (u of retention.due.users; track u.id) { @@ -55,7 +55,7 @@

Due for deletion now ({{ } @if (retention.upcoming.users.length || retention.upcoming.groups.length) { -

Due within 30 days

+

Due within 30 days

@for (u of retention.upcoming.users; track u.id) {
account{{ u.username }} diff --git a/frontend/src/app/pages/connectors/connectors.component.html b/frontend/src/app/pages/connectors/connectors.component.html index 4900607..93f7cc3 100644 --- a/frontend/src/app/pages/connectors/connectors.component.html +++ b/frontend/src/app/pages/connectors/connectors.component.html @@ -1,7 +1,7 @@
-

Connectors

+

Connectors

Pull knowledge from where it already lives. Syncs only re-process documents that changed.

Decision log → @@ -11,7 +11,7 @@

Connectors

-

Add a source

+

Add a source

@for (k of kinds; track k.kind) { @@ -26,7 +26,7 @@

Add a source

@if (formOpen && kind) {
-

{{ editingId ? 'Edit' : 'New' }}: {{ kind.label }}

+

{{ editingId ? 'Edit' : 'New' }}: {{ kind.label }}

-

{{ c.name }}

+

{{ c.name }}

{{ summary(c) }}

@if (c.created_by) {

Added by {{ c.created_by }}

} @if (c.last_sync_at) {

Last sync {{ c.last_sync_at | date:'medium' }}

} diff --git a/frontend/src/app/pages/decisions/decisions.component.html b/frontend/src/app/pages/decisions/decisions.component.html index 3891db9..3bda737 100644 --- a/frontend/src/app/pages/decisions/decisions.component.html +++ b/frontend/src/app/pages/decisions/decisions.component.html @@ -1,7 +1,7 @@
-

{{ register === 'decision' ? 'Decision log' : 'Risk register' }}

+

{{ register === 'decision' ? 'Decision log' : 'Risk register' }}

{{ register === 'decision' ? 'Every decision, newest first, with what replaced it.' : 'Every risk, and whether it was mitigated or actually happened.' }}

@@ -26,7 +26,7 @@

{{ register === 'decision' ? 'Decision log' : 'Risk register' }}

@if (!loading && !error && rows.length === 0) {
-

No {{ register }}s yet

+

No {{ register }}s yet

Ingest meeting notes or connect a source of ADRs, pull requests or issues.

Set up a connector →
@@ -41,9 +41,9 @@

No {{ register }}s yet

@if (r.item.review_status === 'pending') { needs review } @for (t of r.item.tags.slice(0, 3); track t) { #{{ t }} }
-

+

{{ r.item.title }} -

+

@if (r.item.details.why) {

{{ r.item.details.why }}

} @for (x of r.replacedBy; track x.id) {

↳ {{ verb(x.kind) }} by @@ -60,4 +60,10 @@

} + @if (rows.length < total) { +
+ Showing the newest {{ rows.length }} of {{ total }} + +
+ } diff --git a/frontend/src/app/pages/decisions/decisions.component.ts b/frontend/src/app/pages/decisions/decisions.component.ts index 04f86f2..4a13f4f 100644 --- a/frontend/src/app/pages/decisions/decisions.component.ts +++ b/frontend/src/app/pages/decisions/decisions.component.ts @@ -28,6 +28,9 @@ export class DecisionsComponent implements OnInit { query = '' rows: Row[] = [] error = ''; loading = false + /** register entries in all; the page holds the newest `limit` */ + total = 0 + limit = 300 constructor(private http: HttpClient) {} @@ -35,22 +38,28 @@ export class DecisionsComponent implements OnInit { async switchTo(register: Register) { if (register === this.register) return - this.register = register; this.statusFilter = '' + this.register = register; this.statusFilter = ''; this.limit = 300 await this.load() } async load() { this.loading = true; this.error = '' try { - const params = new HttpParams().set('type', this.register) + const params = new HttpParams().set('type', this.register).set('limit', this.limit) const data = await firstValueFrom(this.http.get(`${API_BASE}/knowledge/register`, { params })) this.rows = this.toRows(data) + this.total = data.total ?? data.items.length } catch (e) { this.error = errorMessage(e, 'Could not load the register') } finally { this.loading = false } } + async loadMore() { + this.limit += 300 + await this.load() + } + private toRows(data: RegisterResponse): Row[] { - const titles = new Map(data.items.map(i => [i.id, i.title])) + const titles = new Map([...data.items, ...(data.related ?? [])].map(i => [i.id, i.title])) const ref = (id: string, kind: string): Ref => ({ id, title: titles.get(id) ?? id, kind }) const byTarget = new Map() const bySource = new Map() diff --git a/frontend/src/app/pages/graphrag/graphrag.component.html b/frontend/src/app/pages/graphrag/graphrag.component.html index 286a5e6..64b38b9 100644 --- a/frontend/src/app/pages/graphrag/graphrag.component.html +++ b/frontend/src/app/pages/graphrag/graphrag.component.html @@ -1,7 +1,7 @@
- GraphRAG Assistant +

GraphRAG Assistant

Graph-aware retrieval · answers grounded in your knowledge base

@@ -44,6 +44,10 @@ {{ cite.status }} } {{ cite.title }} + @if (cite.inferred) { + matched + } } diff --git a/frontend/src/app/pages/groups/groups.component.html b/frontend/src/app/pages/groups/groups.component.html index 11b1a62..cda1d66 100644 --- a/frontend/src/app/pages/groups/groups.component.html +++ b/frontend/src/app/pages/groups/groups.component.html @@ -1,7 +1,7 @@
-

Groups

+

Groups

Share knowledge with a group. You also keep your own space, which only you can see.

@@ -12,7 +12,7 @@

Groups

@if (spaces.invitations().length) {
-

Invitations

+

Invitations

@for (inv of spaces.invitations(); track inv.group_id) {
@@ -28,7 +28,7 @@

Invitations

-

Create a group

+

Create a group

@@ -39,7 +39,7 @@

Create a group

-

Your groups

+

Your groups

@if (groups.length === 0) {

You're not in any group yet.

}
@for (g of groups; track g.id) { @@ -62,13 +62,13 @@

Your groups

@if (!selected) { -

Members

+

Members

Select a group to see its members.

} @else { -

{{ selected.name }}

+

{{ selected.name }}

@if (isAdmin) { -

Invite people

+

Invite people

@if (candidates.length) { @@ -87,7 +87,7 @@

Invite people

} } -

Members ({{ active.length }})

+

Members ({{ active.length }})

@for (m of active; track m.user_id) {
@@ -101,7 +101,7 @@

Members ({{ active.length }}

@if (isAdmin && pending.length) { -

Invitations

+

Invitations

@for (m of pending; track m.user_id) {
diff --git a/frontend/src/app/pages/knowledge-detail/knowledge-detail.component.html b/frontend/src/app/pages/knowledge-detail/knowledge-detail.component.html index 275fc00..81c1857 100644 --- a/frontend/src/app/pages/knowledge-detail/knowledge-detail.component.html +++ b/frontend/src/app/pages/knowledge-detail/knowledge-detail.component.html @@ -6,7 +6,7 @@ } @if (!error && !item) { -

{{ data ? 'Knowledge item not found' : 'Loading…' }}

+

{{ ctx || error ? 'Knowledge item not found' : 'Loading…' }}

} @if (!error && item) { @@ -14,7 +14,7 @@
{{ item.type }} @if (!editing) { -

{{ item.title }}

+

{{ item.title }}

} @else { } @@ -155,7 +155,7 @@

Relat

- +

Relationships

diff --git a/frontend/src/app/pages/knowledge-detail/knowledge-detail.component.ts b/frontend/src/app/pages/knowledge-detail/knowledge-detail.component.ts index 97c25bf..30ed580 100644 --- a/frontend/src/app/pages/knowledge-detail/knowledge-detail.component.ts +++ b/frontend/src/app/pages/knowledge-detail/knowledge-detail.component.ts @@ -6,7 +6,7 @@ import { HttpClient } from '@angular/common/http' import { firstValueFrom } from 'rxjs' import { API_BASE } from '../../services/auth.service' import { errorMessage } from '../../services/http-error' -import { Artifact, CrossLink, KnowledgeItem, KnowledgeResponse, Relationship } from '../../models/api' +import { Artifact, CrossLink, ItemContext, ItemsResponse, KnowledgeItem, Relationship } from '../../models/api' import { ItemLineageComponent } from '../../components/item-lineage/item-lineage.component' import { SpaceService } from '../../services/space.service' import { SpaceInfo } from '../../models/api' @@ -21,7 +21,11 @@ type RelatedItem = { item: KnowledgeItem; score: number } styleUrl: './knowledge-detail.component.css' }) export class KnowledgeDetailComponent implements OnInit { - data: KnowledgeResponse | null = null; error = ''; id = '' + /** the item with what its page shows around it (GET /knowledge/items/{id}/context) */ + ctx: ItemContext | null = null; error = ''; id = '' + /** link targets for the lineage panel: the newest LINK_CANDIDATES items */ + candidates: KnowledgeItem[] = [] + private readonly LINK_CANDIDATES = 500 private loadedItem: KnowledgeItem | null = null editing = false; saving = false; deleting = false; crudError = '' editedTitle = '' @@ -34,15 +38,13 @@ export class KnowledgeDetailComponent implements OnInit { private readonly SKIP_KEYS = new Set(['what', 'steps', 'okf_original_id']) get item(): KnowledgeItem | undefined { - return this.loadedItem ?? this.data?.knowledge_items.find(i => i.id === this.id) + return this.loadedItem ?? this.ctx?.item } get artifact(): Artifact | undefined { - return this.data?.artifacts.find(a => a.id === this.item?.artifact_id) + return this.ctx?.artifact ?? undefined } get relationships(): Relationship[] { - const item = this.item - if (!this.data || !item) return [] - return this.data.relationships.filter(e => e.from === item.id || e.to === item.id) + return this.ctx?.relationships ?? [] } // Deduplicated, filtered detail entries @@ -81,14 +83,15 @@ export class KnowledgeDetailComponent implements OnInit { } get relatedItems(): RelatedItem[] { - const data = this.data, item = this.item - if (!data || !item) return [] + const ctx = this.ctx, item = this.item + if (!ctx || !item) return [] return this.crossLinks .filter(l => l.item_id_a === item.id || l.item_id_b === item.id) .map(l => { const otherId = l.item_id_a === item.id ? l.item_id_b : l.item_id_a - const other = data.knowledge_items.find(i => i.id === otherId) - return other ? { item: other, score: l.score } : null + const node = ctx.nodes[otherId] + return node?.kind === 'item' + ? { item: { id: otherId, title: node.title, type: node.type } as KnowledgeItem, score: l.score } : null }) .filter((x): x is RelatedItem => x !== null) .sort((a, b) => b.score - a.score) @@ -97,11 +100,9 @@ export class KnowledgeDetailComponent implements OnInit { // Resolve a relationship edge to a display label + route resolveNode(edge: Relationship): { label: string; route: string[] } | null { const otherId = edge.from === this.item?.id ? edge.to : edge.from - const ki = this.data?.knowledge_items.find(i => i.id === otherId) - if (ki) return { label: ki.title, route: ['/knowledge', ki.id] } - const art = this.data?.artifacts.find(a => a.id === otherId) - if (art) return { label: art.title, route: ['/knowledge'] } - return null + const node = this.ctx?.nodes[otherId] + if (!node) return null + return { label: node.title, route: node.kind === 'item' ? ['/knowledge', otherId] : ['/knowledge'] } } // Highlight evidence phrases from details in the source content @@ -165,13 +166,17 @@ export class KnowledgeDetailComponent implements OnInit { async load() { try { - const [item, links, kr] = await Promise.all([ - firstValueFrom(this.http.get(`${API_BASE}/knowledge/items/${this.id}`)), + const id = encodeURIComponent(this.id) + const [ctx, links, candidates] = await Promise.all([ + firstValueFrom(this.http.get(`${API_BASE}/knowledge/items/${id}/context`)), firstValueFrom(this.http.get(`${API_BASE}/knowledge/links`)).catch((): CrossLink[] => []), - firstValueFrom(this.http.get(`${API_BASE}/knowledge`)), + firstValueFrom(this.http.get(`${API_BASE}/knowledge/items`, + { params: { exclude_rejected: true, limit: this.LINK_CANDIDATES } })) + .catch((): ItemsResponse => ({ items: [], total: 0 })), ]) - this.loadedItem = item - this.data = kr + this.loadedItem = ctx.item + this.ctx = ctx + this.candidates = candidates.items this.crossLinks = links } catch (e) { this.error = errorMessage(e, 'Could not load knowledge item') } } diff --git a/frontend/src/app/pages/knowledge/knowledge.component.html b/frontend/src/app/pages/knowledge/knowledge.component.html index 101553f..6bb74bc 100644 --- a/frontend/src/app/pages/knowledge/knowledge.component.html +++ b/frontend/src/app/pages/knowledge/knowledge.component.html @@ -1,8 +1,8 @@
-

Knowledge Hub

-

{{ data.artifacts.length }} artifacts · {{ data.knowledge_items.length }} items{{ pendingCount > 0 ? ' · ' + pendingCount + ' pending review' : '' }}

+

Knowledge Hub

+

{{ artifactTotal }} artifacts · {{ itemTotal }} items{{ pendingCount > 0 ? ' · ' + pendingCount + ' pending review' : '' }}

@@ -31,7 +31,7 @@

Knowledge Hub

-

Ingest Artifact

+

Ingest Artifact

Paste text, upload a file, fetch a URL, or extract decisions from a meeting transcript / email / Slack thread using AI.

@@ -97,7 +97,7 @@

Ingest Artifact

- @@ -122,7 +122,7 @@

Ingest Artifact

@if (data.artifacts.length > 0) {
-

Artifacts ({{ data.artifacts.length }})

+

Artifacts ({{ data.artifacts.length < artifactTotal ? data.artifacts.length + ' of ' + artifactTotal : artifactTotal }})

@for (a of data.artifacts; track a) {
@@ -156,8 +156,8 @@

Artifacts
-

Knowledge Graph

-

{{ graph.nodes.length }} nodes · {{ graph.edges.length }} relationships

+

Knowledge Graph

+

{{ graph.nodes.length }} nodes · {{ graph.edges.length }} relationships{{ graph.truncated ? ' · the newest ' + graph.nodes.length + ' of ' + graph.total_nodes : '' }}

@if (selectedNode && selectedNode.type !== 'artifact') { @@ -175,7 +175,7 @@

Knowledge Graph

Knowledge Graph

-
} @else {
-

Select a node

+

Select a node

Inspect its relationships and jump into the detail page.

} - +
} @else {
-

No graph yet

+

No graph yet

Ingest an artifact to generate artifact-to-knowledge relationships.

} @@ -237,9 +237,10 @@

No graph yet

- +
- @for (t of itemTypes; track t) { @@ -260,7 +261,7 @@

No graph yet

} {{ item.date | date }}
-

{{ item.title }}

+

{{ item.title }}

by {{ item.author }}

@if (item.review_status === 'pending') { pending review @@ -272,9 +273,20 @@

{{ item.title }}

} @if (filteredItems.length === 0) {
-

No knowledge items yet

-

Add an artifact above to extract decisions, risks, best practices, and checklists.

+ @if (filtering) { +

No matching items

+

Try another word or type, or use Advanced search for the full text of sources.

+ } @else { +

No knowledge items yet

+

Add an artifact above to extract decisions, risks, best practices, and checklists.

+ }
}
+ @if (hasMore) { +
+ Showing {{ filteredItems.length }} of {{ matchingTotal }} items + +
+ }
diff --git a/frontend/src/app/pages/knowledge/knowledge.component.ts b/frontend/src/app/pages/knowledge/knowledge.component.ts index 1119506..4252679 100644 --- a/frontend/src/app/pages/knowledge/knowledge.component.ts +++ b/frontend/src/app/pages/knowledge/knowledge.component.ts @@ -2,7 +2,7 @@ import { Component, OnInit, ElementRef, ViewChild, HostListener } from '@angular import { CommonModule } from '@angular/common' import { FormsModule } from '@angular/forms' import { RouterLink } from '@angular/router' -import { HttpClient } from '@angular/common/http' +import { HttpClient, HttpParams } from '@angular/common/http' import { firstValueFrom } from 'rxjs' import { API_BASE } from '../../services/auth.service' import { errorMessage } from '../../services/http-error' @@ -24,6 +24,10 @@ export class KnowledgeComponent implements OnInit { data: KnowledgeResponse = { artifacts: [], knowledge_items: [], relationships: [], playbooks: [] } graph: GraphResponse = { nodes: [], edges: [], layout: 'force-directed' } query = ''; typeFilter = 'all'; loading = false; error = '' + // the space a page at a time: GET /knowledge returns the newest PAGE_SIZE after `offset` + readonly PAGE_SIZE = 100 + loadingMore = false + private filterTimer: ReturnType | null = null selectedNodeId = ''; ingestMode: IngestMode = 'text' title = ''; author = ''; tags = ''; content = '' fileTitle = ''; fileAuthor = ''; fileTags = ''; selectedFile: File | null = null @@ -99,15 +103,52 @@ export class KnowledgeComponent implements OnInit { ngOnInit() { this.loadKnowledge() } - get filteredItems() { - return this.data.knowledge_items.filter(item => { - const matchQ = `${item.title} ${item.tags.join(' ')}`.toLowerCase().includes(this.query.toLowerCase()) - return matchQ && (this.typeFilter === 'all' || item.type === this.typeFilter) - }) + /** the server filters (title / tags and type), so the page holds only matching items */ + get filteredItems() { return this.data.knowledge_items } + + get itemTypes() { return this.data.item_types ?? [...new Set(this.data.knowledge_items.map(i => i.type))].sort() } + get pendingCount() { + return this.data.totals?.pending_review ?? this.data.knowledge_items.filter(i => i.review_status === 'pending').length + } + get artifactTotal() { return this.data.totals?.artifacts ?? this.data.artifacts.length } + get itemTotal() { return this.data.totals?.knowledge_items ?? this.data.knowledge_items.length } + get matchingTotal() { return this.data.totals?.matching_items ?? this.data.knowledge_items.length } + get filtering() { return !!this.query.trim() || this.typeFilter !== 'all' } + get hasMore() { + return this.data.knowledge_items.length < this.matchingTotal || this.data.artifacts.length < this.artifactTotal + } + + /** Typing in the filter: ask the server again once the typing pauses. */ + onFilterChange() { + if (this.filterTimer) clearTimeout(this.filterTimer) + this.filterTimer = setTimeout(() => this.loadKnowledge(), 300) } - get itemTypes() { return [...new Set(this.data.knowledge_items.map(i => i.type))].sort() } - get pendingCount() { return this.data.knowledge_items.filter(i => i.review_status === 'pending').length } + private pageParams(offset: number): HttpParams { + let params = new HttpParams().set('limit', this.PAGE_SIZE).set('offset', offset) + if (this.query.trim()) params = params.set('q', this.query.trim()) + if (this.typeFilter !== 'all') params = params.set('type', this.typeFilter) + return params + } + + async loadMore() { + this.loadingMore = true + try { + const offset = Math.max(this.data.knowledge_items.length, this.data.artifacts.length) + const more = await firstValueFrom(this.http.get(`${API_BASE}/knowledge`, + { params: this.pageParams(offset) })) + const have = new Set(this.data.relationships.map(r => `${r.from}|${r.to}|${r.type}`)) + this.data = { + ...more, + artifacts: [...this.data.artifacts, ...more.artifacts], + knowledge_items: [...this.data.knowledge_items, ...more.knowledge_items], + relationships: [...this.data.relationships, + ...more.relationships.filter(r => !have.has(`${r.from}|${r.to}|${r.type}`))], + } + } catch (e) { + this.error = errorMessage(e, 'Failed to load more knowledge') + } finally { this.loadingMore = false } + } get selectedNode() { return this.graph.nodes.find(n => n.id === this.selectedNodeId) } get selectedNodeLinks() { if (!this.selectedNode) return [] @@ -241,7 +282,7 @@ export class KnowledgeComponent implements OnInit { async loadKnowledge() { try { const [kr, gr] = await Promise.all([ - firstValueFrom(this.http.get(`${API_BASE}/knowledge`)), + firstValueFrom(this.http.get(`${API_BASE}/knowledge`, { params: this.pageParams(0) })), firstValueFrom(this.http.get(`${API_BASE}/knowledge/graph`)), ]) this.data = kr diff --git a/frontend/src/app/pages/login/login.component.css b/frontend/src/app/pages/login/login.component.css index 5f29731..01e0430 100644 --- a/frontend/src/app/pages/login/login.component.css +++ b/frontend/src/app/pages/login/login.component.css @@ -1,7 +1,7 @@ /* layout comes from .auth-shell / .auth-card in styles.css */ .auth-brand { display: flex; align-items: center; gap: 0.65rem; margin-bottom: 0.25rem; } .auth-brand .brand-mark { width: 38px; height: 38px; border-radius: 11px; } -.auth-card h2 { font-size: 1.35rem; letter-spacing: -0.02em; } +.auth-card h1 { font-size: 1.35rem; letter-spacing: -0.02em; } .auth-sub { color: var(--text-3); font-size: 0.9rem; margin-top: -0.5rem; } .form-stack { margin: 0.5rem 0 0.25rem; } .input-field { width: 100%; min-height: 42px; } diff --git a/frontend/src/app/pages/login/login.component.html b/frontend/src/app/pages/login/login.component.html index 3cf955f..bbeab91 100644 --- a/frontend/src/app/pages/login/login.component.html +++ b/frontend/src/app/pages/login/login.component.html @@ -2,7 +2,7 @@
-

Knowledge Hubs

+

Knowledge Hubs

@if (stage === 'checking') { @@ -14,7 +14,10 @@

Knowledge Hubs

@if (justInstalled) {

✓ {{ justInstalled }} is installed. Sign in to start.

}
- + + @if (mode === 'register') {

{{ passwordRules }}

} @if (mode === 'register' && needsSetupToken) { diff --git a/frontend/src/app/pages/login/login.component.ts b/frontend/src/app/pages/login/login.component.ts index f8ec663..979e9d1 100644 --- a/frontend/src/app/pages/login/login.component.ts +++ b/frontend/src/app/pages/login/login.component.ts @@ -7,6 +7,7 @@ import { ModelCatalog, ModelService } from '../../services/model.service' import { ModelSetupComponent } from '../../components/model-setup/model-setup.component' import { errorMessage } from '../../services/http-error' import { IconComponent } from '../../components/icon/icon.component' +import { PASSWORD_RULES } from '../../services/usernames' /** * Sign-in starts with a model check: one of the three local models must be @@ -29,6 +30,7 @@ export class LoginComponent implements OnInit { username = ''; password = ''; error = ''; loading = false // a fresh server with SETUP_TOKEN set: the first account (the admin) needs it needsSetupToken = false; setupToken = '' + readonly passwordRules = PASSWORD_RULES // set by authInterceptor when the API rejected a stored token readonly sessionExpired: boolean diff --git a/frontend/src/app/pages/model-manager/model-manager.component.css b/frontend/src/app/pages/model-manager/model-manager.component.css index 1320b04..ae32056 100644 --- a/frontend/src/app/pages/model-manager/model-manager.component.css +++ b/frontend/src/app/pages/model-manager/model-manager.component.css @@ -73,7 +73,7 @@ h1 { font-size: 1.6rem; font-weight: 700; letter-spacing: -0.025em; margin-botto .ram-note { color: var(--warning-text); } .model-card-top { display: flex; justify-content: space-between; align-items: flex-start; } -.model-card h3 { font-size: 1rem; margin: 0; } +.model-card h2 { font-size: 1rem; margin: 0; } .model-meta { font-size: 0.8rem; color: var(--text-3); margin-top: 0.2rem; } .badge-group { display: flex; flex-direction: column; align-items: flex-end; gap: 0.3rem; } diff --git a/frontend/src/app/pages/model-manager/model-manager.component.html b/frontend/src/app/pages/model-manager/model-manager.component.html index fb41c1a..e44bcfd 100644 --- a/frontend/src/app/pages/model-manager/model-manager.component.html +++ b/frontend/src/app/pages/model-manager/model-manager.component.html @@ -53,7 +53,7 @@

Model Manager

-

{{ model.name }}

+

{{ model.name }}

{{ model.size }} · needs {{ model.ramRequired }} RAM

@@ -117,7 +117,7 @@

{{ model.name }}

-

Embeddings

+

Embeddings

Model: {{ service.modelStatus()?.embedding?.model || 'all-MiniLM-L6-v2' }}. After changing LOCAL_EMBED_MODEL on the server, re-embed so search and GraphRAG diff --git a/frontend/src/app/pages/onboarding/onboarding.component.html b/frontend/src/app/pages/onboarding/onboarding.component.html index 73d117f..e4472d4 100644 --- a/frontend/src/app/pages/onboarding/onboarding.component.html +++ b/frontend/src/app/pages/onboarding/onboarding.component.html @@ -37,14 +37,28 @@

Create Your Account

- + + {{ passwordRules }}
+ @if (needsSetupToken) { +
+ + + + This is a new hub: the first account becomes its administrator. Enter the + SETUP_TOKEN from the server's configuration. +
+ } @if (setupError) {

{{ setupError }}

}
-
diff --git a/frontend/src/app/pages/onboarding/onboarding.component.spec.ts b/frontend/src/app/pages/onboarding/onboarding.component.spec.ts new file mode 100644 index 0000000..3f12d97 --- /dev/null +++ b/frontend/src/app/pages/onboarding/onboarding.component.spec.ts @@ -0,0 +1,69 @@ +import { ComponentFixture, TestBed } from '@angular/core/testing' +import { provideHttpClient } from '@angular/common/http' +import { HttpTestingController, provideHttpClientTesting } from '@angular/common/http/testing' +import { provideRouter } from '@angular/router' +import { OnboardingComponent } from './onboarding.component' + +const CATALOG = { + ollamaReachable: true, anyInstalled: true, ramGb: 16, recommended: 'llama3.1:8b', + models: [{ id: 'llama3.1:8b', name: 'Llama 3.1 8B', size: '4.7 GB', ramRequired: '8 GB+', minRamGb: 8, + description: '', provider: 'ollama' as const, installed: true, active: false }], +} + +describe('OnboardingComponent first account', () => { + let fixture: ComponentFixture + let http: HttpTestingController + const text = () => (fixture.nativeElement as HTMLElement).textContent ?? '' + const tick = () => new Promise(resolve => setTimeout(resolve)) + + async function openAccountStep(setupTokenRequired: boolean) { + fixture.detectChanges() // ngOnInit → GET /models/catalog, /auth/setup + http.expectOne('/models/catalog').flush(CATALOG) + http.expectOne('/auth/setup').flush({ setup_token_required: setupTokenRequired, registration_open: true }) + await fixture.whenStable() + const started = fixture.componentInstance.start() // Get Started re-checks the catalog + http.expectOne('/models/catalog').flush(CATALOG) + await started + fixture.detectChanges() + } + + async function createAccount(expectedBody: object) { + const c = fixture.componentInstance + c.username = 'priya'; c.password = 'secret123' + const done = c.createAccount() + const register = http.expectOne('/auth/register') + expect(register.request.body).toEqual(expectedBody) + register.flush({ user_id: '1', role: 'admin' }, { status: 201, statusText: 'Created' }) + await tick() + http.expectOne('/auth/token').flush({ access_token: 'x.y.z', token_type: 'bearer' }) + await tick() + http.expectOne('/auth/me').flush({ id: '1', username: 'priya', role: 'admin', permissions: [], llm_model: null }) + await done + expect(c.step).toBe('done') + } + + beforeEach(async () => { + localStorage.removeItem('kh_token') + await TestBed.configureTestingModule({ + imports: [OnboardingComponent], + providers: [provideRouter([]), provideHttpClient(), provideHttpClientTesting()], + }).compileComponents() + fixture = TestBed.createComponent(OnboardingComponent) + http = TestBed.inject(HttpTestingController) + }) + + afterEach(() => { http.verify(); localStorage.removeItem('kh_token') }) + + it('asks a new server for the setup token and sends it', async () => { + await openAccountStep(true) + expect(text()).toContain('SETUP_TOKEN') + fixture.componentInstance.setupToken = 'from-the-server' + await createAccount({ username: 'priya', password: 'secret123', setup_token: 'from-the-server' }) + }) + + it('creates the account without a token when none is needed', async () => { + await openAccountStep(false) + expect(text()).not.toContain('SETUP_TOKEN') + await createAccount({ username: 'priya', password: 'secret123' }) + }) +}) diff --git a/frontend/src/app/pages/onboarding/onboarding.component.ts b/frontend/src/app/pages/onboarding/onboarding.component.ts index 024e9ac..e344f1d 100644 --- a/frontend/src/app/pages/onboarding/onboarding.component.ts +++ b/frontend/src/app/pages/onboarding/onboarding.component.ts @@ -12,7 +12,7 @@ import { errorMessage } from '../../services/http-error' * step, because signing in needs one. */ type Step = 'welcome' | 'model' | 'account' | 'done' -import { USERNAME_RULES, isValidUsername } from '../../services/usernames' +import { PASSWORD_MIN_LENGTH, PASSWORD_RULES, USERNAME_RULES, isValidUsername } from '../../services/usernames' @Component({ selector: 'app-onboarding', @@ -26,14 +26,23 @@ export class OnboardingComponent implements OnInit { checking = true username = '' readonly usernameRules = USERNAME_RULES + readonly passwordRules = PASSWORD_RULES + get passwordOk(): boolean { return this.password.length >= PASSWORD_MIN_LENGTH } get usernameOk(): boolean { return isValidUsername(this.username) } password = '' + // a fresh server with SETUP_TOKEN set: this first account becomes the admin and needs it + needsSetupToken = false + setupToken = '' isCreating = false setupError = '' constructor(private auth: AuthService, private models: ModelService, private router: Router) {} - ngOnInit() { this.refreshCatalog() } + ngOnInit() { + this.refreshCatalog() + this.auth.setupStatus().then(s => this.needsSetupToken = s.setup_token_required) + .catch(() => { /* refreshCatalog reports an unreachable server */ }) + } async refreshCatalog() { this.checking = true @@ -68,7 +77,8 @@ export class OnboardingComponent implements OnInit { this.isCreating = true this.setupError = '' try { - await this.auth.registerAndLogin(this.username, this.password) + await this.auth.registerAndLogin(this.username, this.password, + this.needsSetupToken ? this.setupToken : undefined) this.step = 'done' } catch (e) { this.setupError = errorMessage(e, 'Could not create account') diff --git a/frontend/src/app/pages/playbooks/playbooks.component.html b/frontend/src/app/pages/playbooks/playbooks.component.html index 2590a89..59fe46c 100644 --- a/frontend/src/app/pages/playbooks/playbooks.component.html +++ b/frontend/src/app/pages/playbooks/playbooks.component.html @@ -1,7 +1,7 @@
-

Playbooks

+

Playbooks

Repeatable procedures, built from steps you write or from how-tos and checklists already in this space.

@@ -11,7 +11,7 @@

Playbooks

-

{{ editingId ? 'Edit playbook' : 'New playbook' }}

+

{{ editingId ? 'Edit playbook' : 'New playbook' }}

@@ -56,7 +56,7 @@

{{ editingId ? 'Edit playbook' @if (playbooks.length === 0 && !error) {
-

No playbooks yet

+

No playbooks yet

Write the steps for something your team does repeatedly, like a release or onboarding.

} @@ -66,7 +66,7 @@

No playbooks yet

{{ p.category }} {{ p.steps.length }} step{{ p.steps.length === 1 ? '' : 's' }}
-

{{ p.title }}

+

{{ p.title }}

    @for (s of p.steps; track $index) {
  1. diff --git a/frontend/src/app/pages/playbooks/playbooks.component.ts b/frontend/src/app/pages/playbooks/playbooks.component.ts index 2536b9c..a51bd21 100644 --- a/frontend/src/app/pages/playbooks/playbooks.component.ts +++ b/frontend/src/app/pages/playbooks/playbooks.component.ts @@ -5,7 +5,7 @@ import { HttpClient } from '@angular/common/http' import { firstValueFrom } from 'rxjs' import { API_BASE } from '../../services/auth.service' import { errorMessage } from '../../services/http-error' -import { KnowledgeItem, KnowledgeResponse, Playbook, PlaybookStep } from '../../models/api' +import { ItemsResponse, KnowledgeItem, Playbook, PlaybookStep } from '../../models/api' /** Item types that make sense as playbook steps */ const STEP_TYPES = new Set(['how-to', 'checklist', 'best-practice', 'action-item', 'lesson']) @@ -36,10 +36,11 @@ export class PlaybooksComponent implements OnInit { try { const [playbooks, knowledge] = await Promise.all([ firstValueFrom(this.http.get(`${API_BASE}/knowledge/playbooks`)), - firstValueFrom(this.http.get(`${API_BASE}/knowledge`)), + firstValueFrom(this.http.get(`${API_BASE}/knowledge/items`, + { params: { type: [...STEP_TYPES], exclude_rejected: true, limit: 1000 } })), ]) this.playbooks = playbooks - this.stepItems = knowledge.knowledge_items.filter(i => STEP_TYPES.has(i.type) && i.review_status !== 'rejected') + this.stepItems = knowledge.items } catch (e) { this.error = errorMessage(e, 'Could not load playbooks') } } diff --git a/frontend/src/app/pages/review/review.component.html b/frontend/src/app/pages/review/review.component.html index 5e0db7e..7b7834f 100644 --- a/frontend/src/app/pages/review/review.component.html +++ b/frontend/src/app/pages/review/review.component.html @@ -1,8 +1,8 @@
    -

    Review Queue

    -

    {{ items.length }} pending item{{ items.length === 1 ? '' : 's' }} awaiting review

    +

    Review Queue

    +

    {{ total }} pending item{{ total === 1 ? '' : 's' }} awaiting review{{ total > items.length ? ' · showing the oldest ' + items.length : '' }}

    Back to hub
    @@ -11,7 +11,7 @@

    Review Queue

    @if (items.length === 0 && !error) {
    -

    Queue is clear

    +

    Queue is clear

    All extracted knowledge has been reviewed.

    } @@ -35,7 +35,7 @@

    Queue is clear

} @else { -

{{ item.title }}

+

{{ item.title }}

by {{ item.author }}

@for (t of item.tags; track t) { #{{ t }} }
diff --git a/frontend/src/app/pages/review/review.component.ts b/frontend/src/app/pages/review/review.component.ts index bde6c3d..4950457 100644 --- a/frontend/src/app/pages/review/review.component.ts +++ b/frontend/src/app/pages/review/review.component.ts @@ -18,6 +18,9 @@ type ReviewItem = KnowledgeItem }) export class ReviewComponent implements OnInit { items: ReviewItem[] = []; editingId: string | null = null + /** the whole queue's length (the page holds the oldest PAGE_SIZE) */ + total = 0 + readonly PAGE_SIZE = 50 editTitle = ''; editNote = ''; loading = false; error = '' constructor(private http: HttpClient) {} @@ -26,7 +29,10 @@ export class ReviewComponent implements OnInit { async load() { try { - this.items = await firstValueFrom(this.http.get(`${API_BASE}/knowledge/review`)) + const res = await firstValueFrom(this.http.get(`${API_BASE}/knowledge/review`, + { params: { limit: this.PAGE_SIZE }, observe: 'response' })) + this.items = res.body ?? [] + this.total = Number(res.headers.get('X-Total-Count') ?? this.items.length) } catch (e) { this.error = errorMessage(e, 'Could not load review queue') console.error('review load error:', e) diff --git a/frontend/src/app/pages/search/search.component.html b/frontend/src/app/pages/search/search.component.html index da14da1..fb98f8e 100644 --- a/frontend/src/app/pages/search/search.component.html +++ b/frontend/src/app/pages/search/search.component.html @@ -1,7 +1,7 @@
-

Search Knowledge Base

+

Search Knowledge Base

@if (result) {

{{ result.total }} result{{ result.total === 1 ? '' : 's' }} for "{{ result.query || 'all' }}"

}
@@ -11,11 +11,11 @@

Search Knowledge Base

- @for (t of ITEM_TYPES; track t) { } - @for (s of SOURCE_TYPES; track s) { } @@ -38,7 +38,7 @@

Search Knowledge Base

@if (item.status) { {{ item.status }} } {{ item.date | date }}
-

{{ item.title }}

+

{{ item.title }}

by {{ item.author }}

@for (t of item.tags; track t) { #{{ t }} }
@@ -51,25 +51,37 @@

{{ item.title }}

@for (a of result.artifacts; track a.id) { -
+
{{ a.source_type }} {{ a.created_at | date }}
-

{{ a.title }}

+

{{ a.title }}

by {{ a.author }}

@if (a.snippet) {

“{{ a.snippet }}”

}
@for (t of a.tags; track t) { #{{ t }} }
-
+ + @if (a.first_item_id) { + + + + + } @else { +
+ +

Nothing was extracted from this source.

+
+ } }
} @if (result.total === 0) { -

No results

Try a different query or remove a filter.

+

No results

Try a different query or remove a filter.

} } @if (!result && !loading) { -

Start searching

Enter a query above — results are instant and links are shareable.

+

Start searching

Enter a query above — results are instant and links are shareable.

}
diff --git a/frontend/src/app/services/auth.guard.ts b/frontend/src/app/services/auth.guard.ts index 82c1147..385ad49 100644 --- a/frontend/src/app/services/auth.guard.ts +++ b/frontend/src/app/services/auth.guard.ts @@ -8,3 +8,12 @@ export const authGuard = () => { if (auth.isLoggedIn()) return true return router.createUrlTree(['/login']) } + +/** Pages only server admins use (the API refuses everyone else anyway): others go to the hub. */ +export const adminGuard = async () => { + const auth = inject(AuthService) + const router = inject(Router) + if (!auth.isLoggedIn()) return router.createUrlTree(['/login']) + if (!auth.me()) await auth.loadMe() // a direct visit: the role isn't loaded yet + return auth.can('manage_users') ? true : router.createUrlTree(['/knowledge']) +} diff --git a/frontend/src/app/services/auth.service.ts b/frontend/src/app/services/auth.service.ts index 4894eec..6e021d4 100644 --- a/frontend/src/app/services/auth.service.ts +++ b/frontend/src/app/services/auth.service.ts @@ -51,8 +51,8 @@ export class AuthService { return firstValueFrom(this.http.get(`${API_BASE}/auth/setup`)) } - async registerAndLogin(username: string, password: string): Promise { - await this.register(username, password) + async registerAndLogin(username: string, password: string, setupToken?: string): Promise { + await this.register(username, password, setupToken) await this.login(username, password) } diff --git a/frontend/src/app/services/space.service.spec.ts b/frontend/src/app/services/space.service.spec.ts new file mode 100644 index 0000000..ccefacc --- /dev/null +++ b/frontend/src/app/services/space.service.spec.ts @@ -0,0 +1,49 @@ +import { Component } from '@angular/core' +import { TestBed } from '@angular/core/testing' +import { provideHttpClient } from '@angular/common/http' +import { provideHttpClientTesting } from '@angular/common/http/testing' +import { NavigationEnd, Router, provideRouter } from '@angular/router' +import { SpaceService } from './space.service' +import { routes } from '../app.routes' + +@Component({ template: 'page' }) +class Page {} + +describe('SpaceService.switchTo', () => { + let router: Router + let ended: string[] + + beforeEach(() => { + localStorage.removeItem('kh_space') + // the app's real 'switching-space' stop, with stand-in pages for the hub and search + TestBed.configureTestingModule({ + providers: [ + provideHttpClient(), provideHttpClientTesting(), + provideRouter([...routes.filter(r => r.path === 'switching-space'), + { path: 'knowledge', component: Page }, { path: 'search', component: Page }]), + ], + }) + router = TestBed.inject(Router) + ended = [] + router.events.subscribe(e => { if (e instanceof NavigationEnd) ended.push(e.urlAfterRedirects) }) + }) + + afterEach(() => localStorage.removeItem('kh_space')) + + it('re-opens the hub for the new space', async () => { + await router.navigateByUrl('/knowledge') + ended = [] + await TestBed.inject(SpaceService).switchTo('group-1') + // a real navigation away and back: going via '/' (which redirects to /knowledge) was ignored on the hub + expect(ended).toEqual(['/switching-space', '/knowledge']) + expect(router.url).toBe('/knowledge') + expect(TestBed.inject(SpaceService).activeId()).toBe('group-1') + }) + + it('returns to the page it was on', async () => { + await router.navigateByUrl('/search?q=x') + ended = [] + await TestBed.inject(SpaceService).switchTo('group-1') + expect(ended).toEqual(['/switching-space', '/search?q=x']) + }) +}) diff --git a/frontend/src/app/services/space.service.ts b/frontend/src/app/services/space.service.ts index fb65a41..c70c893 100644 --- a/frontend/src/app/services/space.service.ts +++ b/frontend/src/app/services/space.service.ts @@ -44,7 +44,9 @@ export class SpaceService { if (id === this._activeId()) return this.setActive(id) const url = this.router.url.startsWith('/knowledge/') ? '/knowledge' : this.router.url // items don't cross spaces - await this.router.navigateByUrl('/', { skipLocationChange: true }) + // Not via '/': it redirects to /knowledge, so on the hub both navigations would be to the + // current URL, which the router ignores, and the hub kept showing the previous space. + await this.router.navigateByUrl('/switching-space', { skipLocationChange: true }) await this.router.navigateByUrl(url) } diff --git a/frontend/src/app/services/usernames.ts b/frontend/src/app/services/usernames.ts index e65370a..db17597 100644 --- a/frontend/src/app/services/usernames.ts +++ b/frontend/src/app/services/usernames.ts @@ -5,6 +5,10 @@ */ export const USERNAME_RULES = '3 to 60 characters: letters, numbers, dot, underscore or hyphen. Not case-sensitive.' +/** Mirrors check_new_password in backend/app/auth.py (the server also refuses common passwords). */ +export const PASSWORD_MIN_LENGTH = 8 +export const PASSWORD_RULES = `At least ${PASSWORD_MIN_LENGTH} characters, not a common password and not your username.` + const PATTERN = /^[A-Za-z0-9][A-Za-z0-9._-]{2,59}$/ export function isValidUsername(name: string): boolean { diff --git a/frontend/src/styles.css b/frontend/src/styles.css index 403e0e1..3d26880 100644 --- a/frontend/src/styles.css +++ b/frontend/src/styles.css @@ -18,8 +18,8 @@ --border-strong: #d2d6e0; --text: #0f172a; --text-2: #475569; - --text-3: #64748b; - --text-4: #94a3b8; + --text-3: #5e6d82; /* 4.5:1 or better on every surface (WCAG AA) */ + --text-4: #5b6e89; /* was #94a3b8: 2.3:1, too faint for the text it's used for */ --accent: #4f46e5; --accent-hover: #4338ca; @@ -29,7 +29,7 @@ --on-accent: #ffffff; --focus-ring: 0 0 0 3px rgba(79, 70, 229, 0.22); - --success: #16a34a; --success-soft: #dcfce7; --success-text: #15803d; + --success: #12863d; --success-soft: #dcfce7; --success-text: #147c3b; --warning: #d97706; --warning-soft: #fef3c7; --warning-text: #92400e; --danger: #dc2626; --danger-soft: #fee2e2; --danger-text: #b91c1c; --info: #0284c7; --info-soft: #e0f2fe; --info-text: #075985; @@ -48,8 +48,8 @@ color-scheme: dark; --bg: #0b0d12; --surface: #12151c; --surface-2: #171b23; --surface-3: #1e232d; --border: #252a35; --border-strong: #333a48; - --text: #e6e9f0; --text-2: #b3bac7; --text-3: #8a93a5; --text-4: #626b7d; - --accent: #6366f1; --accent-hover: #7c7ff5; --accent-text: #a5b4fc; + --text: #e6e9f0; --text-2: #b3bac7; --text-3: #8a93a5; --text-4: #838c9e; + --accent: #5e61f1; --accent-hover: #7c7ff5; --accent-text: #a5b4fc; --accent-soft: rgba(99, 102, 241, 0.14); --accent-soft-2: rgba(99, 102, 241, 0.24); --focus-ring: 0 0 0 3px rgba(129, 140, 248, 0.35); --success-soft: rgba(34, 197, 94, 0.14); --success-text: #4ade80; @@ -65,8 +65,8 @@ color-scheme: dark; --bg: #0b0d12; --surface: #12151c; --surface-2: #171b23; --surface-3: #1e232d; --border: #252a35; --border-strong: #333a48; - --text: #e6e9f0; --text-2: #b3bac7; --text-3: #8a93a5; --text-4: #626b7d; - --accent: #6366f1; --accent-hover: #7c7ff5; --accent-text: #a5b4fc; + --text: #e6e9f0; --text-2: #b3bac7; --text-3: #8a93a5; --text-4: #838c9e; + --accent: #5e61f1; --accent-hover: #7c7ff5; --accent-text: #a5b4fc; --accent-soft: rgba(99, 102, 241, 0.14); --accent-soft-2: rgba(99, 102, 241, 0.24); --focus-ring: 0 0 0 3px rgba(129, 140, 248, 0.35); --success-soft: rgba(34, 197, 94, 0.14); --success-text: #4ade80; @@ -93,9 +93,13 @@ body { font-feature-settings: "cv11", "ss01"; } h1, h2, h3, h4 { color: var(--text); line-height: 1.25; letter-spacing: -0.015em; font-weight: 650; } +h1 { font-size: 1.6rem; } h2 { font-size: 1.5rem; } h3 { font-size: 1.05rem; } h4 { font-size: 0.9rem; } +/* the list pages' section headings sit one level under their

, sized as before */ +.knowledge-shell h2 { font-size: 1.05rem; } +.knowledge-shell h3 { font-size: 0.9rem; } a { color: var(--accent-text); text-decoration: none; } a:hover { text-decoration: underline; text-underline-offset: 2px; } code { font-family: var(--font-mono); font-size: 0.85em; background: var(--surface-3); padding: 0.1em 0.35em; border-radius: 4px; } @@ -296,7 +300,7 @@ main { display: flex; justify-content: space-between; align-items: flex-end; gap: 1rem; flex-wrap: wrap; padding: 0 0 0.25rem; } -.toolbar h2 { font-size: 1.6rem; font-weight: 700; letter-spacing: -0.025em; } +.toolbar h1, .toolbar h2 { font-size: 1.6rem; font-weight: 700; letter-spacing: -0.025em; } .toolbar p { color: var(--text-3); font-size: 0.9rem; margin-top: 0.3rem; max-width: 62ch; } .toolbar > div:last-child:not(:first-child) { display: flex; gap: 0.5rem; flex-wrap: wrap; align-items: center; } @@ -309,7 +313,7 @@ main { } .ingest-panel { display: flex; flex-direction: column; gap: 1rem; padding: 1.35rem 1.5rem; } -.ingest-panel h3 { font-size: 1rem; } +.ingest-panel h2, .ingest-panel h3 { font-size: 1rem; } .ingest-panel p, .knowledge-card p, .empty-state p { color: var(--text-3); font-size: 0.875rem; } .ingest-panel textarea { width: 100%; min-height: 170px; } .panel-actions { display: flex; justify-content: space-between; align-items: center; gap: 1rem; flex-wrap: wrap; } @@ -332,12 +336,12 @@ main { min-height: 200px; padding: 2.5rem 2rem; text-align: center; background: var(--surface-2); border-style: dashed; box-shadow: none; } -.empty-state h3 { font-size: 1rem; } +.empty-state h2, .empty-state h3 { font-size: 1rem; } /* ── Cards & grids ─────────────────────────────────────────────────────────── */ .knowledge-grid { display: grid; grid-template-columns: repeat(auto-fill, minmax(280px, 1fr)); gap: 0.9rem; } .knowledge-card { display: flex; flex-direction: column; gap: 0.5rem; padding: 1.1rem 1.15rem; min-height: 150px; } -.knowledge-card h3 { font-size: 0.95rem; font-weight: 600; line-height: 1.4; margin-top: 0.15rem; } +.knowledge-card h2, .knowledge-card h3 { font-size: 0.95rem; font-weight: 600; line-height: 1.4; margin-top: 0.15rem; } .knowledge-card-link { color: inherit; text-decoration: none; transition: border-color 0.15s, box-shadow 0.15s, transform 0.15s; } .knowledge-card-link:hover { text-decoration: none; border-color: var(--accent-soft-2); box-shadow: var(--shadow-md); transform: translateY(-1px); } @@ -349,12 +353,12 @@ main { .review-list { display: flex; flex-direction: column; gap: 0.6rem; } .review-card { padding: 1rem 1.15rem; transition: border-color 0.15s; } .review-card:hover { border-color: var(--border-strong); } -.review-card h3 { font-size: 0.98rem; font-weight: 600; } +.review-card h2, .review-card h3 { font-size: 0.98rem; font-weight: 600; } .review-actions { display: flex; gap: 0.5rem; margin-top: 0.85rem; flex-wrap: wrap; } .connector-grid { display: grid; gap: 0.9rem; grid-template-columns: repeat(auto-fill, minmax(min(100%, 340px), 1fr)); } .connector-card { display: flex; flex-direction: column; gap: 0.6rem; padding: 1.15rem 1.25rem; } -.connector-card h3 { font-size: 1rem; } +.connector-card h2, .connector-card h3 { font-size: 1rem; } .sync-stats { display: flex; flex-wrap: wrap; gap: 0.35rem; } .sync-stats span { background: var(--surface-3); border-radius: 6px; font-size: 0.75rem; padding: 0.2rem 0.5rem; color: var(--text-2); } @@ -460,7 +464,7 @@ main > .space-banner, main > .health-banner { margin-bottom: 1.25rem; } /* ── Item detail ───────────────────────────────────────────────────────────── */ .detail-hero { display: flex; flex-direction: column; gap: 1rem; padding: 1.5rem 1.6rem; } .detail-title { display: flex; flex-direction: column; gap: 0.6rem; } -.detail-title h2 { font-size: 1.55rem; font-weight: 700; line-height: 1.3; letter-spacing: -0.02em; } +.detail-title h1, .detail-title h2 { font-size: 1.55rem; font-weight: 700; line-height: 1.3; letter-spacing: -0.02em; } .detail-grid { display: grid; gap: 1.25rem; grid-template-columns: minmax(0, 1fr) 340px; align-items: start; } .detail-panel { padding: 1.35rem 1.5rem; } .detail-panel h3 { font-size: 0.98rem; margin-bottom: 0.9rem; } @@ -522,7 +526,7 @@ main > .space-banner, main > .health-banner { margin-bottom: 1.25rem; } .graph-node.selected circle, .graph-node:focus circle, .graph-node:hover circle { fill: var(--warning-soft); stroke: var(--warning); stroke-width: 3; } .graph-node text { fill: var(--text-2); font-size: 11px; pointer-events: none; text-anchor: middle; } .graph-inspector { border: 1px solid var(--border); border-radius: var(--radius); padding: 1rem; background: var(--surface-2); } -.graph-inspector h4 { font-size: 0.95rem; margin-top: 0.6rem; } +.graph-inspector h3, .graph-inspector h4 { font-size: 0.95rem; margin-top: 0.6rem; } .relationship-list { display: flex; flex-direction: column; gap: 0.4rem; margin-top: 0.9rem; } .relationship-list button { flex-direction: column; align-items: flex-start; height: auto; padding: 0.55rem 0.7rem; text-align: left; white-space: normal; } .relationship-list span { color: var(--text-3); font-size: 0.68rem; text-transform: uppercase; letter-spacing: 0.04em; } @@ -595,11 +599,11 @@ main > .space-banner, main > .health-banner { margin-bottom: 1.25rem; } main { padding: 1.25rem 1rem 2.5rem; } .form-grid { grid-template-columns: 1fr; } .toolbar { align-items: flex-start; } - .toolbar h2 { font-size: 1.35rem; } + .toolbar h1, .toolbar h2 { font-size: 1.35rem; } .relationship-table > div { grid-template-columns: 1fr; gap: 0.3rem; } .knowledge-graph { height: 340px; } .detail-hero, .detail-panel, .ingest-panel { padding: 1.1rem; } - .detail-title h2 { font-size: 1.3rem; } + .detail-title h1, .detail-title h2 { font-size: 1.3rem; } .chat-container { height: calc(100vh - 7.5rem); } .message { max-width: 94%; } } @@ -609,3 +613,6 @@ main > .space-banner, main > .health-banner { margin-bottom: 1.25rem; } font-size: 0.72rem; font-weight: 650; letter-spacing: 0.07em; text-transform: uppercase; color: var(--text-3); } + +/* links in running text are underlined: colour alone doesn't mark them (WCAG 1.4.1) */ +p a:not(.button-like) { text-decoration: underline; text-underline-offset: 2px; } From 97b4a1287fcb5393c6e837fa1173e724edcbbc49 Mon Sep 17 00:00:00 2001 From: itisar-345 <143781880+itisar-345@users.noreply.github.com> Date: Tue, 20 Oct 2026 14:02:05 +0530 Subject: [PATCH 2/2] Ground GraphRAG answers and cut LLM calls; add LLM evaluation Answers now see item details and source passages, cite numbered entries, and use one fixed reply when nothing covers the question. Query rewrite, LLM rerank and LLM routing are off by default. Extraction uses JSON schemas; Ollama keeps the model loaded. Measured with backend/evals on llama3.1:8b (CPU): answers correct 15 -> 19 of 19, false refusals 4 -> 0, median answer time 78 s -> 13 s. Also updates the README, testing guide and changelog. --- .gitignore | 1 + CHANGELOG.md | 10 + README.md | 8 +- backend/.env.example | 13 +- backend/app/services/graphrag.py | 367 ++++++++++-------- backend/app/services/llm_client.py | 2 +- backend/app/services/llm_extraction.py | 38 +- backend/app/services/providers.py | 15 +- backend/evals/README.md | 62 +++ backend/evals/__init__.py | 1 + backend/evals/corpus.py | 282 ++++++++++++++ backend/evals/run_eval.py | 207 ++++++++++ backend/tests/conftest.py | 3 + backend/tests/test_graphrag.py | 47 ++- backend/tests/test_groups.py | 3 +- backend/tests/test_lineage.py | 14 +- docs/architecture.md | 15 +- docs/configuration.md | 5 +- docs/testing.md | 6 +- .../app/pages/graphrag/graphrag.component.ts | 4 +- 20 files changed, 900 insertions(+), 203 deletions(-) create mode 100644 backend/evals/README.md create mode 100644 backend/evals/__init__.py create mode 100644 backend/evals/corpus.py create mode 100644 backend/evals/run_eval.py diff --git a/.gitignore b/.gitignore index 64eb753..6ef68c6 100644 --- a/.gitignore +++ b/.gitignore @@ -24,6 +24,7 @@ venv/ backend/data/*.db backend/data/*.json deploy/backups/ +backend/evals/results/ # Editor .idea/ diff --git a/CHANGELOG.md b/CHANGELOG.md index a305c53..944ce06 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,7 +21,17 @@ Notable changes to Knowledge Hubs. The format follows [Keep a Changelog](https:/ - `GET /knowledge/items` (filtered, paged items) and `GET /knowledge/items/{id}/context` (what an item's page shows). - GraphRAG answers whose model names its sources without writing their ids get those sources inferred from the wording, marked "matched". +- [`backend/evals/`](backend/evals/): an end-to-end evaluation of extraction and GraphRAG against a real model, with a fixed corpus, answer key and questions. + ### Changed +- **GraphRAG answers are grounded, cited by number, and about 6× faster** (measured with `backend/evals/` on llama3.1:8b, CPU: 15 → 19 of 19 answers correct, 4 → 0 false "not covered" replies, median answer time 78 s → 13 s): + - The model now sees each item's details (who, why, due, steps), its source's title, and the passage it came from. It used to see titles only, and answered "the context doesn't say who" for owners on record. + - Context entries are numbered and cited as `[1]`, `[2]`, mapped back to items; small models copied long ids poorly. + - One fixed reply when nothing in the knowledge base is about the question ("The knowledge base doesn't cover this."), with no sources. A refusal no longer gets sources inferred from the topics it names. + - The LLM question rewrite (sub-queries + HyDE), LLM rerank and LLM routing are off by default (`GRAPHRAG_QUERY_REWRITE`, `GRAPHRAG_RERANK`, `GRAPHRAG_LLM_ROUTE`): each cost about as long as the answer itself, and retrieval found the right source every time without them. + - Shorter answers (at most about 120 words, no preamble or closing summary). +- Extraction replies are constrained to a JSON schema (Ollama structured outputs), and a decision is kept in one item with its reason. +- Ollama keeps the model loaded for 30 minutes between calls (`OLLAMA_KEEP_ALIVE`) instead of unloading it after 5. - **Lists are bounded.** `GET /knowledge` returns a page (default 100) with `totals`, filters items on the server, and leaves out document text unless `include_content=true`; the review queue, decision register, graph and search have limits too. On 3,000 items the Hub's response was 4 MB and, under load, ran ArcadeDB out of memory. The pages show **Load more**. - **Ingestion no longer waits for the summary.** The LLM summary of a new source is built after the response (about 30 s on a CPU). - **App instances run several server processes** (`APP_WORKERS`, default 2), and bcrypt runs on a worker thread. diff --git a/README.md b/README.md index 780279d..fc02e5e 100644 --- a/README.md +++ b/README.md @@ -13,7 +13,9 @@ A **local-first knowledge base** for teams. It turns meeting notes, retros, deci - **Personal and group spaces.** Everyone has a private space and can create or join groups that share one. A switcher in the nav picks where you're working. - **Decisions have a history.** A decision can supersede, reverse or amend an older one, and risks can be mitigated or actually happen. The app tracks each item's status and shows how it evolved. - **Connects only where you point it.** Knowledge can come from folders, git repositories (ADRs), GitHub, Jira and Linear, but only from addresses a user enters, and only when they click **Sync now**. +- **Answers you can check.** GraphRAG answers from the items it retrieves, the details recorded on them and the passages of the sources they came from, cites them by number, and says plainly when the knowledge base doesn't cover a question. A [built-in evaluation](backend/evals/) measures answers, citations and extraction against a real model. - **One command to try it, a resilient stack for production.** `docker compose up` runs everything on one machine. [`deploy/`](deploy/) runs it behind a TLS load balancer, with two app instances, a Celery worker and a three-node ArcadeDB cluster. +- **Ready to face the internet.** In production the first (admin) account needs a setup token, sign-ins are throttled, sessions can be revoked, and the web server sends a strict Content Security Policy. See [Security notes](docs/security.md). | Page | What you do there | |---|---| @@ -23,7 +25,7 @@ A **local-first knowledge base** for teams. It turns meeting notes, retros, deci | [**Decisions**](docs/images/decisions.png) | The decision log and risk register, with each item's status and what replaced it | | [**Playbooks**](docs/images/playbooks.png) | Repeatable procedures, written as steps or built from how-tos and checklists in the space | | [**Sources**](docs/images/sources.png) | Connectors that pull in transcripts, notes, ADRs, pull requests and issues | -| [**GraphRAG**](docs/images/graphrag.png) | Ask questions; answers cite the items they used and flag superseded decisions | +| [**GraphRAG**](docs/images/graphrag.png) | Ask questions; answers cite their sources, flag superseded decisions, and say so when nothing covers the question | | [**Groups**](docs/images/groups.png) | Create groups, invite people and share a space with them | | [**Models**](docs/images/models.png) | Download, switch or remove the local LLMs; re-embed the knowledge base | | [**Users**](docs/images/users.png) *(server admins)* | Create accounts, change roles, see inactive accounts due for deletion | @@ -42,6 +44,8 @@ docker compose up --build Open **http://localhost:8080**, create an account (the first one is the server admin) and pick a model to download. No configuration file is needed: secrets are generated on first start. +The local stack listens on this machine only. Before letting other machines in (`KH_BIND=0.0.0.0`), set `SETUP_TOKEN` so that nobody else can create the admin account; for anything on the internet, use the [production stack](docs/deployment.md). + To use a GPU, an Ollama you already run, or to work on the code with live reload, see [Getting started](docs/getting-started.md). ## Documentation @@ -62,7 +66,9 @@ To use a GPU, an Ollama you already run, or to work on the code with live reload | [API](docs/api.md) | Every endpoint | | [Security notes](docs/security.md) | How accounts, data, credentials and inputs are protected | | [Testing](docs/testing.md) | Running the backend and frontend test suites | +| [LLM evaluation](backend/evals/README.md) | Measuring answer quality, citations, extraction and speed against a real model | | [Known limitations](docs/known-limitations.md) | What it doesn't do (yet) | +| [Changelog](CHANGELOG.md) | What changed, including steps to take when upgrading | ## Contributing diff --git a/backend/.env.example b/backend/.env.example index 745d600..b0dd411 100644 --- a/backend/.env.example +++ b/backend/.env.example @@ -66,9 +66,16 @@ OLLAMA_NUM_CTX=8192 # Max size of an uploaded file or fetched URL, in MB (also caps request bodies) MAX_UPLOAD_MB=20 -# GraphRAG: LLM rerank of retrieved candidates (on|off). It only runs when there -# are more than 2x top_k candidates; "off" always keeps the fusion (RRF) order. -GRAPHRAG_RERANK=on +# GraphRAG: extra LLM calls per question, each about as slow as the answer itself on a +# CPU. Off by default: on evals/ they didn't make answers better (see backend/evals/). +# GRAPHRAG_RERANK: rerank retrieved candidates (only with more than 2x top_k of them) +# GRAPHRAG_QUERY_REWRITE: rewrite the question into sub-queries + a hypothetical answer +# GRAPHRAG_LLM_ROUTE: classify questions the embedding classifier is unsure about +GRAPHRAG_RERANK=off +GRAPHRAG_QUERY_REWRITE=off +GRAPHRAG_LLM_ROUTE=off +# How long Ollama keeps the model loaded between calls (its default unloads it after 5 min) +OLLAMA_KEEP_ALIVE=30m # ── Offline enforcement ─────────────────────────────────────────────────────── TRANSFORMERS_OFFLINE=1 diff --git a/backend/app/services/graphrag.py b/backend/app/services/graphrag.py index f380518..3d0439d 100644 --- a/backend/app/services/graphrag.py +++ b/backend/app/services/graphrag.py @@ -35,35 +35,35 @@ logger = logging.getLogger(__name__) -# ── route → system prompt ──────────────────────────────────────────────────── -_SYSTEM: Dict[str, str] = { - "factual": ( - "You are a precise knowledge assistant. " - "Answer using ONLY the provided context nodes — do not infer, extrapolate, or stitch together implications. " - "Every factual claim MUST be directly stated in a context node and cited as [item_id]. " - "If the context does not explicitly contain the answer, say so. " - "Implying an answer from weak or tangential context is a worse failure than declining to answer." - ), - "exploratory": ( - "You are a knowledge assistant helping with open-ended exploration. " - "Synthesise ONLY what is explicitly stated across the context nodes — do not infer connections that are not directly supported. " - "Cite every claim as [item_id]. " - "Acknowledge gaps honestly. " - "Implying an answer from weak or tangential context is a worse failure than declining to answer." - ), - "comparative": ( - "You are a knowledge assistant specialised in comparison. " - "Compare ONLY what is explicitly stated in the context nodes — do not infer attributes that are not directly present. " - "Cite each point as [item_id]. " - "If the context does not support a direct comparison, say so rather than implying one." - ), - "procedural": ( - "You are a knowledge assistant specialised in step-by-step guidance. " - "Present steps using ONLY content explicitly found in the context nodes. " - "Cite each step as [item_id]. " - "Do not fill gaps with assumed or inferred steps — if a step is missing from the context, say so." - ), +# ── the answer prompt ─────────────────────────────────────────────────────── +# Written for small local models (llama3.1:8b class). Measured on evals/: the +# earlier prompts ("implying an answer is a worse failure than declining") +# made the model deny facts that were in its context, and asking it to copy +# long ids made it cite by title instead; numbered entries fix both. +NO_ANSWER = "The knowledge base doesn't cover this." + +_ANSWER_SYSTEM = f"""\ +You answer questions about a team's knowledge base: its decisions, risks, action items, lessons, +how-tos and best practices. Use only the numbered context entries. + +- State what the entries say, including reasons, owners, dates and what replaced what. Leave out + details the entries don't give: never write "not specified". +- After each statement, cite the entries it comes from by number, like [2] or [1][3]. +- If an entry answers the question even in part, answer with what it says. +- Only if no entry is about the question at all, reply with exactly: {NO_ANSWER} +- An entry marked superseded or reversed is history: give the one that replaced it as current. +- Be brief. Answer first, in at most 120 words. Use a short bullet list only for several separate + points. No preamble and no closing summary.\ +""" + +_ROUTE_HINT: Dict[str, str] = { + "factual": "Answer the specific question directly.", + "exploratory": "Cover the main points the entries make.", + "comparative": "Compare the options point by point.", + "procedural": "If the question asks how to do something, give the steps in order.", } +# kept for route classification and older callers +_SYSTEM: Dict[str, str] = {route: _ANSWER_SYSTEM + "\n" + hint for route, hint in _ROUTE_HINT.items()} _SYSTEM["fallback"] = _SYSTEM["factual"] @@ -95,6 +95,12 @@ def _item_text(item: Dict[str, Any]) -> str: # 2. Query transformation (sub-queries + HyDE) # ───────────────────────────────────────────────────────────────────────────── +# GRAPHRAG_QUERY_REWRITE=on rewrites the question into sub-queries plus a hypothetical +# answer (HyDE) with the LLM before retrieving. Off by default: on evals/ it cost a full +# LLM call (about 25 s on a CPU) and retrieval found the right source without it. +QUERY_REWRITE = os.getenv("GRAPHRAG_QUERY_REWRITE", "off").lower() in ("1", "on", "true", "yes") + + async def transform_query(question: str) -> Dict[str, Any]: prompt = ( "Rewrite the user question below into 3 distinct versions that cover " @@ -207,6 +213,12 @@ def route_query(question: str) -> str: return "exploratory" +# GRAPHRAG_LLM_ROUTE=on asks the LLM to classify questions the embedding classifier is +# unsure about. Off by default: the route only picks a one-line hint in the answer prompt, +# not worth an LLM call (about 5 s on a CPU); the keyword rules decide instead. +LLM_ROUTE = os.getenv("GRAPHRAG_LLM_ROUTE", "off").lower() in ("1", "on", "true", "yes") + + async def route_query_llm(question: str, q_vec: Optional[List[float]] = None) -> str: """q_vec: the question's embedding, if the caller already has it.""" # Phase 3: try embedding-similarity classifier first to save an LLM call @@ -215,6 +227,8 @@ async def route_query_llm(question: str, q_vec: Optional[List[float]] = None) -> embedding_route = await _route_by_embedding(q_vec) if embedding_route: return embedding_route + if not LLM_ROUTE: + return route_query(question) # Ambiguous — fall back to LLM classification prompt = ( @@ -270,8 +284,10 @@ def _rrf(ranked_lists: List[List[Dict[str, Any]]], k: int = 60) -> List[Dict[str # 8. LLM reranking # ───────────────────────────────────────────────────────────────────────────── -# GRAPHRAG_RERANK=off skips the LLM rerank entirely (RRF order is used). -RERANK_ENABLED = os.getenv("GRAPHRAG_RERANK", "on").lower() not in ("0", "off", "false", "no") +# GRAPHRAG_RERANK=on adds an LLM rerank of the fused candidates. Off by default: on +# evals/ it cost about 27 s per question on a CPU, and the fusion (RRF) order already +# put the right source in the context every time. +RERANK_ENABLED = os.getenv("GRAPHRAG_RERANK", "off").lower() in ("1", "on", "true", "yes") # Only worth an LLM call when there is a real choice to make: with at most # top_n * RERANK_MIN_RATIO candidates, the RRF order is kept as-is. RERANK_MIN_RATIO = 2 @@ -328,39 +344,112 @@ async def rerank( # 9. Context assembly # ───────────────────────────────────────────────────────────────────────────── +_DETAIL_SKIP = {"extractor", "confidence", "evidence", "what", "okf_original_id"} + + +def _detail_line(details: Dict[str, Any]) -> str: + parts = [] + for key, value in (details or {}).items(): + if key in _DETAIL_SKIP or value in (None, "", [], {}): + continue + if isinstance(value, list): + value = "; ".join(str(v) for v in value[:8]) + parts.append(f"{key}: {str(value)[:200]}") + return " | ".join(parts) + + def _build_context( summaries: List[Dict[str, Any]], ranked_items: List[Dict[str, Any]], graph_neighbours: List[Dict[str, Any]], -) -> str: +) -> Tuple[str, Dict[int, Dict[str, Any]]]: + """ + The numbered context the model answers from, and what each number refers to. + Items carry their details (who, why, due, steps…) and the passage of the source + they came from: without them the model saw only titles, and answered "the context + doesn't say who" for an action item whose owner was on record. + """ + refs: Dict[int, Dict[str, Any]] = {} parts: List[str] = [] - if summaries: - lines = ["=== ARTIFACT SUMMARIES ==="] - for s in summaries: - lines.append(f"[{s.get('artifact_id','?')}] {s.get('title','')}: {s.get('summary','')}") - parts.append("\n".join(lines)) + + def add(entry: Dict[str, Any]) -> int: + refs[len(refs) + 1] = entry + return len(refs) + + # number every item first, so "replaced by" can point at a later entry + numbers = {id(n): add(n) for n in ranked_items} + number_of = {n.get("id"): numbers[id(n)] for n in ranked_items} if ranked_items: lines = ["=== KNOWLEDGE ITEMS ==="] for n in ranked_items: - nid = n.get("id", "?") - title = n.get("title") or n.get("label", "") - kind = n.get("kind") or n.get("type", "") - score = n.get("rrf_score") or n.get("score") or 0.0 - status = f" status={n['status']}" if n.get("status") else "" - lines.append(f"[{nid}] ({kind}{status}) relevance={score:.3f}: {title}") + num = numbers[id(n)] + kind = n.get("kind") or n.get("type", "") + status = f", {n['status']}" if n.get("status") else "" + lines.append(f"[{num}] {kind}{status}: {n.get('title') or n.get('label', '')}") + details = _detail_line(n.get("details") or {}) + if details: + lines.append(f" {details}") for succ in n.get("replaced_by") or []: - lines.append(f" replaced_by: [{succ['id']}] ({succ['kind']}) {succ.get('title') or ''}") - for k, v in (n.get("details") or {}).items(): - if isinstance(v, str) and v and k not in ("evidence", "confidence"): - lines.append(f" {k}: {v}") + ref = f"[{number_of[succ['id']]}] " if succ.get("id") in number_of else "" + lines.append(f" replaced by: {ref}{succ.get('title') or succ['id']}") + if n.get("passage"): + lines.append(f' from "{n.get("source_title") or "a source"}": "{n["passage"]}"') parts.append("\n".join(lines)) if graph_neighbours: - lines = ["=== RELATED CONTEXT (graph) ==="] + lines = ["=== RELATED ITEMS ==="] for n in graph_neighbours: - status = f" status={n['status']}" if n.get("status") else "" - lines.append(f"[{n.get('id','?')}] ({n.get('kind') or n.get('type','')}{status}): {n.get('title') or n.get('label','')}") + num = add(n) + status = f", {n['status']}" if n.get("status") else "" + lines.append(f"[{num}] {n.get('kind') or n.get('type', '')}{status}: {n.get('title') or n.get('label', '')}") + parts.append("\n".join(lines)) + if summaries: + lines = ["=== SOURCE SUMMARIES ==="] + for sm in summaries: + num = add({"id": sm.get("artifact_id"), "title": sm.get("title"), "kind": "artifact"}) + lines.append(f'[{num}] summary of "{sm.get("title", "")}": {sm.get("summary", "")}') parts.append("\n".join(lines)) - return "\n\n".join(parts) + return "\n\n".join(parts), refs + + +def _sentences(text: str) -> List[str]: + return [p.strip() for p in re.split(r"(?<=[.!?])\s+|\n+", text) if p.strip()] + + +def _passage(content: str, about: str, limit: int = 360) -> str: + """The stretch of a source that best matches `about`: its best sentence and the next ones.""" + sentences = _sentences(content or "") + words = _significant_words(about) + if not sentences or not words: + return "" + scores = [len(_significant_words(sn) & words) for sn in sentences] + best = max(range(len(sentences)), key=lambda i: scores[i]) + if scores[best] == 0: + return "" + passage, i = sentences[best], best + 1 + while i < len(sentences) and len(passage) + len(sentences[i]) < limit: + passage += " " + sentences[i] + i += 1 + return passage[:limit] + + +async def _hydrate(store: "GraphStore", user_id: str, nodes: List[Dict[str, Any]]) -> None: + """Add details, the source title and a source passage to each retrieved item, in place.""" + from app.repositories import ArtifactRepository, KnowledgeItemRepository + ids = [n["id"] for n in nodes if n.get("id") and n.get("kind") != "artifact"] + full = {r["id"]: r for r in await KnowledgeItemRepository(store.session).get_many(user_id, ids)} + artifacts: Dict[str, Dict[str, Any]] = {} + for n in nodes: + item = full.get(n.get("id")) + if not item: + continue + n["details"] = item.get("details") or {} + aid = item.get("artifact_id") + if aid and aid not in artifacts: + artifacts[aid] = await ArtifactRepository(store.session).get_owned(user_id, aid) or {} + art = artifacts.get(aid) or {} + n["source_title"] = art.get("title") + about = " ".join([item.get("title") or ""] + [str(v) for v in n["details"].values() if isinstance(v, str)]) + n["passage"] = _passage(art.get("content") or "", about) # ───────────────────────────────────────────────────────────────────────────── @@ -372,23 +461,9 @@ async def _generate( context: str, route: str, history: Optional[List[Dict[str, str]]] = None, + nodes: Optional[List[Dict[str, Any]]] = None, ) -> str: system = _SYSTEM.get(route, _SYSTEM["factual"]) - system += """ - -When citing sources, copy the exact bracketed id shown before each context entry, e.g. [decision_1a2b3c4d5e6f]. -Never cite by title and never shorten or invent ids. - -Entries marked status=superseded or status=reversed are history, not current policy. -Never present them as the current decision: give the decision that replaced them -(named in "replaced_by") as current, and mention the older one only as background. - -Format your answer as a well-structured response: -- Use bullet points for lists -- Group related information together -- Provide clear, actionable advice -- End with a brief summary -""" user_msg = f"Context:\n{context}\n\nQuestion: {question}" messages = [{"role": "system", "content": system}] if history: @@ -397,106 +472,30 @@ async def _generate( content = await llm_client.chat( messages=messages, - temperature=0.2, - max_tokens=1000, + temperature=0.1, + max_tokens=450, ) if content: return content - - # ✅ Improved fallback - group by type and format nicely - return _format_fallback_answer(question, context) - - -def _format_fallback_answer(question: str, context: str) -> str: - """Format a nice fallback answer when LLM is unavailable.""" - lines = context.splitlines() - - best_practices = [] - how_tos = [] - lessons = [] - decisions = [] - risks = [] - - current_section = None - for line in lines: - line = line.strip() - if not line: - continue - if line == "=== KNOWLEDGE ITEMS ===": - current_section = "items" - continue - if line == "=== ARTIFACT SUMMARIES ===": - current_section = "summaries" - continue - if line == "=== RELATED CONTEXT (graph) ===": - current_section = "graph" - continue - if line.startswith("==="): - current_section = None - continue - - # Parse item lines - if current_section == "items" and "(" in line and ")" in line: - # Extract type and content - parts = line.split(":", 1) - if len(parts) == 2: - meta = parts[0].strip() - content_text = parts[1].strip() - - # Extract type (and lifecycle status, if any) from "[id] (type status=x) relevance=…" - m = re.match(r"\[[^\]]*\]\s*\(([\w-]+)(?:\s+status=(\w+))?\)", meta) - kind, status = (m.group(1), m.group(2)) if m else ("", None) - if status in ("superseded", "reversed"): - content_text = f"{content_text} _({status})_" - if kind == "best-practice": - best_practices.append(content_text) - elif kind == "how-to": - how_tos.append(content_text) - elif kind == "lesson": - lessons.append(content_text) - elif kind == "decision": - decisions.append(content_text) - elif kind == "risk": - risks.append(content_text) - - # Build formatted answer - answer_parts = [] - answer_parts.append(f"Based on the provided context, here are insights about **{question}**:\n") - - if how_tos: - answer_parts.append("### 📋 How-To Steps") - for i, item in enumerate(how_tos[:5], 1): - answer_parts.append(f"{i}. {item}") - answer_parts.append("") - - if best_practices: - answer_parts.append("### ✅ Best Practices") - for item in best_practices[:5]: - answer_parts.append(f"• {item}") - answer_parts.append("") - - if lessons: - answer_parts.append("### 📚 Lessons Learned") - for item in lessons[:5]: - answer_parts.append(f"• {item}") - answer_parts.append("") - - if decisions: - answer_parts.append("### 📝 Key Decisions") - for item in decisions[:5]: - answer_parts.append(f"• {item}") - answer_parts.append("") - - if risks: - answer_parts.append("### ⚠️ Risks to Consider") - for item in risks[:5]: - answer_parts.append(f"• {item}") - answer_parts.append("") - - if not any([how_tos, best_practices, lessons, decisions, risks]): + return _format_fallback_answer(question, nodes or []) + + +def _format_fallback_answer(question: str, nodes: List[Dict[str, Any]]) -> str: + """An answer without the LLM: the retrieved items, grouped by type.""" + groups = [("how-to", "How-to steps"), ("best-practice", "Best practices"), ("lesson", "Lessons learned"), + ("decision", "Decisions"), ("risk", "Risks"), ("action-item", "Action items")] + parts = [f"The local model didn't answer, so here is what the knowledge base holds on **{question}**:\n"] + for kind, heading in groups: + entries = [n for n in nodes if (n.get("kind") or n.get("type")) == kind][:5] + if entries: + parts.append(f"### {heading}") + for n in entries: + status = f" _({n['status']})_" if n.get("status") in ("superseded", "reversed") else "" + parts.append(f"- {n.get('title') or n.get('label', '')}{status}") + parts.append("") + if len(parts) == 1: return f"No specific guidance found for: *{question}*. Please try rephrasing your question." - - return "\n".join(answer_parts) + return "\n".join(parts) # ───────────────────────────────────────────────────────────────────────────── @@ -529,19 +528,25 @@ def _extract_citations( answer: str, nodes: List[Dict[str, Any]], summaries: Optional[List[Dict[str, Any]]] = None, + refs: Optional[Dict[int, Dict[str, Any]]] = None, ) -> List[Dict[str, str]]: """ - Citation format is the full id in brackets — `[item_id]` for knowledge - items, `[artifact_id]` for artifact summaries — exactly as written by - _build_context. Only exact matches against the supplied context count. + Citations are the entry numbers of the context, "[2]" or "[1][3]" or "[1, 3]" + (see _build_context), mapped back through `refs`; a full id in brackets is + accepted too. Only entries of the supplied context count. """ by_id = _citable(nodes, summaries) + for ref in (refs or {}).values(): + if ref.get("id") and ref["id"] not in by_id: + by_id[ref["id"]] = {"id": ref["id"], "title": ref.get("title") or ref["id"], + "type": ref.get("kind") or ref.get("type") or "item"} + numbered = {str(num): ref.get("id") for num, ref in (refs or {}).items()} cited: List[Dict[str, str]] = [] seen_ids = set() for token in re.findall(r"\[([^\]]+)\]", answer): # tolerate "[id1, id2]" groupings for part in token.split(","): - cid = part.strip() + cid = numbered.get(part.strip(), part.strip()) if cid in by_id and cid not in seen_ids: seen_ids.add(cid) cited.append(by_id[cid]) @@ -562,6 +567,17 @@ def _significant_words(text: str) -> set: if len(w) > 2 and w not in _CITE_STOPWORDS} +_NO_ANSWER_MARKERS = ("doesn't cover", "does not cover", "no information", "not mention", "does not contain", + "doesn't contain", "unable to answer", "cannot answer", "no relevant", "not provide") + + +def is_no_answer(answer: str) -> bool: + """A reply saying the knowledge base has nothing on the question (short, with a refusal phrase).""" + a = answer.strip().lower() + return a.startswith(NO_ANSWER.lower().rstrip(".")) or ( + len(a) < 400 and any(m in a for m in _NO_ANSWER_MARKERS)) + + def _infer_citations( answer: str, nodes: List[Dict[str, Any]], @@ -575,6 +591,8 @@ def _infer_citations( when at least half of its title's words, and at least two, appear in the answer. Marked `inferred`, so the UI can say these were matched, not cited. """ + if is_no_answer(answer): + return [] # a refusal names the topics it didn't find; they aren't its sources answer_words = _significant_words(answer) scored = [] for entry in _citable(nodes, summaries).values(): @@ -613,7 +631,11 @@ async def embed_and_route() -> Tuple[Optional[List[float]], str]: # Query transformation (LLM) and routing depend only on the question, so # they run concurrently; routing reuses the question embedding. - transformed, (q_vec, route) = await asyncio.gather(transform_query(question), embed_and_route()) + if QUERY_REWRITE: + transformed, (q_vec, route) = await asyncio.gather(transform_query(question), embed_and_route()) + else: + transformed = {"sub_queries": [question], "hyde_doc": question} + q_vec, route = await embed_and_route() sub_queries = transformed["sub_queries"] hyde_doc = transformed["hyde_doc"] lap("transform_route") @@ -701,12 +723,17 @@ async def embed_and_route() -> Tuple[Optional[List[float]], str]: primary = [n for n in reranked if "graph" not in n.get("retrieved_by", "")] graph_nbr = [n for n in reranked if "graph" in n.get("retrieved_by", "")] + await _hydrate(store, user_id, primary) + lap("hydrate") - context = _build_context(summaries_used, primary, graph_nbr) - answer = await _generate(question, context, route, history) + context, refs = _build_context(summaries_used, primary, graph_nbr) + answer = await _generate(question, context, route, history, nodes=reranked) lap("generate") - citations = (_extract_citations(answer, reranked, summaries_used) - or _infer_citations(answer, reranked, summaries_used)) + if is_no_answer(answer): + citations = [] + else: + citations = (_extract_citations(answer, reranked, summaries_used, refs) + or _infer_citations(answer, reranked, summaries_used)) return { "answer": answer, diff --git a/backend/app/services/llm_client.py b/backend/app/services/llm_client.py index 4fe61a4..ab8ecd7 100644 --- a/backend/app/services/llm_client.py +++ b/backend/app/services/llm_client.py @@ -12,7 +12,7 @@ async def chat( messages: List[Dict[str, str]], temperature: float = 0, max_tokens: int = 1000, - json_mode: bool = False, + json_mode: bool | Dict[str, Any] = False, # True, or a JSON schema the reply must follow workspace: Optional[Any] = None, ) -> Optional[str]: return await get_llm_provider(workspace).chat( diff --git a/backend/app/services/llm_extraction.py b/backend/app/services/llm_extraction.py index 545c049..00796f0 100644 --- a/backend/app/services/llm_extraction.py +++ b/backend/app/services/llm_extraction.py @@ -24,7 +24,8 @@ "risks": [{"risk": "...", "severity": "low|medium|high"}], "summary": "..." } -3. decisions: things explicitly agreed, decided, or confirmed — in any format. +3. decisions: things explicitly agreed, decided, or confirmed — in any format. Keep one decision in one + item with all its parts ("keep X and add Y"), and put its reason in "why". 4. action_items: concrete tasks assigned or volunteered by a person. 5. risks: concerns, blockers, open questions, or uncertainties raised. 6. summary: 2-3 sentences covering the main outcome or thread conclusion. @@ -53,6 +54,7 @@ "risks": [{"risk": "...", "severity": "low|medium|high"}] } 3. decisions: choices the text states were made or agreed — not proposals or options still being considered. + Keep one decision in one item with all its parts ("keep X and add Y"), and put its reason in "why". 4. action_items: concrete tasks assigned to someone or explicitly still to be done. 5. how_tos: procedures with at least two ordered steps. "what" names the goal; steps are short imperative phrases. 6. best_practices: explicit recommendations or rules the text tells the reader to follow. @@ -70,6 +72,36 @@ """ +# The reply shapes as JSON schemas: Ollama then constrains its output to them (structured +# outputs), so a small model can't return malformed JSON, stray keys or wrapped text. +def _entries(*fields: str, lists: tuple = ()) -> Dict[str, Any]: + props: Dict[str, Any] = {f: {"type": "string"} for f in fields} + props.update({f: {"type": "array", "items": {"type": "string"}} for f in lists}) + return {"type": "array", "maxItems": 10, + "items": {"type": "object", "properties": props, "required": list(fields) + list(lists)}} + + +_RISK = {"type": "array", "maxItems": 10, "items": {"type": "object", "properties": { + "risk": {"type": "string"}, "severity": {"type": "string", "enum": ["low", "medium", "high"]}}, + "required": ["risk", "severity"]}} + +_TRANSCRIPT_SCHEMA: Dict[str, Any] = {"type": "object", "properties": { + "decisions": _entries("what", "why", "who"), + "action_items": _entries("task", "owner", "due"), + "risks": _RISK, + "summary": {"type": "string"}, +}, "required": ["decisions", "action_items", "risks", "summary"]} + +_DOCUMENT_SCHEMA: Dict[str, Any] = {"type": "object", "properties": { + "decisions": _entries("what", "why", "who"), + "action_items": _entries("task", "owner", "due"), + "how_tos": _entries("what", lists=("steps",)), + "best_practices": _entries("what", "why"), + "lessons": _entries("what", "why"), + "risks": _RISK, +}, "required": ["decisions", "action_items", "how_tos", "best_practices", "lessons", "risks"]} + + def _extract_json(raw: str) -> Dict[str, Any]: """ Parse JSON from LLM output, tolerating accidental markdown fences @@ -115,7 +147,7 @@ async def extract_from_transcript(text: str, source_type: str = "transcript", wo ], temperature=0, max_tokens=1500, - json_mode=True, + json_mode=_TRANSCRIPT_SCHEMA, workspace=workspace, ) if content: @@ -256,7 +288,7 @@ async def extract_from_document(text: str, workspace: Optional[Any] = None) -> D ], temperature=0, max_tokens=1500, - json_mode=True, + json_mode=_DOCUMENT_SCHEMA, workspace=workspace, ) if content is None and index == 0: diff --git a/backend/app/services/providers.py b/backend/app/services/providers.py index 297fa27..5281c73 100644 --- a/backend/app/services/providers.py +++ b/backend/app/services/providers.py @@ -13,7 +13,7 @@ import threading from contextvars import ContextVar from abc import ABC, abstractmethod -from typing import Any, List, Optional +from typing import Any, Dict, List, Optional logger = logging.getLogger(__name__) @@ -36,6 +36,9 @@ # Ollama's default context (2-4k tokens) silently truncates long prompts; # extraction sends up to ~3k tokens of input plus a 1.5k-token reply. OLLAMA_NUM_CTX = int(os.getenv("OLLAMA_NUM_CTX", "8192")) +# How long Ollama keeps the model in memory after a call. Its default (5 minutes) +# unloads it between quiet spells, and reloading an 8B model costs 30-60 s on a CPU. +OLLAMA_KEEP_ALIVE = os.getenv("OLLAMA_KEEP_ALIVE", "30m") LOCAL_EMBED_MODEL = os.getenv("LOCAL_EMBED_MODEL", "all-MiniLM-L6-v2") @@ -51,7 +54,7 @@ async def chat( messages: List[dict], temperature: float = 0, max_tokens: int = 1000, - json_mode: bool = False, + json_mode: bool | Dict[str, Any] = False, ) -> Optional[str]: ... @property @@ -82,21 +85,25 @@ async def chat( messages: List[dict], temperature: float = 0, max_tokens: int = 1000, - json_mode: bool = False, + json_mode: bool | Dict[str, Any] = False, ) -> Optional[str]: + """json_mode: True for any JSON object, or a JSON schema the reply must follow (structured output).""" try: import httpx payload: dict[str, Any] = { "model": self.model, "messages": messages, "stream": False, + "keep_alive": OLLAMA_KEEP_ALIVE, "options": { "temperature": temperature, "num_predict": max_tokens, "num_ctx": OLLAMA_NUM_CTX, }, } - if json_mode: + if isinstance(json_mode, dict): + payload["format"] = json_mode + elif json_mode: payload["format"] = "json" async with httpx.AsyncClient( timeout=httpx.Timeout(connect=3.0, read=120.0, write=10.0, pool=5.0) diff --git a/backend/evals/README.md b/backend/evals/README.md new file mode 100644 index 0000000..0d1d156 --- /dev/null +++ b/backend/evals/README.md @@ -0,0 +1,62 @@ +# LLM evaluation + +Measures what users get from the local model, end to end: what extraction finds in each kind of source, +whether GraphRAG answers are right and cite the right source, whether it says so when the knowledge base +has nothing, and how long each takes. Run it before and after changing a prompt, a retrieval step or the +model, and keep the change only if the numbers say so. + +It needs a running API with a real LLM (the unit tests use a fake one) and takes about 15–30 minutes on a +CPU. + +## Corpus + +[`corpus.py`](corpus.py): seven fictional sources (a meeting transcript, an email thread, a Slack thread, +a postmortem, a runbook, an ADR and pasted retro notes), each with an answer key of the decisions, +risks, action items, lessons, how-tos and best practices it holds; and 22 questions: 19 with the facts +a correct answer must contain and the source it should cite, and 3 the knowledge base can't answer. + +## Run + +Against a backend started from source (no Redis needed), with its own ArcadeDB: + +```bash +docker run -d --name kh-eval-arcadedb -p 12490:2480 \ + -e JAVA_OPTS="-Darcadedb.server.rootPassword=eval-password" arcadedata/arcadedb:26.9.1 +cd backend +ARCADEDB_URL=http://localhost:12490 ARCADEDB_PASSWORD=eval-password AUTH_RATE_LIMIT=false \ + SECRET_KEY=$(python -c "import secrets; print(secrets.token_urlsafe(48))") \ + uvicorn app.main:app --port 18000 & +python -m evals.run_eval --base http://localhost:18000 --label my-change \ + --arcadedb http://localhost:12490 --arcadedb-auth root:eval-password +``` + +`--arcadedb` lets it wait for the summaries built after ingestion before asking questions, so they don't +compete for the model while answers are timed. Results go to `evals/results/