diff --git a/.env.api.example b/.env.api.example new file mode 100644 index 0000000..567ceae --- /dev/null +++ b/.env.api.example @@ -0,0 +1,2 @@ +# Must match the password portion of NEO4J_AUTH in .env. +NEO4J_PASSWORD=local-knowledge-only diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..33aaed3 --- /dev/null +++ b/.env.example @@ -0,0 +1,2 @@ +# Local development credentials. Change both example-derived files together for shared deployments. +NEO4J_AUTH=neo4j/local-knowledge-only diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..76c1aab --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,109 @@ +name: CI + +on: + pull_request: + branches: [main] + push: + branches: [main] + +permissions: + contents: read + +concurrency: + group: ci-${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + server: + name: Server tests + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - name: Check out repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ github.event.pull_request.head.sha || github.sha }} + persist-credentials: false + - name: Install uv + uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0 + with: + version: "0.12.0" + python-version: "3.13" + - name: Run server tests + run: uv run --frozen --project server pytest server/tests -q + + web: + name: Web tests and build + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - name: Check out repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ github.event.pull_request.head.sha || github.sha }} + persist-credentials: false + - name: Install Node.js + uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 + with: + node-version: "24" + cache: npm + cache-dependency-path: web/package-lock.json + - name: Install dependencies + run: npm ci + working-directory: web + - name: Audit dependencies + run: npm audit --audit-level=high + working-directory: web + - name: Run web tests + run: npm test -- --run + working-directory: web + - name: Build production bundle + run: npm run build + working-directory: web + + compose: + name: Compose config + runs-on: ubuntu-latest + timeout-minutes: 5 + steps: + - name: Check out repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ github.event.pull_request.head.sha || github.sha }} + persist-credentials: false + - name: Validate Compose configuration + run: | + test -f .env || cp .env.example .env + test -f .env.api || cp .env.api.example .env.api + python3 - <<'PY' + from pathlib import Path + + def read_value(path: str, key: str) -> str: + for raw in Path(path).read_text().splitlines(): + line = raw.strip() + if line and not line.startswith("#") and line.startswith(f"{key}="): + return line.split("=", 1)[1] + raise SystemExit(f"missing {key} in {path}") + + auth = read_value(".env.example", "NEO4J_AUTH") + password = read_value(".env.api.example", "NEO4J_PASSWORD") + assert auth.startswith("neo4j/") and auth.split("/", 1)[1] == password + PY + docker compose config --quiet + + release-ready: + name: Release ready + if: ${{ always() }} + needs: [server, web, compose] + runs-on: ubuntu-latest + timeout-minutes: 2 + env: + SERVER_RESULT: ${{ needs.server.result }} + WEB_RESULT: ${{ needs.web.result }} + COMPOSE_RESULT: ${{ needs.compose.result }} + steps: + - name: Require every release gate + run: | + test "$SERVER_RESULT" = success + test "$WEB_RESULT" = success + test "$COMPOSE_RESULT" = success diff --git a/README.ja.md b/README.ja.md index f24cb2e..fa5a177 100644 --- a/README.ja.md +++ b/README.ja.md @@ -42,11 +42,20 @@ OKF Markdown ──> FastAPI cataloger ──> PostgreSQL + pgvector ──> Rea ```bash git clone https://github.com/hungrytech/knowledge-wiki.git cd knowledge-wiki +test -f .env || cp .env.example .env +test -f .env.api || cp .env.api.example .env.api docker compose up -d --build docker compose ps curl -fsS http://localhost:8000/api/health ``` +サンプル認証情報はローカル専用です。共有環境または非ローカル環境へデプロイする前に強いパスワードへ変更し、`.env` の `NEO4J_AUTH` に含まれるパスワード部分と `.env.api` の `NEO4J_PASSWORD` を同じ値に保ってください。 + +```dotenv +NEO4J_AUTH=neo4j/replace-with-a-strong-password +NEO4J_PASSWORD=replace-with-a-strong-password +``` + - Wiki browser: - API: - PostgreSQL: `localhost:5433`（ローカル開発専用） @@ -77,6 +86,57 @@ data/wiki/ `Source` は外部資料と provenance を保存し、`Concept` と `Comparison` は解釈を記録します。この境界により主張をレビュー可能にし、根拠を黙って書き換えることを防ぎます。 +### 型付きコンテキスト関係 + +長期記憶文書は、frontmatter に方向と種類を持つ関係を記録できます。短縮形式は既存 corpus の規約に合い、構造化形式では edge 単位の根拠や時刻も保持できます。 + +```yaml +relations: + - "supported-by [[sources/primary-evidence]]" + - kind: supersedes + target: concepts/previous-decision + context: ADR-0042 + timestamp: 2026-07-26T10:30:00Z +``` + +本文の内部リンクは `links-to`、frontmatter の関係は明示された kind として投影されます。UI の **コンテキスト** グラフは選択文書の incoming/outgoing edge を表示します。Neo4j は再構築可能な派生物であり、Markdown の関係が正本です。 + +### コンテキストが長く使える知識へ昇格する流れ + +グラフに線ができただけでは、信頼できる知識にはなりません。WikiBrain は一時的な観察が暗黙に永続的な事実へ変わらないよう、収集、解釈、利用を分けます。 + +| 段階 | 意味 | 代表的な文書・関係 | +|---|---|---| +| 0. 入口 | 会話、URL、handoff、観察がまだ知識として採用されていない | curator が収集するまでは正本 bundle の外に置く | +| 1. 根拠 | URL、時刻、hash とともに原文を保存する | `Source` | +| 2. 理解 | 主張を簡潔に整理し、根拠と照合する | `supported-by` で結んだ `Concept` または `Comparison` | +| 3. 利用 | レビュー済みの知識を実装や長期的な判断に使う | `Project` または判断用 `Concept` と、`extends`・`adopts` などの明示的な関係 | +| 4. 更新 | 新しい根拠で以前の結論を改めても、履歴は消さない | 新文書から旧文書への `supersedes` | + +現在の昇格は人によるレビューで行います。Curator が出典を確認し、重複を除き、事実と解釈を分けてから関係を明示します。グラフには、長期文書が作られた理由と、何を置き換えたかが残ります。 + +```mermaid +flowchart LR + A[会話、URL、
handoff、観察] -->|curator が収集| B[Source
変更しない原文根拠] + B --> C{出典があり
関連性と可読性が十分か?} + C -- いいえ --> X[長期 corpus の外に置くか
より良い根拠を求める] + C -- はい --> D[Concept または Comparison
整理した理解] + D -->|supported-by| B + D --> E{レビュー済みで
今後も役立つか?} + E -- まだ --> D + E -- はい --> F[Project または長期的な
判断用 Concept] + F -->|extends / adopts| D + G[新しい根拠] --> H[新しい解釈] + H -->|supersedes| D + H --> F +``` + +#### 現在実装されていること + +WikiBrain は型付き関係を解析・保存し、`/api/graph` で返し、Neo4j が有効なら派生グラフへ投影します。UI では選択文書の incoming/outgoing 関係を確認できます。Markdown が正本で、PostgreSQL と Neo4j は再構築できます。 + +現在、会話 session の自動収集、保持期間の計算、人気スコアによる自動昇格、関係種類の推論は行いません。`brainctl remember` コマンドも存在しません。自動昇格は将来の機能です。実装する場合は provenance の保持、競合の可視化、レビュー可能な方針を先に整え、その後に正本 Markdown を変更する必要があります。 + ## 開発 ```bash @@ -89,6 +149,7 @@ npm --prefix web run build - **ローカル利用**を前提としています。API、DB ポート、Compose の認証情報を公衆インターネットへ公開しないでください。 - Compose の PostgreSQL パスワードは開発用プレースホルダーです。共有/本番環境では変更し、シークレット管理を使用してください。 +- URL 取り込みは、検証済みの公開 IPv4 ターゲットにのみ接続します。IPv6-only/NAT64 ターゲット、圧縮レスポンス、5 MiB を超えるレスポンス、全体で 20 秒の deadline を超える取得は拒否します。取り込んだ内容を信頼する前に、出典と本文を確認してください。 - 公開 fork 前に `data/wiki/` に個人メモやライセンス制限のある内容がないか確認してください。 ## ロードマップ diff --git a/README.ko.md b/README.ko.md index bc734c0..1cc7450 100644 --- a/README.ko.md +++ b/README.ko.md @@ -45,11 +45,20 @@ OKF Markdown ──> FastAPI cataloger ──> PostgreSQL + pgvector ──> Rea ```bash git clone https://github.com/hungrytech/knowledge-wiki.git cd knowledge-wiki +test -f .env || cp .env.example .env +test -f .env.api || cp .env.api.example .env.api docker compose up -d --build docker compose ps curl -fsS http://localhost:8000/api/health ``` +예제 자격 증명은 로컬 전용입니다. 공유 환경이나 비로컬 환경에 배포하기 전에는 강한 비밀번호로 교체하고, `.env`의 `NEO4J_AUTH` 비밀번호 부분과 `.env.api`의 `NEO4J_PASSWORD`를 같게 유지하세요. + +```dotenv +NEO4J_AUTH=neo4j/replace-with-a-strong-password +NEO4J_PASSWORD=replace-with-a-strong-password +``` + - Wiki browser: - API: - PostgreSQL: `localhost:5433` — 로컬 개발 전용 @@ -80,6 +89,57 @@ data/wiki/ `Source`는 외부 원문과 provenance를 보관하고, `Concept`와 `Comparison`은 해석을 기록합니다. 이 경계를 유지하면 주장 검토가 쉬우며 근거를 조용히 바꾸지 않게 됩니다. +### 타입이 있는 맥락 관계 + +장기기억 문서는 frontmatter에 방향과 종류가 있는 관계를 기록할 수 있습니다. 짧은 형식은 기존 corpus 관례이고, 구조화 형식은 edge 수준의 근거나 시간을 함께 보존합니다. + +```yaml +relations: + - "supported-by [[sources/primary-evidence]]" + - kind: supersedes + target: concepts/previous-decision + context: ADR-0042 + timestamp: 2026-07-26T10:30:00Z +``` + +본문 내부 링크는 `links-to`로, frontmatter 관계는 명시한 kind로 projection됩니다. UI의 **맥락** 그래프는 선택 문서의 incoming/outgoing edge를 보여 줍니다. Neo4j는 재생성 가능한 파생물이고 Markdown 관계가 정본입니다. + +### 맥락이 오래 쓰는 지식으로 승급되는 과정 + +그래프 선이 생겼다고 곧바로 믿을 만한 지식이 되는 것은 아닙니다. WikiBrain은 임시 관찰이 조용히 영구 사실로 굳지 않도록 수집, 해석, 활용을 나눕니다. + +| 단계 | 의미 | 대표 문서·관계 | +|---|---|---| +| 0. 입구 | 대화, URL, handoff, 관찰이 아직 지식으로 채택되지 않은 상태 | curator가 수집하기 전에는 정본 bundle 밖에 둠 | +| 1. 근거 | URL, 시각, hash와 함께 원문을 보존 | `Source` | +| 2. 이해 | 주장을 짧게 정리하고 근거와 대조 | `supported-by`로 연결한 `Concept` 또는 `Comparison` | +| 3. 활용 | 검토한 지식이 구현이나 오래 유지할 의사결정에 쓰임 | `Project` 또는 의사결정 중심 `Concept`, `extends`·`adopts` 등 명시적 관계 | +| 4. 갱신 | 새 근거가 이전 결론을 바꾸되 과거 기록은 지우지 않음 | 새 문서가 이전 문서를 `supersedes`로 연결 | + +현재 승급은 사람이 검토하는 과정입니다. Curator가 출처를 확인하고, 중복을 제거하고, 사실과 해석을 분리한 뒤 관계를 직접 기록합니다. 그래프는 장기 문서가 왜 생겼고 무엇을 대신했는지 남깁니다. + +```mermaid +flowchart LR + A[대화, URL,
handoff, 관찰] -->|curator가 수집| B[Source
바꾸지 않는 원문 근거] + B --> C{출처가 남아 있고
관련성과 가독성이 충분한가?} + C -- 아니오 --> X[장기 corpus 밖에 두거나
더 나은 근거 요청] + C -- 예 --> D[Concept 또는 Comparison
정제한 이해] + D -->|supported-by| B + D --> E{검토를 마쳤고
나중에도 쓸 만한가?} + E -- 아직 아님 --> D + E -- 예 --> F[Project 또는 오래 유지할
의사결정 Concept] + F -->|extends / adopts| D + G[새 근거] --> H[새 해석] + H -->|supersedes| D + H --> F +``` + +#### 지금 실제로 되는 것 + +WikiBrain은 타입 관계를 읽고 보존하며, `/api/graph`로 반환하고, Neo4j가 켜져 있으면 파생 그래프에 투영합니다. UI에서는 선택 문서의 들어오는 관계와 나가는 관계를 볼 수 있습니다. Markdown이 정본이고 PostgreSQL과 Neo4j는 다시 만들 수 있습니다. + +현재는 대화 session을 자동 수집하거나, 보존 기간을 계산하거나, 인기 점수로 자동 승급하거나, 관계 종류를 추론하지 않습니다. `brainctl remember` 명령도 없습니다. 자동 승급은 향후 기능입니다. 구현하려면 provenance 보존, 충돌 노출, 검토 가능한 정책을 먼저 갖추고 나서 정본 Markdown을 변경해야 합니다. + ## 개발 ```bash @@ -92,6 +152,7 @@ npm --prefix web run build - 이 프로젝트는 **로컬 사용**을 목표로 합니다. API, DB 포트, Compose 자격 증명을 인터넷에 노출하지 마세요. - Compose PostgreSQL 비밀번호는 개발용 placeholder입니다. 공유/운영 배포 전에는 변경하고 secret 관리를 사용하세요. +- URL 수집은 검증된 공개 IPv4 대상에만 연결합니다. IPv6-only/NAT64 대상, 압축 응답, 5 MiB 초과 응답, 전체 20초 deadline을 넘는 수집은 거부합니다. 캡처한 내용을 신뢰하기 전에 출처와 본문을 검토하세요. - 공개 fork 전에 `data/wiki/`에 개인 노트나 라이선스가 제한된 자료가 없는지 검토하세요. ## 로드맵 diff --git a/README.md b/README.md index 0d25d79..83527a9 100644 --- a/README.md +++ b/README.md @@ -69,11 +69,20 @@ Markdown remains canonical; deleting the derived database does not lose knowledg git clone https://github.com/hungrytech/knowledge-wiki.git cd knowledge-wiki +test -f .env || cp .env.example .env +test -f .env.api || cp .env.api.example .env.api docker compose up -d --build docker compose ps curl -fsS http://localhost:8000/api/health ``` +The example credentials are for local use only. Before a shared or non-local deployment, replace them with a strong password and keep the password portion of `NEO4J_AUTH` in `.env` equal to `NEO4J_PASSWORD` in `.env.api`: + +```dotenv +NEO4J_AUTH=neo4j/replace-with-a-strong-password +NEO4J_PASSWORD=replace-with-a-strong-password +``` + Open: - **Wiki browser:** @@ -131,6 +140,57 @@ data/wiki/ A `Source` document preserves external material and provenance. A `Concept` or `Comparison` document adds interpretation. Keeping these separate makes claims reviewable and lets the corpus evolve without silently rewriting source evidence. +### Typed context relations + +Durable context can declare directed, typed relations in frontmatter. The compact form matches the corpus's existing convention; the structured form adds edge-level provenance or time metadata: + +```yaml +relations: + - "supported-by [[sources/primary-evidence]]" + - kind: supersedes + target: concepts/previous-decision + context: ADR-0042 + timestamp: 2026-07-26T10:30:00Z +``` + +Internal Markdown links are projected as `links-to`; frontmatter relations keep their explicit kind. The UI's **Context** graph shows incoming and outgoing edges around the selected document. Neo4j remains a rebuildable projection—the Markdown relation is canonical. + +### How context becomes durable knowledge + +A graph edge does not make a note trustworthy. WikiBrain separates capture, interpretation, and action so a temporary observation does not silently become a durable fact. + +| Stage | What it means | Typical document or relation | +|---|---|---| +| 0. Intake | A conversation, URL, handoff, or observation has not been accepted as knowledge yet | Outside the canonical bundle until a curator captures it | +| 1. Evidence | The original material is captured with its URL, time, and hash | `Source` | +| 2. Understanding | A claim is distilled, checked, and connected to its evidence | `Concept` or `Comparison` with `supported-by` | +| 3. Use | Reviewed knowledge informs an implementation or durable decision | `Project`, or a decision-focused `Concept`, linked with `extends`, `adopts`, or another explicit kind | +| 4. Revision | New evidence changes an older conclusion without rewriting history | A new document with `supersedes`; the old document remains traceable | + +Promotion is review-driven today. A curator checks provenance, removes duplicates, separates fact from interpretation, and writes the relation explicitly. The graph records why the durable document exists and what it replaced. + +```mermaid +flowchart LR + A[Conversation, URL,
handoff, or observation] -->|curator captures| B[Source
immutable evidence] + B --> C{Provenance present?
Relevant and readable?} + C -- No --> X[Keep outside the durable corpus
or request better evidence] + C -- Yes --> D[Concept or Comparison
distilled understanding] + D -->|supported-by| B + D --> E{Reviewed and useful
for future work?} + E -- Not yet --> D + E -- Yes --> F[Project or durable
decision Concept] + F -->|extends / adopts| D + G[New evidence] --> H[New interpretation] + H -->|supersedes| D + H --> F +``` + +#### What is implemented now + +WikiBrain parses and preserves typed relations, returns them from `/api/graph`, projects them into Neo4j when enabled, and renders incoming and outgoing context in the UI. Markdown remains canonical; PostgreSQL and Neo4j can be rebuilt. + +WikiBrain does **not** currently ingest chat sessions automatically, run a retention clock, calculate a popularity score, infer relation kinds, or provide a `brainctl remember` command. Automatic promotion may be added later, but it must retain provenance, expose conflicts, and require a reviewable policy before it can change canonical Markdown. + ## Development ```bash @@ -153,7 +213,7 @@ docker compose exec -T postgres psql -U knowledge -d knowledge_wiki -Atc \ - This project is designed for **local use**. Do not expose the API, database port, or Docker Compose credentials directly to the public internet. - The Compose PostgreSQL password is a development-only placeholder. Replace it and use managed secret handling before any shared or production deployment. -- URL ingestion should remain restricted to public addresses; preserve sources and inspect fetched content before trusting it. +- URL ingestion connects only to validated public IPv4 targets. IPv6-only/NAT64 targets, compressed responses, responses over 5 MiB, and fetches exceeding the 20-second total deadline are rejected. Preserve sources and inspect fetched content before trusting it. - Knowledge may contain personal notes or licensed material. Review `data/wiki/` before making a fork or derivative repository public. ## Roadmap diff --git a/README.zh-CN.md b/README.zh-CN.md index 76eb3b5..6f42241 100644 --- a/README.zh-CN.md +++ b/README.zh-CN.md @@ -42,11 +42,20 @@ OKF Markdown ──> FastAPI cataloger ──> PostgreSQL + pgvector ──> Rea ```bash git clone https://github.com/hungrytech/knowledge-wiki.git cd knowledge-wiki +test -f .env || cp .env.example .env +test -f .env.api || cp .env.api.example .env.api docker compose up -d --build docker compose ps curl -fsS http://localhost:8000/api/health ``` +示例凭据仅适用于本地环境。在共享环境或非本地环境部署之前，请改用强密码，并确保 `.env` 中 `NEO4J_AUTH` 的密码部分与 `.env.api` 中的 `NEO4J_PASSWORD` 保持一致。 + +```dotenv +NEO4J_AUTH=neo4j/replace-with-a-strong-password +NEO4J_PASSWORD=replace-with-a-strong-password +``` + - Wiki browser： - API： - PostgreSQL：`localhost:5433`（仅限本地开发） @@ -77,6 +86,57 @@ data/wiki/ `Source` 保存外部材料和 provenance，`Concept` 与 `Comparison` 记录解释。保留此边界可使主张更易审查，并避免静默重写证据。 +### 带类型的上下文关系 + +长期记忆文档可以在 frontmatter 中声明有方向、有类型的关系。简写形式沿用现有 corpus 约定；结构化形式还能保存 edge 级来源或时间信息。 + +```yaml +relations: + - "supported-by [[sources/primary-evidence]]" + - kind: supersedes + target: concepts/previous-decision + context: ADR-0042 + timestamp: 2026-07-26T10:30:00Z +``` + +正文内部链接会投影为 `links-to`，frontmatter 关系保留显式 kind。UI 的**上下文**图展示所选文档的 incoming/outgoing edge。Neo4j 仍是可重建投影，Markdown 关系才是权威来源。 + +### 上下文晋升为长期知识的过程 + +图中出现一条边，并不代表内容已经可信。WikiBrain 将采集、解释和使用分开，避免临时观察在没有审查的情况下变成永久事实。 + +| 阶段 | 含义 | 典型文档或关系 | +|---|---|---| +| 0. 入口 | 对话、URL、handoff 或观察尚未被接受为知识 | curator 采集之前保留在权威 bundle 之外 | +| 1. 证据 | 连同 URL、时间和 hash 保存原始材料 | `Source` | +| 2. 理解 | 精炼主张，并与证据核对 | 通过 `supported-by` 连接的 `Concept` 或 `Comparison` | +| 3. 使用 | 经审查的知识用于实现或长期决策 | `Project` 或决策型 `Concept`，并使用 `extends`、`adopts` 等显式关系 | +| 4. 修订 | 新证据改变旧结论，但不删除历史 | 新文档通过 `supersedes` 指向旧文档 | + +目前的晋升由人工审查完成。Curator 检查来源、去重、分离事实与解释，然后显式写入关系。图会记录长期文档为何产生，以及它替代了什么。 + +```mermaid +flowchart LR + A[对话、URL、
handoff 或观察] -->|curator 采集| B[Source
不可变的原始证据] + B --> C{来源完整且
相关、可读吗?} + C -- 否 --> X[保留在长期 corpus 之外
或请求更好的证据] + C -- 是 --> D[Concept 或 Comparison
精炼后的理解] + D -->|supported-by| B + D --> E{已经审查并且
今后仍有用吗?} + E -- 尚未 --> D + E -- 是 --> F[Project 或长期保存的
决策型 Concept] + F -->|extends / adopts| D + G[新证据] --> H[新解释] + H -->|supersedes| D + H --> F +``` + +#### 当前已经实现的功能 + +WikiBrain 会解析并保存带类型的关系，通过 `/api/graph` 返回；启用 Neo4j 时，也会投影到派生图中。UI 可以显示所选文档的 incoming/outgoing 关系。Markdown 是权威来源，PostgreSQL 和 Neo4j 都可以重建。 + +目前不会自动采集对话 session、计算保留周期、按流行度评分自动晋升，也不会推断关系类型。项目中也没有 `brainctl remember` 命令。自动晋升属于未来功能；在它能修改权威 Markdown 之前，必须先保留 provenance、显示冲突，并采用可审查的策略。 + ## 开发 ```bash @@ -89,6 +149,7 @@ npm --prefix web run build - 项目面向**本地使用**。不要将 API、数据库端口或 Compose 凭据直接暴露到互联网。 - Compose PostgreSQL 密码只是开发占位符。在共享或生产部署前请替换，并使用密钥管理。 +- URL 采集仅连接到经过验证的公共 IPv4 目标。IPv6-only/NAT64 目标、压缩响应、超过 5 MiB 的响应，以及超过 20 秒总 deadline 的采集都会被拒绝。信任采集内容前，请检查来源和正文。 - 公开 fork 前请检查 `data/wiki/`，避免包含个人笔记或受许可证限制的材料。 ## 路线图 diff --git a/server/Dockerfile b/server/Dockerfile index e2f5ce8..5cb3768 100644 --- a/server/Dockerfile +++ b/server/Dockerfile @@ -1,7 +1,9 @@ FROM python:3.12-slim WORKDIR /app -COPY pyproject.toml ./ -RUN pip install --no-cache-dir --index-url https://download.pytorch.org/whl/cpu torch -RUN pip install --no-cache-dir fastapi "uvicorn[standard]" pyyaml "psycopg[binary]" httpx sentence-transformers "neo4j>=5.26,<6" +COPY pyproject.toml uv.lock ./ +RUN pip install --no-cache-dir uv==0.12.0 \ + && uv export --frozen --no-dev --no-emit-project --prune torch --no-hashes -o /tmp/requirements.txt +RUN pip install --no-cache-dir --index-url https://download.pytorch.org/whl/cpu "torch==2.13.0+cpu" +RUN pip install --no-cache-dir -r /tmp/requirements.txt COPY app ./app CMD ["uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "8000"] diff --git a/server/app/main.py b/server/app/main.py index 39c24e7..5cb688c 100644 --- a/server/app/main.py +++ b/server/app/main.py @@ -17,19 +17,10 @@ from .catalog import semantic_ids, sync_catalog from .neo4j_graph import Neo4jGraphProjection, create_projection from .embeddings import LocalEmbedder -from .okf import Concept, parse_concept, render_concept +from .okf import Concept, load_concepts, parse_concept, parse_relations, render_concept from .sources import fetch_url -def load_concepts(root: Path) -> list[Concept]: - concepts: list[Concept] = [] - for path in root.rglob("*.md"): - if path.name in {"index.md", "log.md"}: - continue - concepts.append(parse_concept(path.relative_to(root).as_posix(), path.read_text(encoding="utf-8"))) - return sorted(concepts, key=lambda item: item.id) - - def concept_slug(title: str) -> str: slug = re.sub(r"[^a-z0-9]+", "-", title.lower()).strip("-") return slug or "untitled" @@ -42,6 +33,7 @@ class IngestRequest(BaseModel): tags: list[str] = [] resource: str | None = None body: str = Field(min_length=1) + relations: list[str] = Field(default_factory=list) class UrlIngestRequest(BaseModel): @@ -98,15 +90,33 @@ def graph() -> dict[str, list[dict] | str]: return graph_store.graph() concepts = load_concepts(root) ids = {concept.id for concept in concepts} + edges = [ + { + "source": concept.id, + "target": target, + "kind": "links-to", + "context": None, + "timestamp": None, + } + for concept in concepts + for target in concept.links + if target in ids + ] + [ + { + "source": concept.id, + "target": relation.target, + "kind": relation.kind, + "context": relation.context, + "timestamp": relation.timestamp, + } + for concept in concepts + for relation in concept.relations + if relation.target in ids + ] return { "source": "filesystem", "nodes": [{"id": concept.id, "label": concept.title, "type": concept.type, "tags": concept.tags} for concept in concepts], - "edges": [ - {"source": concept.id, "target": target} - for concept in concepts - for target in concept.links - if target in ids - ], + "edges": sorted(edges, key=lambda edge: (edge["source"], edge["target"], edge["kind"])), } @app.get("/api/search") @@ -137,6 +147,10 @@ def ingest(payload: IngestRequest) -> dict: target = root / relative if target.exists(): raise HTTPException(status_code=409, detail="A concept with this title already exists") + try: + relations = parse_relations(relative[:-3], payload.relations) + except ValueError as error: + raise HTTPException(status_code=422, detail=str(error)) from error concept = Concept( id=relative[:-3], type=payload.type, @@ -147,6 +161,7 @@ def ingest(payload: IngestRequest) -> dict: timestamp=datetime.now(UTC).replace(microsecond=0).isoformat().replace("+00:00", "Z"), body=payload.body, links=[], + relations=relations, ) target.parent.mkdir(parents=True, exist_ok=True) target.write_text(render_concept(concept), encoding="utf-8") diff --git a/server/app/neo4j_graph.py b/server/app/neo4j_graph.py index ff98e48..5b44029 100644 --- a/server/app/neo4j_graph.py +++ b/server/app/neo4j_graph.py @@ -7,6 +7,8 @@ from __future__ import annotations from collections.abc import Iterable +import hashlib +import json from typing import Any, Protocol from .okf import Concept @@ -24,6 +26,16 @@ class Neo4jDriver(Protocol): def session(self, **kwargs: Any) -> Neo4jSession: ... +def _relation_key(relation: Any) -> str: + """Return a stable identity without leaking context text into the graph key.""" + payload = json.dumps( + [relation.kind, relation.target, relation.context, relation.timestamp], + ensure_ascii=False, + separators=(",", ":"), + ) + return hashlib.sha256(payload.encode("utf-8")).hexdigest() + + def graph_records(concepts: Iterable[Concept]) -> dict[str, list[dict[str, Any]]]: """Return deterministic, metadata-only records with links confined to this corpus.""" ordered = sorted(concepts, key=lambda concept: concept.id) @@ -38,6 +50,17 @@ def graph_records(concepts: Iterable[Concept]) -> dict[str, list[dict[str, Any]] "resource": concept.resource, "timestamp": concept.timestamp, "links": sorted({target for target in concept.links if target in known_ids}), + "relations": [ + { + "key": _relation_key(relation), + "kind": relation.kind, + "target": relation.target, + "context": relation.context, + "timestamp": relation.timestamp, + } + for relation in concept.relations + if relation.target in known_ids + ], } for concept in ordered ] @@ -113,6 +136,18 @@ def sync(self, concepts: Iterable[Concept]) -> None: graph=self.graph_key, documents=documents, ) + session.run( + """ + UNWIND $documents AS doc + UNWIND doc.relations AS relation + MATCH (source:Document {graph: $graph, id: doc.id}) + MATCH (target:Document {graph: $graph, id: relation.target}) + MERGE (source)-[edge:RELATES_TO {graph: $graph, key: relation.key}]->(target) + SET edge.kind = relation.kind, edge.context = relation.context, edge.timestamp = relation.timestamp + """, + graph=self.graph_key, + documents=documents, + ) def graph(self) -> dict[str, list[dict[str, Any]] | str]: with self.driver.session() as session: @@ -128,9 +163,12 @@ def graph(self) -> dict[str, list[dict[str, Any]] | str]: ).data() edges = session.run( """ - MATCH (source:Document {graph: $graph})-[:LINKS_TO]->(target:Document {graph: $graph}) - RETURN source.id AS source, target.id AS target - ORDER BY source, target + MATCH (source:Document {graph: $graph})-[edge]->(target:Document {graph: $graph}) + WHERE type(edge) IN ['LINKS_TO', 'RELATES_TO'] AND edge.graph = $graph + RETURN source.id AS source, target.id AS target, + CASE type(edge) WHEN 'LINKS_TO' THEN 'links-to' ELSE edge.kind END AS kind, + edge.context AS context, edge.timestamp AS timestamp + ORDER BY source, target, kind """, graph=self.graph_key, ).data() diff --git a/server/app/okf.py b/server/app/okf.py index 5d6058a..d569456 100644 --- a/server/app/okf.py +++ b/server/app/okf.py @@ -1,7 +1,9 @@ from __future__ import annotations -from dataclasses import dataclass -from pathlib import PurePosixPath +from dataclasses import dataclass, field +import logging +from pathlib import Path, PurePosixPath +from posixpath import normpath import re from typing import Any @@ -9,6 +11,17 @@ _FRONTMATTER = re.compile(r"\A---\s*\n(.*?)\n---\s*\n?(.*)\Z", re.DOTALL) _LINK = re.compile(r"\[[^\]]+\]\(([^)]+\.md)\)") +_RELATION = re.compile(r"^\s*([a-zA-Z0-9][a-zA-Z0-9._-]*)\s+\[\[([^\]]+)\]\]\s*$") +_NUMBERED_DUPLICATE = re.compile(r"^(.*) \d+$") +_logger = logging.getLogger(__name__) + + +@dataclass(frozen=True) +class Relation: + kind: str + target: str + context: str | None = None + timestamp: str | None = None @dataclass(frozen=True) @@ -25,6 +38,7 @@ class Concept: title_ko: str | None = None description_ko: str | None = None body_ko: str | None = None + relations: list[Relation] = field(default_factory=list) def _concept_id(path: str) -> str: @@ -53,12 +67,76 @@ def _extract_links(concept_id: str, body: str) -> list[str]: return result +def _normalize_relation_target(concept_id: str, raw_target: str) -> str: + target = raw_target.strip().split("|", 1)[0].split("#", 1)[0] + if not target: + raise ValueError("relation target must not be empty") + if "://" in target or "\\" in target: + raise ValueError("relation target must be an internal POSIX document id") + target_path = PurePosixPath(target.lstrip("/")) + if target_path.suffix == ".md": + target_path = target_path.with_suffix("") + top_level = target_path.parts[0] if target_path.parts else "" + if target.startswith("/") or top_level in {"concepts", "sources", "comparisons", "projects"}: + resolved = target_path + else: + resolved = PurePosixPath(concept_id).parent / target_path + normalized = normpath(resolved.as_posix()) + normalized_path = PurePosixPath(normalized) + if normalized in {"", ".", ".."} or (normalized_path.parts and normalized_path.parts[0] == ".."): + raise ValueError("relation target must stay inside the wiki bundle") + return normalized + + +def _optional_string(value: Any) -> str | None: + if value in (None, ""): + return None + if hasattr(value, "isoformat"): + return value.isoformat() + return str(value) + + +def parse_relations(concept_id: str, values: Any) -> list[Relation]: + if values in (None, []): + return [] + if not isinstance(values, list): + raise ValueError("relations must be a YAML list") + relations: list[Relation] = [] + for value in values: + if isinstance(value, str): + match = _RELATION.match(value) + if not match: + raise ValueError("relation strings must use ' [[]]'") + kind, target = match.groups() + context = timestamp = None + elif isinstance(value, dict): + kind = value.get("kind") or value.get("type") + target = value.get("target") + context = _optional_string(value.get("context")) + timestamp = _optional_string(value.get("timestamp")) + else: + raise ValueError("relations entries must be strings or mappings") + if not isinstance(kind, str) or not re.fullmatch(r"[a-zA-Z0-9][a-zA-Z0-9._-]*", kind): + raise ValueError("relation kind must be a slug") + if not isinstance(target, str): + raise ValueError("relation target must be a string") + relation = Relation( + kind=kind.casefold(), + target=_normalize_relation_target(concept_id, target), + context=context, + timestamp=timestamp, + ) + if relation not in relations: + relations.append(relation) + return relations + + def parse_concept(path: str, source: str) -> Concept: match = _FRONTMATTER.match(source) if not match: raise ValueError("OKF concept requires YAML frontmatter") metadata: dict[str, Any] = yaml.safe_load(match.group(1)) or {} - concept_type = metadata.get("type") + concept_type = metadata.get("type") or metadata.get("kind") if not isinstance(concept_type, str) or not concept_type.strip(): raise ValueError("OKF concept requires a non-empty type") body = match.group(2).strip() @@ -68,7 +146,7 @@ def parse_concept(path: str, source: str) -> Concept: concept_id = _concept_id(path) return Concept( id=concept_id, - type=concept_type, + type={"concept": "Concept", "comparison": "Comparison", "project": "Project", "source": "Source"}.get(concept_type.strip().lower(), concept_type.strip()), title=str(metadata.get("title") or concept_id.rsplit("/", 1)[-1]), description=metadata.get("description"), tags=[str(tag) for tag in tags], @@ -76,17 +154,53 @@ def parse_concept(path: str, source: str) -> Concept: timestamp=metadata.get("timestamp"), body=body, links=_extract_links(concept_id, body), + relations=parse_relations(concept_id, metadata.get("relations")), title_ko=metadata.get("title_ko"), description_ko=metadata.get("description_ko"), body_ko=metadata.get("body_ko"), ) +def load_concepts(root: Path) -> list[Concept]: + """Load every non-reserved OKF Markdown document from a bundle root.""" + concepts: list[Concept] = [] + for path in root.rglob("*.md"): + if path.name in {"index.md", "log.md"}: + continue + try: + source = path.read_text(encoding="utf-8") + except OSError as error: + duplicate = _NUMBERED_DUPLICATE.fullmatch(path.stem) + canonical = path.with_name(f"{duplicate.group(1)}.md") if duplicate else None + if canonical is not None and canonical.is_file(): + _logger.warning("Skipping unreadable numbered duplicate %s; canonical sibling %s exists: %s", path, canonical, error) + continue + raise + concepts.append(parse_concept(path.relative_to(root).as_posix(), source)) + return sorted(concepts, key=lambda item: item.id) + + def render_concept(concept: Concept) -> str: metadata: dict[str, Any] = {"type": concept.type, "title": concept.title} for key in ("description", "resource", "tags", "timestamp", "title_ko", "description_ko", "body_ko"): value = getattr(concept, key) if value not in (None, [], ""): metadata[key] = value + if concept.relations: + metadata["relations"] = [ + f"{relation.kind} [[{relation.target}]]" + if relation.context is None and relation.timestamp is None + else { + key: value + for key, value in { + "kind": relation.kind, + "target": relation.target, + "context": relation.context, + "timestamp": relation.timestamp, + }.items() + if value is not None + } + for relation in concept.relations + ] frontmatter = yaml.safe_dump(metadata, allow_unicode=True, sort_keys=False).strip() return f"---\n{frontmatter}\n---\n\n{concept.body.strip()}\n" diff --git a/server/app/sources.py b/server/app/sources.py index 861586a..7f4d67d 100644 --- a/server/app/sources.py +++ b/server/app/sources.py @@ -1,12 +1,21 @@ from __future__ import annotations +import asyncio import ipaddress import re +import time +from collections.abc import Callable from html.parser import HTMLParser -from urllib.parse import urlparse +from urllib.parse import ParseResult, urljoin, urlparse +import dns.exception +import dns.resolver import httpx +_MAX_RESPONSE_BYTES = 5 * 1024 * 1024 +_MAX_ELAPSED_SECONDS = 20.0 +_NETWORK_TIMEOUT_SECONDS = 5.0 + class _TextExtractor(HTMLParser): def __init__(self) -> None: @@ -32,18 +41,88 @@ def handle_data(self, data: str) -> None: self.parts.append(text) -def _validate_public_url(url: str) -> None: +def _dns_addresses_before_deadline(hostname: str, deadline: float, resolver=None) -> tuple[str, ...]: + active_resolver = resolver if resolver is not None else dns.resolver.Resolver(configure=True) + addresses: list[str] = [] + for record_type in ("A", "AAAA"): + remaining = deadline - time.monotonic() + if remaining <= 0: + raise ValueError("URL ingest deadline exceeded") + try: + answer = active_resolver.resolve( + hostname, + record_type, + lifetime=remaining, + search=False, + ) + except dns.resolver.NXDOMAIN as exc: + raise ValueError("URL hostname could not be resolved") from exc + except dns.resolver.NoAnswer: + continue + except dns.resolver.LifetimeTimeout as exc: + raise ValueError("URL ingest deadline exceeded") from exc + except dns.exception.DNSException as exc: + raise ValueError("URL hostname could not be resolved") from exc + addresses.extend(str(record) for record in answer) + return tuple(dict.fromkeys(addresses)) + + +def _resolve_public_addresses( + hostname: str, + port: int, + *, + deadline: float | None = None, + resolver=None, +) -> tuple[str, ...]: + del port # DNS address records do not vary by HTTP destination port. + deadline = deadline if deadline is not None else time.monotonic() + _MAX_ELAPSED_SECONDS + _ensure_before_deadline(deadline, time.monotonic) + try: + literal = ipaddress.ip_address(hostname) + except ValueError: + addresses = _dns_addresses_before_deadline(hostname, deadline, resolver) + else: + addresses = (str(literal),) + + if not addresses: + raise ValueError("URL hostname could not be resolved") + parsed_addresses = tuple(ipaddress.ip_address(address) for address in addresses) + if not parsed_addresses or any(not address.is_global for address in parsed_addresses): + raise ValueError("Private network URLs are not allowed") + if any(address.is_multicast for address in parsed_addresses): + raise ValueError("Public unicast URLs are required") + public_ipv4 = tuple( + original + for original, parsed_address in zip(addresses, parsed_addresses, strict=True) + if isinstance(parsed_address, ipaddress.IPv4Address) + ) + if not public_ipv4: + raise ValueError("IPv6 URL targets are not supported") + return public_ipv4 + + +def _validated_public_target( + url: str, + *, + deadline: float | None = None, +) -> tuple[ParseResult, tuple[str, ...]]: parsed = urlparse(url) if parsed.scheme not in {"http", "https"} or not parsed.hostname: raise ValueError("Only absolute http(s) URLs are supported") - if parsed.hostname == "localhost": - raise ValueError("Local URLs are not allowed") + if parsed.username is not None or parsed.password is not None: + raise ValueError("URL credentials are not allowed") try: - address = ipaddress.ip_address(parsed.hostname) - except ValueError: - return - if not address.is_global: - raise ValueError("Private network URLs are not allowed") + explicit_port = parsed.port + except ValueError as exc: + raise ValueError("URL port is invalid") from exc + port = explicit_port if explicit_port is not None else (443 if parsed.scheme == "https" else 80) + if not 1 <= port <= 65535: + raise ValueError("URL port is invalid") + return parsed, _resolve_public_addresses(parsed.hostname, port, deadline=deadline) + + +def _validate_public_url(url: str) -> None: + _validated_public_target(url) def title_from_markdown(body: str) -> str | None: @@ -51,16 +130,147 @@ def title_from_markdown(body: str) -> str | None: return match.group(1).strip() if match else None -def fetch_url(url: str) -> tuple[str, str]: - _validate_public_url(url) - response = httpx.get(url, follow_redirects=True, timeout=20, headers={"User-Agent": "KnowledgeWiki/0.1"}) - response.raise_for_status() - markdown = response.text - if str(response.url).split("?", 1)[0].endswith(".md") or "text/markdown" in response.headers.get("content-type", ""): - title = title_from_markdown(markdown) or urlparse(str(response.url)).hostname or "Untitled source" +def _ensure_before_deadline(deadline: float, clock: Callable[[], float]) -> None: + if clock() >= deadline: + raise ValueError("URL ingest deadline exceeded") + + +def _remaining_network_timeout(deadline: float, clock: Callable[[], float]) -> float: + remaining = deadline - clock() + if remaining <= 0: + raise ValueError("URL ingest deadline exceeded") + return min(_NETWORK_TIMEOUT_SECONDS, remaining) + + +async def _pinned_request(client: httpx.AsyncClient, url: str, *, deadline: float) -> httpx.Response: + parsed, addresses = _validated_public_target(url, deadline=deadline) + _ensure_before_deadline(deadline, time.monotonic) + assert parsed.hostname is not None + address = addresses[0] + pinned_host = f"[{address}]" if ipaddress.ip_address(address).version == 6 else address + pinned_netloc = f"{pinned_host}:{parsed.port}" if parsed.port is not None else pinned_host + pinned_url = parsed._replace(netloc=pinned_netloc).geturl() + + hostname = parsed.hostname.encode("idna").decode("ascii") + host_header = f"[{hostname}]" if ":" in hostname else hostname + if parsed.port is not None: + host_header = f"{host_header}:{parsed.port}" + network_timeout = _remaining_network_timeout(deadline, time.monotonic) + request = client.build_request( + "GET", + pinned_url, + headers={ + "Accept-Encoding": "identity", + "Host": host_header, + "User-Agent": "KnowledgeWiki/0.1", + }, + extensions={ + "sni_hostname": hostname, + "timeout": { + "connect": network_timeout, + "read": network_timeout, + "write": network_timeout, + "pool": network_timeout, + }, + }, + ) + return await client.send(request, stream=True) + + +async def _read_limited_text( + response: httpx.Response, + max_response_bytes: int, + *, + deadline: float, + clock: Callable[[], float], +) -> str: + content_encoding = response.headers.get("content-encoding", "identity").strip().lower() + if content_encoding not in {"", "identity"}: + await response.aclose() + raise ValueError("Compressed URL responses are not allowed") + + chunks: list[bytes] = [] + size = 0 + try: + _ensure_before_deadline(deadline, clock) + async for chunk in response.aiter_bytes(): + _ensure_before_deadline(deadline, clock) + size += len(chunk) + if size > max_response_bytes: + raise ValueError(f"URL response exceeds {max_response_bytes} bytes") + chunks.append(chunk) + finally: + await response.aclose() + return b"".join(chunks).decode(response.encoding or "utf-8", errors="replace") + + +async def _fetch_url_async( + url: str, + *, + transport: httpx.AsyncBaseTransport | None = None, + max_response_bytes: int = _MAX_RESPONSE_BYTES, +) -> tuple[str, str]: + current_url = url + deadline = time.monotonic() + _MAX_ELAPSED_SECONDS + try: + async with asyncio.timeout(_MAX_ELAPSED_SECONDS): + async with httpx.AsyncClient( + transport=transport, + follow_redirects=False, + timeout=httpx.Timeout(_NETWORK_TIMEOUT_SECONDS), + trust_env=False, + ) as client: + for redirect_count in range(6): + _ensure_before_deadline(deadline, time.monotonic) + response = await _pinned_request(client, current_url, deadline=deadline) + try: + _ensure_before_deadline(deadline, time.monotonic) + except Exception: + await response.aclose() + raise + if not response.has_redirect_location: + break + if redirect_count == 5: + await response.aclose() + raise ValueError("Too many URL redirects") + next_url = urljoin(current_url, response.headers["location"]) + await response.aclose() + current_url = next_url + else: # pragma: no cover - the bounded loop always breaks or raises + raise ValueError("Too many URL redirects") + try: + response.raise_for_status() + markdown = await _read_limited_text( + response, + max_response_bytes, + deadline=deadline, + clock=time.monotonic, + ) + except Exception: + await response.aclose() + raise + except TimeoutError as exc: + raise ValueError("URL ingest deadline exceeded") from exc + if current_url.split("?", 1)[0].endswith(".md") or "text/markdown" in response.headers.get("content-type", ""): + title = title_from_markdown(markdown) or urlparse(current_url).hostname or "Untitled source" return title, markdown parser = _TextExtractor() parser.feed(markdown) - title = parser.title or urlparse(str(response.url)).hostname or "Untitled source" + title = parser.title or urlparse(current_url).hostname or "Untitled source" body = "\n\n".join(parser.parts) return title, body + + +def fetch_url( + url: str, + *, + transport: httpx.AsyncBaseTransport | None = None, + max_response_bytes: int = _MAX_RESPONSE_BYTES, +) -> tuple[str, str]: + return asyncio.run( + _fetch_url_async( + url, + transport=transport, + max_response_bytes=max_response_bytes, + ) + ) diff --git a/server/pyproject.toml b/server/pyproject.toml index f905080..93dbe53 100644 --- a/server/pyproject.toml +++ b/server/pyproject.toml @@ -10,6 +10,7 @@ dependencies = [ "httpx>=0.27,<1", "sentence-transformers>=3.0,<4", "neo4j>=5.26,<6", + "dnspython>=2.7,<3", ] [dependency-groups] diff --git a/server/tests/test_api.py b/server/tests/test_api.py index 74adead..1f1a418 100644 --- a/server/tests/test_api.py +++ b/server/tests/test_api.py @@ -60,13 +60,35 @@ def test_api_lists_documents_exposes_graph_search_and_ingests_okf(tmp_path): assert health.json() == {"status": "ok"} assert [item["id"] for item in documents.json()] == ["concepts/pgvector", "concepts/rag"] - assert graph.json()["edges"] == [{"source": "concepts/rag", "target": "concepts/pgvector"}] + assert graph.json()["edges"] == [{ + "source": "concepts/rag", + "target": "concepts/pgvector", + "kind": "links-to", + "context": None, + "timestamp": None, + }] assert [item["id"] for item in search.json()] == ["concepts/pgvector", "concepts/rag"] assert ingested.status_code == 201 assert ingested.json()["id"] == "sources/okf-launch" assert (tmp_path / "sources/okf-launch.md").exists() +def test_api_rejects_invalid_relation_input_without_creating_a_document(tmp_path): + response = TestClient(create_app(tmp_path)).post( + "/api/documents", + json={ + "type": "Concept", + "title": "Broken relation", + "body": "# Broken", + "relations": ["not valid"], + }, + ) + + assert response.status_code == 422 + assert response.json() == {"detail": "relation strings must use ' [[]]'"} + assert not (tmp_path / "concepts/broken-relation.md").exists() + + def test_api_exposes_okf_korean_translation_extensions(tmp_path): _write_concept( tmp_path, @@ -96,6 +118,31 @@ def test_api_exposes_okf_korean_translation_extensions(tmp_path): assert payload["body_ko"] == "# 개요\n\nKtor는 Kotlin용 비동기 웹 프레임워크입니다." +def test_api_graph_exposes_typed_context_relations_from_markdown(tmp_path): + _write_concept( + tmp_path, + "concepts/decision.md", + """--- +type: Concept +title: Decision +relations: + - "supported-by [[sources/evidence]]" +--- + +See [background](/concepts/background.md). +""", + ) + _write_concept(tmp_path, "concepts/background.md", "---\ntype: Concept\ntitle: Background\n---\n\n# Background") + _write_concept(tmp_path, "sources/evidence.md", "---\ntype: Source\ntitle: Evidence\n---\n\n# Evidence") + + payload = TestClient(create_app(tmp_path)).get("/api/graph").json() + + assert payload["edges"] == [ + {"source": "concepts/decision", "target": "concepts/background", "kind": "links-to", "context": None, "timestamp": None}, + {"source": "concepts/decision", "target": "sources/evidence", "kind": "supported-by", "context": None, "timestamp": None}, + ] + + def test_api_allows_tailnet_web_origins(tmp_path): _write_concept(tmp_path, "concepts/okf.md", "---\ntype: Concept\ntitle: OKF\n---\n\n# OKF") client = TestClient(create_app(tmp_path)) diff --git a/server/tests/test_neo4j_graph.py b/server/tests/test_neo4j_graph.py index c93f309..dc25843 100644 --- a/server/tests/test_neo4j_graph.py +++ b/server/tests/test_neo4j_graph.py @@ -1,11 +1,12 @@ from types import SimpleNamespace +from typing import Any import pytest from app.neo4j_graph import Neo4jGraphProjection, graph_records -def concept(identifier: str, *, links=(), tags=(), type="Concept"): +def concept(identifier: str, *, links=(), relations=(), tags=(), type="Concept") -> Any: return SimpleNamespace( id=identifier, title=identifier.rsplit("/", 1)[-1], @@ -15,6 +16,7 @@ def concept(identifier: str, *, links=(), tags=(), type="Concept"): resource=None, timestamp=None, links=list(links), + relations=list(relations), ) @@ -36,6 +38,7 @@ def test_graph_records_keep_only_internal_known_links_and_metadata(): "resource": None, "timestamp": None, "links": [], + "relations": [], }, { "id": "concepts/rag", @@ -46,10 +49,50 @@ def test_graph_records_keep_only_internal_known_links_and_metadata(): "resource": None, "timestamp": None, "links": ["concepts/pgvector"], + "relations": [], }, ] +def test_graph_records_keep_typed_context_relations_for_known_documents(): + relation = SimpleNamespace( + kind="supported-by", + target="sources/evidence", + context="Primary source", + timestamp="2026-07-25T00:00:00Z", + ) + + records = graph_records([ + concept("concepts/decision", relations=[relation]), + concept("sources/evidence", type="Source"), + ]) + + relation_record = records["documents"][0]["relations"][0] + assert len(relation_record["key"]) == 64 + assert {key: value for key, value in relation_record.items() if key != "key"} == { + "kind": "supported-by", + "target": "sources/evidence", + "context": "Primary source", + "timestamp": "2026-07-25T00:00:00Z", + } + + +def test_relation_keys_keep_distinct_provenance_for_the_same_kind_and_target(): + relations = [ + SimpleNamespace(kind="supported-by", target="sources/evidence", context="Issue", timestamp=None), + SimpleNamespace(kind="supported-by", target="sources/evidence", context="Release notes", timestamp=None), + ] + + records = graph_records([ + concept("concepts/decision", relations=relations), + concept("sources/evidence", type="Source"), + ]) + + projected = records["documents"][0]["relations"] + assert len(projected) == 2 + assert len({relation["key"] for relation in projected}) == 2 + + class RecordingSession: def __init__(self): self.calls = [] @@ -82,8 +125,9 @@ def test_projection_rebuilds_only_its_own_graph_namespace(): query, params = driver.session_instance.calls[0] assert "MATCH (node {graph: $graph}) DETACH DELETE node" in query assert params == {"graph": "knowledge-wiki"} - assert len(driver.session_instance.calls) == 4 + assert len(driver.session_instance.calls) == 5 assert any("[:LINKS_TO {graph: $graph}]" in query for query, _ in driver.session_instance.calls) + assert any("[edge:RELATES_TO {graph: $graph, key: relation.key}]" in query for query, _ in driver.session_instance.calls) def test_projection_graph_uses_neo4j_document_and_link_records(): @@ -99,7 +143,13 @@ def run(self, query, **params): self.calls.append((query, params)) if "RETURN doc.id AS id" in query: return Result([{"id": "concepts/rag", "label": "RAG", "type": "Concept", "tags": ["rag"]}]) - return Result([{"source": "concepts/rag", "target": "concepts/pgvector"}]) + return Result([{ + "source": "concepts/rag", + "target": "concepts/pgvector", + "kind": "links-to", + "context": None, + "timestamp": None, + }]) driver = RecordingDriver() driver.session_instance = ReadSession() @@ -108,7 +158,13 @@ def run(self, query, **params): assert projection.graph() == { "source": "neo4j", "nodes": [{"id": "concepts/rag", "label": "RAG", "type": "Concept", "tags": ["rag"]}], - "edges": [{"source": "concepts/rag", "target": "concepts/pgvector"}], + "edges": [{ + "source": "concepts/rag", + "target": "concepts/pgvector", + "kind": "links-to", + "context": None, + "timestamp": None, + }], } diff --git a/server/tests/test_okf.py b/server/tests/test_okf.py index e704ffc..91ef804 100644 --- a/server/tests/test_okf.py +++ b/server/tests/test_okf.py @@ -1,4 +1,6 @@ -from app.okf import Concept, parse_concept, render_concept +from pathlib import Path + +from app.okf import Concept, load_concepts, parse_concept, parse_relations, render_concept def test_parse_concept_reads_required_type_and_markdown_links(): @@ -23,12 +25,78 @@ def test_parse_concept_reads_required_type_and_markdown_links(): assert concept.links == ["concepts/rag", "concepts/local"] +def test_concept_positional_constructor_remains_backward_compatible(): + concept = Concept( + "concepts/legacy", + "Concept", + "Legacy", + None, + [], + None, + None, + "# Legacy", + [], + "레거시", + "기존 설명", + "# 레거시", + ) + + assert concept.title_ko == "레거시" + assert concept.description_ko == "기존 설명" + assert concept.body_ko == "# 레거시" + assert concept.relations == [] + assert "title_ko: 레거시" in render_concept(concept) + + def test_parse_concept_ignores_external_markdown_urls(): concept = parse_concept("concepts/okf.md", "---\ntype: Concept\n---\n\n[spec](https://example.com/SPEC.md)") assert concept.links == [] +def test_parse_concept_reads_existing_wikilink_relations_and_structured_context(): + concept = parse_concept( + "concepts/temporal-metric.md", + """--- +kind: concept +title: Temporal metric +relations: + - "extends [[concepts/temporal-architecture]]" + - kind: supported-by + target: /sources/temporal-issue.md + context: Issue evidence + timestamp: 2026-07-25T14:57:15Z +--- + +# Metric +""", + ) + + assert concept.title == "Temporal metric" + assert concept.type == "Concept" + assert [relation.__dict__ for relation in concept.relations] == [ + {"kind": "extends", "target": "concepts/temporal-architecture", "context": None, "timestamp": None}, + { + "kind": "supported-by", + "target": "sources/temporal-issue", + "context": "Issue evidence", + "timestamp": "2026-07-25T14:57:15+00:00", + }, + ] + + +def test_relation_targets_normalize_parent_segments_but_cannot_escape_the_bundle(): + assert parse_relations("concepts/nested/decision", ["supported-by [[../../sources/evidence.md]]"])[0].target == "sources/evidence" + + for target in ("../../../outside", "https://example.com/evidence", r"sources\evidence"): + try: + parse_relations("concepts/decision", [f"supported-by [[{target}]]"]) + except ValueError: + pass + else: + raise AssertionError(f"Expected unsafe relation target to be rejected: {target}") + + def test_render_concept_emits_okf_frontmatter_and_body(): concept = Concept( id="concepts/rag", @@ -58,3 +126,32 @@ def test_parse_concept_rejects_missing_type(): assert "type" in str(error) else: raise AssertionError("Expected missing type to be rejected") + + +def test_load_concepts_reads_bundle_documents_but_skips_reserved_files(tmp_path: Path): + (tmp_path / "concepts").mkdir() + (tmp_path / "index.md").write_text("# Index\n", encoding="utf-8") + (tmp_path / "log.md").write_text("# Log\n", encoding="utf-8") + (tmp_path / "concepts" / "entry.md").write_text("---\ntype: Concept\ntitle: Entry\n---\n\nBody", encoding="utf-8") + + concepts = load_concepts(tmp_path) + + assert [concept.id for concept in concepts] == ["concepts/entry"] + + +def test_load_concepts_skips_unreadable_numbered_duplicate_when_canonical_exists(tmp_path: Path, monkeypatch): + (tmp_path / "concepts").mkdir() + canonical = tmp_path / "concepts" / "entry.md" + duplicate = tmp_path / "concepts" / "entry 2.md" + canonical.write_text("---\ntype: Concept\ntitle: Entry\n---\n\nBody", encoding="utf-8") + duplicate.write_text("cloud placeholder", encoding="utf-8") + original_read_text = Path.read_text + + def read_text(path, *args, **kwargs): + if path == duplicate: + raise OSError(11, "Resource deadlock avoided") + return original_read_text(path, *args, **kwargs) + + monkeypatch.setattr(Path, "read_text", read_text) + + assert [concept.id for concept in load_concepts(tmp_path)] == ["concepts/entry"] diff --git a/server/tests/test_sources.py b/server/tests/test_sources.py index db6f3b2..a267e70 100644 --- a/server/tests/test_sources.py +++ b/server/tests/test_sources.py @@ -1,5 +1,296 @@ -from app.sources import title_from_markdown +import asyncio +import socketserver +import threading +import time + +import dns.resolver +import httpx +import pytest + +import app.sources as sources +from app.sources import _read_limited_text, _validate_public_url, fetch_url, title_from_markdown + + +def _install_dns(monkeypatch, addresses_by_host: dict[str, str]): + calls: list[tuple[str, str, float, bool]] = [] + + class FakeResolver: + def __init__(self, *, configure=True): + assert configure is True + + def resolve(self, hostname, record_type, *, lifetime, search): + calls.append((hostname, record_type, lifetime, search)) + if record_type == "AAAA": + return [] + return [addresses_by_host[hostname]] + + monkeypatch.setattr(sources.dns.resolver, "Resolver", FakeResolver) + return calls def test_title_from_markdown_uses_first_heading_for_raw_markdown_sources(): assert title_from_markdown("# Open Knowledge Format (OKF)\n\nSpecification body.") == "Open Knowledge Format (OKF)" + + +def test_validate_public_url_rejects_hostname_resolving_to_loopback(monkeypatch): + _install_dns(monkeypatch, {"attacker.example": "127.0.0.1"}) + + with pytest.raises(ValueError, match="Private network"): + _validate_public_url("http://attacker.example/resource") + + +def test_validate_public_url_rejects_private_answer_in_mixed_dns_set(): + class MixedResolver: + def resolve(self, _hostname, record_type, *, lifetime, search): + assert lifetime > 0 + assert search is False + return ["93.184.216.34"] if record_type == "A" else ["::1"] + + with pytest.raises(ValueError, match="Private network"): + sources._resolve_public_addresses( + "mixed.example", + 80, + deadline=time.monotonic() + 1, + resolver=MixedResolver(), + ) + + +def test_validate_public_url_rejects_multicast_literal(): + with pytest.raises(ValueError, match="Public unicast"): + _validate_public_url("http://224.0.0.1/resource") + + +def test_validate_public_url_rejects_nat64_literal(): + with pytest.raises(ValueError, match="IPv6 URL targets"): + _validate_public_url("http://[64:ff9b::7f00:1]/resource") + + +def test_validate_public_url_rejects_port_zero(monkeypatch): + resolver_factory_calls: list[bool] = [] + monkeypatch.setattr( + sources.dns.resolver, + "Resolver", + lambda *, configure=True: resolver_factory_calls.append(configure), + ) + + with pytest.raises(ValueError, match="port"): + _validate_public_url("http://public.example:0/resource") + + assert resolver_factory_calls == [] + + +def test_dns_resolution_passes_remaining_absolute_deadline(): + lifetimes: list[float] = [] + + class TimingOutResolver: + def resolve(self, _hostname, _record_type, *, lifetime, search): + assert search is False + lifetimes.append(lifetime) + raise dns.resolver.LifetimeTimeout(timeout=lifetime, errors=[]) + + started = time.monotonic() + with pytest.raises(ValueError, match="deadline"): + sources._resolve_public_addresses( + "public.example", + 80, + deadline=started + 0.02, + resolver=TimingOutResolver(), + ) + + assert 0 < lifetimes[0] <= 0.02 + assert time.monotonic() - started < 0.2 + + +def test_repeated_dns_timeouts_do_not_exhaust_global_capacity(): + calls: list[tuple[str, str, float]] = [] + + class TimingOutResolver: + def resolve(self, hostname, record_type, *, lifetime, search): + assert search is False + calls.append((hostname, record_type, lifetime)) + raise dns.resolver.LifetimeTimeout(timeout=lifetime, errors=[]) + + resolver = TimingOutResolver() + for _ in range(5): + with pytest.raises(ValueError, match="deadline"): + sources._resolve_public_addresses( + "public.example", + 80, + deadline=time.monotonic() + 1, + resolver=resolver, + ) + + assert len(calls) == 5 + + +def test_remaining_network_timeout_uses_smaller_deadline_budget(): + assert sources._remaining_network_timeout(20.0, clock=lambda: 19.75) == pytest.approx(0.25) + + +def test_fetch_url_disables_ambient_proxy(monkeypatch): + real_client = httpx.AsyncClient + client_options: dict[str, object] = {} + + class RecordingClient(real_client): + def __init__(self, *args, **kwargs): + client_options.update(kwargs) + super().__init__(*args, **kwargs) + + monkeypatch.setattr(sources.httpx, "AsyncClient", RecordingClient) + _install_dns(monkeypatch, {"public.example": "93.184.216.34"}) + + def respond(_request: httpx.Request) -> httpx.Response: + return httpx.Response(200, headers={"content-type": "text/markdown"}, text="# Direct") + + assert fetch_url("http://public.example/doc.md", transport=httpx.MockTransport(respond))[0] == "Direct" + assert client_options["trust_env"] is False + + +def test_fetch_url_pins_connection_to_validated_address(monkeypatch): + _install_dns(monkeypatch, {"public.example": "93.184.216.34"}) + requests: list[httpx.Request] = [] + + def respond(request: httpx.Request) -> httpx.Response: + requests.append(request) + return httpx.Response(200, headers={"content-type": "text/markdown"}, text="# Pinned") + + title, body = fetch_url("http://public.example/doc.md", transport=httpx.MockTransport(respond)) + + assert (title, body) == ("Pinned", "# Pinned") + assert str(requests[0].url) == "http://93.184.216.34/doc.md" + assert requests[0].headers["host"] == "public.example" + assert requests[0].headers["accept-encoding"] == "identity" + + +def test_fetch_url_rejects_private_redirect_before_following_it(monkeypatch): + _install_dns( + monkeypatch, + { + "public.example": "93.184.216.34", + "internal.example": "127.0.0.1", + }, + ) + requests: list[httpx.Request] = [] + + def respond(request: httpx.Request) -> httpx.Response: + requests.append(request) + return httpx.Response(302, headers={"location": "http://internal.example/admin"}) + + with pytest.raises(ValueError, match="Private network"): + fetch_url("http://public.example/start", transport=httpx.MockTransport(respond)) + + assert len(requests) == 1 + + +def test_fetch_url_rejects_response_over_size_limit(monkeypatch): + _install_dns(monkeypatch, {"public.example": "93.184.216.34"}) + + def respond(_request: httpx.Request) -> httpx.Response: + return httpx.Response(200, headers={"content-type": "text/markdown"}, content=b"123456789") + + with pytest.raises(ValueError, match="response exceeds"): + fetch_url( + "http://public.example/doc.md", + transport=httpx.MockTransport(respond), + max_response_bytes=8, + ) + + +def test_fetch_url_rejects_compressed_response(monkeypatch): + _install_dns(monkeypatch, {"public.example": "93.184.216.34"}) + + def respond(_request: httpx.Request) -> httpx.Response: + return httpx.Response( + 200, + headers={"content-type": "text/markdown", "content-encoding": "gzip"}, + stream=httpx.ByteStream(b"not-a-gzip-stream"), + ) + + with pytest.raises(ValueError, match="Compressed URL responses"): + fetch_url("http://public.example/doc.md", transport=httpx.MockTransport(respond)) + + +def test_fetch_url_cancels_slow_response_headers_at_total_deadline(monkeypatch): + _install_dns(monkeypatch, {"public.example": "93.184.216.34"}) + monkeypatch.setattr(sources, "_MAX_ELAPSED_SECONDS", 0.02) + + class SlowHeaderTransport(httpx.BaseTransport, httpx.AsyncBaseTransport): + async_cancelled = False + + def handle_request(self, request: httpx.Request) -> httpx.Response: + time.sleep(0.1) + return httpx.Response(200, request=request, text="# Too late") + + async def handle_async_request(self, request: httpx.Request) -> httpx.Response: + try: + await asyncio.sleep(0.1) + except asyncio.CancelledError: + self.async_cancelled = True + raise + return httpx.Response(200, request=request, text="# Too late") + + transport = SlowHeaderTransport() + started = time.monotonic() + with pytest.raises(ValueError, match="deadline"): + fetch_url("http://public.example/doc.md", transport=transport) + + assert time.monotonic() - started < 0.08 + assert transport.async_cancelled + + +def test_fetch_url_closes_real_slow_header_socket_at_total_deadline(monkeypatch): + dripping_started = threading.Event() + disconnected = threading.Event() + + class SlowHeaderHandler(socketserver.BaseRequestHandler): + def handle(self): + request_bytes = b"" + while b"\r\n\r\n" not in request_bytes: + chunk = self.request.recv(4096) + if not chunk: + return + request_bytes += chunk + dripping_started.set() + try: + for byte in b"HTTP/1.1 ": + self.request.sendall(bytes([byte])) + time.sleep(0.05) + self.request.settimeout(0.5) + if self.request.recv(1) == b"": + disconnected.set() + except OSError: + disconnected.set() + + class TestServer(socketserver.ThreadingTCPServer): + allow_reuse_address = True + daemon_threads = True + + with TestServer(("127.0.0.1", 0), SlowHeaderHandler) as server: + server_thread = threading.Thread(target=server.serve_forever, daemon=True) + server_thread.start() + monkeypatch.setattr(sources, "_MAX_ELAPSED_SECONDS", 0.3) + monkeypatch.setattr(sources, "_resolve_public_addresses", lambda *_args, **_kwargs: ("127.0.0.1",)) + + started = time.monotonic() + with pytest.raises(ValueError, match="deadline"): + fetch_url(f"http://public.example:{server.server_address[1]}/doc.md") + + assert time.monotonic() - started < 0.6 + assert dripping_started.is_set() + assert disconnected.wait(1) + server.shutdown() + server_thread.join(timeout=1) + assert not server_thread.is_alive() + + +def test_read_limited_text_enforces_total_deadline(): + class OneChunkStream(httpx.AsyncByteStream): + async def __aiter__(self): + yield b"still-streaming" + + response = httpx.Response(200, stream=OneChunkStream()) + + with pytest.raises(ValueError, match="deadline"): + asyncio.run(_read_limited_text(response, 1024, deadline=10.0, clock=lambda: 10.0)) + + assert response.is_closed diff --git a/server/uv.lock b/server/uv.lock index e504d9f..f15046e 100644 --- a/server/uv.lock +++ b/server/uv.lock @@ -218,6 +218,15 @@ nvtx = [ { name = "nvidia-nvtx", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (platform_machine == 'AMD64' and sys_platform == 'win32')" }, ] +[[package]] +name = "dnspython" +version = "2.8.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/8c/8b/57666417c0f90f08bcafa776861060426765fdb422eb10212086fb811d26/dnspython-2.8.0.tar.gz", hash = "sha256:181d3c6996452cb1189c4046c61599b84a5a86e099562ffde77d26984ff26d0f", size = 368251, upload-time = "2025-09-07T18:58:00.022Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ba/5a/18ad964b0086c6e62e2e7500f7edc89e3faa45033c71c1893d34eed2b2de/dnspython-2.8.0-py3-none-any.whl", hash = "sha256:01d9bbc4a2d76bf0db7c1f729812ded6d912bd318d3b1cf81d30c0f845dbf3af", size = 331094, upload-time = "2025-09-07T18:57:58.071Z" }, +] + [[package]] name = "fastapi" version = "0.139.0" @@ -828,6 +837,7 @@ name = "okf-knowledge-api" version = "0.1.0" source = { virtual = "." } dependencies = [ + { name = "dnspython" }, { name = "fastapi" }, { name = "httpx" }, { name = "neo4j" }, @@ -845,6 +855,7 @@ dev = [ [package.metadata] requires-dist = [ + { name = "dnspython", specifier = ">=2.7,<3" }, { name = "fastapi", specifier = ">=0.115,<1" }, { name = "httpx", specifier = ">=0.27,<1" }, { name = "neo4j", specifier = ">=5.26,<6" }, diff --git a/web/package-lock.json b/web/package-lock.json index a3612a1..22e41d7 100644 --- a/web/package-lock.json +++ b/web/package-lock.json @@ -1832,9 +1832,9 @@ } }, "node_modules/nanoid": { - "version": "3.3.15", - "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.15.tgz", - "integrity": "sha512-y7Wygv/7mEOvxTuEQDB8StXdMRBWf1kR/tlhAzBRUFkB2jfcLOAxO/SHmOO2zgz1pVgK29/kyupn059/bCHdjA==", + "version": "3.3.16", + "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.16.tgz", + "integrity": "sha512-bzlKTyNJ7+LdGIIwy8ijFpIqEQIvafahV7eYykJ8Cvh42EdJeODoJ6gUJXpQJvej1BddH8OqTXZNE/KfbWAu8Q==", "funding": [ { "type": "github", @@ -1902,9 +1902,9 @@ } }, "node_modules/postcss": { - "version": "8.5.16", - "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.5.16.tgz", - "integrity": "sha512-vuwillviilfKZsg0VGj5R/YwwcHx4SLsIOI/7K6mQkWx+l5cUHTjj5g0AasTBcyXsbfTgrwsUNmVUb5xVwyPwg==", + "version": "8.5.25", + "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.5.25.tgz", + "integrity": "sha512-DTPx3RWSSnWyzLxQnlH0rJP+EW5ekl16ZU4/psbIhA0e53kJfdgaN5vKM+xP7yJtXVu+nfdVFmlgFDEKAe4Pyw==", "funding": [ { "type": "opencollective", @@ -1921,7 +1921,7 @@ ], "license": "MIT", "dependencies": { - "nanoid": "^3.3.12", + "nanoid": "^3.3.16", "picocolors": "^1.1.1", "source-map-js": "^1.2.1" }, diff --git a/web/src/App.test.tsx b/web/src/App.test.tsx index bc256a8..60560dc 100644 --- a/web/src/App.test.tsx +++ b/web/src/App.test.tsx @@ -49,6 +49,7 @@ test('opens the manual OKF ingestion form', async () => { fireEvent.click(screen.getByRole('button', { name: /문서 추가/i })) expect(await screen.findByLabelText('문서 제목')).toBeInTheDocument() + expect(screen.getByLabelText('맥락 관계')).toHaveAttribute('placeholder', expect.stringContaining('supported-by')) expect(screen.getByLabelText('Markdown 본문')).toBeInTheDocument() }) @@ -116,3 +117,38 @@ test('organizes the sidebar and graph by knowledge domain rather than document t expect(screen.getByText('SYSTEMS', { selector: '.graph-lane-label' })).toBeInTheDocument() expect(screen.getByText('SPRING', { selector: '.graph-lane-label' })).toBeInTheDocument() }) + +test('switches from the domain overview to a focused typed context graph', async () => { + vi.stubGlobal('fetch', vi.fn((url: string) => { + const data = url.includes('/graph') + ? { + nodes: [ + { id: 'concepts/decision', label: 'Architecture decision', type: 'Concept', tags: ['architecture'] }, + { id: 'sources/evidence', label: 'Primary evidence', type: 'Source', tags: ['evidence'] }, + ], + edges: [{ source: 'concepts/decision', target: 'sources/evidence', kind: 'supported-by', context: 'Pinned source', timestamp: null }], + } + : [ + { id: 'concepts/decision', type: 'Concept', title: 'Architecture decision', description: 'Durable context', tags: ['architecture'], body: '# Decision', links: [], relations: [] }, + { id: 'sources/evidence', type: 'Source', title: 'Primary evidence', description: 'Evidence', tags: ['evidence'], body: '# Evidence', links: [], relations: [] }, + ] + return Promise.resolve({ ok: true, json: () => Promise.resolve(data) }) + })) + + render() + const domainMode = await screen.findByRole('button', { name: '도메인 그래프 보기' }) + const contextMode = screen.getByRole('button', { name: '맥락 그래프 보기' }) + expect(domainMode).toHaveAttribute('aria-pressed', 'true') + expect(contextMode).toHaveAttribute('aria-pressed', 'false') + + fireEvent.click(contextMode) + + expect(domainMode).toHaveAttribute('aria-pressed', 'false') + expect(contextMode).toHaveAttribute('aria-pressed', 'true') + expect(screen.getByRole('group', { name: '선택 문서 맥락 그래프' })).toBeInTheDocument() + expect(screen.getByText('supported-by · 근거 있음', { selector: '.context-edge-label' })).toBeInTheDocument() + expect(screen.getByText('Primary evidence', { selector: '.context-node-label' })).toBeInTheDocument() + + fireEvent.keyDown(screen.getByRole('button', { name: 'Primary evidence 문서 열기' }), { key: 'Enter' }) + expect(screen.getByRole('dialog', { name: 'Primary evidence 상세 문서' })).toBeInTheDocument() +}) diff --git a/web/src/App.tsx b/web/src/App.tsx index f2d67d2..ad00e91 100644 --- a/web/src/App.tsx +++ b/web/src/App.tsx @@ -1,11 +1,13 @@ import { useEffect, useMemo, useState, type ReactNode } from 'react' +type Relation = { kind: string; target: string; context?: string | null; timestamp?: string | null } +type GraphEdge = { source: string; target: string; kind?: string; context?: string | null; timestamp?: string | null } type Document = { id: string; type: string; title: string; description?: string | null; tags: string[] - resource?: string | null; timestamp?: string | null; body: string; links: string[] + resource?: string | null; timestamp?: string | null; body: string; links: string[]; relations?: Relation[] title_ko?: string | null; description_ko?: string | null; body_ko?: string | null } -type Graph = { source?: 'neo4j' | 'filesystem'; nodes: { id: string; label: string; type: string; tags: string[] }[]; edges: { source: string; target: string }[] } +type Graph = { source?: 'neo4j' | 'filesystem'; nodes: { id: string; label: string; type: string; tags: string[] }[]; edges: GraphEdge[] } const configuredApi = import.meta.env.VITE_API_URL const isLocalBrowser = ['localhost', '127.0.0.1'].includes(window.location.hostname) @@ -82,6 +84,71 @@ function GraphView({ graph, activeId, select }: { graph: Graph; activeId: string } +const relationLabels: Record = { + 'links-to': '링크', + supports: '근거로 지원', + 'supported-by': '근거 있음', + extends: '확장', + 'extended-by': '확장됨', + supersedes: '대체', + 'superseded-by': '대체됨', + 'decided-by': '결정 근거', + 'handoff-to': '인계', + 'related-to': '관련', +} +const relationLabel = (kind?: string) => `${kind ?? 'links-to'} · ${relationLabels[kind ?? 'links-to'] ?? '관계'}` + +function ContextGraph({ graph, activeId, select }: { graph: Graph; activeId: string | null; select: (id: string) => void }) { + const active = graph.nodes.find(node => node.id === activeId) + if (!active) return
문서를 선택하면 직접 연결된 맥락을 보여 줍니다.
+ const incident = graph.edges.filter(edge => edge.source === active.id || edge.target === active.id) + const nodeById = new Map(graph.nodes.map(node => [node.id, node])) + const neighborIds = [...new Set(incident.map(edge => edge.source === active.id ? edge.target : edge.source))] + const visibleIds = neighborIds.slice(0, 12) + const visibleEdges = incident.filter(edge => visibleIds.includes(edge.source === active.id ? edge.target : edge.source)) + const visibleGroups = [...visibleEdges.reduce((groups, edge) => { + const key = `${edge.source}\u0000${edge.target}` + const group = groups.get(key) ?? { source: edge.source, target: edge.target, edges: [] as GraphEdge[] } + group.edges.push(edge); groups.set(key, group) + return groups + }, new Map()).values()] + const positions = new Map([[active.id, { x: 500, y: 180 }]]) + visibleIds.forEach((id, index) => { + const angle = (Math.PI * 2 * index / Math.max(visibleIds.length, 1)) - Math.PI / 2 + positions.set(id, { x: 500 + Math.cos(angle) * 360, y: 180 + Math.sin(angle) * 120 }) + }) + const openNode = (id: string) => { if (nodeById.has(id)) select(id) } + return <> +
+ + + {visibleGroups.map(group => { + const source = positions.get(group.source); const target = positions.get(group.target) + if (!source || !target) return null + const reverse = visibleGroups.some(candidate => candidate.source === group.target && candidate.target === group.source) + const length = Math.hypot(target.x - source.x, target.y - source.y) || 1 + const perpendicular = { x: -(target.y - source.y) / length, y: (target.x - source.x) / length } + const lineOffset = reverse ? 12 : 0 + const labelOffset = reverse ? 72 : 0 + const label = group.edges.length === 1 ? relationLabel(group.edges[0].kind) : `${group.source === active.id ? 'OUT' : 'IN'} ${group.edges.length}` + const kinds = group.edges.map(edge => relationLabel(edge.kind)).join(', ') + return + {kinds} + + {label} + + })} + {visibleIds.map(id => { const node = nodeById.get(id); const point = positions.get(id); if (!node || !point) return null; return openNode(id)} onKeyDown={event => { if (event.key === 'Enter' || event.key === ' ') { event.preventDefault(); openNode(id) } }}>{node.label.slice(0, 24)} })} + {active.label.slice(0, 26)}선택한 장기기억 + +
+ {incident.length === 0 ?

명시적 관계가 없습니다. 본문 링크나 relations frontmatter를 추가하세요.

:
+ {incident.map((edge, index) => { const outgoing = edge.source === active.id; const neighborId = outgoing ? edge.target : edge.source; const neighbor = nodeById.get(neighborId); return })} + {neighborIds.length > visibleIds.length &&

그래프에는 직접 이웃 12개만 표시했습니다. 관계 목록에는 전체 {incident.length}개가 남습니다.

} +
} + +} + function InlineMarkdown({ text, onInternalLink }: { text: string; onInternalLink?: (id: string) => void }) { const tokens = text.split(/(\[[^\]]+\]\([^)]+\)|\*\*[^*]+\*\*|`[^`]+`)/g) return <>{tokens.map((token, index) => { @@ -150,6 +217,7 @@ function MarkdownView({ body, onInternalLink }: { body: string; onInternalLink?: export default function App() { const [documents, setDocuments] = useState([]) const [graph, setGraph] = useState({ nodes: [], edges: [] }) + const [graphMode, setGraphMode] = useState<'domain' | 'context'>('domain') const [activeId, setActiveId] = useState(null) const [detailId, setDetailId] = useState(null) const [detailLanguage, setDetailLanguage] = useState<'en' | 'ko'>('en') @@ -159,7 +227,7 @@ export default function App() { const [navigationOpen, setNavigationOpen] = useState(false) const [theme, setTheme] = useState<'dark' | 'light'>(() => typeof localStorage !== 'undefined' && typeof localStorage.getItem === 'function' && localStorage.getItem('knowledge-wiki-theme') === 'light' ? 'light' : 'dark') useEffect(() => { document.documentElement.dataset.theme = theme; if (typeof localStorage !== 'undefined' && typeof localStorage.setItem === 'function') localStorage.setItem('knowledge-wiki-theme', theme) }, [theme]) - const [form, setForm] = useState({ type: 'Concept', title: '', description: '', tags: '', resource: '', body: '# 개요\n\n' }) + const [form, setForm] = useState({ type: 'Concept', title: '', description: '', tags: '', resource: '', relations: '', body: '# 개요\n\n' }) useEffect(() => { Promise.all([fetch(`${API}/documents`), fetch(`${API}/graph`)]).then(async ([docs, graphResponse]) => { @@ -171,7 +239,15 @@ export default function App() { async function submitIngest(event: React.FormEvent) { event.preventDefault() - const response = await fetch(`${API}/documents`, { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ ...form, tags: form.tags.split(',').map(tag => tag.trim()).filter(Boolean) }) }) + const response = await fetch(`${API}/documents`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ + ...form, + tags: form.tags.split(',').map(tag => tag.trim()).filter(Boolean), + relations: form.relations.split('\n').map(relation => relation.trim()).filter(Boolean), + }), + }) if (!response.ok) { setError('문서 저장에 실패했습니다. 같은 제목의 문서가 있는지 확인하세요.'); return } setShowIngest(false) window.location.reload() @@ -197,11 +273,11 @@ export default function App() {

LOCAL KNOWLEDGE SYSTEM

지식 작업 공간

문서를 탐색하고, 연결을 확인하고, 로컬 지식을 관리하세요.

전체 문서{documents.length}
문서 연결{graph.edges.length}
문서 형식OKF
검색 인덱스준비됨
{error &&
{error}
} -

{graph.source === 'neo4j' ? 'NEO4J · RELATIONSHIP MAP' : 'RELATIONSHIP MAP'}

{graph.source === 'neo4j' ? 'Neo4j 관계 그래프' : '지식 그래프'}

노드 {graph.nodes.length}개
selectDocument(id, true)}/>

{graph.source === 'neo4j' ? 'Markdown 정본에서 재생성한 Neo4j 관계를 도메인별로 묶어 보여 줍니다.' : '노드를 선택하면 읽기 쉬운 상세 문서가 열립니다.'}

+

{graph.source === 'neo4j' ? 'NEO4J · RELATIONSHIP MAP' : 'RELATIONSHIP MAP'}

{graphMode === 'context' ? '장기기억 맥락 그래프' : graph.source === 'neo4j' ? 'Neo4j 관계 그래프' : '지식 그래프'}

{graphMode === 'context' ? `관계 ${graph.edges.length}개` : `노드 ${graph.nodes.length}개`}
{graphMode === 'context' ? selectDocument(id, true)}/> : selectDocument(id, true)}/>}

{graphMode === 'context' ? '화살표는 관계 방향입니다. Markdown 링크와 typed relations를 구분해 표시합니다.' : graph.source === 'neo4j' ? 'Markdown 정본에서 재생성한 Neo4j 관계를 도메인별로 묶어 보여 줍니다.' : '노드를 선택하면 읽기 쉬운 상세 문서가 열립니다.'}

{active ? <>
{typeLabel(active.type)}

{active.title}

{active.description}

{active.tags.map(tag => #{tag})}

본문, 표, 코드, 메타데이터와 연결 문서를 상세 보기에서 읽을 수 있습니다.

{active.id}.md{active.resource && 원문 보기 ↗}
:
아직 문서가 없습니다. 첫 OKF 문서를 추가하세요.
}

BROWSE

문서 목록

setQuery(event.target.value)} />
{filtered.map(doc => )}
{detail &&
setDetailId(null)}>
event.stopPropagation()}>
{typeLabel(detail.type)}

{detailLanguage === 'ko' && detail.title_ko ? detail.title_ko : detail.title}

{detailLanguage === 'ko' && detail.description_ko ? detail.description_ko : detail.description}

{detail.id}.md{detail.timestamp && 갱신 {detail.timestamp}}{detail.resource && 원문 보기 ↗}
{detail.tags.map(tag => #{tag})}
selectDocument(id, true)}/>
{detail.links.length > 0 &&

연결 문서

{detail.links.map(link => { const linked = documents.find(doc => doc.id === link); return })}
}
} - {showIngest &&

CREATE OKF DOCUMENT

문서 추가