diff --git a/scripts/update_man_pages.py b/scripts/update_man_pages.py index f99363ad0..dc897e364 100644 --- a/scripts/update_man_pages.py +++ b/scripts/update_man_pages.py @@ -53,13 +53,18 @@ # must import them explicitly or every subcommand comes back empty. from basic_memory.cli.app import app import basic_memory.cli.commands.posix # noqa: F401 (registers cat/grep/ls/find/tail/head/tree) +import basic_memory.cli.commands.okf # noqa: F401 (registers OKF commands) import basic_memory.cli.commands.man # noqa: F401 (registers `bm man apropos`) # Section-1 page name -> `bm` command path. Seven pages resolve directly from the # page name (`grep` -> `bm grep`); apropos(1) documents `bm man apropos`, a verb on # the `man` subgroup, so it needs an explicit path. The map lives here rather than # in page frontmatter because man1/*.md is only ever rewritten by this generator. -SECTION1_COMMAND_PATHS: Mapping[str, str] = {"apropos": "man apropos"} +SECTION1_COMMAND_PATHS: Mapping[str, str] = { + "apropos": "man apropos", + "okf-export": "okf export", + "okf-check": "okf check", +} def resolve_cli_command(command_path: str) -> Any: diff --git a/src/basic_memory/cli/app.py b/src/basic_memory/cli/app.py index 5811a9b1c..b0ed453d5 100644 --- a/src/basic_memory/cli/app.py +++ b/src/basic_memory/cli/app.py @@ -41,6 +41,10 @@ def app_callback( ) -> None: """Basic Memory - Local-first personal knowledge management.""" + # OKF check is filesystem-only; export resolves config lazily without source repair. + if ctx.invoked_subcommand == "okf": + return + command_name = ctx.invoked_subcommand or "root" # Host installation only copies packaged resources. Broken DB/config state diff --git a/src/basic_memory/cli/commands/okf.py b/src/basic_memory/cli/commands/okf.py new file mode 100644 index 000000000..638a9e57f --- /dev/null +++ b/src/basic_memory/cli/commands/okf.py @@ -0,0 +1,70 @@ +"""Export and check static OKF v0.2 directory bundles.""" + +from __future__ import annotations + +from pathlib import Path +from typing import TYPE_CHECKING + +import typer + +from basic_memory.cli.app import app + +if TYPE_CHECKING: + from basic_memory.okf.validation import CheckReport + +okf_app = typer.Typer(help="Export and check OKF v0.2-compatible directory bundles") +app.add_typer(okf_app, name="okf") + + +def print_report(report: CheckReport, json_output: bool) -> None: + if json_output: + typer.echo(report.model_dump_json()) + else: + typer.echo( + f"{'Valid' if report.success else 'Invalid'} OKF bundle: {report.concepts} concepts" + ) + for diagnostic in report.diagnostics: + typer.echo(f"{diagnostic.path}: {diagnostic.rule}: {diagnostic.message}") + if not report.success: + raise typer.Exit(1) + + +@okf_app.command("check") +def check( + bundle_path: Path = typer.Argument(..., help="Directory bundle to validate"), + json_output: bool = typer.Option(False, "--json", help="Output machine-readable diagnostics"), +) -> None: + """Validate OKF v0.2 structural rules; exit nonzero on violations.""" + from basic_memory.okf.validation import check_bundle + + print_report(check_bundle(bundle_path), json_output) + + +@okf_app.command("export") +def export( + destination: Path = typer.Argument(..., help="Destination outside the source project"), + project: str = typer.Option(..., "--project", "-p", help="Configured local project to export"), + replace: bool = typer.Option(False, "--replace", help="Replace an existing destination bundle"), + json_output: bool = typer.Option(False, "--json", help="Output machine-readable diagnostics"), +) -> None: + """Stage, validate, and publish a static OKF v0.2-compatible bundle. + + Preserves source files and non-Markdown assets. Links become standard Markdown; + BM semantics use the bm.okf_export extension. History is best-effort recorded history. + """ + from basic_memory.cli.commands.command_utils import run_with_cleanup + from basic_memory.cli.container import get_or_create_container + from basic_memory.db import maybe_install_uvloop + from basic_memory.okf.export import export_project + from basic_memory.okf.validation import CheckReport, Diagnostic + + try: + config = get_or_create_container().config + # PostgreSQL needs the guarded policy before run_with_cleanup creates its loop. + maybe_install_uvloop(config) + report = run_with_cleanup(export_project(config, project, destination, replace=replace)) + except (ValueError, OSError, UnicodeError) as error: + report = CheckReport( + diagnostics=[Diagnostic(path=str(destination), rule="export", message=str(error))] + ) + print_report(report, json_output) diff --git a/src/basic_memory/cli/main.py b/src/basic_memory/cli/main.py index afaa3d4b5..40b5224ba 100644 --- a/src/basic_memory/cli/main.py +++ b/src/basic_memory/cli/main.py @@ -30,6 +30,7 @@ def _version_only_invocation(argv: list[str]) -> bool: install, man, mcp, + okf, orphans, posix, project, diff --git a/src/basic_memory/ignore_utils.py b/src/basic_memory/ignore_utils.py index c3386970c..284362e4f 100644 --- a/src/basic_memory/ignore_utils.py +++ b/src/basic_memory/ignore_utils.py @@ -201,7 +201,9 @@ def load_bmignore_patterns() -> Set[str]: return patterns -def load_gitignore_patterns(base_path: Path, use_gitignore: bool = True) -> Set[str]: +def load_gitignore_patterns( + base_path: Path, use_gitignore: bool = True, *, strict: bool = False +) -> Set[str]: """Load gitignore patterns from .gitignore file and .bmignore. Combines patterns from: @@ -212,10 +214,30 @@ def load_gitignore_patterns(base_path: Path, use_gitignore: bool = True) -> Set[ Args: base_path: The base directory to search for .gitignore file use_gitignore: If False, only load patterns from .bmignore (default: True) + strict: Read without creating files; only missing ignore files permit defaults. Returns: Set of patterns to ignore """ + if strict: + # Publishing a snapshot must never silently include files excluded by unreadable rules. + patterns: set[str] = set() + bmignore_path = get_bmignore_path() + paths = [bmignore_path, base_path / ".gitignore"] if use_gitignore else [bmignore_path] + for path in paths: + try: + content = path.read_text(encoding="utf-8") + except FileNotFoundError: + content = "" + patterns.update( + pattern + for line in content.splitlines() + if (pattern := _parse_ignore_pattern_line(line)) is not None + ) + if path == bmignore_path and not patterns: + patterns.update(DEFAULT_IGNORE_PATTERNS) + return patterns + # Start with patterns from .bmignore patterns = load_bmignore_patterns() diff --git a/src/basic_memory/index/local_project.py b/src/basic_memory/index/local_project.py index 5fd1c228e..84136e156 100644 --- a/src/basic_memory/index/local_project.py +++ b/src/basic_memory/index/local_project.py @@ -4,6 +4,7 @@ import asyncio import os +import stat from collections.abc import Mapping, Sequence from contextlib import nullcontext from dataclasses import dataclass @@ -223,6 +224,7 @@ def scan_local_project_index_files( project_root: Path, *, ignore_patterns: LocalProjectIndexIgnorePatterns | None = None, + strict: bool = False, ) -> LocalProjectIndexScan: """Walk one local project and report eligible files plus unreadable subtrees.""" project_root = project_root.expanduser().resolve() @@ -254,7 +256,7 @@ def _scan_error(error: OSError) -> None: # The root scan failed (onerror re-raised). Never return an empty, # delete-everything snapshot; files discovered before a deeper traversal # error are kept. - if not file_paths: + if strict or not file_paths: raise break @@ -269,16 +271,23 @@ def _scan_error(error: OSError) -> None: for name in filenames: path = root_path / name - try: - if path.is_symlink() or not path.is_file(): - continue - except OSError: - continue relative_path = path.relative_to(project_root).as_posix() if local_relative_path_is_filtered(relative_path): continue if should_ignore_path(path, project_root, active_ignore_patterns): continue + try: + # Export requires every eligible file: lstat propagates errors that + # pathlib predicates may suppress, without following symlinks. + if strict: + if not stat.S_ISREG(path.lstat().st_mode): + continue + elif path.is_symlink() or not path.is_file(): + continue + except OSError: + if strict: + raise + continue file_paths.append(relative_path) return LocalProjectIndexScan( diff --git a/src/basic_memory/man/__init__.py b/src/basic_memory/man/__init__.py index b8695dcb4..106f381ae 100644 --- a/src/basic_memory/man/__init__.py +++ b/src/basic_memory/man/__init__.py @@ -468,7 +468,11 @@ def _render_cli_form( alternatives = [_synopsis_option_token(present_longs[opt])[1:-1] for opt in pair] tokens.append("[" + " | ".join(alternatives) + "]") else: - tokens.append(_synopsis_option_token(param, required=param.name in required_options)) + tokens.append( + _synopsis_option_token( + param, required=param.required or param.name in required_options + ) + ) prefix = f"bm {command_path}" indent = " " * (len(prefix) + 1) diff --git a/src/basic_memory/man/man1/okf-check(1).md b/src/basic_memory/man/man1/okf-check(1).md new file mode 100644 index 000000000..9dd1c7f04 --- /dev/null +++ b/src/basic_memory/man/man1/okf-check(1).md @@ -0,0 +1,56 @@ +--- +title: okf-check(1) +type: manpage +section: 1 +name: okf-check +summary: check OKF v0.2 structural conformance without a project database +generated: cli +--- + +# okf-check(1) + +## NAME + +**okf-check** — check OKF v0.2 structural conformance without a project database + +## SYNOPSIS + +``` +bm okf check BUNDLE_PATH [--json] +``` + +## DESCRIPTION + +Walk a directory bundle without Basic Memory configuration, indexing, or ignore +rules. Every non-reserved .md file must be UTF-8, have parseable YAML mapping +frontmatter, and carry a non-empty string `type`. Only root index.md may have +frontmatter, containing only `okf_version`. Index sections have headings and +entries use standard Markdown links. log.md has no frontmatter and groups +recorded entries under `## YYYY-MM-DD` headings, newest first. + +Diagnostics identify the file, rule, and problem. Exit status is 0 for a valid +bundle and 1 for violations or unreadable files. JSON contains `concepts` and +`diagnostics`; reserved files and assets are not counted as concepts. + +Unknown types, unknown keys, missing optional fields, broken cross-links, +missing indexes, and non-Markdown assets are accepted. Version declarations +are advisory. Symlinks are diagnosed as non-portable. This checks the structural +contract, not trust, attestation execution, or every optional field convention. + +The contract follows OKF v0.2 §11: +https://github.com/GoogleCloudPlatform/knowledge-catalog/blob/e6d34fd29c1c6c75ec23078e7a8191a9c8209620/okf/SPEC.md + +## OPTIONS + +- **--json** — Output machine-readable diagnostics + +## EXAMPLES + +``` +bm okf check ~/exports/research +bm okf check ~/exports/research --json +``` + +## SEE ALSO + +- see_also [[okf-export(1)]] diff --git a/src/basic_memory/man/man1/okf-export(1).md b/src/basic_memory/man/man1/okf-export(1).md new file mode 100644 index 000000000..8394eee8f --- /dev/null +++ b/src/basic_memory/man/man1/okf-export(1).md @@ -0,0 +1,88 @@ +--- +title: okf-export(1) +type: manpage +section: 1 +name: okf-export +summary: export a local project as an OKF v0.2-compatible bundle +generated: cli +--- + +# okf-export(1) + +## NAME + +**okf-export** — export a local project as an OKF v0.2-compatible bundle + +## SYNOPSIS + +``` +bm okf export DESTINATION --project PROJECT [--replace] [--json] +``` + +## DESCRIPTION + +Export a configured local project to a static directory outside the project. +Source files remain unchanged. Cloud projects must first be pulled locally. +The export follows Basic Memory's project ignore rules; non-Markdown assets +such as PDFs retain their relative paths. Symlinks are not exported. +Concept filenames must use lowercase `.md`; supported BM alternatives such as +`.markdown` or `.MD` must be renamed before export so OKF readers cannot skip them. + +Concept frontmatter is retained, with `type` using BM's canonical string value +and defaulting to `note` when absent or null. Absent `tags` default to an empty +list. Other metadata values are preserved. Wikilinks become standard Markdown links. +Exact file paths, titles, and permalinks resolve within the exported snapshot; +unresolved links remain broken links. Ambiguous aliases are not guessed. +Code examples retain literal wikilinks. + +Generated index.md files contain standard links and only the root carries +`okf_version: "0.2"` frontmatter. The root log.md has no frontmatter and +records accepted Basic Memory journal history under ISO date headings. +File materialization may lag recorded acceptance; the log does not claim +every recorded version is represented by the exported files. +It does not reconstruct offline edits. Live Wiki bytes are not copied. +Unmarked files at reserved filenames must be renamed before export, even without +frontmatter; only recognized Wiki artifacts or marked OKF indexes are replaced. +Databases predating the accepted-change journal produce an empty history without +being migrated by export. + +The destination is staged and checked before publication. Existing destinations +are refused unless `--replace` is explicit. A failed publication restores the +previous bundle; if restoration also fails, its bytes remain in a sibling +`.NAME.bm-okf-backup-*` directory. Source changes detected during export cause +failure. Export is intended for a quiescent project, not as a transaction over +concurrent filesystem edits. Unchanged project state produces identical bytes. + +## BM EXTENSION + +The YAML `bm.okf_export` mapping has `version: 1` and `relations`, an ordered +list of original BM wikilink relations with `type`, `target`, and `context`. +This preserves typed edges and authored target spelling after links become +ordinary Markdown. Existing `bm` keys are preserved; an existing `okf_export` +key or non-mapping `bm` is a collision and fails export. + +Categorized observations retain their human-readable `[category] content` +syntax, tags, context, and temporal qualifiers in the body. They are not copied +into a second metadata list. The extension declares this BM interpretation of +the body; generic OKF consumers can read it as ordinary Markdown. Relations +in metadata are authoritative for recovering BM edge types; ordinary Markdown +links alone only express untyped edges. This command does not add an importer +or switch Basic Memory's canonical syntax. + +## OPTIONS + +- **-p, --project** — Configured local project to export +- **--replace** — Replace an existing destination bundle +- **--json** — Output machine-readable diagnostics + +## EXAMPLES + +``` +bm okf export ~/exports/research --project research +bm okf export ~/exports/research --project research --replace --json +bm okf check ~/exports/research +``` + +## SEE ALSO + +- see_also [[okf-check(1)]] diff --git a/src/basic_memory/okf/__init__.py b/src/basic_memory/okf/__init__.py new file mode 100644 index 000000000..0a5e973a4 --- /dev/null +++ b/src/basic_memory/okf/__init__.py @@ -0,0 +1 @@ +"""Static Open Knowledge Format export and filesystem conformance checks.""" diff --git a/src/basic_memory/okf/export.py b/src/basic_memory/okf/export.py new file mode 100644 index 000000000..cc8380b20 --- /dev/null +++ b/src/basic_memory/okf/export.py @@ -0,0 +1,194 @@ +"""Local project snapshot and staged publication of a validated OKF bundle.""" + +from pathlib import Path, PurePosixPath +import shutil +from tempfile import TemporaryDirectory +from uuid import uuid4 + +from basic_memory.config import APP_DATABASE_NAME, BasicMemoryConfig, DatabaseBackend, ProjectMode +from basic_memory.okf.render import ExportFile, ExportSnapshot, RecordedChange, render_bundle +from basic_memory.okf.validation import CheckReport, check_bundle, parse_document +from basic_memory.utils import generate_permalink + + +async def recorded_history( + config: BasicMemoryConfig, project_name: str, root: Path +) -> tuple[RecordedChange, ...]: + """Read available journal evidence without indexing or repairing source files.""" + from basic_memory import db + from basic_memory.models.project import Project + from basic_memory.repository.project_repository import ProjectRepository + from basic_memory.utils import ensure_timezone_aware + from sqlalchemy import inspect, select + + # database_path creates an empty DB as a side effect; a read-only export must + # distinguish absent history before opening the existing database. + database_path = config.data_dir_path / APP_DATABASE_NAME + if config.database_backend == DatabaseBackend.SQLITE and not database_path.exists(): + return () + engine, session_maker = await db.get_or_create_db( + db_path=database_path, + db_type=db.DatabaseType.FILESYSTEM, + config=config, + ensure_migrations=False, + ) + # An older database has no accepted-change journal. Export remains read-only + # and reports no recorded history instead of migrating or querying missing columns. + async with engine.connect() as connection: + has_journal = await connection.run_sync( + lambda sync: ( + inspect(sync).has_table("accepted_project_note_change") + and any( + column["name"] == "partition_position" + for column in inspect(sync).get_columns("project") + ) + ) + ) + if not has_journal: + return () + repository = ProjectRepository() + async with db.scoped_session(session_maker) as session: + # Select only journal identity fields: later project metadata migrations + # must not make already-recorded history unreadable after an upgrade. + project = ( + await session.execute( + select(Project.id, Project.path, Project.partition_position).where( + Project.name == project_name + ) + ) + ).one_or_none() + if project is None: + return () + project_id, project_path, partition_position = project + if Path(project_path).resolve() != root: + raise ValueError("Configured project path differs from the recorded project path") + changes = await repository.list_accepted_note_changes( + session, project_id, through_position=partition_position + ) + return tuple( + RecordedChange( + change.partition_position, + change.file_path, + change.operation, + ensure_timezone_aware(change.accepted_at), + ) + for change in changes + if change.source != "wiki_projector" + ) + + +def snapshot_files(root: Path) -> tuple[ExportFile, ...]: + from basic_memory.ignore_utils import load_gitignore_patterns + from basic_memory.index.local_project import scan_local_project_index_files + from basic_memory.index.local_wiki_projection import _is_projector_owned + from basic_memory.runtime.storage import runtime_file_path_is_markdown_note + + scan = scan_local_project_index_files( + root, ignore_patterns=load_gitignore_patterns(root, strict=True), strict=True + ) + if scan.unreadable_directories: + raise OSError("Incomplete project scan: " + ", ".join(scan.unreadable_directories)) + files = [] + for path in scan.file_paths: + name = PurePosixPath(path).name + if any( + part.casefold() in {"index.md", "log.md"} for part in PurePosixPath(path).parts[:-1] + ): + raise ValueError(f"{path}: reserved OKF directory name; rename it first") + if name.casefold() in {"index.md", "log.md"} and name not in {"index.md", "log.md"}: + raise ValueError(f"{path}: reserved filename casing collides with generated OKF files") + if runtime_file_path_is_markdown_note(path) and PurePosixPath(path).suffix != ".md": + raise ValueError( + f"{path}: OKF concepts require a lowercase .md suffix; rename it first" + ) + content = (root / path).read_bytes() + if PurePosixPath(path).name in {"index.md", "log.md"}: + document = parse_document(content.decode("utf-8"), source=True) + # Never silently discard user-authored concepts at reserved names. + if not _is_projector_owned(path, content) and not ( + path == "index.md" and set(document.metadata) == {"okf_version"} + ): + raise ValueError( + f"{path}: reserved OKF filename contains a concept; rename it first" + ) + continue + files.append(ExportFile(path, content)) + return tuple(files) + + +async def export_project( + config: BasicMemoryConfig, project: str, destination: Path, *, replace: bool = False +) -> CheckReport: + if project not in config.projects: + requested_permalink = generate_permalink(project) + project = next( + (name for name in config.projects if generate_permalink(name) == requested_permalink), + project, + ) + entry = config.projects.get(project) + if entry is None or entry.mode != ProjectMode.LOCAL: + raise ValueError( + "Export requires a configured local project; pull cloud files locally first" + ) + if not Path(entry.path).expanduser().is_absolute(): + raise ValueError("Local project path must be absolute") + root = Path(entry.path).expanduser().resolve() + if not root.is_dir(): + raise ValueError(f"Project directory does not exist: {root}") + destination = destination.expanduser().absolute() + resolved = destination.resolve() + # Path spelling does not establish containment on case-insensitive filesystems. + # Compare existing directory identities in both directions before any writes. + if any( + ancestor.exists() and ancestor.samefile(root) for ancestor in (resolved, *resolved.parents) + ) or (resolved.exists() and any(resolved.samefile(ancestor) for ancestor in root.parents)): + raise ValueError("Destination must be outside, and must not contain, the source project") + if destination.is_symlink(): + raise ValueError("Destination must not be a symlink") + if destination.exists() and (not replace or not destination.is_dir()): + raise ValueError("Destination exists; use --replace to replace a directory bundle") + history = await recorded_history(config, project, root) + files = snapshot_files(root) + snapshot = ExportSnapshot( + project, files, history, permalinks_include_project=config.permalinks_include_project + ) + rendered = render_bundle(snapshot) + destination.parent.mkdir(parents=True, exist_ok=True) + # A sibling staging directory keeps rename on the same filesystem. On a failed + # replacement, restore the previous bundle before propagating the I/O failure. + with TemporaryDirectory(prefix=".bm-okf-", dir=destination.parent) as temporary: + staging = Path(temporary) / "bundle" + staging.mkdir() + for file in rendered: + target = staging / file.path + target.parent.mkdir(parents=True, exist_ok=True) + # Destination filesystems may collapse distinct source spellings. + # Exclusive creation detects collisions before any bundle is published. + with target.open("xb") as output: + output.write(file.content) + report = check_bundle(staging) + if not report.success: + return report + if ( + snapshot_files(root) != files + or await recorded_history(config, project, root) != history + ): + raise ValueError("Project changed during export; retry with unchanged source files") + # Keep rollback bytes outside automatic staging cleanup, including when + # restoring the destination itself fails. + backup = destination.with_name(f".{destination.name}.bm-okf-backup-{uuid4().hex}") + if destination.is_symlink(): + raise ValueError("Destination became a symlink during export") + if destination.exists(): + if not replace: + raise ValueError("Destination appeared during export; refusing to replace it") + destination.rename(backup) + try: + staging.rename(destination) + except OSError: + if backup.exists(): + backup.rename(destination) + raise + if backup.exists(): + shutil.rmtree(backup) + return report diff --git a/src/basic_memory/okf/render.py b/src/basic_memory/okf/render.py new file mode 100644 index 000000000..3c0d795ec --- /dev/null +++ b/src/basic_memory/okf/render.py @@ -0,0 +1,398 @@ +"""Pure export values and Markdown rendering; never renders live Wiki bytes.""" + +from dataclasses import dataclass +from datetime import datetime +from pathlib import PurePosixPath +from urllib.parse import quote +from uuid import UUID + +from markdown_it import MarkdownIt +from markdown_it.rules_inline import StateInline +import yaml + +from basic_memory.file_utils import has_frontmatter, parse_frontmatter +from basic_memory.markdown.entity_parser import ( + _coerce_to_string, + normalize_frontmatter_value, + parse, +) +from basic_memory.markdown.path_links import markdown_link_target +from basic_memory.markdown.plugins import _is_escaped +from basic_memory.repository.entity_repository import file_path_alias +from basic_memory.services.bulk_link_resolver import RelationTargetReference +from basic_memory.services.link_resolver import normalize_link_text +from basic_memory.utils import ( + build_canonical_permalink, + build_permalink_resolution_candidates, + generate_permalink, +) + +from basic_memory.okf.validation import Document, parse_document + + +class ExportDumper(yaml.SafeDumper): + """Retain YAML mapping order while making unordered sets deterministic.""" + + +def represent_set(dumper: ExportDumper, values: set[object]) -> yaml.nodes.MappingNode: + ordered = sorted(values, key=lambda value: yaml.safe_dump(value, sort_keys=True)) + return dumper.represent_mapping("tag:yaml.org,2002:set", [(value, None) for value in ordered]) + + +ExportDumper.add_representer(set, represent_set) + + +def has_unordered_values(value: object) -> bool: + """Identity strings cannot use hash-dependent representations of nested sets.""" + if isinstance(value, set): + return True + if isinstance(value, dict): + return any(has_unordered_values(item) for item in value.values()) + if isinstance(value, (list, tuple)): + return any(has_unordered_values(item) for item in value) + return False + + +@dataclass(frozen=True) +class ExportFile: + path: str + content: bytes + + +@dataclass(frozen=True) +class RecordedChange: + position: int + path: str + operation: str + accepted_at: datetime + + +@dataclass(frozen=True) +class ExportSnapshot: + project: str + files: tuple[ExportFile, ...] + changes: tuple[RecordedChange, ...] = () + permalinks_include_project: bool = True + + +def markdown_link(label: str, path: str, fragment: str = "") -> str: + escaped = label.replace("\\", "\\\\").replace("[", "\\[").replace("]", "\\]") + escaped = " ".join(escaped.splitlines()) + href = quote(path, safe="/") + if fragment: + href += "#" + quote(fragment, safe="") + return f"[{escaped}]({href})" + + +def convert_wikilinks( + body: str, + source: str, + targets: dict[str, str], + project: str, + *, + include_project: bool = True, + ambiguous_aliases: frozenset[str] = frozenset(), + permalinks: dict[str, str] | None = None, + title_targets: dict[str, str] | None = None, +) -> str: + """Normalize export line endings to LF and rewrite only prose wikilink spans.""" + body = body.replace("\r\n", "\n").replace("\r", "\n") + project = generate_permalink(project) + replacements: list[tuple[int, int, str]] = [] + inline_source = "" + path_aliases: dict[str, list[str]] = {} + for path in sorted(set(targets.values())): + path_aliases.setdefault(file_path_alias(path), []).append(path) + + def wikilink(state: StateInline, silent: bool) -> bool: + start = state.pos + if not state.src.startswith("[[", start): + return False + # Image labels use a nested source with different offsets; keep them literal. + if silent or state.linkLevel or state.src != inline_source: + return False + depth = 1 + end = start + 2 + while end < len(state.src) - 1: + # Escaped brackets belong to the target, matching the canonical scanner. + if _is_escaped(state.src, end): + end += 1 + continue + pair = state.src[end : end + 2] + if pair == "[[": + depth += 1 + elif pair == "]]": + depth -= 1 + if depth == 0: + break + end += 2 if pair in {"[[", "]]"} else 1 + if depth: + return False + raw = state.src[start + 2 : end] + target, alias = normalize_link_text(raw) + target, _, fragment = target.partition("#") + label = alias or target or fragment + reference = RelationTargetReference.parse(target) + if reference.explicitly_qualified: + prefix, remainder = reference.project_path() + # Explicit foreign-project references cannot bind to this bundle's aliases. + if prefix is None or generate_permalink(prefix) != project: + return False + target = remainder + # External IDs outrank semantic aliases but are not carried by filesystem bytes. + # Keep their references literal rather than binding to a lower-priority name. + try: + UUID(target) + except ValueError: + pass + else: + return False + # Network-path URIs would turn unresolved file identities into external links. + if target.startswith("//"): + return False + rooted = target.startswith("/") + resolved = None + # Explicit relative links bind to their source directory before semantic aliases. + # Wikilink paths are literal identifiers, so URL decoding must round-trip them. + relative = markdown_link_target(quote(target, safe="/"), source) + # A root-relative URI with escaping dot segments could normalize to a real note. + # Keep that unresolved reference literal rather than inventing a portable edge. + if relative is None and ".." in PurePosixPath(target).parts: + return False + if reference.explicitly_qualified: + relative = None + if ( + include_project + and "/" in target + and generate_permalink(target.partition("/")[0]) == project + ): + relative = None + if not rooted and "/" in target and relative: + resolved = targets.get(relative.lstrip("/")) + if resolved is None: + resolved = targets.get(relative.lstrip("/") + ".md") + if not rooted and resolved is None and permalinks: + # Semantic addresses precede title/path aliases, even when they look like filenames. + for candidate in build_permalink_resolution_candidates( + target, project, include_project + ): + if target in ambiguous_aliases and candidate != target: + break + if candidate in permalinks: + resolved = permalinks[candidate] + break + if not rooted and resolved is None and title_targets: + resolved = title_targets.get(target) + if not rooted and resolved is None: + resolved = targets.get(target) + if ( + not rooted + and resolved is None + and ( + target not in ambiguous_aliases + or "/" in target + or target.casefold().endswith(".md") + ) + ): + # Forgiving filename spelling is a last resort after exact identities. + candidates = ([relative] if relative and "/" in target else []) + [target] + for candidate in candidates: + path = candidate.lstrip("/") + if not path.casefold().endswith(".md"): + path += ".md" + matches = path_aliases.get(file_path_alias(path), []) + if len(matches) == 1: + resolved = matches[0] + break + # A missing target stays a broken link, not a guessed edge to another concept. + # Rooted links already name exact portable file paths; never infer an extension. + href = target if rooted else "/" + (resolved or target or source) + replacements.append((start, end + 2, markdown_link(label, href, fragment))) + state.push("text", "", 0).content = raw + state.pos = end + 2 + return True + + parser = MarkdownIt() + parser.inline.ruler.before("link", "okf_wikilink", wikilink) + # MarkdownIt substitutes U+FFFD for NUL one-for-one, preserving source offsets. + lines = body.replace("\0", "\ufffd").split("\n") + line_offsets = [0] + for line in lines: + line_offsets.append(line_offsets[-1] + len(line) + 1) + document_replacements: list[tuple[int, int, str]] = [] + # Each inline block is parsed separately: backticks in another paragraph must + # not turn this paragraph into code. Map token text back past list/quote prefixes. + environment: dict[str, object] = {} + for token in MarkdownIt().parse(body, environment): + if token.type != "inline" or not token.map or "[[" not in token.content: + continue + replacements.clear() + inline_source = token.content + # Retain document references so link labels remain existing Markdown links. + parser.parseInline(token.content, environment) + if not replacements: + continue + offsets: list[int] = [] + first, last = token.map + source_line = first + for text in token.content.split("\n"): + # MarkdownIt expands continuation indentation tabs. Match the text + # after indentation while keeping replacement endpoints in raw bytes. + stripped = text.lstrip(" \t") + indentation = len(text) - len(stripped) + column = -1 + while source_line < last: + column = lines[source_line].find(stripped) + if column >= 0: + break + source_line += 1 + if source_line == last: + raise ValueError(f"{source}: cannot locate wikilink source span") + offset = line_offsets[source_line] + column + offsets.extend([offset] * indentation) + offsets.extend(range(offset, offset + len(text) - indentation + 1)) + source_line += 1 + for start, end, replacement in replacements: + document_replacements.append((offsets[start], offsets[end - 1] + 1, replacement)) + for start, end, replacement in reversed(document_replacements): + body = body[:start] + replacement + body[end:] + + return body + + +def render_bundle(snapshot: ExportSnapshot) -> tuple[ExportFile, ...]: + """Preserve frontmatter and prose; attach only semantics lost by link conversion.""" + targets: dict[str, str] = {} + permalinks: dict[str, str] = {} + title_targets: dict[str, str] = {} + ambiguous: set[str] = set() + documents: dict[str, Document] = {} + titles: dict[str, str] = {} + note_types: dict[str, str] = {} + for file in snapshot.files: + if PurePosixPath(file.path).suffix != ".md": + continue + content = file.content.decode("utf-8") + document = parse_document(content, source=True) + # Malformed notes can retain an indexed identity absent from their moved file. + # A filesystem snapshot cannot infer that identity safely. + if has_frontmatter(content) and not document.has_frontmatter: + raise ValueError(f"{file.path}: repair malformed frontmatter before export") + documents[file.path] = document + # Resolve using BM's normalized title, but retain authored YAML values in output. + source_metadata = ( + parse_frontmatter(file.content.decode("utf-8")) if document.has_frontmatter else {} + ) + for field in ("title", "type"): + if has_unordered_values(source_metadata.get(field)): + raise ValueError(f"{file.path}: {field} cannot contain an unordered YAML set") + title = _coerce_to_string(normalize_frontmatter_value(source_metadata.get("title"))) + title = title if title and title != "None" else PurePosixPath(file.path).stem + titles[file.path] = title + note_type = source_metadata.get("type") + note_types[file.path] = ( + _coerce_to_string(normalize_frontmatter_value(note_type)) + if note_type is not None + else "note" + ) + permalink = normalize_frontmatter_value(source_metadata.get("permalink")) + if not isinstance(permalink, str) or not permalink: + permalink = build_canonical_permalink( + snapshot.project, file.path, include_project=snapshot.permalinks_include_project + ) + if isinstance(permalink, str) and permalink: + # Offline files can violate the indexed uniqueness rule; never choose a winner. + if permalink in permalinks: + raise ValueError( + f"{file.path}: duplicate permalink {permalink!r} also declared by " + f"{permalinks[permalink]}" + ) + permalinks[permalink] = file.path + # Resources use their full filename as the canonical indexed title. + for file in snapshot.files: + title = titles.get(file.path, PurePosixPath(file.path).name) + if title in title_targets and title_targets[title] != file.path: + ambiguous.add(title) + else: + title_targets[title] = file.path + for alias in ambiguous: + title_targets.pop(alias) + # Relative path resolution uses exact file identities, separate from semantic names. + targets.update({file.path: file.path for file in snapshot.files}) + + output: list[ExportFile] = [] + directories = {PurePosixPath(".")} + for file in snapshot.files: + path = PurePosixPath(file.path) + directories.update(path.parents) + if path.suffix != ".md": + output.append(file) + continue + document = documents[file.path] + metadata = dict(document.metadata) + metadata["type"] = note_types[file.path] + metadata.setdefault("tags", []) + semantic_setting = metadata.get("bm_parse_semantics") + if not ( + semantic_setting is False + or (isinstance(semantic_setting, str) and semantic_setting.lower() == "false") + ): + semantics = parse(document.body) + # Observation syntax remains intact and is documented by the profile. + # Typed relation metadata is needed because a standard link is untyped. + relations = [relation.model_dump() for relation in semantics.relations] + else: + relations = [] + existing = metadata.get("bm", {}) + if not isinstance(existing, dict) or "okf_export" in existing: + raise ValueError(f"{file.path}: bm.okf_export extension collision") + metadata["bm"] = { + **existing, + "okf_export": {"version": 1, "relations": relations}, + } + body = convert_wikilinks( + document.body, + file.path, + targets, + snapshot.project, + include_project=snapshot.permalinks_include_project, + ambiguous_aliases=frozenset(ambiguous), + permalinks=permalinks, + title_targets=title_targets, + ) + content = "---\n" + yaml.dump( + metadata, Dumper=ExportDumper, allow_unicode=True, sort_keys=False + ) + content += "---\n" + body + output.append(ExportFile(file.path, content.encode("utf-8"))) + + for directory in sorted(directories): + index_path = str(directory / "index.md") + lines = ['---\nokf_version: "0.2"\n---\n'] if directory == PurePosixPath(".") else [] + lines.append("# " + (snapshot.project if str(directory) == "." else directory.name) + "\n") + for file in sorted(snapshot.files, key=lambda file: file.path): + path = PurePosixPath(file.path) + if path.parent == directory: + label = titles.get(file.path, path.stem) + lines.append("- " + markdown_link(label, path.name)) + for child in sorted(directories): + if child != directory and child.parent == directory: + lines.append("- " + markdown_link(child.name, child.name + "/index.md")) + output.append(ExportFile(index_path, ("\n".join(lines) + "\n").encode("utf-8"))) + + log = [ + "# Recorded Basic Memory history\n", + "Best-effort accepted changes recorded by Basic Memory. " + "File materialization may lag recorded acceptance. Offline edits are not reconstructed.\n", + ] + day = "" + for change in sorted( + snapshot.changes, key=lambda change: (change.accepted_at, change.position), reverse=True + ): + change_day = change.accepted_at.date().isoformat() + if change_day != day: + log.append(f"\n## {change_day}\n") + day = change_day + log.append(f"- {change.operation}: {markdown_link(change.path, '/' + change.path)}") + output.append(ExportFile("log.md", ("\n".join(log) + "\n").encode("utf-8"))) + return tuple(sorted(output, key=lambda file: file.path)) diff --git a/src/basic_memory/okf/validation.py b/src/basic_memory/okf/validation.py new file mode 100644 index 000000000..34f09822d --- /dev/null +++ b/src/basic_memory/okf/validation.py @@ -0,0 +1,199 @@ +"""OKF v0.2 structural conformance, independent of Basic Memory's index.""" + +from dataclasses import dataclass +from datetime import date +import os +from pathlib import Path +import re + +from markdown_it import MarkdownIt +from pydantic import BaseModel, Field +import yaml + +from basic_memory.file_utils import ParseError, has_frontmatter, parse_frontmatter, strip_bom + + +class Diagnostic(BaseModel): + path: str + rule: str + message: str + + +class CheckReport(BaseModel): + concepts: int = 0 + diagnostics: list[Diagnostic] = Field(default_factory=list) + + @property + def success(self) -> bool: + return not self.diagnostics + + +@dataclass(frozen=True) +class Document: + metadata: dict[str, object] + body: str + has_frontmatter: bool + + +class FrontmatterLoader(yaml.SafeLoader): + """Keep authored timestamp spelling, including ISO T/Z semantics, intact.""" + + +FrontmatterLoader.yaml_implicit_resolvers = { + character: [ + (tag, expression) for tag, expression in resolvers if tag != "tag:yaml.org,2002:timestamp" + ] + for character, resolvers in yaml.SafeLoader.yaml_implicit_resolvers.items() +} + + +def parse_document(content: str, *, source: bool = False) -> Document: + """Require a mapping when a YAML fence is present; preserve YAML value types.""" + if source: + # BM treats unmatched fences and malformed YAML as authored body text. + # Reuse that classification, then load valid metadata without coercing dates. + if not has_frontmatter(content): + return Document({}, content, False) + try: + parse_frontmatter(content) + except ParseError: + return Document({}, content, False) + lines = (strip_bom(content) if source else content).splitlines(keepends=True) + # BM accepts a BOM and leading blank lines; OKF check keeps its on-disk boundary. + if source: + while lines and not lines[0].strip(): + lines.pop(0) + if not lines or (lines[0].rstrip(" \t\r\n") if source else lines[0].strip()) != "---": + return Document({}, content, False) + for end in range(1, len(lines)): + if (lines[end].rstrip(" \t\r\n") if source else lines[end].strip()) == "---": + break + else: + raise ValueError("Unterminated YAML frontmatter") + try: + metadata = yaml.load("".join(lines[1:end]), Loader=FrontmatterLoader) + except yaml.YAMLError as error: + raise ValueError(f"Invalid YAML frontmatter: {error}") from error + if metadata is None: + metadata = {} + if not isinstance(metadata, dict): + raise ValueError("Frontmatter must be a YAML mapping") + return Document(metadata, "".join(lines[end + 1 :]), True) + + +def check_document(path: str, content: str) -> list[Diagnostic]: + """Validate only the structural contract in OKF §11, not optional families.""" + diagnostics: list[Diagnostic] = [] + + def fail(rule: str, message: str) -> None: + diagnostics.append(Diagnostic(path=path, rule=rule, message=message)) + + try: + document = parse_document(content) + except ValueError as error: + fail("frontmatter", str(error)) + return diagnostics + name = Path(path).name + if name not in {"index.md", "log.md"}: + if not document.has_frontmatter: + fail("concept.frontmatter", "Concept requires YAML frontmatter (OKF §4)") + note_type = document.metadata.get("type") + if not isinstance(note_type, str) or not note_type.strip(): + fail("concept.type", "Concept requires a non-empty string type (OKF §4.1)") + return diagnostics + + if document.has_frontmatter and ( + name == "log.md" or path != "index.md" or set(document.metadata) != {"okf_version"} + ): + fail("reserved.frontmatter", "Only root index.md may carry okf_version (OKF §8–9)") + + tokens = MarkdownIt().parse(document.body) + if name == "index.md": + if not any(token.type == "heading_open" for token in tokens): + fail("index.heading", "Index requires a section heading (OKF §8)") + # Links are structural entries; prose and code examples are not entries. + entries: list[bool] = [] + for token in tokens: + if token.type == "list_item_open": + entries.append(False) + elif token.type == "list_item_close": + if not entries.pop(): + fail("index.link", "Index entries require standard Markdown links (OKF §8)") + elif entries and token.type == "inline": + if any(child.type == "link_open" for child in token.children or []): + # A link anywhere in an entry satisfies it, including nested entries. + entries = [True] * len(entries) + else: + previous: date | None = None + in_date_group = False + for index, token in enumerate(tokens): + if token.type == "list_item_open" and not in_date_group: + fail("log.group", "Log entries require a preceding date heading (OKF §9)") + if token.type != "heading_open": + continue + in_date_group = False + if token.tag == "h1": + continue + heading = tokens[index + 1].content + try: + if token.tag != "h2" or not re.fullmatch(r"\d{4}-\d{2}-\d{2}", heading): + raise ValueError("Invalid date heading") + day = date.fromisoformat(heading) + except ValueError: + fail("log.date", "Log date headings must be ## YYYY-MM-DD (OKF §9)") + continue + if previous is not None and day >= previous: + fail("log.order", "Log date groups must be unique and newest first (OKF §9)") + previous = day + in_date_group = True + return diagnostics + + +def check_bundle(root: Path) -> CheckReport: + """Walk all files without project ignore rules or database initialization.""" + report = CheckReport() + if not root.is_dir() or root.is_symlink(): + report.diagnostics.append( + Diagnostic(path=".", rule="bundle.directory", message="Bundle must be a directory") + ) + return report + + def scan_error(error: OSError) -> None: + report.diagnostics.append( + Diagnostic(path=str(error.filename), rule="filesystem.read", message=str(error)) + ) + + for directory, directories, files in os.walk(root, onerror=scan_error, followlinks=False): + for name in sorted([*directories, *files]): + path = Path(directory) / name + relative = path.relative_to(root).as_posix() + if path.is_symlink(): + report.diagnostics.append( + Diagnostic( + path=relative, rule="filesystem.symlink", message="Symlink not portable" + ) + ) + continue + if name not in files or path.suffix != ".md": + continue + if not path.is_file(): + report.diagnostics.append( + Diagnostic( + path=relative, + rule="filesystem.regular_file", + message="Markdown must be a regular file", + ) + ) + continue + if name not in {"index.md", "log.md"}: + report.concepts += 1 + try: + content = path.read_text(encoding="utf-8") + except (OSError, UnicodeError) as error: + report.diagnostics.append( + Diagnostic(path=relative, rule="filesystem.read", message=str(error)) + ) + continue + report.diagnostics.extend(check_document(relative, content)) + report.diagnostics.sort(key=lambda diagnostic: (diagnostic.path, diagnostic.rule)) + return report diff --git a/test-int/test_okf_integration.py b/test-int/test_okf_integration.py new file mode 100644 index 000000000..d78301ac9 --- /dev/null +++ b/test-int/test_okf_integration.py @@ -0,0 +1,203 @@ +"""Real OKF producer/consumer boundary, including pinned upstream compatibility.""" + +import importlib.util +import json +import os +import subprocess +from importlib.machinery import SourceFileLoader +from pathlib import Path +import shutil +import sys +from datetime import UTC, datetime +from types import ModuleType + +import pytest + +from basic_memory import db +from basic_memory.index.local_project import ( + LocalProjectIndexRuntimeFactory, + run_local_project_index_for_project, +) +from basic_memory.okf.validation import check_bundle, check_document, parse_document +from basic_memory.okf.render import ExportFile, ExportSnapshot, render_bundle +from basic_memory.repository.entity_repository import EntityRepository +from basic_memory.repository.relation_repository import RelationRepository +from basic_memory.schemas.search import SearchQuery + +FIXTURES = Path(__file__).parents[1] / "tests/fixtures/okf" + + +def upstream_parser() -> ModuleType: + # Load the unmodified pinned parser only in tests. Explicit counts ensure a + # permissive consumer cannot hide malformed concepts by silently skipping them. + spec = importlib.util.spec_from_loader( + "okf_upstream_document", + SourceFileLoader("okf_upstream_document", str(FIXTURES / "upstream_document.py.txt")), + ) + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +def upstream_concept_count(root: Path) -> int: + module = upstream_parser() + count = 0 + for path in root.rglob("*.md"): + if path.name in {"index.md", "log.md"}: + continue + document = module.OKFDocument.parse(path.read_text(encoding="utf-8")) + document.validate() + count += 1 + return count + + +@pytest.mark.asyncio +async def test_export_real_project(tmp_path): + root = tmp_path / "source" + root.mkdir() + config_dir = tmp_path / "config" + config_dir.mkdir() + (config_dir / "config.json").write_text( + json.dumps( + { + "projects": {"My Project": {"path": str(root)}}, + "default_project": "My Project", + "semantic_search_enabled": False, + "index_changes": False, + "env": "dev", + } + ) + ) + env = {key: value for key, value in os.environ.items() if not key.startswith("BASIC_MEMORY_")} + env.update( + BASIC_MEMORY_CONFIG_DIR=str(config_dir), + BASIC_MEMORY_NO_PROMOS="1", + BASIC_MEMORY_CLI_AUTO_UPDATE="false", + HOME=str(tmp_path), + ) + + def cli(*arguments): + result = subprocess.run( + [sys.executable, "-m", "basic_memory.cli.main", *arguments], + env=env, + capture_output=True, + text=True, + timeout=60, + ) + assert result.returncode == 0, result.stdout + result.stderr + return result.stdout + + (root / "notes").mkdir() + (root / "notes/source.md").write_text( + "---\ntitle: Source\npermalink: source\ntype: note\ntags: [interop]\n" + "sources: [{resource: /references/paper.pdf}]\n---\n# Source\n" + "- [fact] Knowledge stays portable (evidence)\n" + "- depends_on [[Target]] (design)\n" + "See [[Target|the target]] and [paper](../references/paper.pdf).\n", + encoding="utf-8", + ) + cli( + "tool", + "write-note", + "--title", + "Target", + "--folder", + ".", + "--content", + "# Target\n", + "--project", + "My Project", + "--local", + ) + (root / "references").mkdir() + pdf = b"%PDF-1.4\n1 0 obj<>endobj\n%%EOF\n" + (root / "references/paper.pdf").write_bytes(pdf) + (root / "index.md").write_text( + "---\nbm: {profile: wiki/1}\ngenerated: {by: Basic Memory Wiki Projector}\n---\n[[Live Wiki]]\n", + encoding="utf-8", + ) + source_paths = ["notes/source.md", "Target.md", "references/paper.pdf", "index.md"] + before = {path: (root / path).read_bytes() for path in source_paths} + destination = tmp_path / "bundle" + report = json.loads(cli("okf", "export", str(destination), "--project", "my-project", "--json")) + assert report["diagnostics"] == [] + assert report["concepts"] == upstream_concept_count(destination) == 2 + assert (destination / "references/paper.pdf").read_bytes() == pdf + source = parse_document((destination / "notes/source.md").read_text()) + assert "[the target](/Target.md)" in source.body + assert "- depends_on [Target](/Target.md) (design)" in source.body + assert "- [fact] Knowledge stays portable (evidence)" in source.body + assert "bm" in source.metadata + log = (destination / "log.md").read_text() + assert not log.startswith("---") + assert "\n## " in log and "Target.md" in log + assert parse_document((destination / "index.md").read_text()).metadata == {"okf_version": "0.2"} + # The normal write command persists the canonical project name in config. + assert "# my-project" in (destination / "index.md").read_text() + assert not parse_document((destination / "notes/index.md").read_text()).has_frontmatter + assert before == {path: (root / path).read_bytes() for path in source_paths} + first = { + p.relative_to(destination): p.read_bytes() for p in destination.rglob("*") if p.is_file() + } + cli("okf", "export", str(destination), "--project", "my-project", "--replace", "--json") + assert first == { + p.relative_to(destination): p.read_bytes() for p in destination.rglob("*") if p.is_file() + } + + +@pytest.mark.asyncio +async def test_upstream_sample_is_indexed_and_retrievable( + test_project, engine_factory, search_service +): + upstream = FIXTURES / "crypto_bitcoin" + report = check_bundle(upstream) + assert report.success, report.diagnostics + assert report.concepts == upstream_concept_count(upstream) == 9 + root = Path(test_project.path) + shutil.copytree(upstream, root, dirs_exist_ok=True) + result = await run_local_project_index_for_project( + test_project, runtime_factory=LocalProjectIndexRuntimeFactory(), force_full=True + ) + assert sum(batch.failed_files for batch in result.batch_results) == 0 + _, session_maker = engine_factory + async with db.scoped_session(session_maker) as session: + entities = await EntityRepository(project_id=test_project.id).find_all(session) + concepts = [ + entity + for entity in entities + if Path(entity.file_path).name not in {"index.md", "log.md"} + ] + assert len(concepts) == 9 + edges = await RelationRepository(project_id=test_project.id).find_by_type( + session, "links_to" + ) + assert any(edge.to_id is not None and "transactions" in edge.to_name for edge in edges) + results = await search_service.search(SearchQuery(text="Bitcoin")) + assert results + assert any("transactions" in result.file_path for result in results) + target = root / "tables/transactions.md" + assert "transaction" in target.read_text(encoding="utf-8").lower() + + +def test_upstream_sample_log_is_not_conformance_authority(): + text = (FIXTURES / "upstream_invalid_log.md").read_text() + assert {diagnostic.rule for diagnostic in check_document("log.md", text)} == { + "reserved.frontmatter" + } + + +def test_upstream_timestamp_behavior_survives_export(): + module = upstream_parser() + source = "--- \t\ntype: note\nstale_after: 2026-06-30T14:00:00Z\n---\t\n# A\n" + exported = next( + file.content.decode() + for file in render_bundle(ExportSnapshot("p", (ExportFile("a.md", source.encode()),))) + if file.path == "a.md" + ) + for content in (source, exported): + document = module.OKFDocument.parse(content) + document.validate() + assert module.is_stale(document.frontmatter, datetime(2026, 7, 1, tzinfo=UTC)) + assert not module.is_stale(document.frontmatter, datetime(2026, 6, 1, tzinfo=UTC)) diff --git a/tests/cli/test_man_command.py b/tests/cli/test_man_command.py index dbbdd3dec..dca2c7cae 100644 --- a/tests/cli/test_man_command.py +++ b/tests/cli/test_man_command.py @@ -159,8 +159,21 @@ def fake_run(*args, **kwargs): # Section 1 is the CLI's own documentation, so a page and its command must not # drift: an option a user can type that the page never names is undocumented. # find(1) went stale exactly this way when `bm find` grew --meta/--fields. -PAGES_WITHOUT_TOP_LEVEL_COMMANDS = {"apropos"} # `bm man apropos`, not `bm apropos` -DOCUMENTED_VERBS = {"cat", "find", "grep", "head", "ls", "tail", "tree"} +# Match the nested paths used by scripts/update_man_pages.py. +NESTED_COMMAND_PATHS = { + "apropos": ("man", "apropos"), + "okf-check": ("okf", "check"), + "okf-export": ("okf", "export"), +} +DOCUMENTED_VERBS = { + "cat", + "find", + "grep", + "head", + "ls", + "tail", + "tree", +} | NESTED_COMMAND_PATHS.keys() def test_section_1_pages_document_every_option_of_their_command(): @@ -171,10 +184,14 @@ def test_section_1_pages_document_every_option_of_their_command(): checked: set[str] = set() for page in bundled_pages(): - if page.section != 1 or page.name in PAGES_WITHOUT_TOP_LEVEL_COMMANDS: + if page.section != 1: continue body = page.body() - for param in cli.commands[page.name].params: + command = cli + for segment in NESTED_COMMAND_PATHS.get(page.name, (page.name,)): + assert isinstance(command, TyperGroup) + command = command.commands[segment] + for param in command.params: for option in param.opts: if not option.startswith("--"): continue diff --git a/tests/fixtures/okf/LICENSE.md b/tests/fixtures/okf/LICENSE.md new file mode 100644 index 000000000..6b0b1270f --- /dev/null +++ b/tests/fixtures/okf/LICENSE.md @@ -0,0 +1,203 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + diff --git a/tests/fixtures/okf/PIN.md b/tests/fixtures/okf/PIN.md new file mode 100644 index 000000000..b6eb4f7b6 --- /dev/null +++ b/tests/fixtures/okf/PIN.md @@ -0,0 +1,12 @@ +# Upstream OKF compatibility fixture + +Source: https://github.com/GoogleCloudPlatform/knowledge-catalog/tree/e6d34fd29c1c6c75ec23078e7a8191a9c8209620/okf + +Revision: `e6d34fd29c1c6c75ec23078e7a8191a9c8209620`. Apache-2.0 license in LICENSE.md. +`upstream_document.py.txt` is an unmodified test-only copy of +`okf/src/reference_agent/bundle/document.py`. `crypto_bitcoin/` is the upstream +sample bundle, excluding the generated viewer HTML. Expected concepts: 9. +No upstream code is imported by the runtime package. + +`upstream_invalid_log.md` is the unchanged Acme sample log: its frontmatter +violates §9 and is deliberately a negative validation fixture. diff --git a/tests/fixtures/okf/crypto_bitcoin/datasets/crypto_bitcoin.md b/tests/fixtures/okf/crypto_bitcoin/datasets/crypto_bitcoin.md new file mode 100644 index 000000000..3636a98f8 --- /dev/null +++ b/tests/fixtures/okf/crypto_bitcoin/datasets/crypto_bitcoin.md @@ -0,0 +1,74 @@ +--- +type: BigQuery Dataset +resource: https://bigquery.googleapis.com/v2/projects/bigquery-public-data/datasets/crypto_bitcoin +title: Bitcoin Blockchain Dataset +description: A public Google BigQuery dataset containing the complete transaction + ledger and block history of the Bitcoin blockchain. +tags: +- bitcoin +- blockchain +- crypto +- public-data +generated: + by: reference_agent/gemini-3.5-flash + at: '2026-07-10T23:14:11+00:00' +sources: +- title: BigQuery Dataset Metadata - crypto_bitcoin + resource: https://bigquery.googleapis.com/v2/projects/bigquery-public-data/datasets/crypto_bitcoin + id: bq-crypto-bitcoin-meta +--- + +The `crypto_bitcoin` dataset is a public Google BigQuery dataset containing the entire blockchain transaction history for Bitcoin. It is updated continuously and provides a highly structured, queryable format of block and transaction data from the genesis block onwards. + +The dataset contains four primary tables: +- [blocks](../tables/blocks.md) representing Bitcoin blocks, including hashes, sizes, transaction counts, and block rewards. +- [transactions](../tables/transactions.md) containing top-level transaction details such as inputs/outputs totals, fees, and cryptographic signatures. +- [inputs](../tables/inputs.md) containing the transaction inputs (spending previous outputs) representing the source of funds. +- [outputs](../tables/outputs.md) containing the transaction outputs representing the destinations of funds (addresses and values). + +This dataset is widely used for blockchain forensics, macroeconomic analysis of transaction volumes, wallet balance tracking, and research into mining activities. + +# Schema + +As a BigQuery Dataset, `crypto_bitcoin` acts as a namespace and container for the following tables: + +| Table ID | Description | +| :--- | :--- | +| **[blocks](../tables/blocks.md)** | Blocks containing transactions that have been validated and written to the ledger. | +| **[transactions](../tables/transactions.md)** | Individual ledger entries where value is transferred between participants. | +| **[inputs](../tables/inputs.md)** | References to UTXOs (Unspent Transaction Outputs) being spent in transactions. | +| **[outputs](../tables/outputs.md)** | Outputs created by transactions that become new UTXOs. | + +# Common query patterns + +### 1. Count of blocks and average transaction count per block by month +This query calculates the monthly volume of blocks and the average number of transactions included per block. + +```sql +SELECT + TIMESTAMP_TRUNC(timestamp, MONTH) AS month, + COUNT(1) AS total_blocks, + AVG(transaction_count) AS avg_transactions_per_block +FROM + `bigquery-public-data.crypto_bitcoin.blocks` +GROUP BY + month +ORDER BY + month DESC +LIMIT 12; +``` + +### 2. Transaction fee statistics (in Satoshis) over the last 30 days +This query explores transaction fee distributions across recent transactions. + +```sql +SELECT + MIN(fee) AS min_fee, + MAX(fee) AS max_fee, + AVG(fee) AS avg_fee, + APPROX_QUANTILES(fee, 2)[OFFSET(1)] AS median_fee +FROM + `bigquery-public-data.crypto_bitcoin.transactions` +WHERE + block_timestamp >= TIMESTAMP_SUB(CURRENT_TIMESTAMP(), INTERVAL 30 DAY); +``` diff --git a/tests/fixtures/okf/crypto_bitcoin/datasets/index.md b/tests/fixtures/okf/crypto_bitcoin/datasets/index.md new file mode 100644 index 000000000..f8c1e3e3b --- /dev/null +++ b/tests/fixtures/okf/crypto_bitcoin/datasets/index.md @@ -0,0 +1,3 @@ +# BigQuery Dataset + +* [Bitcoin Blockchain Dataset](crypto_bitcoin.md) - A public Google BigQuery dataset containing the complete transaction ledger and block history of the Bitcoin blockchain. diff --git a/tests/fixtures/okf/crypto_bitcoin/index.md b/tests/fixtures/okf/crypto_bitcoin/index.md new file mode 100644 index 000000000..375dcc279 --- /dev/null +++ b/tests/fixtures/okf/crypto_bitcoin/index.md @@ -0,0 +1,5 @@ +# Subdirectories + +* [datasets](datasets/index.md) - A public Google BigQuery dataset containing the complete transaction ledger and block history of the Bitcoin blockchain. +* [references](references/index.md) - The references directory contains join paths for tracing transaction details and an anomaly detection metric for identifying historical duplicate transactions across blocks. +* [tables](tables/index.md) - This directory contains tables documenting Bitcoin blockchain data, including blocks, transactions, and transaction inputs and outputs. diff --git a/tests/fixtures/okf/crypto_bitcoin/references/index.md b/tests/fixtures/okf/crypto_bitcoin/references/index.md new file mode 100644 index 000000000..eba584fda --- /dev/null +++ b/tests/fixtures/okf/crypto_bitcoin/references/index.md @@ -0,0 +1,4 @@ +# Subdirectories + +* [joins](joins/index.md) - This directory contains join paths linking blocks to transactions and transactions to their corresponding inputs and outputs to trace transaction details. +* [metrics](metrics/index.md) - An anomaly detection metric to find historical duplicate transactions across different blocks. diff --git a/tests/fixtures/okf/crypto_bitcoin/references/joins/blocks___transactions.md b/tests/fixtures/okf/crypto_bitcoin/references/joins/blocks___transactions.md new file mode 100644 index 000000000..4c31c1389 --- /dev/null +++ b/tests/fixtures/okf/crypto_bitcoin/references/joins/blocks___transactions.md @@ -0,0 +1,34 @@ +--- +type: Reference +resource: https://github.com/blockchain-etl/bitcoin-etl +title: Blocks to Transactions Join Path +description: Join relationship linking blocks to their corresponding transactions + by block height / block number. +tags: +- join +- bitcoin +generated: + by: reference_agent/gemini-3.5-flash + at: '2026-07-10T23:15:51+00:00' +sources: +- id: bitcoin-etl + resource: https://github.com/blockchain-etl/bitcoin-etl + title: Bitcoin ETL Parser +--- + +This join path represents the link between a block and all the transactions included in that block. This is useful for analyzing block density, mining fee shares, and validating transaction confirmation times relative to block production. + +```sql +SELECT + b.number AS block_height, + b.hash AS block_hash, + b.timestamp AS block_timestamp, + t.hash AS transaction_hash, + t.fee AS transaction_fee +FROM + `bigquery-public-data.crypto_bitcoin.blocks` AS b +JOIN + `bigquery-public-data.crypto_bitcoin.transactions` AS t +ON + b.number = t.block_number; +``` diff --git a/tests/fixtures/okf/crypto_bitcoin/references/joins/index.md b/tests/fixtures/okf/crypto_bitcoin/references/joins/index.md new file mode 100644 index 000000000..527f72a79 --- /dev/null +++ b/tests/fixtures/okf/crypto_bitcoin/references/joins/index.md @@ -0,0 +1,5 @@ +# Reference + +* [Blocks to Transactions Join Path](blocks___transactions.md) - Join relationship linking blocks to their corresponding transactions by block height / block number. +* [Transactions to Inputs Join Path](inputs___transactions.md) - Join path between transactions and inputs to trace fund consumption details. +* [Transactions to Outputs Join Path](outputs___transactions.md) - Join path between transactions and outputs to audit target recipient distribution. diff --git a/tests/fixtures/okf/crypto_bitcoin/references/joins/inputs___transactions.md b/tests/fixtures/okf/crypto_bitcoin/references/joins/inputs___transactions.md new file mode 100644 index 000000000..7d27deecc --- /dev/null +++ b/tests/fixtures/okf/crypto_bitcoin/references/joins/inputs___transactions.md @@ -0,0 +1,34 @@ +--- +type: Reference +resource: https://github.com/blockchain-etl/bitcoin-etl +title: Transactions to Inputs Join Path +description: Join path between transactions and inputs to trace fund consumption details. +tags: +- join +- bitcoin +generated: + by: reference_agent/gemini-3.5-flash + at: '2026-07-10T23:15:56+00:00' +sources: +- id: bitcoin-etl + title: Bitcoin ETL Parser + resource: https://github.com/blockchain-etl/bitcoin-etl +--- + +This join path connects transactions to their inputs. In the Unspent Transaction Output (UTXO) database structure, joining the main `transactions` table with the flat `inputs` table lets analysts audit the historical origin of funds being consumed in a transaction. + +```sql +SELECT + t.hash AS transaction_hash, + t.block_timestamp AS transaction_timestamp, + i.index AS input_index, + i.spent_transaction_hash, + i.spent_output_index, + i.value AS input_value_satoshis +FROM + `bigquery-public-data.crypto_bitcoin.transactions` AS t +JOIN + `bigquery-public-data.crypto_bitcoin.inputs` AS i +ON + t.hash = i.transaction_hash; +``` diff --git a/tests/fixtures/okf/crypto_bitcoin/references/joins/outputs___transactions.md b/tests/fixtures/okf/crypto_bitcoin/references/joins/outputs___transactions.md new file mode 100644 index 000000000..d11e2c45f --- /dev/null +++ b/tests/fixtures/okf/crypto_bitcoin/references/joins/outputs___transactions.md @@ -0,0 +1,34 @@ +--- +type: Reference +resource: https://github.com/blockchain-etl/bitcoin-etl +title: Transactions to Outputs Join Path +description: Join path between transactions and outputs to audit target recipient + distribution. +tags: +- join +- bitcoin +generated: + by: reference_agent/gemini-3.5-flash + at: '2026-07-10T23:15:59+00:00' +sources: +- title: Bitcoin ETL Parser + resource: https://github.com/blockchain-etl/bitcoin-etl + id: bitcoin-etl +--- + +This join path relates a transaction to its generated outputs. Joining `transactions` with `outputs` is useful for tracking how funds are distributed (split or forwarded) from a parent transaction into target addresses. + +```sql +SELECT + t.hash AS transaction_hash, + t.block_timestamp AS transaction_timestamp, + o.index AS output_index, + o.addresses, + o.value AS output_value_satoshis +FROM + `bigquery-public-data.crypto_bitcoin.transactions` AS t +JOIN + `bigquery-public-data.crypto_bitcoin.outputs` AS o +ON + t.hash = o.transaction_hash; +``` diff --git a/tests/fixtures/okf/crypto_bitcoin/references/metrics/duplicate_transactions.md b/tests/fixtures/okf/crypto_bitcoin/references/metrics/duplicate_transactions.md new file mode 100644 index 000000000..2a4d95700 --- /dev/null +++ b/tests/fixtures/okf/crypto_bitcoin/references/metrics/duplicate_transactions.md @@ -0,0 +1,37 @@ +--- +type: Reference +resource: https://cloud.google.com/blog/topics/public-datasets/bitcoin-in-bigquery-blockchain-analytics-on-public-data +title: Duplicate Transactions Metric +description: An anomaly detection metric to find historical duplicate transactions + across different blocks. +tags: +- metric +- anomaly-detection +- bitcoin +generated: + by: reference_agent/gemini-3.5-flash + at: '2026-07-10T23:15:47+00:00' +sources: +- id: gcp-blog + title: 'Bitcoin in BigQuery: blockchain analytics on public data' + resource: https://cloud.google.com/blog/topics/public-datasets/bitcoin-in-bigquery-blockchain-analytics-on-public-data +--- + +The anomaly query pattern identifies transactions that appear in multiple blocks. Historically, in the Bitcoin blockchain, transactions could be duplicated due to a behavior in the original BerkeleyDB database engine that allowed non-unique keys. This was later addressed by implementing Bitcoin Improvement Proposal [BIP-0030](https://github.com/bitcoin/bips/blob/master/bip-0030.mediawiki) and transitioning to LevelDB. + +### standardSQL +```sql +SELECT + transaction_id, + COUNT(transaction_id) AS dup_transaction_count +FROM ( + SELECT + hash AS transaction_id + FROM + `bigquery-public-data.crypto_bitcoin.transactions` +) +GROUP BY + transaction_id +HAVING + dup_transaction_count > 1; +``` diff --git a/tests/fixtures/okf/crypto_bitcoin/references/metrics/index.md b/tests/fixtures/okf/crypto_bitcoin/references/metrics/index.md new file mode 100644 index 000000000..817630d20 --- /dev/null +++ b/tests/fixtures/okf/crypto_bitcoin/references/metrics/index.md @@ -0,0 +1,3 @@ +# Reference + +* [Duplicate Transactions Metric](duplicate_transactions.md) - An anomaly detection metric to find historical duplicate transactions across different blocks. diff --git a/tests/fixtures/okf/crypto_bitcoin/tables/blocks.md b/tests/fixtures/okf/crypto_bitcoin/tables/blocks.md new file mode 100644 index 000000000..00bda0a89 --- /dev/null +++ b/tests/fixtures/okf/crypto_bitcoin/tables/blocks.md @@ -0,0 +1,111 @@ +--- +type: BigQuery Table +resource: https://bigquery.googleapis.com/v2/projects/bigquery-public-data/datasets/crypto_bitcoin/tables/blocks +title: Bitcoin Blocks Table +description: All blocks from the Bitcoin blockchain, including block headers, transaction + counts, sizes, and timestamps. +tags: +- bitcoin +- blockchain +- crypto +generated: + by: reference_agent/gemini-3.5-flash + at: '2026-07-10T23:16:06+00:00' +sources: +- resource: https://github.com/blockchain-etl/bitcoin-etl + title: Bitcoin ETL Export Tool + id: bitcoin-etl +- id: bip-141 + resource: https://github.com/bitcoin/bips/blob/master/bip-0141.mediawiki + title: BIP-141 Segregated Witness (Consensus layer) +--- + +The `blocks` table contains structured records for every block in the Bitcoin blockchain [^bitcoin-etl]. Each row in this table represents a single block, captured with detailed block header attributes such as hash, size, transaction count, nonce, difficulty bits, and the Merkle root of all transactions contained within that block. + +The dataset is continually exported from live nodes and represents a complete historical index of Bitcoin blocks starting from the genesis block in January 2009. The table is partitioned by month using the `timestamp_month` column to optimize query performance and lower data scanning costs when filtering blocks by date. + +The table can be joined with [transactions](transactions.md) to drill down into individual payments or to aggregate block-level statistics like total transaction fees, transaction densities, and witness data weights. + +# Schema + +| Field Name | Type | Mode | Description | +| :--- | :--- | :--- | :--- | +| **hash** | STRING | REQUIRED | Unique block hash that identifies the block. | +| **size** | INTEGER | NULLABLE | Total size of the block data in bytes. | +| **stripped_size** | INTEGER | NULLABLE | The size of block data in bytes excluding witness data. | +| **weight** | INTEGER | NULLABLE | Three times the base size plus the total size as defined in BIP-141 [^bip-141]. | +| **number** | INTEGER | REQUIRED | The sequential height of the block. | +| **version** | INTEGER | NULLABLE | Protocol version specified in the block header. | +| **merkle_root** | STRING | NULLABLE | The root node of a Merkle tree, where leaves are transaction hashes. | +| **timestamp** | TIMESTAMP | REQUIRED | Block creation timestamp specified in the block header. | +| **timestamp_month** | DATE | REQUIRED | Month of the block creation timestamp (used as the partitioning key). | +| **nonce** | STRING | NULLABLE | Difficulty solution specified in the block header. | +| **bits** | STRING | NULLABLE | Difficulty threshold specified in the block header. | +| **coinbase_param** | STRING | NULLABLE | Data specified in the coinbase transaction of this block. | +| **transaction_count** | INTEGER | NULLABLE | Number of transactions included in this block. | + +# Common query patterns + +### 1. Daily block counts and average transactions per block +Find out how many blocks are mined each day and the average number of transactions per block for a specific month. + +```sql +SELECT + DATE(timestamp) AS block_date, + COUNT(1) AS blocks_mined, + AVG(transaction_count) AS avg_transactions_per_block, + SUM(transaction_count) AS total_transactions +FROM + `bigquery-public-data.crypto_bitcoin.blocks` +WHERE + timestamp_month = '2023-10-01' +GROUP BY + block_date +ORDER BY + block_date ASC; +``` + +### 2. Retrieve details for a specific block height +Lookup a single block's metadata and structure using its height number. + +```sql +SELECT + number, + hash, + timestamp, + size, + transaction_count, + version, + coinbase_param +FROM + `bigquery-public-data.crypto_bitcoin.blocks` +WHERE + number = 800000; +``` + +### 3. Calculate monthly average block size and weight +Analyze the adoption and impact of SegWit over time by analyzing the trends in block size, stripped size, and SegWit weight [^bip-141]. + +```sql +SELECT + timestamp_month, + COUNT(1) AS blocks_mined, + AVG(size) AS avg_block_size_bytes, + AVG(stripped_size) AS avg_stripped_size_bytes, + AVG(weight) AS avg_weight +FROM + `bigquery-public-data.crypto_bitcoin.blocks` +WHERE + timestamp_month >= '2020-01-01' +GROUP BY + timestamp_month +ORDER BY + timestamp_month DESC; +``` + +# Joins + +- [transactions](../references/joins/blocks___transactions.md) — Connects blocks to all included transactions to trace block validation times, miner fee revenue, or transaction densities. + +[^bitcoin-etl]: https://github.com/blockchain-etl/bitcoin-etl +[^bip-141]: https://github.com/bitcoin/bips/blob/master/bip-0141.mediawiki diff --git a/tests/fixtures/okf/crypto_bitcoin/tables/index.md b/tests/fixtures/okf/crypto_bitcoin/tables/index.md new file mode 100644 index 000000000..029cbcb4d --- /dev/null +++ b/tests/fixtures/okf/crypto_bitcoin/tables/index.md @@ -0,0 +1,6 @@ +# BigQuery Table + +* [Bitcoin Blocks Table](blocks.md) - All blocks from the Bitcoin blockchain, including block headers, transaction counts, sizes, and timestamps. +* [Bitcoin Outputs Table](outputs.md) - Outputs from all Bitcoin transactions, including script details and values in Satoshis. +* [Bitcoin Transaction Inputs](inputs.md) - Bitcoin transaction inputs detailing UTXOs spent. +* [Bitcoin Transactions Table](transactions.md) - All Bitcoin transactions containing inputs, outputs, block metadata, and fee structures. diff --git a/tests/fixtures/okf/crypto_bitcoin/tables/inputs.md b/tests/fixtures/okf/crypto_bitcoin/tables/inputs.md new file mode 100644 index 000000000..94981c76f --- /dev/null +++ b/tests/fixtures/okf/crypto_bitcoin/tables/inputs.md @@ -0,0 +1,102 @@ +--- +type: BigQuery Table +resource: https://bigquery.googleapis.com/v2/projects/bigquery-public-data/datasets/crypto_bitcoin/tables/inputs +title: Bitcoin Transaction Inputs +description: Bitcoin transaction inputs detailing UTXOs spent. +tags: +- bitcoin +- crypto +- blockchain +- utxo +- inputs +generated: + by: reference_agent/gemini-3.5-flash + at: '2026-07-10T23:16:22+00:00' +sources: +- resource: https://github.com/blockchain-etl/bitcoin-etl + title: Bitcoin ETL GitHub Repository + id: bitcoin-etl +--- + +The `inputs` table contains details of all transaction inputs (UTXOs spent) on the Bitcoin blockchain. Each row represents a single input that was consumed to fund a transaction[^bitcoin-etl]. Because Bitcoin uses an Unspent Transaction Output (UTXO) model, every transaction consumes existing outputs (which become "inputs" in the new transaction) and creates new outputs[^bitcoin-etl]. + +This table is particularly useful for tracking the flow of funds, analyzing spending behavior, and tracing transaction lineage. By linking the `spent_transaction_hash` and `spent_output_index` of an input back to the [outputs](outputs.md) table, analysts can fully reconstruct the transaction graph. + +Data is exported from the blockchain using the open-source [bitcoin-etl](https://github.com/blockchain-etl/bitcoin-etl) tool[^bitcoin-etl] and is housed in the [crypto_bitcoin](../datasets/crypto_bitcoin.md) dataset. + +# Schema + +| Field Name | Type | Mode | Description | +| :--- | :--- | :--- | :--- | +| **transaction_hash** | STRING | NULLABLE | Hash of the transaction containing this input | +| **block_hash** | STRING | NULLABLE | Hash of the block containing this transaction | +| **block_number** | INTEGER | NULLABLE | Height of the block containing this transaction | +| **block_timestamp** | TIMESTAMP | NULLABLE | Timestamp of the block containing this transaction | +| **index** | INTEGER | NULLABLE | 0-based index of this input within the transaction | +| **spent_transaction_hash** | STRING | NULLABLE | Hash of the transaction containing the output spent by this input | +| **spent_output_index** | INTEGER | NULLABLE | Index of the output spent by this input in the original transaction | +| **script_asm** | STRING | NULLABLE | Symbolic representation of the input's script (scriptSig) | +| **script_hex** | STRING | NULLABLE | Hexadecimal representation of the input's script (scriptSig) | +| **sequence** | INTEGER | NULLABLE | Transaction input sequence number | +| **required_signatures** | INTEGER | NULLABLE | Number of signatures required to spend (if applicable) | +| **type** | STRING | NULLABLE | Type of script (e.g., `witness_v1_taproot`, `pubkeyhash`) | +| **addresses** | STRING | REPEATED | List of addresses associated with this input | +| **value** | NUMERIC | NULLABLE | Value of the spent output in Satoshis | + +# Common query patterns + +### 1. Identify the largest transaction inputs in a given period +This query retrieves the largest inputs consumed on a specific day, demonstrating how to find massive UTXO consolidations or large-value transfers. + +```sql +SELECT + block_timestamp, + transaction_hash, + value / 100000000 AS value_btc, + addresses +FROM `bigquery-public-data.crypto_bitcoin.inputs` +WHERE block_timestamp >= '2024-04-17 00:00:00 UTC' + AND block_timestamp < '2024-04-18 00:00:00 UTC' +ORDER BY value DESC +LIMIT 10; +``` + +### 2. Track input types over time +Analyze the adoption of modern Bitcoin script types (like Taproot) by counting inputs grouped by their transaction script type. + +```sql +SELECT + DATE(block_timestamp) AS block_date, + type, + COUNT(1) AS input_count, + SUM(value) / 100000000 AS total_value_btc +FROM `bigquery-public-data.crypto_bitcoin.inputs` +WHERE block_timestamp >= '2024-01-01 00:00:00 UTC' +GROUP BY block_date, type +ORDER BY block_date DESC, input_count DESC; +``` + +### 3. Trace provenance by joining inputs and outputs +To find where funds spent in a transaction came from, you can join the inputs table to the outputs table using the spent transaction keys. + +```sql +SELECT + inp.transaction_hash AS spending_tx, + inp.block_timestamp AS spend_time, + out.transaction_hash AS source_tx, + out.block_timestamp AS source_time, + inp.value / 100000000 AS value_btc +FROM `bigquery-public-data.crypto_bitcoin.inputs` AS inp +JOIN `bigquery-public-data.crypto_bitcoin.outputs` AS out + ON inp.spent_transaction_hash = out.transaction_hash + AND inp.spent_output_index = out.index +WHERE inp.block_timestamp >= '2024-04-17 00:00:00 UTC' + AND inp.block_timestamp < '2024-04-17 01:00:00 UTC' +LIMIT 10; +``` + +# Joins + +- [transactions](../references/joins/inputs___transactions.md) — Connects this spent input to the parent transaction record which spent it. + +[^bitcoin-etl]: https://github.com/blockchain-etl/bitcoin-etl diff --git a/tests/fixtures/okf/crypto_bitcoin/tables/outputs.md b/tests/fixtures/okf/crypto_bitcoin/tables/outputs.md new file mode 100644 index 000000000..667ef3eb4 --- /dev/null +++ b/tests/fixtures/okf/crypto_bitcoin/tables/outputs.md @@ -0,0 +1,99 @@ +--- +type: BigQuery Table +resource: https://bigquery.googleapis.com/v2/projects/bigquery-public-data/datasets/crypto_bitcoin/tables/outputs +title: Bitcoin Outputs Table +description: Outputs from all Bitcoin transactions, including script details and values + in Satoshis. +tags: +- bitcoin +- blockchain +- crypto +- utxo +generated: + by: reference_agent/gemini-3.5-flash + at: '2026-07-10T23:16:28+00:00' +sources: +- id: bitcoin-etl + resource: https://github.com/blockchain-etl/bitcoin-etl + title: Bitcoin ETL Export Tool +- title: BigQuery Bitcoin Outputs Table Metadata + resource: https://bigquery.googleapis.com/v2/projects/bigquery-public-data/datasets/crypto_bitcoin/tables/outputs + id: outputs-table +--- + +The `outputs` table contains structured data representing all transaction outputs (also known as UTXOs or Unspent Transaction Outputs before they are spent) in the Bitcoin blockchain. Each row represents a single output generated by a transaction, which specifies an amount of satoshis (value) and the cryptographic conditions (locking script) required to spend it. + +Data is extracted and exported from the blockchain ledger using the open-source `bitcoin-etl` tool[^bitcoin-etl]. This table belongs to the [crypto_bitcoin](../datasets/crypto_bitcoin.md) dataset and can be joined with sibling tables such as [transactions](transactions.md) and [inputs](inputs.md) to reconstruct full transaction lineages and trace the movement of funds across the network. + +### Grain and Interpretation + +The table grain is **one row per transaction output**, uniquely identified by the combination of `transaction_hash` and output `index`. The `value` field represents the output amount in Satoshis (where 1 BTC = 100,000,000 Satoshis), represented as a high-precision `NUMERIC` type. + +# Schema + +| Field Name | Type | Mode | Description | +|---|---|---|---| +| `transaction_hash` | STRING | NULLABLE | Hash of the transaction containing this output. | +| `block_hash` | STRING | NULLABLE | Hash of the block containing this transaction. | +| `block_number` | INTEGER | NULLABLE | The block number/height. | +| `block_timestamp` | TIMESTAMP | NULLABLE | Timestamp of when the block was mined. | +| `index` | INTEGER | NULLABLE | The zero-based index of the output within the transaction. | +| `script_asm` | STRING | NULLABLE | Symbolic representation (Assembly) of the locking script. | +| `script_hex` | STRING | NULLABLE | Hexadecimal representation of the locking script. | +| `required_signatures` | INTEGER | NULLABLE | Number of signatures required to spend this output (typically 1 for common addresses). | +| `type` | STRING | NULLABLE | Type of the script (e.g., `pubkeyhash`, `scripthash`). | +| `addresses` | STRING | REPEATED | List of Bitcoin addresses associated with this output (normally contains a single address). | +| `value` | NUMERIC | NULLABLE | The value of the output in Satoshis (1 BTC = 100,000,000 Satoshis). | + +# Common query patterns + +### 1. Calculate the total value of transaction outputs generated on a specific day +The following query aggregates the output values for a given day to find the total volume minted, converting Satoshis to Bitcoin (BTC). + +```sql +SELECT + DATE(block_timestamp) AS date, + SUM(value) / 100000000.0 AS total_btc_volume, + COUNT(1) AS output_count +FROM `bigquery-public-data.crypto_bitcoin.outputs` +WHERE block_timestamp >= '2023-01-01 00:00:00 UTC' + AND block_timestamp < '2023-01-02 00:00:00 UTC' +GROUP BY 1; +``` + +### 2. Find the largest transaction outputs for a given block +This query lists the highest-value outputs in block number 301641, along with the receiving addresses. + +```sql +SELECT + transaction_hash, + `index`, + addresses, + value / 100000000.0 AS btc_value, + type +FROM `bigquery-public-data.crypto_bitcoin.outputs` +WHERE block_number = 301641 +ORDER BY value DESC +LIMIT 5; +``` + +### 3. Analyze output types over a specific time range +This query identifies the popularity of different script locking types (such as `pubkeyhash` or `scripthash`) over a weekly timeframe. + +```sql +SELECT + type, + COUNT(1) AS output_count, + SUM(value) / 100000000.0 AS total_btc +FROM `bigquery-public-data.crypto_bitcoin.outputs` +WHERE block_timestamp >= '2023-06-01 00:00:00 UTC' + AND block_timestamp < '2023-06-08 00:00:00 UTC' +GROUP BY type +ORDER BY output_count DESC; +``` + +# Joins + +- [transactions](../references/joins/outputs___transactions.md) — Connects this output back to the parent transaction record that created it. + +[^bitcoin-etl]: Blockchain ETL Bitcoin Extractor: https://github.com/blockchain-etl/bitcoin-etl diff --git a/tests/fixtures/okf/crypto_bitcoin/tables/transactions.md b/tests/fixtures/okf/crypto_bitcoin/tables/transactions.md new file mode 100644 index 000000000..de61137d5 --- /dev/null +++ b/tests/fixtures/okf/crypto_bitcoin/tables/transactions.md @@ -0,0 +1,142 @@ +--- +type: BigQuery Table +resource: https://bigquery.googleapis.com/v2/projects/bigquery-public-data/datasets/crypto_bitcoin/tables/transactions +title: Bitcoin Transactions Table +description: All Bitcoin transactions containing inputs, outputs, block metadata, + and fee structures. +tags: +- bitcoin +- crypto +- blockchain +- transactions +generated: + by: reference_agent/gemini-3.5-flash + at: '2026-07-10T23:16:14+00:00' +sources: +- title: Bitcoin ETL Parser + resource: https://github.com/blockchain-etl/bitcoin-etl + id: bitcoin-etl +- resource: https://bigquery.googleapis.com/v2/projects/bigquery-public-data/datasets/crypto_bitcoin/tables/transactions + id: bq-metadata + title: BigQuery transactions Table Schema +- resource: https://cloud.google.com/blog/topics/public-datasets/bitcoin-in-bigquery-blockchain-analytics-on-public-data + id: gcp-blog + title: 'Bitcoin in BigQuery: blockchain analytics on public data' +--- + +This table contains all Bitcoin transactions since the genesis block in January 2009. The data is exported from the Bitcoin blockchain using the open-source `bitcoin-etl` utility.[^bitcoin-etl] The grain of this table is one row per transaction. + +Each transaction contains structural data such as its hash, size, coinbase indicator, block metadata (like block number, hash, and timestamp), fees, and nested records representing its spending inputs and resulting outputs. To perform cost-effective queries, the table is partitioned on the `block_timestamp_month` column. + +This table links directly to several sibling tables in the [crypto_bitcoin](../datasets/crypto_bitcoin.md) dataset, such as [blocks](blocks.md). While inputs and outputs are nested as repeated records here, they are also flattened into dedicated sibling tables: [inputs](inputs.md) and [outputs](outputs.md). + +# Schema + +| Field Name | Type | Mode | Description | +|---|---|---|---| +| **hash** | STRING | REQUIRED | The unique SHA-256 hash of this transaction | +| **size** | INTEGER | NULLABLE | The size of this transaction in bytes | +| **virtual_size** | INTEGER | NULLABLE | The virtual transaction size (differs from size for SegWit/witness transactions) | +| **version** | INTEGER | NULLABLE | Protocol version specified in the block which contained this transaction | +| **lock_time** | INTEGER | NULLABLE | Earliest time/block height that miners can include the transaction | +| **block_hash** | STRING | REQUIRED | Hash of the block which contains this transaction | +| **block_number** | INTEGER | REQUIRED | Number of the block which contains this transaction | +| **block_timestamp** | TIMESTAMP | REQUIRED | Timestamp of the block which contains this transaction | +| **block_timestamp_month** | DATE | REQUIRED | Partitioning column; month of the block which contains this transaction | +| **input_count** | INTEGER | NULLABLE | The number of inputs in the transaction | +| **output_count** | INTEGER | NULLABLE | The number of outputs in the transaction | +| **input_value** | NUMERIC | NULLABLE | Total value of inputs in the transaction | +| **output_value** | NUMERIC | NULLABLE | Total value of outputs in the transaction | +| **is_coinbase** | BOOLEAN | NULLABLE | True if this transaction is a coinbase transaction (mined block reward) | +| **fee** | NUMERIC | NULLABLE | The transaction fee paid to miners (input_value - output_value) | +| **inputs** | RECORD | REPEATED | Nested array of transaction inputs | +| *inputs.***index** | INTEGER | REQUIRED | 0-indexed number of an input within a transaction | +| *inputs.***spent_transaction_hash** | STRING | NULLABLE | The hash of the transaction containing the output that this input spends | +| *inputs.***spent_output_index** | INTEGER | NULLABLE | The index of the output this input spends | +| *inputs.***script_asm** | STRING | NULLABLE | Symbolic representation of the script signature | +| *inputs.***script_hex** | STRING | NULLABLE | Hexadecimal representation of the script signature | +| *inputs.***sequence** | INTEGER | NULLABLE | Sequence number for locktime modifications | +| *inputs.***required_signatures** | INTEGER | NULLABLE | The number of signatures required to authorize the spent output | +| *inputs.***type** | STRING | NULLABLE | The address type of the spent output (e.g. pubkeyhash, scripthash) | +| *inputs.***addresses** | STRING | REPEATED | Array of addresses which own the spent output | +| *inputs.***value** | NUMERIC | NULLABLE | The value in base currency (satoshis) attached to the spent output | +| **outputs** | RECORD | REPEATED | Nested array of transaction outputs | +| *outputs.***index** | INTEGER | REQUIRED | 0-indexed number of the output used to reference this specific output later | +| *outputs.***script_asm** | STRING | NULLABLE | Symbolic representation of the script pubkey | +| *outputs.***script_hex** | STRING | NULLABLE | Hexadecimal representation of the script pubkey | +| *outputs.***required_signatures** | INTEGER | NULLABLE | The number of signatures required to authorize spending of this output | +| *outputs.***type** | STRING | NULLABLE | The address type of the output | +| *outputs.***addresses** | STRING | REPEATED | Array of addresses which own this output | +| *outputs.***value** | NUMERIC | NULLABLE | The value in base currency (satoshis) attached to this output | + +# Common query patterns + +### 1. Calculate average transaction fees and size over a month +The following query aggregates daily transaction volumes, average fees, and average sizes for a specific partitioned month. + +```sql +SELECT + DATE(block_timestamp) AS transaction_date, + COUNT(1) AS transaction_count, + AVG(fee) AS avg_fee_satoshis, + AVG(size) AS avg_size_bytes +FROM + `bigquery-public-data.crypto_bitcoin.transactions` +WHERE + block_timestamp_month = '2023-10-01' +GROUP BY + transaction_date +ORDER BY + transaction_date; +``` + +### 2. Identify the highest-value transactions in a given month +This query retrieves the largest transactions by output value, excluding coinbase transactions (block rewards). + +```sql +SELECT + hash, + block_number, + block_timestamp, + output_count, + output_value +FROM + `bigquery-public-data.crypto_bitcoin.transactions` +WHERE + block_timestamp_month = '2023-10-01' + AND is_coinbase = FALSE +ORDER BY + output_value DESC +LIMIT 10; +``` + +### 3. Analyze output types and values (unnesting repeated records) +To analyze the distribution of different Bitcoin address types (such as `scripthash` or `witness_v0_keyhash`), you must unnest the `outputs` repeated record. + +```sql +SELECT + out.type AS address_type, + COUNT(1) AS output_count, + SUM(out.value) AS total_value +FROM + `bigquery-public-data.crypto_bitcoin.transactions`, + UNNEST(outputs) AS out +WHERE + block_timestamp_month = '2023-10-01' +GROUP BY + address_type +ORDER BY + total_value DESC; +``` + +# Metrics + +- [Duplicate transactions across blocks](../references/metrics/duplicate_transactions.md) — An anomaly detection query to spot old duplicate transaction IDs prior to the BIP-0030 implementation. + +# Joins + +- [blocks](../references/joins/blocks___transactions.md) — Links block metadata to find which block mined this transaction. +- [inputs](../references/joins/inputs___transactions.md) — Links a transaction to its UTXO spending sources. +- [outputs](../references/joins/outputs___transactions.md) — Links a transaction to its output receipts. + +[^bitcoin-etl]: Blockchain ETL on GitHub: https://github.com/blockchain-etl/bitcoin-etl diff --git a/tests/fixtures/okf/upstream_document.py.txt b/tests/fixtures/okf/upstream_document.py.txt new file mode 100644 index 000000000..a804ecfd0 --- /dev/null +++ b/tests/fixtures/okf/upstream_document.py.txt @@ -0,0 +1,133 @@ +from __future__ import annotations + +from dataclasses import dataclass, field +from datetime import datetime, timezone +from typing import Any + +import yaml + +# OKF v0.2 §11: `type` is the only always-required frontmatter key. +REQUIRED_FRONTMATTER_KEYS = ("type",) + +_FRONTMATTER_DELIM = "---" + + +class _Loader(yaml.SafeLoader): + """SafeLoader that leaves timestamps as the text the author wrote. + + PyYAML implements YAML 1.1, whose implicit resolvers turn a value like + `2026-06-30T14:00:00Z` into a `datetime`. Dumping it back yields + `2026-06-30 14:00:00+00:00`, so a parse/serialize round-trip silently + rewrites the author's frontmatter. Dropping the resolver keeps every + value a string, matching the YAML 1.2 core schema. + """ + + +_Loader.yaml_implicit_resolvers = { + ch: [(tag, regexp) for tag, regexp in resolvers if tag != "tag:yaml.org,2002:timestamp"] + for ch, resolvers in yaml.SafeLoader.yaml_implicit_resolvers.items() +} + + +class OKFDocumentError(ValueError): + pass + + +@dataclass +class OKFDocument: + frontmatter: dict[str, Any] = field(default_factory=dict) + body: str = "" + + @classmethod + def parse(cls, text: str) -> "OKFDocument": + lines = text.splitlines() + if not lines or lines[0].strip() != _FRONTMATTER_DELIM: + return cls(frontmatter={}, body=text) + + end_idx = None + for i in range(1, len(lines)): + if lines[i].strip() == _FRONTMATTER_DELIM: + end_idx = i + break + if end_idx is None: + raise OKFDocumentError("Unterminated YAML frontmatter block") + + fm_text = "\n".join(lines[1:end_idx]) + try: + fm = yaml.load(fm_text, Loader=_Loader) or {} + except yaml.YAMLError as e: + raise OKFDocumentError(f"Invalid YAML in frontmatter: {e}") from e + if not isinstance(fm, dict): + raise OKFDocumentError("Frontmatter must be a YAML mapping") + + body = "\n".join(lines[end_idx + 1:]) + if body.startswith("\n"): + body = body[1:] + return cls(frontmatter=fm, body=body) + + def serialize(self) -> str: + fm_text = yaml.safe_dump( + self.frontmatter, sort_keys=False, allow_unicode=True + ).rstrip() + body = self.body if self.body.endswith("\n") else self.body + "\n" + return f"{_FRONTMATTER_DELIM}\n{fm_text}\n{_FRONTMATTER_DELIM}\n\n{body}" + + def validate(self) -> None: + missing = [k for k in REQUIRED_FRONTMATTER_KEYS if not self.frontmatter.get(k)] + if missing: + raise OKFDocumentError( + f"Missing required frontmatter keys: {', '.join(missing)}" + ) + + +def normalize_verified(frontmatter: dict[str, Any]) -> list[dict[str, Any]]: + """Return the `verified` events as a list (OKF v0.2 §5.2). + + A single verifier MAY be written as one `{ by, at }` mapping without the + list dash; consumers MUST treat a bare mapping as a one-element list. + """ + verified = frontmatter.get("verified") + if verified is None: + return [] + if isinstance(verified, dict): + return [verified] + if isinstance(verified, list): + return [v for v in verified if isinstance(v, dict)] + return [] + + +def trust_tier(frontmatter: dict[str, Any]) -> str: + """Derive a concept's trust tier from `verified` (OKF v0.2 §5.3). + + - No `verified` key ⇒ "unverified". + - `verified` by non-`human:` actors only ⇒ "machine-confirmed". + - `verified` by a `human:` actor ⇒ "human-reviewed". + """ + events = normalize_verified(frontmatter) + if not events: + return "unverified" + for event in events: + by = str(event.get("by") or "") + if by.startswith("human:"): + return "human-reviewed" + return "machine-confirmed" + + +def is_stale(frontmatter: dict[str, Any], now: datetime | None = None) -> bool: + """Whether a concept is stale per `stale_after` (OKF v0.2 §5.5). + + A concept is stale when `now >= stale_after`. Returns False when + `stale_after` is absent, or is not an ISO 8601 datetime with an explicit + UTC offset: a date-only `2026-12-31` names a different instant in every + timezone, so it is ignored rather than guessed at. + """ + raw = str(frontmatter.get("stale_after") or "") + if "T" not in raw: + return False + try: + stale_after = datetime.fromisoformat(raw) + except ValueError: + return False + if stale_after.tzinfo is None: + return False + return (now or datetime.now(timezone.utc)) >= stale_after diff --git a/tests/fixtures/okf/upstream_invalid_log.md b/tests/fixtures/okf/upstream_invalid_log.md new file mode 100644 index 000000000..1f93436b3 --- /dev/null +++ b/tests/fixtures/okf/upstream_invalid_log.md @@ -0,0 +1,22 @@ +--- +type: Log +title: Acme Retail bundle history +--- + +# Bundle history + +## 2026-07-01 + +- **Verified** the full bundle for OKF v0.2 conformance. `human:kliu@acme` reviewed all `verified` and `sources` entries. + +## 2026-06-30 + +- **Re-generated** `metrics/revenue.md`, `computations/revenue-ytd.md`, and `policies/revenue-recognition.md` after Finance published the FY2026 revenue recognition policy addendum. Updated `stale_after` on both revenue concepts to `2026-12-31T00:00:00Z`. + +## 2026-04-15 + +- **Deprecated** the legacy gross-margin definition. Original file moved to `metrics/gross-margin-legacy.md` with `status: deprecated`. New definition at `metrics/gross-margin.md` implements the FY2026 Cost Allocation Standard (includes fulfillment and shipping costs in COGS). + +## 2026-02-10 + +- **Bundle bootstrapped** by `reference_agent/gemini-2.5-pro` from the BigQuery `INFORMATION_SCHEMA` and a 90-day sample of `region-us.INFORMATION_SCHEMA.JOBS_BY_PROJECT`. Initial trust tier: machine-confirmed across the board; finance-critical concepts flagged for human review. diff --git a/tests/index/test_local_project_scan_parity.py b/tests/index/test_local_project_scan_parity.py index 17d7fe779..c64cada71 100644 --- a/tests/index/test_local_project_scan_parity.py +++ b/tests/index/test_local_project_scan_parity.py @@ -159,7 +159,8 @@ def test_local_project_index_file_paths_aborts_when_root_unreadable(tmp_path: Pa local_project_index_file_paths(missing_root, ignore_patterns=set()) -def test_local_project_index_file_paths_skips_symlinked_files(tmp_path: Path) -> None: +@pytest.mark.parametrize("strict", [False, True]) +def test_local_project_index_file_paths_skips_symlinked_files(tmp_path: Path, strict: bool) -> None: """Symlinked files must not be indexed (their target may be outside the project).""" project_root = (tmp_path / "project").resolve() project_root.mkdir() @@ -171,7 +172,21 @@ def test_local_project_index_file_paths_skips_symlinked_files(tmp_path: Path) -> except (OSError, NotImplementedError): pytest.skip("symlinks not supported on this platform") - assert local_project_index_file_paths(project_root, ignore_patterns=set()) == ("keep.md",) + assert scan_local_project_index_files( + project_root, ignore_patterns=set(), strict=strict + ).file_paths == ("keep.md",) + + +def test_strict_scan_rejects_partial_walk(tmp_path: Path, monkeypatch) -> None: + (tmp_path / "a.md").write_text("# A", encoding="utf-8") + + def partial_walk(*args, **kwargs): + yield str(tmp_path), [], ["a.md"] + raise PermissionError("walk failed after a file") + + monkeypatch.setattr(local_project.os, "walk", partial_walk) + with pytest.raises(PermissionError, match="walk failed after a file"): + scan_local_project_index_files(tmp_path, ignore_patterns=set(), strict=True) @pytest.mark.asyncio diff --git a/tests/okf/test_export_failures.py b/tests/okf/test_export_failures.py new file mode 100644 index 000000000..ee5267e82 --- /dev/null +++ b/tests/okf/test_export_failures.py @@ -0,0 +1,436 @@ +"""Publication fault injection and read-only journal boundaries.""" + +from datetime import UTC, datetime +from pathlib import Path + +import pytest + +from basic_memory import db +from basic_memory.config import BasicMemoryConfig, ProjectEntry +from basic_memory.models.project import AcceptedProjectNoteChange +from basic_memory.okf.export import export_project, recorded_history, snapshot_files +from basic_memory.okf.render import ExportFile, ExportSnapshot, render_bundle +from basic_memory.okf.validation import check_bundle + + +@pytest.fixture +def source_config(config_home): + root = config_home / "source" + root.mkdir() + (root / "a.md").write_text("---\ntype: note\n---\nA") + return BasicMemoryConfig(projects={"export": ProjectEntry(path=str(root))}) + + +@pytest.mark.asyncio +async def test_source_and_destination_changes_abort(source_config, tmp_path, monkeypatch): + import basic_memory.okf.export as exporting + + root = Path(source_config.projects["export"].path) + destination = tmp_path / "bundle" + original_check = exporting.check_bundle + + def mutate_source(staging): + (root / "a.md").write_text("changed") + return original_check(staging) + + monkeypatch.setattr(exporting, "check_bundle", mutate_source) + with pytest.raises(ValueError, match="Project changed"): + await export_project(source_config, "export", destination) + assert not destination.exists() + + def create_destination(staging): + destination.mkdir() + (destination / "keep").write_text("keep") + return original_check(staging) + + monkeypatch.setattr(exporting, "check_bundle", create_destination) + with pytest.raises(ValueError, match="appeared"): + await export_project(source_config, "export", destination) + assert (destination / "keep").read_text() == "keep" + + +@pytest.mark.asyncio +@pytest.mark.parametrize("fail_restore", [False, True]) +async def test_publish_failure_preserves_previous_bundle( + source_config, tmp_path, monkeypatch, fail_restore +): + destination = tmp_path / "bundle" + destination.mkdir() + (destination / "keep").write_text("keep") + rename = Path.rename + + def fail_publish(path, target): + if path.name == "bundle" and path != destination: + raise OSError("publish failed") + if fail_restore and ".bm-okf-backup-" in path.name: + raise OSError("restore failed") + return rename(path, target) + + monkeypatch.setattr(Path, "rename", fail_publish) + with pytest.raises(OSError, match="failed"): + await export_project(source_config, "export", destination, replace=True) + if fail_restore: + backups = list(tmp_path.glob(".bundle.bm-okf-backup-*")) + assert len(backups) == 1 + assert (backups[0] / "keep").read_text() == "keep" + else: + assert (destination / "keep").read_text() == "keep" + assert not list(tmp_path.glob(".bm-okf-*")) + + +@pytest.mark.asyncio +async def test_symlink_and_missing_source_are_rejected(source_config, tmp_path, monkeypatch): + root = Path(source_config.projects["export"].path) + destination = tmp_path / "bundle" + other = tmp_path / "other" + other.mkdir() + destination.symlink_to(other, target_is_directory=True) + with pytest.raises(ValueError, match="symlink"): + await export_project(source_config, "export", destination) + destination.unlink() + import basic_memory.okf.export as exporting + + original_check = exporting.check_bundle + + def create_symlink(staging): + destination.symlink_to(other, target_is_directory=True) + return original_check(staging) + + monkeypatch.setattr(exporting, "check_bundle", create_symlink) + with pytest.raises(ValueError, match="became a symlink"): + await export_project(source_config, "export", destination, replace=True) + (root / "a.md").unlink() + root.rmdir() + with pytest.raises(ValueError, match="does not exist"): + await export_project(source_config, "export", destination) + source_config.projects["export"].path = "relative" + with pytest.raises(ValueError, match="absolute"): + await export_project(source_config, "export", destination) + + +def test_unreadable_subtree_is_a_failure(tmp_path, monkeypatch): + import os + + def failed_walk(root, *, onerror, **kwargs): + onerror(PermissionError(13, "denied", str(root / "sub"))) + yield str(root), [], [] + + monkeypatch.setattr(os, "walk", failed_walk) + assert check_bundle(tmp_path).diagnostics[0].rule == "filesystem.read" + with pytest.raises(OSError, match="Incomplete project scan"): + snapshot_files(tmp_path) + + +@pytest.mark.parametrize("bm", ["broken", "{okf_export: {version: 1}}"]) +def test_extension_collision_is_not_overwritten(bm): + snapshot = ExportSnapshot("p", (ExportFile("a.md", f"---\nbm: {bm}\n---\n".encode()),)) + with pytest.raises(ValueError, match="extension collision"): + render_bundle(snapshot) + + +def test_ambiguous_alias_and_semantic_opt_out(): + snapshot = ExportSnapshot( + "p", + ( + ExportFile("one/a.md", b"---\ntitle: Same\n---\n"), + ExportFile("two/a.md", b"---\ntitle: Same\n---\n"), + ExportFile("source.md", b"---\nbm_parse_semantics: false\n---\n[[Same]]"), + ), + ) + output = {file.path: file.content for file in render_bundle(snapshot)} + assert b"[Same](/Same)" in output["source.md"] + assert b"relations: []" in output["source.md"] + assert b"[one](one/index.md)" in output["index.md"] + + +@pytest.mark.asyncio +async def test_journal_materialization_and_identity(app_config, test_project, engine_factory): + _, session_maker = engine_factory + root = Path(test_project.path).resolve() + assert await recorded_history(app_config, "missing", root) == () + with pytest.raises(ValueError, match="differs"): + await recorded_history(app_config, test_project.name, root / "wrong") + accepted_at = datetime(2026, 9, 14, tzinfo=UTC) + async with db.scoped_session(session_maker) as session: + project = await session.get(type(test_project), test_project.id) + assert project is not None + project.partition_position = 1 + session.add( + AcceptedProjectNoteChange( + project_id=project.id, + project_external_id=project.external_id, + partition_position=1, + entity_id=1, + note_external_id="note", + permalink="a", + title="A", + operation="create", + file_path="a.md", + accepted_at=accepted_at, + source="cli", + ) + ) + # Journal acceptance remains useful even when its materialization marker lags. + assert len(await recorded_history(app_config, test_project.name, root)) == 1 + async with db.scoped_session(session_maker) as session: + from sqlalchemy import select + + change = (await session.execute(select(AcceptedProjectNoteChange))).scalar_one() + change.materialized_at = accepted_at + history = await recorded_history(app_config, test_project.name, root) + assert len(history) == 1 + assert history[0].path == "a.md" + assert history[0].accepted_at.date() == accepted_at.date() + + +@pytest.mark.asyncio +@pytest.mark.parametrize("partial_journal", [False, True]) +async def test_pre_journal_database_exports_without_migration( + source_config, tmp_path, monkeypatch, partial_journal +): + from sqlalchemy import text + from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine + from basic_memory.config import APP_DATABASE_NAME, DatabaseBackend + + source_config.database_backend = DatabaseBackend.SQLITE + database_path = source_config.data_dir_path / APP_DATABASE_NAME + database_path.parent.mkdir(parents=True, exist_ok=True) + engine = create_async_engine(f"sqlite+aiosqlite:///{database_path}") + try: + async with engine.begin() as connection: + await connection.execute(text("CREATE TABLE project (id INTEGER PRIMARY KEY)")) + if partial_journal: + await connection.execute( + text("CREATE TABLE accepted_project_note_change (id INTEGER PRIMARY KEY)") + ) + + async def existing_db(**kwargs): + assert kwargs["ensure_migrations"] is False + return engine, async_sessionmaker(engine) + + monkeypatch.setattr(db, "get_or_create_db", existing_db) + before = database_path.read_bytes() + destination = tmp_path / "bundle" + report = await export_project(source_config, "export", destination) + assert report.success and report.concepts == 1 + assert database_path.read_bytes() == before + assert "\n## " not in (destination / "log.md").read_text() + finally: + await engine.dispose() + + +@pytest.mark.asyncio +async def test_history_reads_only_legacy_project_identity(source_config, monkeypatch): + from sqlalchemy import text + from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine + from basic_memory.config import APP_DATABASE_NAME, DatabaseBackend + + source_config.database_backend = DatabaseBackend.SQLITE + path = source_config.data_dir_path / APP_DATABASE_NAME + path.parent.mkdir(parents=True, exist_ok=True) + root = Path(source_config.projects["export"].path) + engine = create_async_engine(f"sqlite+aiosqlite:///{path}") + try: + async with engine.begin() as connection: + await connection.execute( + text( + "CREATE TABLE project (id INTEGER PRIMARY KEY, name TEXT, path TEXT, partition_position INTEGER)" + ) + ) + await connection.execute( + text("INSERT INTO project VALUES (1, 'export', :path, 1)"), {"path": str(root)} + ) + await connection.run_sync(AcceptedProjectNoteChange.metadata.create_all) + session_maker = async_sessionmaker(engine) + async with session_maker.begin() as session: + session.add( + AcceptedProjectNoteChange( + project_id=1, + project_external_id="p", + partition_position=1, + entity_id=1, + note_external_id="n", + permalink="a", + title="A", + operation="create", + file_path="a.md", + accepted_at=datetime(2026, 1, 1, tzinfo=UTC), + source="cli", + ) + ) + + async def existing_db(**kwargs): + return engine, session_maker + + monkeypatch.setattr(db, "get_or_create_db", existing_db) + before = path.read_bytes() + history = await recorded_history(source_config, "export", root) + assert len(history) == 1 and history[0].path == "a.md" + assert path.read_bytes() == before + finally: + await engine.dispose() + + +@pytest.mark.parametrize("directory", ["index.md", "log.md", "nested/Index.md"]) +def test_reserved_directory_is_rejected_before_staging(source_config, directory): + root = Path(source_config.projects["export"].path) + parent = root / directory + parent.mkdir(parents=True) + (parent / "a.md").write_text("---\ntype: note\n---\n# A") + with pytest.raises(ValueError, match="reserved OKF directory name; rename it first"): + snapshot_files(root) + + +@pytest.mark.asyncio +async def test_file_stat_failure_preserves_replacement_destination( + source_config, tmp_path, monkeypatch +): + root = Path(source_config.projects["export"].path) + source = root / "a.md" + destination = tmp_path / "bundle" + destination.mkdir() + (destination / "keep").write_bytes(b"previous bundle") + original_lstat = Path.lstat + + def failing_lstat(path, *args, **kwargs): + if path == source: + raise PermissionError(13, "stat unavailable", str(path)) + return original_lstat(path, *args, **kwargs) + + monkeypatch.setattr(Path, "lstat", failing_lstat) + with pytest.raises(PermissionError, match="stat unavailable"): + await export_project(source_config, "export", destination, replace=True) + assert (destination / "keep").read_bytes() == b"previous bundle" + + +@pytest.mark.asyncio +async def test_staging_path_collision_preserves_destination(source_config, tmp_path, monkeypatch): + import basic_memory.okf.export as exporting + + destination = tmp_path / "bundle" + destination.mkdir() + (destination / "keep").write_bytes(b"previous bundle") + original_open = Path.open + + def case_insensitive_staging_open(path, mode="r", *args, **kwargs): + if mode == "xb": + path = path.with_name(path.name.lower()) + return original_open(path, mode, *args, **kwargs) + + monkeypatch.setattr(Path, "open", case_insensitive_staging_open) + monkeypatch.setattr( + exporting, + "render_bundle", + lambda snapshot: (ExportFile("A.md", b"first"), ExportFile("a.md", b"second")), + ) + with pytest.raises(FileExistsError): + await export_project(source_config, "export", destination, replace=True) + assert (destination / "keep").read_bytes() == b"previous bundle" + + +@pytest.mark.asyncio +@pytest.mark.parametrize("relation", ["same", "child", "parent"]) +async def test_filesystem_identical_source_containment_is_rejected( + source_config, tmp_path, monkeypatch, relation +): + root = Path(source_config.projects["export"].path).resolve() + alias = root.with_name(root.name.upper()) + destination = {"same": alias, "child": alias / "new" / "bundle", "parent": alias.parent}[ + relation + ] + if relation == "parent": + destination = root.parent.with_name(root.parent.name.upper()) + alias = destination + actual = root.parent + else: + actual = root + original_stat = Path.stat + + def case_insensitive_stat(path, *args, **kwargs): + if path.is_relative_to(alias): + path = actual / path.relative_to(alias) + return original_stat(path, *args, **kwargs) + + def case_insensitive_exists(path): + try: + case_insensitive_stat(path) + except FileNotFoundError: + return False + return True + + monkeypatch.setattr(Path, "stat", case_insensitive_stat) + # Python 3.14 exists() uses os.path.exists directly, bypassing Path.stat. + monkeypatch.setattr(Path, "exists", case_insensitive_exists) + before = (root / "a.md").read_bytes() + with pytest.raises(ValueError, match="Destination must be outside"): + await export_project(source_config, "export", destination, replace=True) + assert (root / "a.md").read_bytes() == before + assert not list(tmp_path.rglob("*.bm-okf-backup-*")) + + +@pytest.mark.asyncio +async def test_exact_project_name_precedes_normalized_alias(source_config, tmp_path): + other = tmp_path / "other-project" + other.mkdir() + (other / "chosen.md").write_text("# Exact project", encoding="utf-8") + source_config.projects = { + "my-project": source_config.projects["export"], + "My Project": ProjectEntry(path=str(other)), + } + destination = tmp_path / "bundle" + report = await export_project(source_config, "My Project", destination) + assert report.success and report.concepts == 1 + assert (destination / "chosen.md").is_file() + assert not (destination / "a.md").exists() + + +@pytest.mark.asyncio +@pytest.mark.parametrize("ignore_name", [".gitignore", ".bmignore"]) +async def test_unreadable_ignore_file_aborts_publication( + source_config, tmp_path, monkeypatch, ignore_name +): + from basic_memory.ignore_utils import get_bmignore_path + + root = Path(source_config.projects["export"].path) + ignore = root / ignore_name if ignore_name == ".gitignore" else get_bmignore_path() + ignore.parent.mkdir(parents=True, exist_ok=True) + ignore.write_text("credentials.json\n", encoding="utf-8") + (root / "credentials.json").write_text("excluded bytes", encoding="utf-8") + destination = tmp_path / "bundle" + destination.mkdir() + (destination / "keep").write_bytes(b"previous bundle") + original_read = Path.read_text + + def denied_read(path, *args, **kwargs): + if path == ignore: + raise PermissionError(13, "ignore rules unreadable", str(path)) + return original_read(path, *args, **kwargs) + + monkeypatch.setattr(Path, "read_text", denied_read) + with pytest.raises(PermissionError, match="ignore rules unreadable"): + await export_project(source_config, "export", destination, replace=True) + assert (destination / "keep").read_bytes() == b"previous bundle" + monkeypatch.setattr(Path, "read_text", original_read) + assert (await export_project(source_config, "export", destination, replace=True)).success + assert not (destination / "credentials.json").exists() + + +@pytest.mark.parametrize("empty_bmignore", [False, True]) +def test_strict_ignore_defaults_do_not_create_files(tmp_path, monkeypatch, empty_bmignore): + from basic_memory.ignore_utils import ( + DEFAULT_IGNORE_PATTERNS, + create_default_bmignore, + load_gitignore_patterns, + ) + + bmignore = tmp_path / ".bmignore" + monkeypatch.setattr("basic_memory.ignore_utils.get_bmignore_path", lambda: bmignore) + if empty_bmignore: + bmignore.write_text("# no custom rules\n", encoding="utf-8") + assert load_gitignore_patterns(tmp_path, use_gitignore=False, strict=True) == ( + DEFAULT_IGNORE_PATTERNS + ) + assert bmignore.exists() is empty_bmignore + if empty_bmignore: + create_default_bmignore() + assert bmignore.read_text(encoding="utf-8") == "# no custom rules\n" diff --git a/tests/okf/test_okf.py b/tests/okf/test_okf.py new file mode 100644 index 000000000..525dcf293 --- /dev/null +++ b/tests/okf/test_okf.py @@ -0,0 +1,931 @@ +"""Structural rules, source-preserving rendering, and staged export failures.""" + +from datetime import UTC, datetime +import json +from pathlib import Path + +import pytest +from typer.testing import CliRunner + +from basic_memory.cli.main import app +from basic_memory.cli.container import CliContainer, set_container +from basic_memory.config import BasicMemoryConfig, ProjectEntry +from basic_memory.okf.export import export_project, snapshot_files +from basic_memory.okf.render import ( + ExportFile, + ExportSnapshot, + RecordedChange, + convert_wikilinks, + render_bundle, +) +from basic_memory.okf.validation import check_bundle, check_document, parse_document +from basic_memory.runtime.mode import RuntimeMode + + +@pytest.mark.parametrize( + "path,content,rule", + [ + ("a.md", "body", "concept.frontmatter"), + ("a.md", "---\ntype: [\n---\n", "frontmatter"), + ("a.md", "---\ntype: note\n", "frontmatter"), + ("a.md", "---\n- note\n---\n", "frontmatter"), + ("a.md", "---\ntype: ' '\n---\n", "concept.type"), + ("a.md", "---\ntype: 2\n---\n", "concept.type"), + ("index.md", "---\ntype: note\n---\n# Index", "reserved.frontmatter"), + ("sub/index.md", "---\nokf_version: '0.2'\n---\n# Index", "reserved.frontmatter"), + ("log.md", "---\nokf_version: '0.2'\n---\n# Log", "reserved.frontmatter"), + ("index.md", "- [[Target]]", "index.link"), + ("index.md", "[Target](target.md)", "index.heading"), + ("log.md", "## Yesterday", "log.date"), + ("log.md", "## 2026-02-31", "log.date"), + ("log.md", "### 2026-01-01", "log.date"), + ("log.md", "## 2026-01-01\n## 2026-02-01", "log.order"), + ], +) +def test_structural_diagnostics(path, content, rule): + diagnostics = check_document(path, content) + assert any(item.path == path and item.rule == rule for item in diagnostics) + + +@pytest.mark.parametrize( + "path,content", + [ + ( + "odd.md", + "---\ntype: Unregistered\nunknown: [1, true]\nverified: {by: human:a}\n---\n[missing](/gone.md)", + ), + ("index.md", "---\nokf_version: '9.9'\n---\n# Index\n- [Missing](gone/)"), + ("sub/index.md", "# Index\n- [Missing](gone.md)\n\nOrdinary prose."), + ("log.md", "# Log\n## 2026-02-01\n- Added.\n## 2026-01-01\n- Started."), + ("log.md", "# Log\nNo history recorded."), + ], +) +def test_soft_guidance_is_not_rejected(path, content): + assert check_document(path, content) == [] + + +def test_filesystem_check_includes_hidden_files_and_assets(tmp_path): + (tmp_path / "a.md").write_text("---\ntype: note\n---\n") + (tmp_path / "asset.pdf").write_bytes(b"%PDF-1.4") + (tmp_path / ".hidden.md").write_bytes(b"\xff") + (tmp_path / "link.md").symlink_to(tmp_path / "a.md") + report = check_bundle(tmp_path) + assert report.concepts == 2 + assert {item.rule for item in report.diagnostics} == {"filesystem.read", "filesystem.symlink"} + assert not check_bundle(tmp_path / "absent").success + + +@pytest.mark.parametrize( + "source,expected", + [ + ("See [[A|alias]] and [[missing]].", "See [alias](/a.md) and [missing](/missing)."), + ("[[../a#heading]]", "[../a](/a.md#heading)"), + ("[[/a]]", "[/a](/a)"), + ("[[#here]]", "[here](/folder/source.md#here)"), + ("[[broken", "[[broken"), + (r"\[[A]]", r"\[[A]]"), + ("`[[A]]`\n\n```md\n[[A]]\n```\n\n[[A]]", "`[[A]]`\n\n```md\n[[A]]\n```\n\n[A](/a.md)"), + (" [[A]]\n\n[[A]]", " [[A]]\n\n[A](/a.md)"), + ("[already](a.md)", "[already](a.md)"), + ("> [[A]]\n- [[A]]", "> [A](/a.md)\n- [A](/a.md)"), + ], +) +def test_link_conversion(source, expected): + assert ( + convert_wikilinks(source, "folder/source.md", {"A": "a.md", "a.md": "a.md"}, "test") + == expected + ) + + +def test_render_preserves_frontmatter_and_semantics(): + source = "---\ntitle: A\npermalink: a\ntype: custom\ntags: [one]\nbm: {other: true}\nsources: [{resource: /paper.pdf}]\n---\n# A\n- [fact] Categorized claim #tag (why)\n- depends_on [[B]] (because)\n" + snapshot = ExportSnapshot( + "project", + ( + ExportFile("a.md", source.encode()), + ExportFile("b.md", b"---\ntitle: B\n---\n# B\n"), + ExportFile("paper.pdf", b"%PDF-1.4"), + ), + (RecordedChange(1, "a.md", "create", datetime(2026, 9, 14, tzinfo=UTC)),), + ) + output = {file.path: file.content for file in render_bundle(snapshot)} + doc = parse_document(output["a.md"].decode()) + assert doc.metadata["type"] == "custom" + assert doc.metadata["tags"] == ["one"] + assert doc.metadata["sources"] == [{"resource": "/paper.pdf"}] + assert doc.metadata["bm"] == { + "other": True, + "okf_export": { + "version": 1, + "relations": [{"type": "depends_on", "target": "B", "context": "because"}], + }, + } + assert "- [fact] Categorized claim #tag (why)" in doc.body + assert "- depends_on [B](/b.md) (because)" in doc.body + assert output["paper.pdf"] == b"%PDF-1.4" + assert "## 2026-09-14" in output["log.md"].decode() + assert not output["log.md"].startswith(b"---") + assert "[[" not in output["index.md"].decode() + assert render_bundle(snapshot) == render_bundle(snapshot) + + +@pytest.fixture +def export_config(config_home): + root = config_home / "source" + root.mkdir() + (root / "a.md").write_text("---\ntype: note\n---\n# A\n") + return BasicMemoryConfig(projects={"export": ProjectEntry(path=str(root))}) + + +@pytest.mark.asyncio +async def test_export_replace_and_source_safety(export_config, tmp_path): + root = Path(export_config.projects["export"].path) + original = (root / "a.md").read_bytes() + destination = tmp_path / "bundle" + assert (await export_project(export_config, "export", destination)).success + with pytest.raises(ValueError, match="Destination exists"): + await export_project(export_config, "export", destination) + (destination / "old.pdf").write_bytes(b"old") + assert (await export_project(export_config, "export", destination, replace=True)).success + assert not (destination / "old.pdf").exists() + assert (root / "a.md").read_bytes() == original + for unsafe in (root, root / "bundle", root.parent): + with pytest.raises(ValueError, match="outside"): + await export_project(export_config, "export", unsafe, replace=True) + with pytest.raises(ValueError, match="configured local"): + await export_project(export_config, "absent", destination) + + +@pytest.mark.asyncio +async def test_failed_validation_leaves_destination_intact(export_config, tmp_path): + root = Path(export_config.projects["export"].path) + (root / "a.md").write_text("---\ntype: ''\n---\n") + destination = tmp_path / "bundle" + destination.mkdir() + (destination / "keep").write_bytes(b"keep") + report = await export_project(export_config, "export", destination, replace=True) + assert not report.success + assert (destination / "keep").read_bytes() == b"keep" + assert not list(tmp_path.glob(".bm-okf-*")) + + +def test_reserved_source_and_ignore_rules(export_config): + root = Path(export_config.projects["export"].path) + (root / "index.md").write_text( + "---\nbm: {profile: wiki/1}\ngenerated: {by: Basic Memory Wiki Projector}\n---\n[[Live Wiki]]" + ) + (root / "log.md").write_text( + "---\nbm: {profile: wiki/1}\ngenerated: {by: Basic Memory Wiki Projector}\n---\n# old log" + ) + (root / ".secret").write_text("secret") + (root / "paper.pdf").write_bytes(b"pdf") + assert {file.path for file in snapshot_files(root)} == {"a.md", "paper.pdf"} + (root / "index.md").write_text("---\ntype: note\n---\nUser concept") + with pytest.raises(ValueError, match="rename it first"): + snapshot_files(root) + + +def test_cli_check_without_config(tmp_path, monkeypatch): + monkeypatch.setattr( + "basic_memory.cli.app.CliContainer.create", lambda: pytest.fail("config read") + ) + (tmp_path / "a.md").write_text("---\ntype: note\n---\n") + runner = CliRunner() + result = runner.invoke(app, ["okf", "check", str(tmp_path), "--json"]) + assert result.exit_code == 0, result.output + assert json.loads(result.stdout)["concepts"] == 1 + (tmp_path / "a.md").write_text("broken") + result = runner.invoke(app, ["okf", "check", str(tmp_path)]) + assert result.exit_code == 1 + assert "a.md: concept.frontmatter" in result.stdout + + +def test_cli_export(export_config, tmp_path): + set_container(CliContainer(export_config, RuntimeMode.TEST)) + runner = CliRunner() + result = runner.invoke( + app, ["okf", "export", str(tmp_path / "bundle"), "--project", "export", "--json"] + ) + assert result.exit_code == 0, result.output + assert json.loads(result.stdout)["concepts"] == 1 + result = runner.invoke( + app, ["okf", "export", str(tmp_path / "bundle"), "--project", "export", "--json"] + ) + assert result.exit_code == 1 + assert json.loads(result.stdout)["diagnostics"][0]["rule"] == "export" + + +@pytest.mark.parametrize( + "body,expected", + [ + ("[outer [[A]]](url)", "[outer [[A]]](url)"), + ("`open\n\n[[A]]\n\nclose`", "`open\n\n[A](/a.md)\n\nclose`"), + ("[[A]]\r\n[[A]]", "[A](/a.md)\n[A](/a.md)"), + ("[[outer [[inner]]]]", r"[outer \[\[inner\]\]](/outer%20%5B%5Binner%5D%5D)"), + ("- continuation\n [[A]]", "- continuation\n [A](/a.md)"), + ], +) +def test_conversion_respects_inline_block_boundaries(body, expected): + assert convert_wikilinks(body, "source.md", {"A": "a.md"}, "p") == expected + + +def test_nul_normalization_preserves_raw_source_and_link_offsets(): + assert convert_wikilinks("before\0 [[A]]", "source.md", {"A": "a.md"}, "p") == ( + "before\0 [A](/a.md)" + ) + + +def test_unrecognized_parser_normalization_fails_without_rewriting_wrong_span(monkeypatch): + from markdown_it import MarkdownIt + + original_parse = MarkdownIt.parse + + def altered_parse(parser, source, env=None): + tokens = original_parse(parser, source, env) + for token in tokens: + if token.type == "inline": + token.content = "unmappable " + token.content + return tokens + + monkeypatch.setattr(MarkdownIt, "parse", altered_parse) + with pytest.raises(ValueError, match="cannot locate wikilink source span"): + convert_wikilinks("[[A]]", "source.md", {"A": "a.md"}, "p") + + +def test_asset_links_and_reserved_casing(export_config): + root = Path(export_config.projects["export"].path) + (root / "Index.md").write_text("---\ntype: note\n---\n") + with pytest.raises(ValueError, match="casing collides"): + snapshot_files(root) + files = render_bundle( + ExportSnapshot( + "p", + ( + ExportFile("refs/paper.pdf", b"pdf"), + ExportFile("notes/source.md", b"---\ntype: note\n---\n[[../refs/paper.pdf]]"), + ), + ) + ) + assert b"[../refs/paper.pdf](/refs/paper.pdf)" in next( + file.content for file in files if file.path == "notes/source.md" + ) + + +def test_log_entries_require_date_group(): + assert check_document("log.md", "# Log\n- Undated entry")[0].rule == "log.group" + + +def test_source_relative_path_precedes_root_path(): + targets = {"nested/a.md": "nested/a.md", "folder/nested/a.md": "folder/nested/a.md"} + assert ( + convert_wikilinks("[[nested/a]]", "folder/source.md", targets, "p") + == "[nested/a](/folder/nested/a.md)" + ) + + +def test_filename_default_title_resolves_nested_note(): + files = render_bundle( + ExportSnapshot( + "p", + ( + ExportFile("guides/Guide.md", b"---\ntype: note\n---\n# Guide"), + ExportFile("source.md", b"[[Guide]]"), + ), + ) + ) + source = next(file.content for file in files if file.path == "source.md") + assert b"[Guide](/guides/Guide.md)" in source + + +def test_timestamp_and_fence_whitespace_survive_export(): + content = ( + "--- \t\ntype: note\ntitle: A\npermalink: a\ntags: [tag]\n" + "stale_after: 2026-06-30T14:00:00Z\ncreated: 2026-06-01\n---\t\n# A\n" + ) + assert check_document("a.md", content) == [] + expected = parse_document(content) + assert expected.metadata["stale_after"] == "2026-06-30T14:00:00Z" + assert expected.metadata["created"] == "2026-06-01" + files = render_bundle(ExportSnapshot("p", (ExportFile("a.md", content.encode()),))) + document = parse_document(next(file.content.decode() for file in files if file.path == "a.md")) + assert document.body == expected.body + assert {key: document.metadata[key] for key in expected.metadata} == expected.metadata + + +@pytest.mark.parametrize("name", ["index.md", "log.md"]) +def test_unmarked_reserved_notes_are_not_discarded(export_config, name): + root = Path(export_config.projects["export"].path) + path = root / name + path.write_text("# My authored note\nImportant facts") + with pytest.raises(ValueError, match="rename it first"): + snapshot_files(root) + assert path.read_text() == "# My authored note\nImportant facts" + + +def test_check_rejects_fifo_without_opening_it(tmp_path, monkeypatch): + import os + + if os.name == "nt": + pytest.skip("Windows does not support POSIX FIFOs") + path = tmp_path / "blocked.md" + os.mkfifo(path) + monkeypatch.setattr(Path, "read_text", lambda *args, **kwargs: pytest.fail("opened FIFO")) + report = check_bundle(tmp_path) + assert report.diagnostics[0].path == "blocked.md" + assert report.diagnostics[0].rule == "filesystem.regular_file" + + +def test_export_installs_event_loop_policy_before_async_work(export_config, tmp_path, monkeypatch): + import basic_memory.cli.commands.command_utils as command_utils + + set_container(CliContainer(export_config, RuntimeMode.TEST)) + events = [] + run_with_cleanup = command_utils.run_with_cleanup + + def install(config): + assert config is export_config + events.append("policy") + + def run(coroutine): + assert events == ["policy"] + return run_with_cleanup(coroutine) + + monkeypatch.setattr("basic_memory.db.maybe_install_uvloop", install) + monkeypatch.setattr(command_utils, "run_with_cleanup", run) + result = CliRunner().invoke( + app, ["okf", "export", str(tmp_path / "bundle"), "--project", "export"] + ) + assert result.exit_code == 0, result.output + assert events == ["policy"] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("suffix", [".markdown", ".MD", ".Markdown"]) +async def test_export_rejects_non_okf_markdown_suffix(export_config, tmp_path, suffix): + source = Path(export_config.projects["export"].path) / ("note" + suffix) + source.write_text("---\ntype: note\n---\n[[A]]") + destination = tmp_path / "bundle" + with pytest.raises(ValueError, match="lowercase .md suffix"): + await export_project(export_config, "export", destination) + assert not destination.exists() + assert source.read_text() == "---\ntype: note\n---\n[[A]]" + + +def test_index_entry_link_applies_to_whole_item(): + assert check_document("index.md", "# Index\n- [A](a.md)\n\n A description.") == [] + assert check_document("index.md", "# Index\n- Group\n - [A](a.md)\n\n Description.") == [] + diagnostics = check_document("index.md", "# Index\n- [A](a.md)\n - Missing link") + assert [item.rule for item in diagnostics] == ["index.link"] + + +def test_log_heading_ends_date_group(): + text = "## 2026-01-01\n- Recorded\n# Appendix\n- Undated" + assert [item.rule for item in check_document("log.md", text)] == ["log.group"] + + +@pytest.mark.parametrize("prefix", ["\ufeff", "\n \t\n", "\ufeff\n\n"]) +def test_source_frontmatter_prefix_preserves_metadata(prefix): + content = prefix + "---\ntitle: A\npermalink: a\ntags: [tag]\ntype: custom\n---\n# A" + files = render_bundle(ExportSnapshot("p", (ExportFile("a.md", content.encode()),))) + document = parse_document(next(file.content.decode() for file in files if file.path == "a.md")) + assert document.metadata["title"] == "A" + assert document.metadata["permalink"] == "a" + assert document.metadata["tags"] == ["tag"] + assert document.metadata["type"] == "custom" + assert document.body == "# A" + assert check_document("a.md", content)[0].rule == "concept.frontmatter" + + +def test_empty_frontmatter_gets_export_defaults(): + files = render_bundle(ExportSnapshot("p", (ExportFile("a.md", b"---\n---\n# A"),))) + document = parse_document(next(file.content.decode() for file in files if file.path == "a.md")) + assert document.metadata["type"] == "note" + assert document.metadata["tags"] == [] + assert document.body == "# A" + assert check_document("a.md", "---\n---\n# A")[0].rule == "concept.type" + + +@pytest.mark.parametrize("setting", ['"False"', '"FALSE"', '"fAlSe"']) +def test_mixed_case_semantic_opt_out(setting): + content = f"---\nbm_parse_semantics: {setting}\n---\n- depends_on [[A]]" + files = render_bundle(ExportSnapshot("p", (ExportFile("a.md", content.encode()),))) + document = parse_document(next(file.content.decode() for file in files if file.path == "a.md")) + assert document.metadata["bm"] == {"okf_export": {"version": 1, "relations": []}} + + +def test_indented_source_fence_is_body_not_metadata(): + source = " ---\ntype: custom\ntitle: Authored text\n---\n# Body" + files = render_bundle(ExportSnapshot("p", (ExportFile("a.md", source.encode()),))) + document = parse_document(next(file.content.decode() for file in files if file.path == "a.md")) + assert document.metadata["type"] == "note" + assert "title" not in document.metadata + assert document.body == source + assert check_document("a.md", source) == [] + unmatched = "---\ntype: custom\n ---\nBody" + assert parse_document(unmatched, source=True).body == unmatched + + +def test_unique_filename_alias_follows_exact_identity(): + targets = {"My_Note.md": "My_Note.md"} + assert convert_wikilinks("[[my-note]]", "source.md", targets, "p") == "[my-note](/My_Note.md)" + targets["my-note"] = "specific.md" + assert convert_wikilinks("[[my-note]]", "source.md", targets, "p") == "[my-note](/specific.md)" + targets.pop("my-note") + targets["MY-NOTE.md"] = "MY-NOTE.md" + assert convert_wikilinks("[[my-note]]", "source.md", targets, "p") == "[my-note](/my-note)" + targets["folder/My_Note.md"] = "folder/My_Note.md" + assert ( + convert_wikilinks("[[./my-note.md]]", "folder/source.md", targets, "p") + == "[./my-note.md](/folder/My_Note.md)" + ) + + +@pytest.mark.parametrize("source", ["---\n# Thematic break"]) +def test_non_frontmatter_source_blocks_remain_body(source): + files = render_bundle(ExportSnapshot("p", (ExportFile("a.md", source.encode()),))) + exported = next(file.content.decode() for file in files if file.path == "a.md") + document = parse_document(exported) + assert document.metadata["type"] == "note" + assert document.body == source + assert check_document("a.md", exported) == [] + assert check_document("a.md", source)[0].rule == "frontmatter" + + +@pytest.mark.parametrize("prefix", ["my-project", "My Project"]) +def test_project_permalink_prefix_is_not_source_relative(prefix): + targets = { + "my-project/foo": "foo.md", + "folder/my-project/foo.md": "folder/my-project/foo.md", + } + assert ( + convert_wikilinks( + f"[[{prefix}/foo]]", + "folder/source.md", + targets, + "My Project", + permalinks={"my-project/foo": "foo.md"}, + ) + == f"[{prefix}/foo](/foo.md)" + ) + + +def test_cli_export_resolves_configured_display_name(export_config, tmp_path): + export_config.projects = {"My Project": export_config.projects["export"]} + set_container(CliContainer(export_config, RuntimeMode.TEST)) + destination = tmp_path / "bundle" + result = CliRunner().invoke( + app, ["okf", "export", str(destination), "--project", "my-project", "--json"] + ) + assert result.exit_code == 0, result.output + assert json.loads(result.stdout)["concepts"] == 1 + assert "# My Project" in (destination / "index.md").read_text() + + +@pytest.mark.parametrize( + "body", ["- item\n\t[[A]]", "> quote\n>\t[[A]]", "- item\n\t[[A]] and [[A]]"] +) +def test_tab_indented_links_keep_source_indentation(body): + assert convert_wikilinks(body, "source.md", {"A": "a.md"}, "p") == body.replace( + "[[A]]", "[A](/a.md)" + ) + + +@pytest.mark.parametrize("path", ["log.md", "nested/index.md"]) +def test_okf_version_only_exempts_root_index(export_config, path): + root = Path(export_config.projects["export"].path) + target = root / path + target.parent.mkdir(exist_ok=True) + target.write_text("---\nokf_version: '0.2'\n---\nAuthored content") + with pytest.raises(ValueError, match="rename it first"): + snapshot_files(root) + + +@pytest.mark.asyncio +async def test_disabled_project_prefix_policy_reaches_export(export_config, tmp_path): + root = Path(export_config.projects["export"].path) + (root / "folder/export").mkdir(parents=True) + (root / "folder/export/foo.md").write_text("---\ntype: note\n---\nRelative") + (root / "a.md").write_text("---\npermalink: export/foo\n---\nSemantic") + (root / "folder/source.md").write_text("[[export/foo]]") + export_config.permalinks_include_project = False + destination = tmp_path / "bundle" + assert (await export_project(export_config, "export", destination)).success + assert "[export/foo](/folder/export/foo.md)" in (destination / "folder/source.md").read_text() + + +def test_ambiguous_bare_title_does_not_fall_through_to_filename(): + snapshot = ExportSnapshot( + "p", + ( + ExportFile("Same.md", b"---\ntitle: Same\n---\n"), + ExportFile("other.md", b"---\ntitle: Same\npermalink: same\n---\n"), + ExportFile("source.md", b"[[Same]] and [[Same.md]] and [[./Same.md]]"), + ), + ) + source = next(file.content for file in render_bundle(snapshot) if file.path == "source.md") + assert b"[Same](/Same) and [Same.md](/other.md) and [./Same.md](/Same.md)" in source + + +@pytest.mark.parametrize( + "yaml_title,target", + [ + ("123", "123"), + ("[My, Note]", "My, Note"), + ("false", "False"), + ("2026-01-01T00:00:00Z", "2026-01-01T00:00:00+00:00"), + ], +) +def test_source_title_normalization_preserves_authored_metadata(yaml_title, target): + authored = f"---\ntitle: {yaml_title}\n---\n# Note" + snapshot = ExportSnapshot( + "p", + ( + ExportFile("note.md", authored.encode()), + ExportFile("source.md", f"[[{target}]]".encode()), + ), + ) + files = {file.path: file.content.decode() for file in render_bundle(snapshot)} + assert f"[{target}](/note.md)" in files["source.md"] + assert ( + parse_document(files["note.md"]).metadata["title"] + == parse_document(authored).metadata["title"] + ) + assert f"[{target}](note.md)" in files["index.md"] + + +@pytest.mark.parametrize("separator", ["\u2028", "\u2029", "\f"]) +def test_unicode_separators_are_not_markdown_line_boundaries(separator): + body = f"Prose{separator}[[A]] and [[A]]" + assert convert_wikilinks(body, "source.md", {"A": "a.md"}, "p") == body.replace( + "[[A]]", "[A](/a.md)" + ) + + +@pytest.mark.parametrize( + "yaml_type,expected", + [("123", "123"), ("false", "False"), ("[My, Type]", "My, Type"), ("null", "note")], +) +def test_export_uses_canonical_bm_type(yaml_type, expected): + snapshot = ExportSnapshot( + "p", (ExportFile("a.md", f"---\ntype: {yaml_type}\n---\nBody".encode()),) + ) + content = next(file.content.decode() for file in render_bundle(snapshot) if file.path == "a.md") + assert parse_document(content).metadata["type"] == expected + assert check_document("a.md", content) == [] + + +@pytest.mark.parametrize("name", ["index.md", "log.md"]) +def test_wiki_profile_alone_does_not_establish_ownership(export_config, name): + root = Path(export_config.projects["export"].path) + source = root / name + source.write_text("---\nbm: {profile: wiki/1}\n---\nAuthored body") + with pytest.raises(ValueError, match="rename it first"): + snapshot_files(root) + assert "Authored body" in source.read_text() + + +def test_bare_filename_alias_does_not_prefer_source_directory(): + targets = {"folder/My_Note.md": "folder/My_Note.md", "other/My_Note.md": "other/My_Note.md"} + assert ( + convert_wikilinks("[[my-note]]", "folder/source.md", targets, "p") == "[my-note](/my-note)" + ) + assert ( + convert_wikilinks("[[./my-note]]", "folder/source.md", targets, "p") + == "[./my-note](/folder/My_Note.md)" + ) + + +@pytest.mark.parametrize( + "label", + ["![alt [[A]]](img)", "[caption [[A]]][ref]", "![alt [[A]]][ref]"], +) +def test_image_and_reference_labels_keep_literal_wikilinks(label): + body = label + " and [[A]]\n\n[ref]: /existing.md" + expected = label + " and [A](/a.md)\n\n[ref]: /existing.md" + assert convert_wikilinks(body, "source.md", {"A": "a.md"}, "p") == expected + + +def test_relative_wikilink_percent_sequences_are_literal(): + targets = {path: path for path in ("folder/sub/A%20B.md", "folder/sub/A B.md")} + assert ( + convert_wikilinks("[[sub/A%20B.md]]", "folder/source.md", targets, "p") + == "[sub/A%20B.md](/folder/sub/A%2520B.md)" + ) + + +@pytest.mark.parametrize("yaml_permalink,target", [("123", "123"), ("false", "False")]) +def test_scalar_permalink_aliases_preserve_authored_metadata(yaml_permalink, target): + authored = f"---\npermalink: {yaml_permalink}\n---\n# Note" + snapshot = ExportSnapshot( + "p", + ( + ExportFile("note.md", authored.encode()), + ExportFile("source.md", f"[[{target}]]".encode()), + ), + ) + files = {file.path: file.content.decode() for file in render_bundle(snapshot)} + assert f"[{target}](/note.md)" in files["source.md"] + assert ( + parse_document(files["note.md"]).metadata["permalink"] + == parse_document(authored).metadata["permalink"] + ) + + +@pytest.mark.parametrize( + "body,target,expected", + [ + (r"[[A\]]B]]", r"A\]]B", r"[A\\\]\]B](/note.md)"), + (r"[[A\[[B]]", r"A\[[B", r"[A\\\[\[B](/note.md)"), + (r"[[A\\]] tail", r"A\\", r"[A\\\\](/note.md) tail"), + ], +) +def test_wikilink_delimiters_use_canonical_escape_rules(body, target, expected): + assert convert_wikilinks(body, "source.md", {target: "note.md"}, "p") == expected + + +@pytest.mark.parametrize("permalink", ["foo.md", "foo"]) +def test_permalink_precedes_file_alias_but_explicit_relative_path_stays_a_path( + permalink, test_project +): + from basic_memory.models import Entity + from basic_memory.services.bulk_link_resolver import ProjectEntityIdentityIndex + + owner = Entity(title="Owner", file_path="owner.md", permalink=permalink) + index = ProjectEntityIdentityIndex.from_entities( + test_project, [Entity(title="foo", file_path="foo.md"), owner] + ) + assert ( + index.resolve_strict( + "foo.md", include_project_permalinks=True, workspace_permalink=None + ).entity + is owner + ) + snapshot = ExportSnapshot( + "p", + ( + ExportFile("foo.md", b"# File"), + ExportFile("owner.md", f"---\npermalink: {permalink}\n---\n# Permalink owner".encode()), + ExportFile("source.md", b"[[foo.md]] and [[./foo.md]]"), + ), + ) + source = next(file.content for file in render_bundle(snapshot) if file.path == "source.md") + assert b"[foo.md](/owner.md) and [./foo.md](/foo.md)" in source + + +@pytest.mark.asyncio +async def test_duplicate_offline_permalinks_fail_before_replacing_bundle(export_config, tmp_path): + root = Path(export_config.projects["export"].path) + for name in ("a.md", "b.md"): + (root / name).write_text("---\npermalink: same\n---\n[[same]]") + destination = tmp_path / "bundle" + destination.mkdir() + (destination / "keep").write_bytes(b"previous bundle") + with pytest.raises(ValueError, match="b.md: duplicate permalink 'same' also declared by a.md"): + await export_project(export_config, "export", destination, replace=True) + assert (destination / "keep").read_bytes() == b"previous bundle" + + +def test_unique_title_precedes_filename_and_rooted_links_remain_literal(test_project): + from basic_memory.models import Entity + from basic_memory.services.bulk_link_resolver import ProjectEntityIdentityIndex + + owner = Entity(title="foo.md", file_path="owner.md") + index = ProjectEntityIdentityIndex.from_entities( + test_project, [Entity(title="foo", file_path="foo.md", permalink="custom"), owner] + ) + assert ( + index.resolve_strict( + "foo.md", include_project_permalinks=True, workspace_permalink=None + ).entity + is owner + ) + snapshot = ExportSnapshot( + "p", + ( + ExportFile("foo.md", b"---\npermalink: custom\n---\n# File"), + ExportFile("owner.md", b"---\ntitle: foo.md\n---\n# Title owner"), + ExportFile("source.md", b"[[foo.md]] [[/foo]] [[/foo.md]]"), + ), + ) + source = next(file.content for file in render_bundle(snapshot) if file.path == "source.md") + assert b"[foo.md](/owner.md) [/foo](/foo) [/foo.md](/foo.md)" in source + + +def test_permalink_compatibility_candidates_are_not_filename_aliases(test_project): + from basic_memory.models import Entity + from basic_memory.services.bulk_link_resolver import ProjectEntityIdentityIndex + + test_project.name = "p" + index = ProjectEntityIdentityIndex.from_entities( + test_project, [Entity(title="foo", file_path="foo.md", permalink="custom")] + ) + assert ( + index.resolve_strict( + "p/foo", include_project_permalinks=True, workspace_permalink=None + ).entity + is None + ) + snapshot = ExportSnapshot( + "p", + ( + ExportFile("foo.md", b"---\npermalink: custom\n---\n# File"), + ExportFile("source.md", b"[[p/foo]] and [[foo]]"), + ), + ) + source = next(file.content for file in render_bundle(snapshot) if file.path == "source.md") + assert b"[p/foo](/p/foo) and [foo](/foo.md)" in source + + +def test_escaping_wikilink_stays_literal_instead_of_normalizing_to_a_real_note(): + body = "[[../../a.md]] and [[../a.md]]" + assert convert_wikilinks(body, "folder/source.md", {"a.md": "a.md"}, "p") == ( + "[[../../a.md]] and [../a.md](/a.md)" + ) + + +def test_ambiguous_slash_title_still_allows_exact_filename_inference(test_project): + from basic_memory.models import Entity + from basic_memory.services.bulk_link_resolver import ProjectEntityIdentityIndex + + owner = Entity(title="p/foo", file_path="p/foo.md") + index = ProjectEntityIdentityIndex.from_entities( + test_project, [owner, Entity(title="p/foo", file_path="other.md")] + ) + assert ( + index.resolve_strict( + "p/foo", include_project_permalinks=True, workspace_permalink=None + ).entity + is owner + ) + snapshot = ExportSnapshot( + "p", + ( + ExportFile("p/foo.md", b"---\ntitle: p/foo\n---\n"), + ExportFile("other.md", b"---\ntitle: p/foo\n---\n"), + ExportFile("source.md", b"[[p/foo]]"), + ), + ) + source = next(file.content for file in render_bundle(snapshot) if file.path == "source.md") + assert b"[p/foo](/p/foo.md)" in source + + +def test_yaml_sets_are_deterministic_across_processes(): + import os + import subprocess + import sys + + source = b"---\ntype: note\ncustom: !!set {alpha: null, beta: null, gamma: null}\n---\n" + expected = next( + file.content.decode() + for file in render_bundle(ExportSnapshot("p", (ExportFile("a.md", source),))) + if file.path == "a.md" + ) + script = ( + "from basic_memory.okf.render import ExportFile, ExportSnapshot, render_bundle\n" + f"source = {source!r}\n" + "print(next(f.content.decode() for f in render_bundle(ExportSnapshot('p', " + "(ExportFile('a.md', source),))) if f.path == 'a.md'))\n" + ) + outputs = [ + subprocess.check_output( + [sys.executable, "-c", script], env={**os.environ, "PYTHONHASHSEED": seed}, text=True + ) + for seed in ("1", "2") + ] + assert outputs[0] == outputs[1] == expected + "\n" + assert parse_document(outputs[0]).metadata["custom"] == {"alpha", "beta", "gamma"} + assert outputs[0].index("type: note") < outputs[0].index("custom:") + + +def test_explicit_project_qualifiers_cannot_bind_to_foreign_local_aliases(): + snapshot = ExportSnapshot( + "p", + ( + ExportFile("trap.md", b"---\npermalink: q/foo\n---\n"), + ExportFile("target.md", b"---\npermalink: foo\n---\n"), + ExportFile("folder/target.md", b"---\npermalink: local\n---\n"), + ExportFile("folder/source.md", b"[[q::foo]] [[p::foo]] [[p::target.md]]"), + ), + ) + source = next( + file.content for file in render_bundle(snapshot) if file.path == "folder/source.md" + ) + assert b"[[q::foo]] [p::foo](/target.md) [p::target.md](/target.md)" in source + + +@pytest.mark.parametrize("field", ["title", "type"]) +@pytest.mark.parametrize( + "value", + ["!!set {alpha: null, beta: null}", "[!!set {alpha: null}]", "{nested: !!set {a: null}}"], +) +def test_unordered_identity_metadata_fails_explicitly(field, value): + snapshot = ExportSnapshot("p", (ExportFile("a.md", f"---\n{field}: {value}\n---\n".encode()),)) + with pytest.raises(ValueError, match=f"a.md: {field} cannot contain an unordered YAML set"): + render_bundle(snapshot) + + +@pytest.mark.parametrize("include_project", [False, True]) +def test_missing_authored_permalink_uses_canonical_generated_address(include_project): + snapshot = ExportSnapshot( + "p", + ( + ExportFile("folder/foo.md", b"# Foo"), + ExportFile("source.md", b"[[p/folder/foo]]"), + ), + permalinks_include_project=include_project, + ) + source = next(file.content for file in render_bundle(snapshot) if file.path == "source.md") + assert b"[p/folder/foo](/folder/foo.md)" in source + + +@pytest.mark.asyncio +@pytest.mark.parametrize("source", ["---\n- scalar\n---\nBody", "---\nbad: [\n---\nBody"]) +async def test_malformed_note_identity_fails_before_replacing_bundle( + export_config, tmp_path, source +): + root = Path(export_config.projects["export"].path) + note = root / "new.md" + note.write_text(source, encoding="utf-8") + (root / "a.md").write_text("[[export/old]] and [[export/new]]", encoding="utf-8") + destination = tmp_path / "bundle" + destination.mkdir() + (destination / "keep").write_bytes(b"previous bundle") + with pytest.raises(ValueError, match="new.md: repair malformed frontmatter before export"): + await export_project(export_config, "export", destination, replace=True) + assert (destination / "keep").read_bytes() == b"previous bundle" + assert note.read_text(encoding="utf-8") == source + + +def test_network_path_wikilinks_remain_literal(): + body = "[[//example.com/a]] [[//example.com/a|alias]] [[/a.md]]" + assert convert_wikilinks(body, "source.md", {"a.md": "a.md"}, "p") == ( + "[[//example.com/a]] [[//example.com/a|alias]] [/a.md](/a.md)" + ) + + +@pytest.mark.parametrize("duplicate", [False, True]) +def test_nested_resource_titles_resolve_with_ambiguity_handling(test_project, duplicate): + from basic_memory.models import Entity + from basic_memory.services.bulk_link_resolver import ProjectEntityIdentityIndex + + owner = Entity(title="paper.pdf", file_path="refs/paper.pdf") + entities = [owner] + files = [ExportFile("refs/paper.pdf", b"pdf"), ExportFile("source.md", b"[[paper.pdf]]")] + if duplicate: + entities.append(Entity(title="paper.pdf", file_path="other/paper.pdf")) + files.append(ExportFile("other/paper.pdf", b"other pdf")) + index = ProjectEntityIdentityIndex.from_entities(test_project, entities) + resolved = index.resolve_strict( + "paper.pdf", include_project_permalinks=True, workspace_permalink=None + ).entity + assert resolved is (None if duplicate else owner) + source = next( + file.content + for file in render_bundle(ExportSnapshot("p", tuple(files))) + if file.path == "source.md" + ) + assert (b"[paper.pdf](/paper.pdf)" if duplicate else b"[paper.pdf](/refs/paper.pdf)") in source + + +@pytest.mark.parametrize( + "identifier", + [ + "24a6c931-e246-4a48-947d-99b1bcfab3b5", + "24A6C931-E246-4A48-947D-99B1BCFAB3B5", + "24a6c931e2464a48947d99b1bcfab3b5", + "{24a6c931-e246-4a48-947d-99b1bcfab3b5}", + ], +) +def test_external_id_links_cannot_bind_to_semantic_aliases(identifier, test_project): + from basic_memory.models import Entity + from basic_memory.services.bulk_link_resolver import ( + BulkLinkResolutionSnapshot, + ProjectEntityIdentityIndex, + ProjectReferenceIndex, + RelationTargetReference, + ) + + owner = Entity( + external_id="24a6c931-e246-4a48-947d-99b1bcfab3b5", title="Owner", file_path="owner.md" + ) + trap = Entity(title=identifier, permalink=identifier, file_path="trap.md") + index = ProjectEntityIdentityIndex.from_entities(test_project, [owner, trap]) + resolver = BulkLinkResolutionSnapshot( + current_project_id=test_project.id, + projects=ProjectReferenceIndex.from_projects([test_project]), + entity_indexes={test_project.id: index}, + include_project_permalinks=True, + workspace_permalink=None, + ) + assert resolver.resolve(RelationTargetReference.parse(identifier)) is owner + body = f"[[{identifier}]] [[{identifier}|owner]] [[p::{identifier}]]" + files = ( + ExportFile("trap.md", f"---\npermalink: '{identifier}'\n---\n".encode()), + ExportFile("source.md", body.encode()), + ) + source = next( + file.content.decode() + for file in render_bundle(ExportSnapshot("p", files)) + if file.path == "source.md" + ) + assert parse_document(source).body == body diff --git a/tests/test_man_pages.py b/tests/test_man_pages.py index 1e771e9e2..543ea8910 100644 --- a/tests/test_man_pages.py +++ b/tests/test_man_pages.py @@ -452,6 +452,15 @@ def test_render_cli_synopsis_renders_the_shell_form() -> None: assert render_cli_synopsis(apropos_path, apropos).startswith("bm man apropos QUERY") +def test_export_synopsis_requires_project_but_keeps_flags_optional() -> None: + command_path, command = _cli_command(find_page(PageRef("okf-export", 1))) + synopsis = render_cli_synopsis(command_path, command) + assert "--project PROJECT" in synopsis + assert "[--project PROJECT]" not in synopsis + assert "[--replace]" in synopsis + assert "[--json]" in synopsis + + def test_render_options_includes_shared_and_global_flags() -> None: # D2: OPTIONS is the COMPLETE public option list, including the shared output # and routing flags the hand-written blocks left out — grep(1) grows from four