Add entangled weave command for transforming Markdown

README.md

@@ -440,6 +440,21 @@ You can use `${variable}` syntax inside Brei tasks just as you would in a stand-
 
 Entangled has been used successfully with the following document generators. Note that some of these examples were built using older versions of Entangled, but they should work just the same.
 
+### Weaving
+
+The `entangled weave` command transforms a Markdown document into Markdown that renders well in a documentation tool, by applying one or more filters to the code blocks:
+
+```shell
+entangled weave --list-filters                          # show available filters
+entangled weave docs/index.md                           # apply the `label` filter, print to stdout
+entangled weave -f title docs/index.md -o out/index.md  # MkDocs/pymdownx style titles
+```
+
+- `label` wraps each named code block in a `<div class="entangled-code-block">` with a label showing its name or target file. This renders in most Markdown previews (GitHub, Pandoc, Python-Markdown with `md_in_html`).
+- `title` adds a `title` attribute to the code block header, as understood by `pymdownx.superfences`.
+
+Filters can be combined by repeating `-f`; they are applied in the given order. The same transformation is available from Python through `entangled.weave.transform_markdown`, so that documentation tools only need a thin adapter.
+
 ### Pandoc
 
 [Pandoc](https://pandoc.org/) is a very versatile tool for converting documents in any format. It specifically has very wide support for different forms of Markdown syntax out in the wild, including a filter system that lets you extend the workings of Pandoc. Those filters can be written in any language through an API, for instance Python filters can be written using `panflute`, but there is also native support for Lua.

docs/man-pages/english.md

@@ -67,6 +67,10 @@ Each command has its own arguments and flags that can be inspected using `entang
 
 : Keep a loop running, watching for changes in the file system.
 
+`weave [-f filter]... [-o output] [input]`
+
+: Transform Markdown for rendering by documentation tools. Use `--list-filters` to see available filters.
+
 WRITING PROGRAMS
 ================
 

entangled/commands/__init__.py

@@ -8,7 +8,8 @@
 from .status import status
 from .brei import brei
 from .serve import serve
+from .weave import weave
 
 __all__ = [
-    "main", "stitch", "tangle", "new", "reset", "sync", "watch", "status", "brei", "serve"
+    "main", "stitch", "tangle", "new", "reset", "sync", "watch", "status", "brei", "serve", "weave"
 ]

entangled/commands/weave.py

@@ -0,0 +1,50 @@
+from pathlib import Path, PurePath
+import sys
+
+import rich_click as click
+
+from .main import main
+
+from ..config import read_config
+from ..interface import Context
+from ..weave import filters, filter_description, transform_markdown
+
+
+def print_filters():
+    width = max(map(len, filters))
+    for name, f in filters.items():
+        print(f"{name:<{width}}  {filter_description(f)}")
+
+
+@main.command()
+@click.argument("input", type=click.Path(dir_okay=False, allow_dash=True, path_type=Path),
+                default="-")
+@click.option("-o", "--output", type=click.Path(dir_okay=False, allow_dash=True, path_type=Path),
+              default="-", help="output file, defaults to standard output")
+@click.option("-f", "--filter", "filter_names", type=click.Choice(list(filters)), multiple=True,
+              help="filter to apply, can be repeated; defaults to `label`")
+@click.option("--list-filters", is_flag=True, help="list available filters and exit")
+def weave(*, input: Path, output: Path, filter_names: tuple[str, ...] = (),
+          list_filters: bool = False):
+    """Transform Markdown for rendering by documentation tools."""
+    if list_filters:
+        print_filters()
+        return
+
+    context = Context()
+    context |= read_config(context.fs)
+
+    if input == Path("-"):
+        text = sys.stdin.read()
+        filename = PurePath("-")
+    else:
+        text = input.read_text()
+        filename = input
+
+    selected = [filters[n] for n in (filter_names or ("label",))]
+    result = "".join(transform_markdown(context, text, *selected, filename=filename))
+
+    if output == Path("-"):
+        _ = sys.stdout.write(result)
+    else:
+        _ = output.write_text(result)

entangled/logging.py

@@ -20,7 +20,7 @@ def _render_buffer(self, *args, **kwargs):
         return "\n".join(line.rstrip() for line in rendered.splitlines()) + newline_char
 
 # Global rich console object
-console: Console = WhitespaceStrippingConsole()
+console: Console = WhitespaceStrippingConsole(stderr=True)
 
 def logger():
     return logging.getLogger("entangled")

entangled/weave/__init__.py

@@ -0,0 +1,9 @@
+from .base import (
+    ContentFilter, CodeBlockFilter, codeblock_filter, compose_filters, iter_bind, transform_markdown
+)
+from .filters import filters, filter_description, block_title
+
+__all__ = [
+    "ContentFilter", "CodeBlockFilter", "codeblock_filter", "compose_filters", "iter_bind",
+    "transform_markdown", "filters", "filter_description", "block_title"
+]

entangled/weave/base.py

@@ -0,0 +1,66 @@
+"""
+Generic machinery for weaving: transforming the Markdown source of a page,
+code block by code block, into Markdown that renders well in some target.
+"""
+
+from collections.abc import Callable, Iterable, Iterator
+from functools import partial, reduce, wraps
+from itertools import chain
+from pathlib import PurePath
+
+from ..interface import Context, markdown
+from ..iterators import numbered_lines
+from ..model import Content, PlainText, ReferenceId, ReferenceMap, content_to_text
+
+
+type ContentFilter = Callable[[ReferenceMap, Content], Iterable[Content]]
+type CodeBlockFilter = Callable[[ReferenceMap, ReferenceId], Iterable[Content]]
+
+
+def codeblock_filter(f: CodeBlockFilter) -> ContentFilter:
+    """Lift a filter on code blocks to a filter on content, passing plain text
+    through unchanged."""
+    @wraps(f)
+    def _filter(rm: ReferenceMap, c: Content) -> Iterable[Content]:
+        match c:
+            case PlainText():
+                return [c]
+            case ReferenceId():
+                return f(rm, c)
+    return _filter
+
+
+def iter_bind[T, U](lst: Iterable[T], f: Callable[[T], Iterable[U]]) -> Iterator[U]:
+    return chain.from_iterable(map(f, lst))
+
+
+def compose_filter(a: ContentFilter, b: ContentFilter) -> ContentFilter:
+    def _joined(rm: ReferenceMap, c: Content) -> Iterable[Content]:
+        return iter_bind(a(rm, c), partial(b, rm))
+    return _joined
+
+
+def compose_filters(*args: ContentFilter) -> ContentFilter:
+    """Compose filters left to right: the first filter is applied first."""
+    identity: ContentFilter = lambda _, c: [c]
+    return reduce(compose_filter, args, identity)
+
+
+def transform_markdown(
+    context: Context,
+    text: str,
+    *filters: ContentFilter,
+    filename: PurePath = PurePath("-"),
+    reference_map: ReferenceMap | None = None,
+) -> Iterator[str]:
+    """
+    Read a Markdown document, pass its content through the given filters,
+    and yield the resulting Markdown text.
+
+    Filters may modify code blocks in the reference map in place. If no
+    `reference_map` is given, a fresh one is used for this document only.
+    """
+    refs = ReferenceMap() if reference_map is None else reference_map
+    content = markdown(context, refs, numbered_lines(filename, text))
+    for c in iter_bind(content, partial(compose_filters(*filters), refs)):
+        yield content_to_text(refs, c)[0]

entangled/weave/filters.py

@@ -0,0 +1,89 @@
+"""
+Filters for `entangled weave`. Each filter is registered by name in `filters`;
+the first line of its docstring serves as description for `--list-filters`.
+"""
+
+from html import escape
+import re
+
+from ..model import CodeBlock, Content, PlainText, ReferenceId, ReferenceMap
+from ..model.properties import Attribute, Class, Id
+from .base import ContentFilter, codeblock_filter
+
+
+def block_title(code_block: CodeBlock) -> str | None:
+    """Compute a human readable title for a code block from its id and `file`
+    attribute. Returns `None` for anonymous code blocks."""
+    ids = [p.value for p in code_block.properties if isinstance(p, Id)]
+    filenames = [p.value for p in code_block.properties  # pyright: ignore[reportAny]
+                 if isinstance(p, Attribute) and p.key == "file"]
+
+    if len(ids) > 1 or len(filenames) > 1:
+        return "error: ambiguous code block title"
+    if ids and filenames:
+        return f"#{ids[0]} / file: {filenames[0]}"
+    if ids:
+        return f"#{ids[0]}"
+    if filenames:
+        return f"file: {filenames[0]}"
+    return None
+
+
+def first_line(text: str) -> tuple[str, str]:
+    head, nl, rest = text.partition("\n")
+    return head, nl + rest
+
+
+@codeblock_filter
+def title(reference_map: ReferenceMap, r: ReferenceId) -> list[Content]:
+    """Add a `title` attribute to the code block header (MkDocs/pymdownx style)."""
+    code_block = reference_map[r]
+    text = block_title(code_block)
+    ids = [p.value for p in code_block.properties if isinstance(p, Id)]
+    classes = [p.value for p in code_block.properties if isinstance(p, Class)]
+
+    head, rest = first_line(code_block.open_line)
+    fence_match = re.match(r"\s*(`{3,}|~{3,})", head)
+    fence = fence_match.group(1) if fence_match else "```"
+
+    attrs: list[str] = []
+    if ids:
+        attrs.append(str(Id(ids[0])))
+    attrs.extend(str(Class(c)) for c in classes[1:])
+    if text:
+        attrs.append(str(Attribute("title", text)))
+
+    head = fence + (classes[0] if classes else "")
+    if attrs:
+        head += " {" + " ".join(attrs) + "}"
+
+    code_block.open_line = head + (rest or "\n")
+    return [r]
+
+
+@codeblock_filter
+def label(reference_map: ReferenceMap, r: ReferenceId) -> list[Content]:
+    """Wrap code blocks in a `<div>` with a label; renders in most Markdown previews."""
+    code_block = reference_map[r]
+    text = block_title(code_block)
+    if text is None:
+        return [r]
+
+    ind = code_block.indent
+    header = (f"{ind}<div class=\"entangled-code-block\" markdown=\"1\">\n"
+              f"{ind}<div class=\"entangled-code-title\"><code>{escape(text)}</code></div>\n\n")
+    # The closing fence may lack a newline at the end of a file. The blank
+    # lines around `</div>` end the HTML block in CommonMark.
+    eol = "" if code_block.close_line.endswith("\n") else "\n"
+    footer = f"{eol}\n{ind}</div>\n\n"
+    return [PlainText(header), r, PlainText(footer)]
+
+
+filters: dict[str, ContentFilter] = {
+    "label": label,
+    "title": title,
+}
+
+
+def filter_description(f: ContentFilter) -> str:
+    return (f.__doc__ or "").strip().partition("\n")[0]

man/entangled.1

@@ -51,6 +51,10 @@ Tangle codes from the documentation.
 .TP
 \f[CR]watch\f[R]
 Keep a loop running, watching for changes in the file system.
+.TP
+\f[CR]weave [\-f filter]... [\-o output] [input]\f[R]
+Transform Markdown for rendering by documentation tools.
+Use \f[CR]\-\-list\-filters\f[R] to see available filters.
 .SH WRITING PROGRAMS
 Entangled enables a \f[B]documentation first\f[R] approach to
 programming.

test/weave/test_weave.py

@@ -0,0 +1,75 @@
+from pathlib import Path
+from contextlib import chdir
+
+from entangled.commands import main
+from entangled.interface import Context
+from entangled.weave import filters, transform_markdown
+
+import pytest
+
+
+md_in = """# Example
+
+``` {.python file=hello.py}
+<<main>>
+```
+
+``` {.python #main .extra}
+print("hello")
+```
+
+``` {.python}
+anonymous = True
+```
+"""
+
+
+def weave(text: str, *names: str) -> str:
+    return "".join(transform_markdown(Context(), text, *(filters[n] for n in names)))
+
+
+def test_identity():
+    assert weave(md_in) == md_in
+
+
+def test_title():
+    out = weave(md_in, "title")
+    assert '```python {title="file: hello.py"}\n' in out
+    assert '```python {#main .extra title="#main"}\n' in out
+    assert "```python\nanonymous = True" in out
+    assert "<<main>>" in out
+
+
+def test_label():
+    out = weave(md_in, "label")
+    assert out.count('<div class="entangled-code-block" markdown="1">') == 2
+    assert '<code>file: hello.py</code>' in out
+    assert '<code>#main</code>' in out
+    assert "\n\n``` {.python file=hello.py}\n<<main>>\n```\n\n</div>\n\n" in out
+    assert "\n\n``` {.python}\nanonymous = True\n```\n" in out
+
+
+def test_label_indented_without_trailing_newline():
+    text = "- item\n\n    ``` {.python #a}\n    x = 1\n    ```"
+    out = weave(text, "label")
+    assert out.endswith("    ```\n\n    </div>\n\n")
+    assert "    <div class=\"entangled-code-block\" markdown=\"1\">\n" in out
+
+
+def test_label_and_title_compose():
+    out = weave(md_in, "label", "title")
+    assert '<code>#main</code></div>\n\n```python {#main .extra title="#main"}\n' in out
+
+
+def test_weave_command(tmp_path: Path, capsys: pytest.CaptureFixture[str]):
+    with chdir(tmp_path):
+        Path("input.md").write_text(md_in)
+        main(args=["weave", "-f", "title", "-o", "output.md", "input.md"], standalone_mode=False)
+        assert '```python {title="file: hello.py"}' in Path("output.md").read_text()
+
+        main(args=["weave", "input.md"], standalone_mode=False)
+        assert '<code>#main</code>' in capsys.readouterr().out
+
+        main(args=["weave", "--list-filters"], standalone_mode=False)
+        out = capsys.readouterr().out
+        assert all(name in out for name in filters)