diff --git a/deploy/huggingface/README.md b/deploy/huggingface/README.md index 52ff4e8b..808800da 100644 --- a/deploy/huggingface/README.md +++ b/deploy/huggingface/README.md @@ -8,6 +8,7 @@ app_port: 7860 pinned: false license: mit short_description: Ask a sample database in English, with your own API key +thumbnail: https://raw.githubusercontent.com/nadeem4/nl2sql/main/docs/assets/social-card.png --- # nl2sql playground @@ -45,6 +46,12 @@ Everything the Space needs is in this one folder: the `Dockerfile` builds the image with no build context from the repository, and the front-matter above is the Space configuration. Nothing here holds a secret. +The front-matter's `short_description` and `thumbnail` are what this Space's +own link preview is made of; the thumbnail is read from raw GitHub, so the card +works before the Space has built. The image is generated -- regenerate it with +`python scripts/render_social_card.py`, described in +[the hosted demo docs](https://github.com/nadeem4/nl2sql/blob/main/docs/deployment/hosted-demo.md). + **Normally this is automatic.** `.github/workflows/publish_space.yml` in the repository creates the Space if it is missing, mirrors this folder onto its root, and waits for the build -- on every push to `main` that touches this diff --git a/docs/assets/social-card.png b/docs/assets/social-card.png new file mode 100644 index 00000000..108b3dcb Binary files /dev/null and b/docs/assets/social-card.png differ diff --git a/docs/deployment/hosted-demo.md b/docs/deployment/hosted-demo.md index af18ba56..19734377 100644 --- a/docs/deployment/hosted-demo.md +++ b/docs/deployment/hosted-demo.md @@ -351,6 +351,55 @@ If step 3 is refused because the Space has commits of its own, replaces the Space's history with this folder's. The workflow above never needs that, which is why it exists. +## The link preview + +Two different links go around, and each unfurls from a different place: + +| Pasted link | The card comes from | +| --- | --- | +| | the Open Graph and Twitter tags in the page the app serves | +| | `short_description` and `thumbnail` in `deploy/huggingface/README.md`'s front-matter | + +Both show the same 1200x630 card: + +![The link preview card: the nl2sql playground wordmark, the line "Ask a database in plain English, the model plans, the code writes the SQL", and a fragment of a plan turning into SQL](../assets/social-card.png) + +**The app's tags** are added to the built page's `` by +[`preview.py`][preview] as each request goes out, not baked into the bundle. A +crawler runs no JavaScript, so a tag React adds on mount is a tag nobody sees; +and `og:image` and `og:url` have to be absolute, while the same page is the +Space, a container and `http://127.0.0.1:8000`. So the host comes off the +request: `X-Forwarded-Proto` and `X-Forwarded-Host` when a proxy set them +(which is what the Space does), the `Host` header otherwise, and the URL the +app itself saw if neither is a host. The card is served by the app at +`/social-card.png` and ships in the wheel, so a `pip install` serves it too. + +**The Space's card** reads `thumbnail` over raw GitHub rather than from the +Space, so it works before the Space has built and while it is asleep. + +### Regenerating the card + +The card is rendered from `scripts/social_card.html`, which uses the +playground's own colours and both of its typefaces. Edit that file, then: + +```bash +cd web/playground && npm ci && cd ../.. # the fonts the card borrows +python scripts/render_social_card.py +``` + +Headless Chrome shoots it at 1200x630 and the script writes the same bytes to +both places that need them: `docs/assets/social-card.png`, which the Space's +`thumbnail` reads, and +`packages/nl2sql/src/nl2sql/cli/demo/playground/assets/social-card.png`, which +the app serves. A test holds the two byte-identical, so regenerating into only +one of them fails rather than going out half-changed. + +A card validator needs a public URL, so the check that the card really unfurls +can only be run against the deployed Space, with X's card validator or +. What the tests check is everything up to that: the +tags are in the served HTML, their URLs are absolute, and the image is served +as `image/png`. + ## Running it locally instead Hosted mode exists to show the engine on our sample data. To ask questions of @@ -364,5 +413,6 @@ nl2sql demo See [Demo](../getting_started/demo.md). [key-module]: https://github.com/nadeem4/nl2sql/blob/main/packages/nl2sql/src/nl2sql/llm/request_key.py +[preview]: https://github.com/nadeem4/nl2sql/blob/main/packages/nl2sql/src/nl2sql/cli/demo/playground/preview.py [space-readme]: https://github.com/nadeem4/nl2sql/blob/main/deploy/huggingface/README.md [workflow]: https://github.com/nadeem4/nl2sql/blob/main/.github/workflows/publish_space.yml diff --git a/packages/nl2sql/pyproject.toml b/packages/nl2sql/pyproject.toml index 3c3e5360..8f8dc9b5 100644 --- a/packages/nl2sql/pyproject.toml +++ b/packages/nl2sql/pyproject.toml @@ -101,5 +101,7 @@ include = ["nl2sql*"] "nl2sql.evaluation" = ["datasets/*.yaml"] "nl2sql.evaluation.presets" = ["*.yaml"] # The committed Vite build of web/playground: the wheel must carry the finished -# page so `pip install "nl2sql-engine[demo]"` needs no Node toolchain. -"nl2sql.cli.demo.playground" = ["static/*"] +# page so `pip install "nl2sql-engine[demo]"` needs no Node toolchain. The card +# the link preview names sits beside it rather than in `static/`, which the +# Vite build empties on every run. +"nl2sql.cli.demo.playground" = ["static/*", "assets/*"] diff --git a/packages/nl2sql/src/nl2sql/cli/demo/playground/app.py b/packages/nl2sql/src/nl2sql/cli/demo/playground/app.py index 0202180e..f241633f 100644 --- a/packages/nl2sql/src/nl2sql/cli/demo/playground/app.py +++ b/packages/nl2sql/src/nl2sql/cli/demo/playground/app.py @@ -1,8 +1,10 @@ """The playground FastAPI app. -Fifteen routes: +Sixteen routes: -``GET /`` the built React page +``GET /`` the built React page, with the link preview's + tags in its head (see :mod:`preview`) +``GET /social-card.png`` the 1200x630 card those tags name ``GET /api/meta`` mode, dataset, every registered datasource, the guided questions (flat, and grouped by datasource), the roles and how many guided questions have replay recordings @@ -58,6 +60,7 @@ from nl2sql.auth.models import UserContext from nl2sql.cli.demo.playground.hosted import FEEDBACK_MESSAGE, REBUILD_MESSAGE, Hosted from nl2sql.cli.demo.playground.index_panel import IndexPanel +from nl2sql.cli.demo.playground.preview import CARD, CARD_ROUTE, with_preview from nl2sql.cli.demo.playground.settings import SettingsPanel from nl2sql.common.settings import settings from nl2sql.feedback import NOTE_MAX_CHARS, FeedbackStore, run_record, run_signals @@ -304,8 +307,18 @@ async def _invalid_request(request: Request, exc: RequestValidationError) -> JSO app.mount("/static", StaticFiles(directory=str(STATIC_DIR)), name="static") @app.get("/", response_class=HTMLResponse) - def index() -> str: - return page + def index(request: Request) -> str: + # The preview tags are added here rather than baked into the built + # page: they carry absolute URLs, and the host they name is the one + # this request arrived on. See :mod:`preview`. + return with_preview(page, request) + + @app.get(CARD_ROUTE, include_in_schema=False) + def social_card() -> FileResponse: + """The card the preview tags name. It ships with the package.""" + if not CARD.is_file(): + raise HTTPException(status_code=404, detail="No preview card in this install.") + return FileResponse(CARD, media_type="image/png") # One group per datasource that has guided questions, in the order given. groups = [{"datasource": ds, "questions": list(qs)} diff --git a/packages/nl2sql/src/nl2sql/cli/demo/playground/assets/social-card.png b/packages/nl2sql/src/nl2sql/cli/demo/playground/assets/social-card.png new file mode 100644 index 00000000..108b3dcb Binary files /dev/null and b/packages/nl2sql/src/nl2sql/cli/demo/playground/assets/social-card.png differ diff --git a/packages/nl2sql/src/nl2sql/cli/demo/playground/preview.py b/packages/nl2sql/src/nl2sql/cli/demo/playground/preview.py new file mode 100644 index 00000000..c3442d04 --- /dev/null +++ b/packages/nl2sql/src/nl2sql/cli/demo/playground/preview.py @@ -0,0 +1,101 @@ +"""The link preview: what a pasted link to the playground unfurls into. + +Three things have to be true for a card to appear, and each is why a piece of +this module exists: + +1. **The tags are in the HTML the server sends.** A crawler runs no + JavaScript, so a tag the React bundle adds on mount is a tag nobody sees. + :func:`with_preview` puts them in the ```` of the built page as it + goes out. +2. **``og:image`` and ``og:url`` are absolute.** A relative path is not + resolved by most unfurlers, and the one absolute URL that would work + everywhere does not exist: the same page is the Space, a container, and + ``http://127.0.0.1:8000``. :func:`origin` reads the host the visitor typed + off the request instead. +3. **The image is served by the app.** The card ships in the package, so a + ``pip install "nl2sql-engine[demo]"`` serves it too; ``docs/assets`` holds + the same bytes for the Space's own card, which is read from GitHub before + the Space has built. + +The card itself is generated: ``scripts/social_card.html`` is the source and +``python scripts/render_social_card.py`` renders it. +""" + +from __future__ import annotations + +import html +import pathlib +import re +from importlib.resources import files + +CARD = pathlib.Path(str(files("nl2sql.cli.demo.playground") / "assets" / "social-card.png")) + +# Where the app serves the card. A path of its own rather than one under +# ``/static``, because the build empties that directory on every `npm run +# build` and the card is not the bundle's to write. +CARD_ROUTE = "/social-card.png" + +TITLE = "nl2sql playground" + +# The card says the same thing in the same words: whatever is changed here is +# changed in `scripts/social_card.html` too. ``—`` is an em dash; the rest +# of this repository's source is ASCII and this keeps it that way. +DESCRIPTION = ( + "Ask a database in plain English — the model plans, the code writes the SQL. " + "Bring your own API key and ask the sample databases: the plan, the checks, " + "the SQL, the rows and the cost, step by step." +) + +# A host is a name and an optional port and nothing else. Anything else is +# either a proxy this code does not understand or a forged header trying to +# write a URL of its own into the page, and both fall back. +_HOST = re.compile(r"[A-Za-z0-9.\-]{1,253}(:[0-9]{1,5})?") + + +def _first(value: str | None) -> str | None: + """The first entry of a forwarded header; proxies chain them with commas.""" + return value.split(",")[0].strip() if value else None + + +def origin(request) -> str: + """The scheme and host to build the preview's absolute URLs from. + + A Space serves this page behind a proxy: the request the app sees is plain + HTTP to an internal address, while the visitor typed + ``https://nadeem4nk-nl2sql-demo.hf.space``. The forwarded headers carry + that, so they decide; with no proxy in front they are absent and the URL + the app itself saw is already right. + """ + scheme = _first(request.headers.get("x-forwarded-proto")) or request.url.scheme + host = _first(request.headers.get("x-forwarded-host")) or request.headers.get("host") or "" + if scheme in ("http", "https") and _HOST.fullmatch(host): + return f"{scheme}://{host}" + return str(request.base_url).rstrip("/") + + +def head_tags(site: str) -> str: + """The preview's ```` tags, for a page served from ``site``.""" + page, image = f"{site}/", f"{site}{CARD_ROUTE}" + tags = [ + ("name", "description", DESCRIPTION), + ("property", "og:type", "website"), + ("property", "og:site_name", TITLE), + ("property", "og:title", TITLE), + ("property", "og:description", DESCRIPTION), + ("property", "og:url", page), + ("property", "og:image", image), + ("property", "og:image:width", "1200"), + ("property", "og:image:height", "630"), + ("property", "og:image:alt", "The nl2sql playground card: ask a database in plain English."), + ("name", "twitter:card", "summary_large_image"), + ("name", "twitter:title", TITLE), + ("name", "twitter:description", DESCRIPTION), + ("name", "twitter:image", image), + ] + return "".join(f'' + for kind, key, value in tags) + + +def with_preview(page: str, request) -> str: + """The built page with the preview tags in its ````.""" + return page.replace("", head_tags(origin(request)) + "", 1) diff --git a/packages/nl2sql/src/nl2sql/cli/demo/playground/static/index.html b/packages/nl2sql/src/nl2sql/cli/demo/playground/static/index.html index 1720a682..dc75a479 100644 --- a/packages/nl2sql/src/nl2sql/cli/demo/playground/static/index.html +++ b/packages/nl2sql/src/nl2sql/cli/demo/playground/static/index.html @@ -3,7 +3,10 @@ - + nl2sql playground diff --git a/packages/nl2sql/tests/unit/test_hosted_space_definition.py b/packages/nl2sql/tests/unit/test_hosted_space_definition.py index d6216f2b..a8e3f3f8 100644 --- a/packages/nl2sql/tests/unit/test_hosted_space_definition.py +++ b/packages/nl2sql/tests/unit/test_hosted_space_definition.py @@ -32,6 +32,30 @@ def test_the_space_front_matter_declares_a_public_docker_space(): assert front["license"] == "mit" +def test_the_space_card_has_a_sentence_and_a_thumbnail(): + """What the Space's own link preview is made of. + + A Space card shows the `short_description` and the `thumbnail`, and has + nothing else to show: the README's body is the page, not the card. The + thumbnail is read from raw GitHub rather than from the Space, so the card + works before the Space has built and while it is sleeping. + """ + front = _front_matter() + + assert front["short_description"] + assert len(front["short_description"]) <= 60, "Hugging Face truncates a longer one" + assert front["thumbnail"] == ( + "https://raw.githubusercontent.com/nadeem4/nl2sql/main/docs/assets/social-card.png" + ) + + +def test_the_thumbnail_names_a_card_this_repository_holds(): + card = SPACE.parents[1] / "docs" / "assets" / "social-card.png" + + assert card.is_file(), "deploy/huggingface/README.md points its thumbnail at a missing file" + assert card.read_bytes().startswith(b"\x89PNG\r\n\x1a\n") + + def test_the_declared_port_is_the_one_the_container_listens_on(): dockerfile = (SPACE / "Dockerfile").read_text(encoding="utf-8") compose = yaml.safe_load((SPACE / "docker-compose.yml").read_text(encoding="utf-8")) diff --git a/packages/nl2sql/tests/unit/test_playground_app.py b/packages/nl2sql/tests/unit/test_playground_app.py index 00ce101b..324c94d4 100644 --- a/packages/nl2sql/tests/unit/test_playground_app.py +++ b/packages/nl2sql/tests/unit/test_playground_app.py @@ -1,4 +1,7 @@ """The playground FastAPI app: meta, ask, schema and the served page.""" +import pathlib +import re + import pytest fastapi = pytest.importorskip("fastapi") @@ -314,3 +317,99 @@ def test_trace_route_never_resolves_a_file_outside_the_directory(tmp_path): client = _trace_client(tmp_path) # "outside" is a valid-looking id, but its file sits one level up. assert client.get("/api/trace/outside").status_code == 404 + + +# ---------- the link preview ---------- +# +# A crawler runs no JavaScript and resolves no relative path, and both of those +# are the point here: the tags are in the HTML the server sends, and the URLs +# in them are absolute and name the host the visitor typed. + +REPO = pathlib.Path(__file__).resolve().parents[4] +CARD_COPIES = ( + REPO / "docs" / "assets" / "social-card.png", + REPO / "packages" / "nl2sql" / "src" / "nl2sql" / "cli" / "demo" / "playground" / "assets" / "social-card.png", +) + + +def _served_page(**headers) -> str: + client = TestClient(build_app(_Engine(), questions=[], roles=["admin"], mode="live", dataset="chinook")) + response = client.get("/", headers=headers) + assert response.status_code == 200 + return response.text + + +def _tag(html: str, key: str) -> str: + match = re.search(rf'') + assert _tag(html, "og:type") == "website" + assert _tag(html, "og:title") == "nl2sql playground" + assert "plain English" in _tag(html, "og:description") + assert _tag(html, "twitter:card") == "summary_large_image" + assert _tag(html, "twitter:title") == _tag(html, "og:title") + assert _tag(html, "twitter:description") == _tag(html, "og:description") + assert _tag(html, "twitter:image") == _tag(html, "og:image") + # One description, and it says what the card says. + assert _tag(html, "description") == _tag(html, "og:description") + assert html.count('name="description"') == 1 + + +def test_the_preview_urls_are_absolute_and_name_the_host_the_visitor_typed(): + html = _served_page(**{"x-forwarded-proto": "https", + "x-forwarded-host": "nadeem4nk-nl2sql-demo.hf.space"}) + + assert _tag(html, "og:url") == "https://nadeem4nk-nl2sql-demo.hf.space/" + assert _tag(html, "og:image") == "https://nadeem4nk-nl2sql-demo.hf.space/social-card.png" + assert _tag(html, "og:image:width") == "1200" + assert _tag(html, "og:image:height") == "630" + + +def test_without_a_proxy_the_preview_urls_are_the_ones_the_app_was_reached_on(): + html = _served_page() + + # TestClient asks for http://testserver/, which is what a local run looks + # like: absolute, and right for whoever can reach this playground. + assert _tag(html, "og:url") == "http://testserver/" + assert _tag(html, "og:image") == "http://testserver/social-card.png" + + +@pytest.mark.parametrize("host", ['evil">', "not a host", "", " "]) +def test_a_forged_host_header_cannot_write_a_url_into_the_page(host): + html = _served_page(**{"x-forwarded-host": host}) + + assert "" not in html + assert _tag(html, "og:image").startswith("http://") + assert _tag(html, "og:image").endswith("/social-card.png") + + +def test_the_app_serves_the_card_as_a_png(): + client = TestClient(build_app(_Engine(), questions=[], roles=["admin"], mode="live", dataset="chinook")) + response = client.get("/social-card.png") + + assert response.status_code == 200 + assert response.headers["content-type"] == "image/png" + assert response.content.startswith(b"\x89PNG\r\n\x1a\n") + + +def test_the_card_the_app_serves_is_the_one_the_space_reads_from_github(): + """Two copies, one file. + + The Space's `thumbnail:` reads the copy in `docs/` over raw GitHub, so the + Space's own card works before the Space has built; the playground serves + the copy in the package, so a pip install has it too. + `scripts/render_social_card.py` writes both, and a card regenerated into + only one of them is the drift this catches. + """ + docs, packaged = (path.read_bytes() for path in CARD_COPIES) + + assert docs == packaged + # Small enough that an unfurler fetches it rather than giving up. + assert len(docs) < 200 * 1024 diff --git a/packages/nl2sql/tests/unit/test_playground_settings.py b/packages/nl2sql/tests/unit/test_playground_settings.py index 3ec6227e..5dd765be 100644 --- a/packages/nl2sql/tests/unit/test_playground_settings.py +++ b/packages/nl2sql/tests/unit/test_playground_settings.py @@ -191,6 +191,7 @@ def _every_route_response(client, trace_id="0b8f7d2e-1111-4222-8333-944455556666 """One request per route; the test fails if a route is added and not listed.""" requests = { ("GET", "/"): lambda: client.get("/"), + ("GET", "/social-card.png"): lambda: client.get("/social-card.png"), ("GET", "/api/meta"): lambda: client.get("/api/meta"), ("GET", "/api/schema"): lambda: client.get("/api/schema"), ("POST", "/api/ask"): lambda: client.post("/api/ask", json={"question": "q1", "role": "admin"}), diff --git a/scripts/render_social_card.py b/scripts/render_social_card.py new file mode 100644 index 00000000..29de6d87 --- /dev/null +++ b/scripts/render_social_card.py @@ -0,0 +1,115 @@ +"""Renders ``scripts/social_card.html`` into the link preview card. + +The card is the 1200x630 image a link to the hosted demo unfurls into: the +playground's Open Graph tags name it, and the Hugging Face Space's front-matter +points its ``thumbnail`` at the copy in ``docs/``. + +Two copies are written, and a test holds them byte-identical: + +``docs/assets/social-card.png`` + what the Space's ``thumbnail:`` reads over raw.githubusercontent.com, so + the Space card works before the Space has built, and what the docs show. + +``packages/nl2sql/src/nl2sql/cli/demo/playground/assets/social-card.png`` + what the playground itself serves at ``/social-card.png``. It ships in the + wheel, so ``pip install "nl2sql-engine[demo]"`` has it too. + +The card borrows the playground's bundled fonts, so run ``npm ci`` in +``web/playground`` first, then:: + + python scripts/render_social_card.py + +Headless Chrome does the rendering. The fonts are inlined as ``data:`` URIs +into a copy of the page first, because Chrome refuses to load a font from a +``file:`` URL into a ``file:`` page. Chrome is found on PATH, in the usual +Windows and macOS locations, or named with ``--chrome``. +""" + +from __future__ import annotations + +import argparse +import base64 +import mimetypes +import pathlib +import re +import shutil +import subprocess +import sys +import tempfile + +ROOT = pathlib.Path(__file__).resolve().parents[1] +SOURCE = ROOT / "scripts" / "social_card.html" +OUTPUTS = ( + ROOT / "docs" / "assets" / "social-card.png", + ROOT / "packages" / "nl2sql" / "src" / "nl2sql" / "cli" / "demo" / "playground" / "assets" / "social-card.png", +) +WIDTH, HEIGHT = 1200, 630 + +# Every local file the stylesheet reaches for, so none is left to Chrome. +_URL = re.compile(r"""url\(\s*["']?([^"')]+)["']?\s*\)""") + +_CHROME_CANDIDATES = ( + "chrome", "google-chrome", "chromium", "chromium-browser", "msedge", + r"C:\Program Files\Google\Chrome\Application\chrome.exe", + r"C:\Program Files (x86)\Google\Chrome\Application\chrome.exe", + "/Applications/Google Chrome.app/Contents/MacOS/Google Chrome", +) + + +def find_chrome(named: str | None) -> str: + if named: + return named + for candidate in _CHROME_CANDIDATES: + found = shutil.which(candidate) or (candidate if pathlib.Path(candidate).exists() else None) + if found: + return found + raise SystemExit("No Chrome found. Pass one with --chrome.") + + +def inline_assets(html: str, base: pathlib.Path) -> str: + """Replaces every ``url(...)`` pointing at a local file with a data URI.""" + + def replace(match: re.Match[str]) -> str: + reference = match.group(1) + if reference.startswith(("data:", "http:", "https:")): + return match.group(0) + path = (base / reference).resolve() + if not path.is_file(): + raise SystemExit(f"{reference} is missing. Run `npm ci` in web/playground first.") + media = mimetypes.guess_type(path.name)[0] or "application/octet-stream" + return f'url("data:{media};base64,{base64.b64encode(path.read_bytes()).decode("ascii")}")' + + return _URL.sub(replace, html) + + +def render(chrome: str, html: str) -> bytes: + with tempfile.TemporaryDirectory() as work: + page = pathlib.Path(work) / "card.html" + page.write_text(html, encoding="utf-8") + shot = pathlib.Path(work) / "card.png" + subprocess.run( + [chrome, "--headless=new", "--disable-gpu", "--hide-scrollbars", + "--force-device-scale-factor=1", f"--window-size={WIDTH},{HEIGHT}", + "--virtual-time-budget=4000", f"--screenshot={shot}", page.as_uri()], + check=True, capture_output=True, + ) + if not shot.is_file(): + raise SystemExit("Chrome wrote no screenshot.") + return shot.read_bytes() + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument("--chrome", help="the Chrome or Chromium binary to render with") + args = parser.parse_args(argv) + + png = render(find_chrome(args.chrome), inline_assets(SOURCE.read_text(encoding="utf-8"), SOURCE.parent)) + for out in OUTPUTS: + out.parent.mkdir(parents=True, exist_ok=True) + out.write_bytes(png) + print(f"{out.relative_to(ROOT)} {len(png) / 1024:.0f} KB") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/social_card.html b/scripts/social_card.html new file mode 100644 index 00000000..0664eb2f --- /dev/null +++ b/scripts/social_card.html @@ -0,0 +1,175 @@ + + + + + nl2sql playground link preview card + + + + +
+
+ + +

nl2sql playground

+
+ +
+

Ask a database in plain English — the model plans, the code writes the SQL

+

Bring your own API key · Sample data · Open source

+
+ +
+ group_by: country · sum(total) + → + SELECT country, SUM(total) … +
+
+ + diff --git a/web/playground/index.html b/web/playground/index.html index 2318c389..b3420956 100644 --- a/web/playground/index.html +++ b/web/playground/index.html @@ -3,7 +3,10 @@ - + nl2sql playground