diff --git a/app/pages/index.vue b/app/pages/index.vue index cafe9bf..9a545ef 100644 --- a/app/pages/index.vue +++ b/app/pages/index.vue @@ -55,24 +55,27 @@ if (content.value.mode === 'prod') { description: fm.value.seo?.description || fm.value.description, }) - // Optional schema.org SoftwareApplication identity, configured through - // `docs.schemaOrg` in app.config; nothing is emitted when unset. + // Optional schema.org identity, configured through `docs.schemaOrg` in app.config; nothing is + // emitted when unset. `organization` becomes its own top-level node (with contactPoint/address it + // is what agents check to verify the business); everything else describes the SoftwareApplication. const { seo, docs } = useAppConfig() - const schemaOrg = docs?.schemaOrg as Record | undefined - if (schemaOrg && Object.keys(schemaOrg).length) { + const { organization, ...softwareApp } = (docs?.schemaOrg ?? {}) as Record & { + organization?: Record + } + const identity = { name: seo?.siteName, url: site.url } + const nodes: Record[] = [] + if (Object.keys(softwareApp).length) { + nodes.push({ '@type': 'SoftwareApplication', ...identity, ...softwareApp }) + } + if (organization && Object.keys(organization).length) { + nodes.push({ '@type': 'Organization', ...identity, ...organization }) + } + if (nodes.length) { useHead({ - script: [ - { - type: 'application/ld+json', - innerHTML: jsonLd({ - '@context': 'https://schema.org', - '@type': 'SoftwareApplication', - name: seo?.siteName, - url: site.url, - ...schemaOrg, - }), - }, - ], + script: nodes.map((node) => ({ + type: 'application/ld+json', + innerHTML: jsonLd({ '@context': 'https://schema.org', ...node }), + })), }) } } diff --git a/modules/markdown-rewrite.ts b/modules/markdown-rewrite.ts new file mode 100644 index 0000000..44fa6f0 --- /dev/null +++ b/modules/markdown-rewrite.ts @@ -0,0 +1,35 @@ +import { readFile, writeFile } from 'node:fs/promises' +import { defineNuxtModule, useLogger } from '@nuxt/kit' +import { resolve } from 'pathe' +import { buildMarkdownRewriteRoutes } from '../utils/markdown-rewrite' + +const logger = useLogger('comark-docs') + +/** + * Serve raw markdown to agents on the *page* URLs: `Accept: text/markdown` (or a curl user-agent) on + * `/getting-started/installation` returns `/raw/getting-started/installation.md`, and `/` returns + * `/llms.txt`. Implemented as Vercel routing-layer rewrites written into `.vercel/output/config.json` + * after Nitro compiles — rewriting at the edge keeps the negotiation out of the ISR cache, which is + * keyed per dest path and so can't serve the HTML variant to a markdown request (or vice versa). + */ +export default defineNuxtModule({ + meta: { + name: 'comark-docs/markdown-rewrite', + }, + setup(_options, nuxt) { + nuxt.hooks.hook('nitro:init', (nitro) => { + if (nitro.options.dev || !nitro.options.preset.includes('vercel')) return + + nitro.hooks.hook('compiled', async () => { + const configPath = resolve(nitro.options.output.dir, 'config.json') + const config = JSON.parse(await readFile(configPath, 'utf8')) + + const routes = buildMarkdownRewriteRoutes() + config.routes.unshift(...routes) + + await writeFile(configPath, JSON.stringify(config, null, 2), 'utf8') + logger.info(`Injected ${routes.length} markdown content-negotiation routes into ${configPath}`) + }) + }) + }, +}) diff --git a/nuxt.schema.ts b/nuxt.schema.ts index e465564..ce9863f 100644 --- a/nuxt.schema.ts +++ b/nuxt.schema.ts @@ -71,12 +71,21 @@ export default defineNuxtSchema({ mark: 'wordmark', }, llms: { - /** Description emitted under the llms.txt heading. */ + /** + * Blockquote summary emitted under the llms.txt heading; empty = site description. + * Good place for "when to use" guidance: name the jobs the product is right for, + * so an agent knows when to reach for it before reading the index. + */ description: '', /** Extra links appended to llms.txt. */ links: [], }, - /** schema.org SoftwareApplication identity, emitted as JSON-LD on the landing page. Empty = none. */ + /** + * schema.org SoftwareApplication identity, emitted as JSON-LD on the landing page. Empty = none. + * The `organization` sub-key is emitted as a separate top-level Organization node — give it + * `contactPoint` (with `contactType` and an email or phone) and `address` (a `PostalAddress`) + * so agents can verify the business behind the site. + */ schemaOrg: {}, /** Extra links appended to the docs page aside. */ asideLinks: [], diff --git a/playground/content/1.getting-started/1.introduction.md b/playground/content/1.getting-started/1.introduction.md index 08d0468..1f28bb1 100644 --- a/playground/content/1.getting-started/1.introduction.md +++ b/playground/content/1.getting-started/1.introduction.md @@ -26,7 +26,7 @@ A GitHub webhook notifies the site on push, which purges the cached pages. Becau - **Versioned previews** — browse any branch or commit of your docs through versioned URLs. - **Docs UI** built with [Nuxt UI](https://ui.nuxt.com): sidebar navigation, search (`⌘K`), table of contents, prev/next links, and a version history panel. - **SEO out of the box** — sitemap, robots, canonical URLs, OG images, and JSON-LD structured data. -- **AI-native** — `llms.txt`, raw Markdown mirrors (`/raw/**`), an MCP server (`/mcp`), an optional "Ask AI" assistant, and [Agent Skills](https://agentskills.io) discovery. +- **AI-native** — `llms.txt`, raw Markdown mirrors (`/raw/**`), Markdown [content negotiation](/concepts/architecture#markdown-for-agents) on every page URL, an MCP server (`/mcp`), an optional "Ask AI" assistant, and [Agent Skills](https://agentskills.io) discovery. ## Keyboard shortcuts diff --git a/playground/content/1.getting-started/3.configuration.md b/playground/content/1.getting-started/3.configuration.md index d0a1c07..2a5397a 100644 --- a/playground/content/1.getting-started/3.configuration.md +++ b/playground/content/1.getting-started/3.configuration.md @@ -107,14 +107,40 @@ export default defineAppConfig({ | `github.owner` / `github.name` | inferred | Repository owner and name (inferred from git when unset). | | `docs.rss.title` | `''` | RSS feed title; empty renders `${siteName} Documentation`. | | `docs.ogImage` | — | `{ accent, tagline, mark }` for the generated OG images. | -| `docs.llms` | — | `{ description, links }` emitted in `llms.txt`. | -| `docs.schemaOrg` | `{}` | schema.org `SoftwareApplication` identity, emitted as JSON-LD on the landing page. | +| `docs.llms` | — | `{ description, links }` emitted in `llms.txt`. `description` becomes the blockquote summary under the heading (empty falls back to the site description). | +| `docs.schemaOrg` | `{}` | schema.org `SoftwareApplication` identity, emitted as JSON-LD on the landing page. The `organization` sub-key is emitted as a separate `Organization` node — see below. | | `docs.asideLinks` | `[]` | Extra links appended to the docs page aside. | ::note Nuxt merges `app.config.ts` across layers with [defu](https://github.com/unjs/defu), which **concatenates arrays**. Your list is appended to the layer's, not substituted for it — which is why every array default in the layer is empty. :: +### Agent metadata + +Two optional keys help AI agents understand and verify your site: + +- `docs.llms.description` is emitted as the blockquote summary of `llms.txt` — the place the [llms.txt spec](https://llmstxt.org) reserves for key context. Use it to tell agents *when* to reach for your product: name the jobs it is right for and how an agent calls it — specific guidance, not marketing copy. +- `docs.schemaOrg.organization` is emitted as a top-level `Organization` JSON-LD node on the landing page. Give it a `contactPoint` (and optionally an `address`, a [PostalAddress](https://schema.org/PostalAddress)) so agents can verify the business behind the site. + +```ts [app.config.ts] +export default defineAppConfig({ + docs: { + llms: { + description: + 'Use My Project to build documentation sites where Markdown is served at request time. ' + + 'Fetch any page as raw markdown at `/raw/.md`, or request any page URL with `Accept: text/markdown`.', + }, + schemaOrg: { + applicationCategory: 'DeveloperApplication', + organization: { + contactPoint: { '@type': 'ContactPoint', contactType: 'customer support', email: 'support@example.com' }, + sameAs: ['https://github.com/my-org'], + }, + }, + }, +}) +``` + ## Environment variables | Variable | Purpose | diff --git a/playground/content/3.concepts/1.architecture.md b/playground/content/3.concepts/1.architecture.md index ada1ef5..f757f10 100644 --- a/playground/content/3.concepts/1.architecture.md +++ b/playground/content/3.concepts/1.architecture.md @@ -67,6 +67,19 @@ The handler verifies the webhook signature with `WEBHOOK_SECRET`, resolves the n Without the webhook, the site still updates: ISR entries expire on their own after the `isr` window. The webhook just makes it immediate. +## Markdown for agents + +Every production documentation page is mirrored as raw Markdown at `/raw/.md` ([versioned previews](/concepts/versioned-previews) serve HTML only). The mirrors carry the same ISR caching as the HTML pages. + +On Vercel, agents don't need to know the mirror URLs. The layer injects rewrites into the build output, ahead of the ISR cache: + +- A request for any page URL with `Accept: text/markdown` (or a curl user-agent) is rewritten to its `/raw/**` mirror. +- A request for `/` is rewritten to `/llms.txt`. + +Because the rewrite happens at the routing layer, the HTML and Markdown variants are cached under different paths and can't poison each other's cache entries. + +A request for a page that doesn't exist returns a real HTTP 404 with a short Markdown body pointing at `/llms.txt`, `/llms-full.txt`, and the sitemap, so an agent that guesses a URL wrong can recover. + ## No redeploys for content Since content never ships in the build, a content-only push doesn't need a deployment at all. On Vercel, an [Ignored Build Step](/deployment/vercel#setup-skip-builds-for-content-pushes) cancels builds for pushes that only touch `content/` — the webhook handles those. Code pushes build and deploy as usual. diff --git a/playground/content/3.concepts/2.versioned-previews.md b/playground/content/3.concepts/2.versioned-previews.md index 0c7b8c0..d054546 100644 --- a/playground/content/3.concepts/2.versioned-previews.md +++ b/playground/content/3.concepts/2.versioned-previews.md @@ -42,6 +42,6 @@ Every other SHA answers 404, and `/tree/` rejects GitHub's hidden `pull//head ## Good to know - Previews are public, like the rest of the site, but send `noindex` robots headers and canonicalize to the production URL — they won't compete with your real pages in search. -- The raw Markdown mirrors work in previews too: `/tree/my-branch/raw/.md`. +- The [raw Markdown mirrors](/concepts/architecture#markdown-for-agents) cover production pages only — preview pages serve HTML. - "Edit this page on GitHub" targets the previewed branch on `/tree/` pages, and is disabled on `/blob/` and `/pr/` pages — a commit can't be edited, and a PR may come from a fork branch the site can't link an editor to. - Preview instances are kept in a small LRU pool per server instance; evicted versions rebuild on demand, and their parsed pages survive in the per-SHA cache. diff --git a/playground/skills/preview-versions/SKILL.md b/playground/skills/preview-versions/SKILL.md index 7e4e3ed..fadb9e2 100644 --- a/playground/skills/preview-versions/SKILL.md +++ b/playground/skills/preview-versions/SKILL.md @@ -19,7 +19,7 @@ Content is served at request time. Production is pinned to a commit SHA; any bra Commit and PR previews are authorized: a `/blob/` SHA must be in production history or belong to a PR (same-repo, or a fork PR carrying the `preview:enabled` label), and `/pr/` applies the same rule. Unauthorized refs answer 404. -Raw markdown mirrors exist at `/raw/**` (and under `/tree/.../raw/` / `/blob/.../raw/`). +Raw markdown mirrors exist at `/raw/**` for production pages only — preview pages serve HTML. Two cache tiers: ISR-cached page HTML at the edge, and a per-parser-version, per-content-SHA runtime cache for parsed Markdown bodies. A GitHub push to the production branch hits `/api/revalidate` and purges ISR. diff --git a/server/routes/raw/[...slug].md.get.ts b/server/routes/raw/[...slug].md.get.ts index 18475de..214ce5a 100644 --- a/server/routes/raw/[...slug].md.get.ts +++ b/server/routes/raw/[...slug].md.get.ts @@ -1,21 +1,18 @@ -import { withLeadingSlash } from 'ufo' - export default defineEventHandler(async (event) => { const slug = getRouterParams(event)['slug.md'] if (!slug?.endsWith('.md')) { - throw createError({ statusCode: 404, statusMessage: 'Page not found' }) + return notFoundMarkdown(event, event.path) } const content = await getProdContent() - const stripped = slug.replace(/\.md$/, '') - const path = stripped === 'index' ? '/' : withLeadingSlash(stripped) - + const path = pagePathFromRawSlug(slug) const markdown = await renderPageMarkdown(content, path) if (!markdown) { - throw createError({ statusCode: 404, statusMessage: 'Page not found' }) + return notFoundMarkdown(event, path) } setHeader(event, 'Content-Type', 'text/markdown; charset=utf-8') + setHeader(event, 'Vary', 'Accept') return markdown }) diff --git a/server/utils/not-found.ts b/server/utils/not-found.ts new file mode 100644 index 0000000..6165ed2 --- /dev/null +++ b/server/utils/not-found.ts @@ -0,0 +1,34 @@ +import type { H3Event } from 'h3' +import { setHeader, setResponseStatus } from 'h3' + +/** + * A 404 with a short markdown body instead of the app shell or a JSON error: agents that land on a + * missing page get pointers to the machine-readable indexes so they can recover instead of guessing. + */ +export function notFoundMarkdown(event: H3Event, path?: string): string { + setResponseStatus(event, 404, 'Page not found') + setHeader(event, 'Content-Type', 'text/markdown; charset=utf-8') + setHeader(event, 'Vary', 'Accept') + + return [ + '# Page not found', + '', + path ? `\`${path}\` does not exist on this site.` : 'This page does not exist on this site.', + '', + 'Where to look next:', + '', + '- [/llms.txt](/llms.txt) — index of every documentation page, with raw markdown links', + '- [/llms-full.txt](/llms-full.txt) — the full documentation as a single markdown file', + '- [/raw/index.md](/raw/index.md) — the landing page as markdown', + '- [/sitemap.xml](/sitemap.xml) — sitemap of the rendered pages', + '', + 'Every documentation page is mirrored as raw markdown at `/raw/.md`.', + '', + ].join('\n') +} + +/** `/raw/**` slug (`getting-started/installation.md`) → content path (`/getting-started/installation`). */ +export function pagePathFromRawSlug(slug: string): string { + const stripped = slug.replace(/\.md$/, '') + return stripped === 'index' ? '/' : `/${stripped}` +} diff --git a/test/markdown-rewrite.test.ts b/test/markdown-rewrite.test.ts new file mode 100644 index 0000000..86fa92c --- /dev/null +++ b/test/markdown-rewrite.test.ts @@ -0,0 +1,80 @@ +import { describe, expect, it } from 'vitest' +import { buildMarkdownRewriteRoutes, type VercelRoute } from '../utils/markdown-rewrite' + +// Vercel resolves `$n` in `dest` from the capture groups of `src` — replicate that to assert on the +// final rewritten path rather than on regex internals. +function rewrite(routes: VercelRoute[], path: string): VercelRoute & { resolved: string } | null { + for (const route of routes) { + const match = path.match(new RegExp(route.src)) + if (!match) continue + const resolved = route.dest.replace(/\$(\d+)/g, (_, n) => match[Number(n)] ?? '') + return { ...route, resolved } + } + return null +} + +const routes = buildMarkdownRewriteRoutes() + +describe('buildMarkdownRewriteRoutes', () => { + it('pairs every rewrite with an Accept matcher and a curl matcher', () => { + expect(routes.length % 2).toBe(0) + const conditions = routes.map((route) => route.has?.[0]) + expect(conditions.filter((c) => c?.key === 'accept').length).toBe(routes.length / 2) + expect(conditions.filter((c) => c?.key === 'user-agent').length).toBe(routes.length / 2) + }) + + it('sets the markdown content type and varies on Accept', () => { + for (const route of routes) { + expect(route.headers?.['content-type']).toBe('text/markdown; charset=utf-8') + expect(route.headers?.vary).toBe('Accept') + } + }) + + it('sends the landing page to llms.txt', () => { + expect(rewrite(routes, '/')?.resolved).toBe('/llms.txt') + }) + + it('sends pages to their raw markdown mirror', () => { + expect(rewrite(routes, '/getting-started/installation')?.resolved).toBe( + '/raw/getting-started/installation.md' + ) + expect(rewrite(routes, '/getting-started/installation/')?.resolved).toBe( + '/raw/getting-started/installation.md' + ) + expect(rewrite(routes, '/writing')?.resolved).toBe('/raw/writing.md') + }) + + it('never rewrites versioned previews (no raw mirrors, HTML only)', () => { + for (const path of [ + '/tree/main', + '/tree/release%2Fv1.2/writing/pages', + '/blob/a1b2c3d', + '/blob/a1b2c3d/getting-started/introduction', + '/pr/28', + '/pr/28/getting-started/introduction', + ]) { + expect(rewrite(routes, path), path).toBeNull() + } + }) + + it('never rewrites the mirrors, APIs, internals or dotted paths', () => { + for (const path of [ + '/raw/getting-started/installation.md', + '/api/content/search-sections', + '/api/assistant', + '/mcp', + '/logos', + '/_nuxt/entry.js', + '/__nuxt_island/foo', + '/llms.txt', + '/llms-full.txt', + '/sitemap.xml', + '/rss.xml', + '/robots.txt', + '/favicon.ico', + '/.well-known/skills/index.json', + ]) { + expect(rewrite(routes, path), path).toBeNull() + } + }) +}) diff --git a/utils/markdown-rewrite.ts b/utils/markdown-rewrite.ts new file mode 100644 index 0000000..91efc09 --- /dev/null +++ b/utils/markdown-rewrite.ts @@ -0,0 +1,43 @@ +// Vercel Build Output routes that serve raw markdown to agents asking for it. Injected ahead of the +// generated routing table (see `modules/markdown-rewrite.ts`), so the rewrite happens at the edge — +// *before* the ISR cache — and the HTML and markdown variants can never poison each other's cache +// entries. Same approach as Docus (nuxt-content/docus `markdown-rewrite`), but with generic patterns: +// comark-docs reads content at request time, so the page list isn't known at build time. + +export interface VercelRoute { + src: string + dest: string + headers?: Record + has?: Array<{ type: 'header'; key: string; value?: string }> +} + +const MARKDOWN_HEADERS = { + 'content-type': 'text/markdown; charset=utf-8', + // acceptmarkdown.com: negotiated responses must vary on Accept so shared caches key both variants. + 'vary': 'Accept', +} + +// A rewrite fires when the client either negotiates markdown or is curl (agents shell out to it). +const MATCHERS: NonNullable[] = [ + [{ type: 'header', key: 'accept', value: '(.*)text/markdown(.*)' }], + [{ type: 'header', key: 'user-agent', value: 'curl/.*' }], +] + +// `src`/`dest` pairs, expanded per matcher below. Order matters: first match wins. +const REWRITES: Array> = [ + // Landing page → the full docs index. + { src: '^/$', dest: '/llms.txt' }, + // Every other extensionless page → its raw markdown mirror. Excluded: the mirrors themselves, API + // routes, versioned previews (`/tree`, `/blob`, `/pr` serve HTML only), Nuxt/Nitro internals + // (`_nuxt`, `__nuxt_island`, …), the MCP endpoint and the layer-owned `/logos` page (not + // content-derived, so it has no mirror). `[^.]` also skips every dotted path: `llms.txt`, + // `sitemap.xml`, `robots.txt`, `favicon.ico`, `/.well-known/**`, … + { src: '^/(?!raw/|api/|tree/|blob/|pr/|mcp$|logos$|_)([^.]+?)/?$', dest: '/raw/$1.md' }, +] + +/** The full route list to prepend to `.vercel/output/config.json`. */ +export function buildMarkdownRewriteRoutes(): VercelRoute[] { + return REWRITES.flatMap((rewrite) => + MATCHERS.map((has) => ({ ...rewrite, headers: MARKDOWN_HEADERS, has })) + ) +}