diff --git a/README.md b/README.md index a8bcda8ed..53b361462 100644 --- a/README.md +++ b/README.md @@ -32,6 +32,24 @@ npm run dev 文档构建为 `dist/public` 下的静态站点，并托管在 Cloudflare Pages。部署前会运行类型检查、文档契约测试及完整静态构建。 +### Agent 可读导出 + +`npm run build` 会在 `/docs` 基路径和站点地图处理完成后生成 Agent 文本。 +每个已发布页面都有 `.md` 副本，HTML 使用 `rel="alternate" type="text/markdown"` +声明该地址，不改变可视界面。 + +- `/docs/llms.txt`、`/docs/llms-full.txt`：全部语言的索引与完整正文。 +- `/docs/{en,zh,ja}/llms.txt`、`llms-full.txt`：对应语言的索引与正文。 +- 各目录的 `llm.txt` 是 `llms.txt` 的兼容别名。 +- `/docs/openapi/service-api-{en,zh,ja}.json`：完整 API 规范。 + +生成器直接解析源 MDX 的 Markdown AST，保留提示标题和图片地址；每个 API +操作包含请求、响应、鉴权及递归引用的 schema。遇到未支持的语义 JSX 或可执行 +表达式会中止构建，避免静默丢失正文。请修改 MDX/OpenAPI 源文件，不要编辑 +`dist/public` 中的生成结果。`npm test` 检查转换逻辑，构建后的 +`npm run test:static` 检查站点地图覆盖、HTML 发现链接、内部链接与 API 契约。 +已有完成基路径处理的构建可使用 `npm run generate:agent-docs` 单独更新文本。 + ### 一些规范化标准 - 文件夹和文件的命名：**一律使用全小写，单词直接`-`隔开，如**`plugin-intro.mdx` diff --git a/README_EN.md b/README_EN.md index 048cdbfb9..90372fb2c 100644 --- a/README_EN.md +++ b/README_EN.md @@ -32,6 +32,27 @@ Place images in the `images` directory, then reference them using the absolute p The documentation is built as a static site in `dist/public` and hosted on Cloudflare Pages. Type checking, documentation contract tests, and a full static build run before deployment. +### Agent-readable exports + +`npm run build` generates agent text after finalizing the `/docs` base path and +sitemaps. Every published page has a canonical `.md` companion, advertised by an +HTML `rel="alternate" type="text/markdown"` link without changing the visible UI. + +- `/docs/llms.txt` and `/docs/llms-full.txt`: all-language index and complete text. +- `/docs/{en,zh,ja}/llms.txt` and `llms-full.txt`: language-specific exports. +- `llm.txt` is an identical compatibility alias for each `llms.txt` index. +- `/docs/openapi/service-api-{en,zh,ja}.json`: complete API specifications. + +The generator reads canonical MDX with a Markdown AST (not compiled image +variables), preserves callout titles and literal image URLs, and exports every +API operation with its request/response contract, security and transitive schema +references. Unknown semantic JSX or executable expressions fail the build rather +than silently losing content. Generated files stay in `dist/public`; edit source +MDX/OpenAPI, not generated Markdown. `npm test` covers conversion, and +`npm run test:static` verifies sitemap coverage, discovery, links and API contracts +after a build. Run `npm run generate:agent-docs` to regenerate text in an existing +finalized build. + ### Some Standardization Guidelines - Folder and file naming: **Use all lowercase, separate words with `-`, such as** `plugin-intro.mdx` diff --git a/package-lock.json b/package-lock.json index c8b69da64..af49897aa 100644 --- a/package-lock.json +++ b/package-lock.json @@ -26,9 +26,15 @@ "@types/node": "^24.10.0", "@types/react": "^19.2.2", "@types/react-dom": "^19.2.2", + "remark-gfm": "^4.0.1", + "remark-mdx": "^3.1.1", + "remark-parse": "^11.0.0", + "remark-stringify": "^11.0.0", "tailwindcss": "^4.3.3", "typescript": "^5.9.3", - "vite": "^8.2.2" + "unified": "^11.0.5", + "vite": "^8.2.2", + "yaml": "^2.9.0" }, "engines": { "node": ">=24" diff --git a/package.json b/package.json index ac80748e8..74ae513ab 100644 --- a/package.json +++ b/package.json @@ -11,14 +11,15 @@ "prepare:fumapress": "node scripts/prepare-fumapress.mjs", "build": "fumapress build", "prebuild": "npm run prepare:fumapress", - "postbuild": "npm run finalize:base-path && npm run generate:sitemap-alternates && npm run normalize:static-paths && npm run test:static", + "postbuild": "npm run finalize:base-path && npm run generate:sitemap-alternates && npm run normalize:static-paths && npm run generate:agent-docs && npm run test:static", + "generate:agent-docs": "node scripts/generate-agent-docs.mjs", "typecheck": "tsc --noEmit", "normalize:static-paths": "node scripts/normalize-static-paths.mjs", - "test:static": "node --test tests/fumapress-static-build.test.mjs", + "test:static": "node --test tests/fumapress-static-build.test.mjs tests/agent-static-build.test.mjs", "generate:sitemap-alternates": "node scripts/generate-alternate-sitemap.mjs", "sync:articles": "node scripts/sync-blog-articles.mjs", "check:articles": "node scripts/sync-blog-articles.mjs --check", - "test": "node --test tests/blog-articles.test.mjs tests/fumapress-migration.test.mjs tests/seo.test.mjs && python3 -m unittest discover -s tests -p 'test_*.py'", + "test": "node --test tests/agent-docs.test.mjs tests/blog-articles.test.mjs tests/fumapress-migration.test.mjs tests/seo.test.mjs && python3 -m unittest discover -s tests -p 'test_*.py'", "finalize:base-path": "node scripts/finalize-base-path.mjs" }, "dependencies": { @@ -42,8 +43,14 @@ "@types/node": "^24.10.0", "@types/react": "^19.2.2", "@types/react-dom": "^19.2.2", + "remark-gfm": "^4.0.1", + "remark-mdx": "^3.1.1", + "remark-parse": "^11.0.0", + "remark-stringify": "^11.0.0", "tailwindcss": "^4.3.3", "typescript": "^5.9.3", - "vite": "^8.2.2" + "unified": "^11.0.5", + "vite": "^8.2.2", + "yaml": "^2.9.0" } } diff --git a/press.config.tsx b/press.config.tsx index 962cad66a..1011d87f8 100644 --- a/press.config.tsx +++ b/press.config.tsx @@ -379,6 +379,7 @@ export default defineConfig({ const defaultLocale = alternates.includes("en") ? "en" : alternates[0]; return <> + {alternates.map((candidate) => ( value.split("/").map(encodeURIComponent).join("/"); +const slug = (value) => value.replace(/\s+/g, "-").toLowerCase(); +const pageRoute = (file) => file.replace(/\.mdx$/, "").replace(/\/index$/, ""); +const text = (value) => ({ type: "text", value }); +const paragraph = (children) => ({ type: "paragraph", children }); +const parser = unified().use(remarkParse).use(remarkMdx).use(remarkGfm); +const printer = unified().use(remarkStringify, { fences: true, bullet: "-" }).use(remarkGfm); + +export function splitFrontmatter(source) { + const match = source.match(/^---\r?\n([\s\S]*?)\r?\n---(?:\r?\n|$)/); + if (!match) throw new Error("Documentation source requires frontmatter"); + return { metadata: parseYaml(match[1]), body: source.slice(match[0].length) }; +} + +export function canonicalLink(value, document, routes, redirects = [], seen = new Set()) { + if (!value || value.startsWith("#")) return value; + let local = value; + const absolute = /^(?:[a-z][a-z\d+.-]*:|\/\/)/i.test(value); + if (absolute) { + // Example URLs can contain prose punctuation; only parse our docs hosts. + if (!/^(?:https?:)?\/\/(?:docs\.langbot\.app|langbot\.app)(?:\/|$)/i.test(value)) return value; + const url = new URL(value, BASE); + if (url.hostname === "docs.langbot.app") local = `${url.pathname}${url.search}${url.hash}`; + else if (url.origin === "https://langbot.app" && (url.pathname === "/docs" || url.pathname.startsWith("/docs/"))) { + local = `${url.pathname.slice(5) || "/"}${url.search}${url.hash}`; + } else return value; // Marketing, API hosts, mailto and other external links. + } + local = local.replace(/^\/docs(?=\/|$)/, ""); + const suffixAt = local.search(/[?#]/); + const suffix = suffixAt < 0 ? "" : local.slice(suffixAt); + let pathname = decodeURIComponent(suffixAt < 0 ? local : local.slice(0, suffixAt)); + pathname = pathname.startsWith("/") ? pathname.slice(1) : path.posix.join(path.posix.dirname(document), pathname); + pathname = path.posix.normalize(pathname).replace(/^\.\//, "").replace(/\/$/, ""); + if (pathname === "." || pathname === "" || LOCALES.includes(pathname)) pathname = `${LOCALES.includes(pathname) ? pathname : document.split("/")[0]}/insight/guide`; + const candidate = pathname.replace(/\.(?:html|mdx|md)$/, "").replace(/\/index$/, ""); + if (routes.has(candidate)) return `${BASE}/${encode(candidate)}.md${suffix}`; + // Honor the same legacy aliases as the human site, without inventing + // locale fallbacks for documents that have no translation. + for (const redirect of redirects) { + const wildcard = redirect.source.match(/^(.*\/):([A-Za-z][A-Za-z0-9_]*)\*$/); + const matches = redirect.source === `/${candidate}` || (wildcard && `/${candidate}`.startsWith(wildcard[1])); + if (matches && redirect.destination !== redirect.source) { + if (seen.has(candidate)) throw new Error(`Redirect cycle: ${candidate}`); + seen.add(candidate); + const destination = wildcard + ? redirect.destination.replace(`:${wildcard[2]}*`, `/${candidate}`.slice(wildcard[1].length)) + : redirect.destination; + return canonicalLink(`${destination}${suffix}`, document, routes, redirects, seen); + } + } + return `${BASE}/${encode(pathname)}${suffix}`; +} + +export function renderMdx(source, document, routes, redirects = []) { + const { metadata, body } = splitFrontmatter(source); + const tree = parser.parse(normalizeMdxContent(body, document)); + const link = (value) => canonicalLink(value, document, routes, redirects); + function transform(node) { + if (["mdxjsEsm", "mdxFlowExpression", "mdxTextExpression"].includes(node.type)) { + if (/^\s*\/\*[\s\S]*\*\/\s*$/.test(node.value)) return []; + // Never silently drop an executable source value that might contain docs. + throw new Error(`${document}: unsupported ${node.type}: ${node.value}`); + } + if (node.children) node.children = node.children.flatMap(transform); + if (["link", "image", "definition"].includes(node.type)) node.url = link(node.url); + if (!node.type.startsWith("mdxJsx")) return [node]; + const attributes = Object.fromEntries((node.attributes ?? []) + .filter((attribute) => attribute.type === "mdxJsxAttribute") + .map((attribute) => [attribute.name, attribute.value])); + const children = node.children ?? []; + if (node.name === "img") { + if (typeof attributes.src !== "string") throw new Error(`${document}: image requires a literal src`); + const image = { type: "image", url: link(attributes.src), alt: attributes.alt ?? "", title: attributes.title ?? null }; + return [node.type === "mdxJsxFlowElement" ? paragraph([image]) : image]; + } + if (node.name === "a" && typeof attributes.href === "string") { + const anchor = { type: "link", url: link(attributes.href), children }; + return [node.type === "mdxJsxFlowElement" ? paragraph([anchor]) : anchor]; + } + if (node.name === "br") return [node.type === "mdxJsxFlowElement" ? paragraph([text("")]) : { type: "break" }]; + if (node.name === "summary") return [paragraph([{ type: "strong", children }])]; + if (node.name === "Callout") { + return [{ type: "blockquote", children: [paragraph([text(`${attributes.title ?? attributes.type ?? "Note"}:`)]), ...children] }]; + } + if (["Card", "Step", "Tab"].includes(node.name)) { + if (typeof attributes.title !== "string") throw new Error(`${document}: ${node.name} requires a literal title`); + const title = typeof attributes.href === "string" + ? { type: "link", url: link(attributes.href), children: [text(attributes.title)] } + : { type: "strong", children: [text(attributes.title)] }; + return [paragraph([title]), ...children]; + } + // Audited source wrappers; fail closed when new semantic components appear. + if (["div", "details", "span", "Steps", "Tabs", "CardGroup"].includes(node.name)) return children; + throw new Error(`${document}: unsupported JSX component ${node.name}`); + } + tree.children = tree.children.flatMap(transform); + const title = metadata.title ?? pageRoute(document); + const url = `${BASE}/${encode(pageRoute(document))}`; + const description = metadata.description ? `${metadata.description}\n\n` : ""; + return { title, description: metadata.description ?? "", url, markdown: `# ${title}\n\nSource: ${url}\n\n${description}${printer.stringify(tree)}` }; +} + +export function operationBundle(spec, apiPath, method) { + const item = spec.paths[apiPath]; + const operation = item[method]; + const common = Object.fromEntries(Object.entries(item).filter(([key]) => !METHODS.test(key))); + const bundle = { + openapi: spec.openapi, + info: spec.info, + servers: operation.servers ?? item.servers ?? spec.servers ?? [], + security: operation.security ?? spec.security ?? [], + paths: { [apiPath]: { ...common, [method]: operation } }, + components: {}, + }; + const visited = new Set(); + function include(ref) { + if (visited.has(ref)) return; + visited.add(ref); + if (!ref.startsWith("#/components/")) throw new Error(`Unsupported OpenAPI reference: ${ref}`); + const keys = ref.slice(2).split("/").map((key) => key.replaceAll("~1", "/").replaceAll("~0", "~")); + const value = keys.reduce((node, key) => node?.[key], spec); + if (value === undefined) throw new Error(`Missing OpenAPI reference: ${ref}`); + let parent = bundle; + for (const key of keys.slice(0, -1)) parent = parent[key] ??= {}; + parent[keys.at(-1)] = value; + visit(value); + } + function visit(value) { + if (!value || typeof value !== "object") return; + if (value.$ref) include(value.$ref); + for (const child of Object.values(value)) visit(child); + } + visit(bundle); + for (const alternative of bundle.security) for (const name of Object.keys(alternative)) { + include(`#/components/securitySchemes/${name.replaceAll("~", "~0").replaceAll("/", "~1")}`); + } + return bundle; +} + +export function renderOperation(spec, apiPath, method, locale) { + const operation = spec.paths[apiPath][method]; + if (!operation.summary) throw new Error(`Missing summary for ${method} ${apiPath}`); + const route = `${locale}/api-reference/${slug(operation.tags?.[0] ?? "unknown")}/${slug(operation.summary)}`; + const url = `${BASE}/${encode(route)}`; + const bundle = operationBundle(spec, apiPath, method); + const auth = bundle.security.length === 0 ? "No authentication required." : + `Authentication (OpenAPI security alternatives; OR between entries, AND within each entry): ${JSON.stringify(bundle.security)}. See components.securitySchemes below for header names, schemes and scopes.`; + const markdown = `# ${operation.summary}\n\nSource: ${url}\n\n## ${method.toUpperCase()} ${apiPath}\n\n${operation.description ?? ""}\n\n${auth}\n\nServers: ${bundle.servers.map((server) => server.url).join(", ") || "Relative to your LangBot instance"}. Use your own LangBot instance, not the documentation host.\n\n[Complete ${locale} OpenAPI specification](${BASE}/openapi/service-api-${locale}.json)\n\n## Operation contract\n\nThe OpenAPI subset below preserves path/operation parameters, request bodies, response codes, media types, examples and schemas. All referenced components are included transitively; resolve local JSON pointers against this document. The operation is preserved verbatim, including security overrides.\n\n\`\`\`json\n${JSON.stringify(bundle, null, 2)}\n\`\`\`\n`; + return { route, title: operation.summary, description: `${method.toUpperCase()} ${apiPath}`, url, markdown }; +} + +function indexText(pages, locale) { + const languages = locale ? [locale] : LOCALES; + const prefix = locale ? `${locale}/` : ""; + const lines = ["# LangBot Documentation", "", "> Canonical documentation for LangBot: deployment, messaging integrations, plugins, development and HTTP API reference.", "", "## Agent resources", "", `- [Complete documentation text](${BASE}/${prefix}llms-full.txt)`, `- [All-language index](${BASE}/llms.txt)`, ""]; + for (const language of languages) { + lines.push(`## ${language}`, "", `- [${language} documentation index](${BASE}/${language}/llms.txt)`, `- [${language} complete documentation text](${BASE}/${language}/llms-full.txt)`, `- [${language} OpenAPI specification](${BASE}/openapi/service-api-${language}.json)`, ""); + for (const page of pages.filter((page) => page.route.startsWith(`${language}/`))) { + const label = page.title.replace(/[\[\]\\]/g, "\\$&").replace(/\s+/g, " "); + lines.push(`- [${label}](${page.url}.md)${page.description ? `: ${page.description.replace(/\s+/g, " ")}` : ""}`); + } + lines.push(""); + } + return `${lines.join("\n")}\n`; +} + +export async function generateAgentDocs({ root = ROOT, output = path.join(root, "dist/public") } = {}) { + const documents = await collectMdxDocuments(root); + const docs = JSON.parse(await readFile(path.join(root, "docs.json"), "utf8")); + const pages = []; + for (const locale of LOCALES) { + const spec = JSON.parse(await readFile(path.join(root, `openapi/service-api-${locale}.json`), "utf8")); + for (const [apiPath, item] of Object.entries(spec.paths)) for (const method of Object.keys(item).filter((key) => METHODS.test(key))) { + pages.push(renderOperation(spec, apiPath, method, locale)); + } + } + const routes = new Set([...documents.map(pageRoute), ...pages.map((page) => page.route)]); + for (const document of documents) { + pages.push({ route: pageRoute(document), ...renderMdx(await readFile(path.join(root, document), "utf8"), document, routes, docs.redirects) }); + } + pages.sort((a, b) => a.route.localeCompare(b.route, "en")); + if (new Set(pages.map((page) => page.route)).size !== pages.length) throw new Error("Duplicate agent page routes"); + // Use the real static sitemap as the acceptance boundary, not a guessed count. + const sitemap = await readFile(path.join(output, "sitemap.xml"), "utf8"); + const actual = [...sitemap.matchAll(/([^<]+)<\/loc>/g)].map((match) => decodeURIComponent(match[1].replace(/\/$/, ""))).sort(); + const expected = pages.map((page) => decodeURIComponent(page.url)).sort(); + if (JSON.stringify(actual) !== JSON.stringify(expected)) throw new Error("Agent sources do not match the exported sitemap"); + for (const page of pages) { + const target = path.join(output, `${page.route}.md`); + await mkdir(path.dirname(target), { recursive: true }); + await writeFile(target, page.markdown); + } + for (const locale of [null, ...LOCALES]) { + const prefix = locale ? `${locale}/` : ""; + const selected = pages.filter((page) => !locale || page.route.startsWith(prefix)); + const index = indexText(selected, locale); + await writeFile(path.join(output, `${prefix}llms.txt`), index); + await writeFile(path.join(output, `${prefix}llm.txt`), index); + await writeFile(path.join(output, `${prefix}llms-full.txt`), `${selected.map((page) => page.markdown.trim()).join("\n\n---\n\n")}\n`); + } + return { pages: pages.length, documents: documents.length, operations: pages.length - documents.length }; +} + +if (process.argv[1] && path.resolve(process.argv[1]) === fileURLToPath(import.meta.url)) { + console.log(await generateAgentDocs()); +} diff --git a/tests/agent-docs.test.mjs b/tests/agent-docs.test.mjs new file mode 100644 index 000000000..0f283d1e0 --- /dev/null +++ b/tests/agent-docs.test.mjs @@ -0,0 +1,58 @@ +import assert from "node:assert/strict"; +import test from "node:test"; +import { canonicalLink, renderMdx, operationBundle, BASE } from "../scripts/generate-agent-docs.mjs"; +import { unified } from "unified"; +import remarkParse from "remark-parse"; + +const routes = new Set(["zh/insight/guide", "zh/usage/models/readme"]); +const document = "zh/insight/guide.mdx"; +test("canonical links preserve external/example URLs without trying to repair them", () => { + for (const url of ["http://127.0.0.1:5300。", "mailto:user@example.org", "https://space.langbot.app/cloud", "https://langbot.app/zh/roadmap"]) { + assert.equal(canonicalLink(url, document, routes), url); + } +}); +test("canonical links handle base paths, Unicode, legacy hosts, relative routes and images", () => { + for (const url of ["/zh/usage/models/readme", "/docs/zh/usage/models/readme", "../usage/models/readme.html", "https://docs.langbot.app/zh/usage/models/readme.html", `${BASE}/zh/usage/models/readme.md`]) { + assert.equal(canonicalLink(`${url}?q=1#model`, document, routes), `${BASE}/zh/usage/models/readme.md?q=1#model`); + } + assert.equal(canonicalLink("../../images/测试.png", document, routes), `${BASE}/images/%E6%B5%8B%E8%AF%95.png`); + assert.equal(canonicalLink("#local", document, routes), "#local"); +}); +test("MDX exports preserve prose, wrapper content, literal images and code samples", () => { + const source = '---\ntitle: "Test"\ndescription: "Description"\n---\n\n\nDanger **now**.\n\n\n\nMore text.\n\n\nExample\n\n[Models](/zh/usage/models/readme.html)\n\n```python\nprint("/images/foo", "/zh/usage/models/readme.html", "")\n```\n'; + const result = renderMdx(source, document, routes).markdown; + assert.match(result, /Danger \*\*now\*\*/); + assert.match(result, /\*\*More\*\*/); + assert.match(result, /More text/); + assert.ok(result.includes(`![Example](${BASE}/images/example.png)`)); + assert.ok(result.includes(`[Models](${BASE}/zh/usage/models/readme.md)`)); + const tree = unified().use(remarkParse).parse(result); + assert.equal(tree.children.find((node) => node.type === "code").value, 'print("/images/foo", "/zh/usage/models/readme.html", "")'); + assert.doesNotMatch(result, /<\/?(?:Warning|Accordion)|__img/); +}); +test("semantic JSX preserves titles and card links; unknown components fail closed", () => { + const source = '---\ntitle: Test\n---\n\nRead this.\n\n\n\nConfigure models.\n\n\n\nRun setup.\n\n\n\nUse shell.\n\n'; + const result = renderMdx(source, document, routes).markdown; + for (const title of ["Important", "Install", "Linux"]) assert.ok(result.includes(title)); + assert.ok(result.includes(`[Models](${BASE}/zh/usage/models/readme.md)`)); + assert.throws(() => renderMdx('---\ntitle: Test\n---\n', document, routes), /unsupported JSX/); +}); +test("fixed redirect chains resolve with cycle detection", () => { + const redirects = [{ source: "/old", destination: "/older" }, { source: "/older", destination: "/zh/usage/models/readme" }]; + assert.equal(canonicalLink("/old#section", document, routes, redirects), `${BASE}/zh/usage/models/readme.md#section`); + assert.throws(() => canonicalLink("/old", document, routes, [...redirects.slice(0, 1), { source: "/older", destination: "/old" }]), /Redirect cycle/); +}); +test("legacy wildcard redirects resolve through multiple rules", () => { + const redirects = [{ source: "/deploy/:slug*", destination: "/zh/deploy/:slug*" }, { source: "/zh/deploy/:slug*", destination: "/zh/usage/:slug*" }]; + assert.equal(canonicalLink("/deploy/models/readme#model", document, routes, redirects), `${BASE}/zh/usage/models/readme.md#model`); +}); +test("OpenAPI closure preserves recursive refs, path parameters and security alternatives", () => { + const spec = { openapi: "3.0.3", info: { title: "Test", version: "1" }, security: [{ Key: [] }], components: { securitySchemes: { Key: { type: "apiKey", in: "header", name: "X-Key" } }, schemas: { Node: { type: "object", properties: { next: { $ref: "#/components/schemas/Node" } } } } }, paths: { "/items/{id}": { parameters: [{ name: "id", in: "path", required: true, schema: { type: "string" } }], get: { responses: { 200: { description: "OK", content: { "application/json": { schema: { $ref: "#/components/schemas/Node" } } } } } } } } }; + const bundle = operationBundle(spec, "/items/{id}", "get"); + assert.deepEqual(bundle.paths, spec.paths); + assert.deepEqual(bundle.components, spec.components); + assert.deepEqual(bundle.security, spec.security); + const bad = structuredClone(spec); + delete bad.components.schemas.Node; + assert.throws(() => operationBundle(bad, "/items/{id}", "get"), /Missing OpenAPI reference/); +}); diff --git a/tests/agent-static-build.test.mjs b/tests/agent-static-build.test.mjs new file mode 100644 index 000000000..d38caa413 --- /dev/null +++ b/tests/agent-static-build.test.mjs @@ -0,0 +1,107 @@ +import assert from "node:assert/strict"; +import { readFile, stat } from "node:fs/promises"; +import path from "node:path"; +import test from "node:test"; +import { unified } from "unified"; +import remarkParse from "remark-parse"; + +const root = path.resolve(import.meta.dirname, ".."); +const output = process.env.AGENT_TEST_OUTPUT ?? path.join(root, "dist/public"); +const base = "https://langbot.app/docs"; +const locales = ["en", "zh", "ja"]; +const read = (file) => readFile(path.join(output, file), "utf8"); +const slug = (value) => value.replace(/\s+/g, "-").toLowerCase(); +const encode = (value) => value.split("/").map(encodeURIComponent).join("/"); +const sitemap = await read("sitemap.xml"); +const urls = [...sitemap.matchAll(/([^<]+)<\/loc>/g)].map((m) => m[1].replace(/\/$/, "")); +const relative = (url) => decodeURIComponent(url.slice(base.length + 1)); +function links(node, result = []) { + if (["link", "image", "definition"].includes(node.type)) result.push(node.url); + for (const child of node.children ?? []) links(child, result); + return result; +} + +test("every sitemap page has meaningful canonical Markdown and HTML discovery", async () => { + assert.equal(urls.length, 479); + for (const url of urls) { + const route = relative(url); + const text = await read(`${route}.md`); + assert.ok(text.length > 100, `${route}: empty Markdown`); + assert.ok(text.includes(url), `${route}: missing canonical URL`); + assert.doesNotMatch(text, /__img\d+/, route); + const html = await read(`${route}/index.html`); + assert.ok(html.includes(`rel="alternate" type="text/markdown" href="${url}.md"`), `${route}: no Markdown discovery`); + } +}); + +test("all index and full-text exports cover exactly the published pages", async () => { + for (const locale of [null, ...locales]) { + const prefix = locale ? `${locale}/` : ""; + const index = await read(`${prefix}llms.txt`); + const full = await read(`${prefix}llms-full.txt`); + assert.equal(await read(`${prefix}llm.txt`), index); + const expected = urls.filter((url) => !locale || url.startsWith(`${base}/${locale}/`)); + const actual = links(unified().use(remarkParse).parse(index)).filter((url) => url.endsWith(".md")); + assert.deepEqual(actual.sort(), expected.map((url) => `${url}.md`).sort()); + for (const url of expected) assert.ok(full.includes((await read(`${relative(url)}.md`)).trim()), `${prefix}full export missing ${url}`); + for (const lang of locale ? [locale] : locales) assert.ok(index.includes(`${base}/openapi/service-api-${lang}.json`)); + assert.doesNotMatch(index, /https:\/\/docs\.langbot\.app/); + assert.ok((await stat(path.join(output, `${prefix}llms-full.txt`))).size < 25 * 1024 * 1024); + } +}); + +test("every API operation preserves the exact OpenAPI contract and auth override", async () => { + let count = 0; + for (const locale of locales) { + const source = JSON.parse(await readFile(path.join(root, `openapi/service-api-${locale}.json`), "utf8")); + assert.deepEqual(JSON.parse(await read(`openapi/service-api-${locale}.json`)), source); + for (const [apiPath, pathItem] of Object.entries(source.paths)) { + for (const [method, operation] of Object.entries(pathItem)) { + if (!/^(get|put|post|delete|patch|options|head|trace)$/.test(method)) continue; + count++; + const route = `${locale}/api-reference/${slug(operation.tags?.[0] ?? "unknown")}/${slug(operation.summary)}`; + const text = await read(`${route}.md`); + assert.ok(text.includes(`${method.toUpperCase()} ${apiPath}`), route); + assert.ok(text.includes(`${base}/${encode(route)}`), route); + assert.ok(text.includes(`${base}/openapi/service-api-${locale}.json`), route); + const bundle = JSON.parse(text.match(/```json\n([\s\S]*?)\n```/)?.[1] ?? "null"); + assert.ok(bundle, `${route}: missing operation bundle`); + assert.deepEqual(bundle.paths[apiPath][method], operation, route); + assert.deepEqual(bundle.security, operation.security ?? source.security ?? [], route); + assert.deepEqual(bundle.servers, operation.servers ?? pathItem.servers ?? source.servers ?? [], route); + if (bundle.security.length === 0) assert.match(text, /No authentication required/); + else for (const alternative of bundle.security) for (const scheme of Object.keys(alternative)) { + assert.deepEqual(bundle.components.securitySchemes[scheme], source.components.securitySchemes[scheme]); + } + function checkRefs(value) { + if (!value || typeof value !== "object") return; + if (value.$ref?.startsWith("#/")) { + const keys = value.$ref.slice(2).split("/").map((key) => key.replaceAll("~1", "/").replaceAll("~0", "~")); + const resolve = (document) => keys.reduce((node, key) => node?.[key], document); + assert.ok(resolve(bundle), `${route}: unresolved ${value.$ref}`); + assert.deepEqual(resolve(bundle), resolve(source)); + } + for (const child of Object.values(value)) checkRefs(child); + } + checkRefs(bundle); + } + } + } + assert.equal(count, 177); +}); + +test("Markdown links and images resolve inside the embedded output", async () => { + const failures = []; + for (const url of urls) { + const text = await read(`${relative(url)}.md`); + for (const link of links(unified().use(remarkParse).parse(text))) { + if (!link.startsWith(base) && !/^\/(?:en|zh|ja|images|docs)\//.test(link)) continue; + if (!link.startsWith(base)) { failures.push([url, link, "unscoped URL"]); continue; } + const target = new URL(link); + const file = decodeURIComponent(target.pathname.replace(/^\/docs\//, "")); + try { await stat(path.join(output, file.includes(".") ? file : `${file}/index.html`)); } + catch { failures.push([url, link, "missing target"]); } + } + } + assert.deepEqual(failures, []); +});