fix(seo): improve docs indexing and localization (#103)

Co-authored-by: dadachann <[email protected]>
2026-10-28 19:35:27 +08:00 · 2026-07-25 14:40:37 +08:00
parent 38a21145d9
commit 35a2843ac2
7 changed files with 2381 additions and 180 deletions
@@ -228,6 +228,291 @@
    {
      "source": "/app",
      "destination": "/en/insight/guide"
+    },
+    {
+      "source": "/en/api-reference/system/获取系统信息",
+      "destination": "/en/api-reference/system/get-system-information",
+      "permanent": true
+    },
+    {
+      "source": "/en/api-reference/model-providers/获取所有供应商",
+      "destination": "/en/api-reference/model-providers/list-all-model-providers",
+      "permanent": true
+    },
+    {
+      "source": "/en/api-reference/model-providers/创建供应商",
+      "destination": "/en/api-reference/model-providers/create-a-model-provider",
+      "permanent": true
+    },
+    {
+      "source": "/en/api-reference/model-providers/获取单个供应商",
+      "destination": "/en/api-reference/model-providers/get-a-model-provider",
+      "permanent": true
+    },
+    {
+      "source": "/en/api-reference/model-providers/更新供应商",
+      "destination": "/en/api-reference/model-providers/update-a-model-provider",
+      "permanent": true
+    },
+    {
+      "source": "/en/api-reference/model-providers/删除供应商",
+      "destination": "/en/api-reference/model-providers/delete-a-model-provider",
+      "permanent": true
+    },
+    {
+      "source": "/en/api-reference/llm-models/获取所有-llm-模型",
+      "destination": "/en/api-reference/llm-models/list-all-llm-models",
+      "permanent": true
+    },
+    {
+      "source": "/en/api-reference/llm-models/创建-llm-模型",
+      "destination": "/en/api-reference/llm-models/create-an-llm-model",
+      "permanent": true
+    },
+    {
+      "source": "/en/api-reference/llm-models/获取单个-llm-模型",
+      "destination": "/en/api-reference/llm-models/get-an-llm-model",
+      "permanent": true
+    },
+    {
+      "source": "/en/api-reference/llm-models/更新-llm-模型",
+      "destination": "/en/api-reference/llm-models/update-an-llm-model",
+      "permanent": true
+    },
+    {
+      "source": "/en/api-reference/llm-models/删除-llm-模型",
+      "destination": "/en/api-reference/llm-models/delete-an-llm-model",
+      "permanent": true
+    },
+    {
+      "source": "/en/api-reference/llm-models/测试-llm-模型",
+      "destination": "/en/api-reference/llm-models/test-an-llm-model",
+      "permanent": true
+    },
+    {
+      "source": "/en/api-reference/embedding-models/获取所有向量模型",
+      "destination": "/en/api-reference/embedding-models/list-all-embedding-models",
+      "permanent": true
+    },
+    {
+      "source": "/en/api-reference/embedding-models/创建向量模型",
+      "destination": "/en/api-reference/embedding-models/create-an-embedding-model",
+      "permanent": true
+    },
+    {
+      "source": "/en/api-reference/embedding-models/获取单个向量模型",
+      "destination": "/en/api-reference/embedding-models/get-an-embedding-model",
+      "permanent": true
+    },
+    {
+      "source": "/en/api-reference/embedding-models/更新向量模型",
+      "destination": "/en/api-reference/embedding-models/update-an-embedding-model",
+      "permanent": true
+    },
+    {
+      "source": "/en/api-reference/embedding-models/删除向量模型",
+      "destination": "/en/api-reference/embedding-models/delete-an-embedding-model",
+      "permanent": true
+    },
+    {
+      "source": "/en/api-reference/embedding-models/测试向量模型",
+      "destination": "/en/api-reference/embedding-models/test-an-embedding-model",
+      "permanent": true
+    },
+    {
+      "source": "/en/api-reference/bots/获取所有机器人",
+      "destination": "/en/api-reference/bots/list-all-bots",
+      "permanent": true
+    },
+    {
+      "source": "/en/api-reference/bots/创建机器人",
+      "destination": "/en/api-reference/bots/create-a-bot",
+      "permanent": true
+    },
+    {
+      "source": "/en/api-reference/bots/获取单个机器人",
+      "destination": "/en/api-reference/bots/get-a-bot",
+      "permanent": true
+    },
+    {
+      "source": "/en/api-reference/bots/更新机器人",
+      "destination": "/en/api-reference/bots/update-a-bot",
+      "permanent": true
+    },
+    {
+      "source": "/en/api-reference/bots/删除机器人",
+      "destination": "/en/api-reference/bots/delete-a-bot",
+      "permanent": true
+    },
+    {
+      "source": "/en/api-reference/bots/获取机器人事件日志",
+      "destination": "/en/api-reference/bots/get-bot-event-logs",
+      "permanent": true
+    },
+    {
+      "source": "/en/api-reference/bots/通过机器人发送消息",
+      "destination": "/en/api-reference/bots/send-a-message-through-a-bot",
+      "permanent": true
+    },
+    {
+      "source": "/en/api-reference/pipelines/获取所有流水线",
+      "destination": "/en/api-reference/pipelines/list-all-pipelines",
+      "permanent": true
+    },
+    {
+      "source": "/en/api-reference/pipelines/创建流水线",
+      "destination": "/en/api-reference/pipelines/create-a-pipeline",
+      "permanent": true
+    },
+    {
+      "source": "/en/api-reference/pipelines/获取流水线元数据",
+      "destination": "/en/api-reference/pipelines/get-pipeline-metadata",
+      "permanent": true
+    },
+    {
+      "source": "/en/api-reference/pipelines/获取单个流水线",
+      "destination": "/en/api-reference/pipelines/get-a-pipeline",
+      "permanent": true
+    },
+    {
+      "source": "/en/api-reference/pipelines/更新流水线",
+      "destination": "/en/api-reference/pipelines/update-a-pipeline",
+      "permanent": true
+    },
+    {
+      "source": "/en/api-reference/pipelines/删除流水线",
+      "destination": "/en/api-reference/pipelines/delete-a-pipeline",
+      "permanent": true
+    },
+    {
+      "source": "/en/api-reference/pipelines/复制流水线",
+      "destination": "/en/api-reference/pipelines/copy-a-pipeline",
+      "permanent": true
+    },
+    {
+      "source": "/en/api-reference/pipelines/获取流水线扩展配置",
+      "destination": "/en/api-reference/pipelines/get-pipeline-extension-configuration",
+      "permanent": true
+    },
+    {
+      "source": "/en/api-reference/pipelines/更新流水线扩展配置",
+      "destination": "/en/api-reference/pipelines/update-pipeline-extension-configuration",
+      "permanent": true
+    },
+    {
+      "source": "/en/api-reference/knowledge-bases/获取所有知识库",
+      "destination": "/en/api-reference/knowledge-bases/list-all-knowledge-bases",
+      "permanent": true
+    },
+    {
+      "source": "/en/api-reference/knowledge-bases/创建知识库",
+      "destination": "/en/api-reference/knowledge-bases/create-a-knowledge-base",
+      "permanent": true
+    },
+    {
+      "source": "/en/api-reference/knowledge-bases/获取单个知识库",
+      "destination": "/en/api-reference/knowledge-bases/get-a-knowledge-base",
+      "permanent": true
+    },
+    {
+      "source": "/en/api-reference/knowledge-bases/更新知识库",
+      "destination": "/en/api-reference/knowledge-bases/update-a-knowledge-base",
+      "permanent": true
+    },
+    {
+      "source": "/en/api-reference/knowledge-bases/删除知识库",
+      "destination": "/en/api-reference/knowledge-bases/delete-a-knowledge-base",
+      "permanent": true
+    },
+    {
+      "source": "/en/api-reference/knowledge-bases/获取知识库文件列表",
+      "destination": "/en/api-reference/knowledge-bases/list-knowledge-base-files",
+      "permanent": true
+    },
+    {
+      "source": "/en/api-reference/knowledge-bases/将文件存入知识库",
+      "destination": "/en/api-reference/knowledge-bases/add-a-file-to-a-knowledge-base",
+      "permanent": true
+    },
+    {
+      "source": "/en/api-reference/knowledge-bases/从知识库删除文件",
+      "destination": "/en/api-reference/knowledge-bases/remove-a-file-from-a-knowledge-base",
+      "permanent": true
+    },
+    {
+      "source": "/en/api-reference/knowledge-bases/检索知识库",
+      "destination": "/en/api-reference/knowledge-bases/retrieve-from-a-knowledge-base",
+      "permanent": true
+    },
+    {
+      "source": "/en/api-reference/knowledge-engines/获取可用知识引擎列表",
+      "destination": "/en/api-reference/knowledge-engines/list-available-knowledge-engines",
+      "permanent": true
+    },
+    {
+      "source": "/en/api-reference/knowledge-engines/获取知识引擎创建配置模式",
+      "destination": "/en/api-reference/knowledge-engines/get-the-knowledge-engine-creation-schema",
+      "permanent": true
+    },
+    {
+      "source": "/en/api-reference/knowledge-engines/获取知识引擎检索配置模式",
+      "destination": "/en/api-reference/knowledge-engines/get-the-knowledge-engine-retrieval-schema",
+      "permanent": true
+    },
+    {
+      "source": "/en/api-reference/parsers/获取可用解析器列表",
+      "destination": "/en/api-reference/parsers/list-available-parsers",
+      "permanent": true
+    },
+    {
+      "source": "/en/api-reference/files/上传图片",
+      "destination": "/en/api-reference/files/upload-an-image",
+      "permanent": true
+    },
+    {
+      "source": "/en/api-reference/files/上传文档",
+      "destination": "/en/api-reference/files/upload-a-document",
+      "permanent": true
+    },
+    {
+      "source": "/en/api-reference/plugins/获取已安装插件列表",
+      "destination": "/en/api-reference/plugins/list-installed-plugins",
+      "permanent": true
+    },
+    {
+      "source": "/en/api-reference/plugins/获取插件详情",
+      "destination": "/en/api-reference/plugins/get-plugin-details",
+      "permanent": true
+    },
+    {
+      "source": "/en/api-reference/plugins/删除插件",
+      "destination": "/en/api-reference/plugins/delete-a-plugin",
+      "permanent": true
+    },
+    {
+      "source": "/en/api-reference/plugins/升级插件",
+      "destination": "/en/api-reference/plugins/upgrade-a-plugin",
+      "permanent": true
+    },
+    {
+      "source": "/en/api-reference/plugins/获取插件配置",
+      "destination": "/en/api-reference/plugins/get-plugin-configuration",
+      "permanent": true
+    },
+    {
+      "source": "/en/api-reference/plugins/更新插件配置",
+      "destination": "/en/api-reference/plugins/update-plugin-configuration",
+      "permanent": true
+    },
+    {
+      "source": "/en/api-reference/plugins/从-marketplace-安装插件",
+      "destination": "/en/api-reference/plugins/install-a-plugin-from-marketplace",
+      "permanent": true
+    },
+    {
+      "source": "/en/api-reference/plugins/从-github-安装插件",
+      "destination": "/en/api-reference/plugins/install-a-plugin-from-github",
+      "permanent": true
    }
  ],
  "navigation": {
@@ -1,6 +1,7 @@
 {
  "scripts": {
    "dev": "npx mintlify dev",
+    "generate:sitemap-alternates": "node scripts/generate-alternate-sitemap.mjs",
    "test": "node --test tests/*.test.mjs"
  },
  "dependencies": {
@@ -0,0 +1,6 @@
+User-agent: *
+Allow: /
+Allow: /_next/
+
+Sitemap: https://docs.langbot.app/sitemap.xml
+Sitemap: https://docs.langbot.app/sitemap-alternates.xml
@@ -0,0 +1,75 @@
+#!/usr/bin/env node
+
+import { writeFile } from "node:fs/promises";
+
+const base = "https://docs.langbot.app";
+const source = process.env.SITEMAP_SOURCE ?? `${base}/sitemap.xml`;
+const output = new URL("../sitemap-alternates.xml", import.meta.url);
+const locales = ["en", "zh", "ja"];
+
+function escapeXml(value) {
+  return value
+    .replaceAll("&", "&amp;")
+    .replaceAll("<", "&lt;")
+    .replaceAll(">", "&gt;")
+    .replaceAll('"', "&quot;")
+    .replaceAll("'", "&apos;");
+}
+
+const response = await fetch(source, {
+  headers: { "user-agent": "LangBot docs alternate-sitemap generator" },
+});
+if (!response.ok) {
+  throw new Error(`Unable to fetch ${source}: ${response.status}`);
+}
+
+const sitemap = await response.text();
+const urls = [...sitemap.matchAll(/<loc>([^<]+)<\/loc>/g)].map((match) =>
+  match[1].replaceAll("&amp;", "&"),
+);
+const groups = new Map();
+
+for (const url of urls) {
+  for (const locale of locales) {
+    const prefix = `${base}/${locale}/`;
+    if (!url.startsWith(prefix)) continue;
+    const suffix = url.slice(prefix.length);
+    const group = groups.get(suffix) ?? {};
+    group[locale] = url;
+    groups.set(suffix, group);
+    break;
+  }
+}
+
+const translatedGroups = [...groups.values()]
+  .filter((group) => group.en && group.zh)
+  .sort((left, right) => left.en.localeCompare(right.en));
+const lines = [
+  '<?xml version="1.0" encoding="UTF-8"?>',
+  '<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9" xmlns:xhtml="http://www.w3.org/1999/xhtml">',
+];
+
+for (const group of translatedGroups) {
+  const alternates = [
+    ["en", group.en],
+    ["zh-CN", group.zh],
+    ...(group.ja ? [["ja", group.ja]] : []),
+    ["x-default", group.zh],
+  ];
+  for (const locale of locales) {
+    if (!group[locale]) continue;
+    lines.push("  <url>", `    <loc>${escapeXml(group[locale])}</loc>`);
+    for (const [language, href] of alternates) {
+      lines.push(
+        `    <xhtml:link rel="alternate" hreflang="${language}" href="${escapeXml(href)}" />`,
+      );
+    }
+    lines.push("  </url>");
+  }
+}
+lines.push("</urlset>");
+
+await writeFile(output, `${lines.join("\n")}\n`);
+console.log(
+  `Wrote ${translatedGroups.length} reciprocal language groups to ${output.pathname}`,
+);
@@ -3,6 +3,7 @@ import { readFile } from "node:fs/promises";
 import test from "node:test";

 const repoRoot = new URL("../", import.meta.url);
+const cjk = /[\u3400-\u4DBF\u4E00-\u9FFF\u3040-\u30FF\uAC00-\uD7AF]/u;

 test("repository-only English README is excluded from Mintlify publishing", async () => {
  const mintignore = await readFile(new URL(".mintignore", repoRoot), "utf8");
@@ -30,3 +31,90 @@ test("the historical README_EN URL keeps its permanent canonical redirect", asyn
  assert.equal(redirects[0].destination, "/en/insight/guide");
  assert.notEqual(redirects[0].permanent, false);
 });
+
+test("custom robots policy is served from the Mintlify project root", async () => {
+  const robots = await readFile(new URL("robots.txt", repoRoot), "utf8");
+  assert.match(robots, /^User-agent: \*$/m);
+  assert.match(robots, /^Allow: \/_next\/$/m);
+  assert.match(
+    robots,
+    /^Sitemap: https:\/\/docs\.langbot\.app\/sitemap\.xml$/m,
+  );
+  assert.match(
+    robots,
+    /^Sitemap: https:\/\/docs\.langbot\.app\/sitemap-alternates\.xml$/m,
+  );
+});
+
+test("alternate sitemap declares reciprocal en, zh-CN, ja and x-default links", async () => {
+  const sitemap = await readFile(
+    new URL("sitemap-alternates.xml", repoRoot),
+    "utf8",
+  );
+  const blocks = [...sitemap.matchAll(/<url>([\s\S]*?)<\/url>/g)].map(
+    (match) => match[1],
+  );
+  assert.ok(blocks.length > 0);
+
+  const groups = new Map();
+  for (const block of blocks) {
+    const loc = block.match(/<loc>([^<]+)<\/loc>/)?.[1];
+    assert.ok(loc);
+    const links = Object.fromEntries(
+      [...block.matchAll(/hreflang="([^"]+)" href="([^"]+)"/g)].map(
+        (match) => [match[1], match[2]],
+      ),
+    );
+    assert.ok(links.en);
+    assert.ok(links["zh-CN"]);
+    assert.equal(links["x-default"], links["zh-CN"]);
+    const reciprocalUrls = [links.en, links["zh-CN"], links.ja].filter(Boolean);
+    const key = reciprocalUrls.join("\n");
+    groups.set(key, [...(groups.get(key) ?? []), loc]);
+  }
+
+  assert.ok(groups.size > 0);
+  for (const [key, locations] of groups) {
+    assert.deepEqual(locations.sort(), key.split("\n").sort());
+  }
+});
+
+test("English Service API specification contains no CJK copy", async () => {
+  const source = await readFile(
+    new URL("openapi/service-api-en.json", repoRoot),
+    "utf8",
+  );
+  assert.doesNotThrow(() => JSON.parse(source));
+  const matches = [...source.matchAll(new RegExp(cjk.source, "gu"))];
+  assert.equal(matches.length, 0, `found ${matches.length} CJK code points`);
+});
+
+test("translated English API routes preserve every historical Chinese URL", async () => {
+  const docs = JSON.parse(await readFile(new URL("docs.json", repoRoot), "utf8"));
+  const redirects = docs.redirects.filter((item) =>
+    item.source.startsWith("/en/api-reference/") && cjk.test(item.source),
+  );
+  cjk.lastIndex = 0;
+
+  assert.equal(redirects.length, 57);
+  assert.equal(new Set(redirects.map((item) => item.source)).size, 57);
+  for (const redirect of redirects) {
+    assert.equal(redirect.permanent, true);
+    assert.match(redirect.destination, /^\/en\/api-reference\//);
+    assert.doesNotMatch(redirect.destination, cjk);
+    cjk.lastIndex = 0;
+  }
+});
+
+test("every Markdown image in the English guide has descriptive alt text", async () => {
+  const guide = await readFile(
+    new URL("en/insight/guide.mdx", repoRoot),
+    "utf8",
+  );
+  const images = [...guide.matchAll(/!\[([^\]]*)\]\(([^)]+)\)/g)];
+  assert.ok(images.length > 0, "expected at least one Markdown image");
+
+  for (const [, alt, src] of images) {
+    assert.ok(alt.trim().length >= 8, `${src} is missing descriptive alt text`);
+  }
+});