mirror of
https://github.com/langbot-app/LangBot.git
synced 2026-10-29 20:05:25 +08:00
ci: sync docs to GitHub Wiki (#104)
Co-authored-by: dadachann <[email protected]>
This commit is contained in:
Hyu
@@ -0,0 +1,87 @@
|
||||
name: Sync GitHub Wiki
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [main]
|
||||
paths:
|
||||
- "en/**"
|
||||
- "zh/**"
|
||||
- "ja/**"
|
||||
- "scripts/build-github-wiki.py"
|
||||
- ".github/workflows/sync-github-wiki.yml"
|
||||
pull_request:
|
||||
paths:
|
||||
- "en/**"
|
||||
- "zh/**"
|
||||
- "ja/**"
|
||||
- "scripts/build-github-wiki.py"
|
||||
- "tests/test_build_github_wiki.py"
|
||||
- ".github/workflows/sync-github-wiki.yml"
|
||||
workflow_dispatch:
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
concurrency:
|
||||
group: sync-langbot-github-wiki
|
||||
cancel-in-progress: false
|
||||
|
||||
jobs:
|
||||
build:
|
||||
name: Build Wiki mirror
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 5
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: "3.12"
|
||||
- name: Run converter tests
|
||||
run: python -m unittest tests/test_build_github_wiki.py
|
||||
- name: Build Wiki pages
|
||||
run: python scripts/build-github-wiki.py --output dist/github-wiki
|
||||
- name: Validate generated pages
|
||||
run: |
|
||||
test -f dist/github-wiki/Home.md
|
||||
test -f dist/github-wiki/_Sidebar.md
|
||||
test "$(find dist/github-wiki -maxdepth 1 -name '*.md' | wc -l)" -gt 250
|
||||
- name: Upload generated Wiki
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: github-wiki
|
||||
path: dist/github-wiki
|
||||
if-no-files-found: error
|
||||
|
||||
sync:
|
||||
name: Publish to LangBot Wiki
|
||||
if: github.event_name != 'pull_request'
|
||||
needs: build
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 5
|
||||
steps:
|
||||
- uses: actions/download-artifact@v4
|
||||
with:
|
||||
name: github-wiki
|
||||
path: generated
|
||||
- name: Clone target Wiki
|
||||
env:
|
||||
WIKI_PUSH_TOKEN: ${{ secrets.LANGBOT_WIKI_PUSH_TOKEN }}
|
||||
run: |
|
||||
test -n "$WIKI_PUSH_TOKEN"
|
||||
git clone "https://x-access-token:${WIKI_PUSH_TOKEN}@github.com/langbot-app/LangBot.wiki.git" target
|
||||
- name: Replace and publish Wiki contents
|
||||
env:
|
||||
SOURCE_SHA: ${{ github.sha }}
|
||||
run: |
|
||||
find target -mindepth 1 -maxdepth 1 ! -name .git -exec rm -rf {} +
|
||||
cp -a generated/. target/
|
||||
cd target
|
||||
git add -A
|
||||
if git diff --cached --quiet; then
|
||||
echo "GitHub Wiki is already up to date."
|
||||
exit 0
|
||||
fi
|
||||
git config user.name "langbot-wiki-sync[bot]"
|
||||
git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
|
||||
git commit -m "docs: sync from langbot-wiki@${SOURCE_SHA::7}"
|
||||
git push origin HEAD:master
|
||||
+12
-8
@@ -1,8 +1,12 @@
|
||||
node_modules/
|
||||
.mintlify/
|
||||
.DS_Store
|
||||
/.omc
|
||||
.mintlify-last/
|
||||
|
||||
# Per-user Claude Code settings (skill symlinks under .claude/skills/ stay tracked)
|
||||
.claude/settings.local.json
|
||||
node_modules/
|
||||
.mintlify/
|
||||
.DS_Store
|
||||
/.omc
|
||||
.mintlify-last/
|
||||
|
||||
# Per-user Claude Code settings (skill symlinks under .claude/skills/ stay tracked)
|
||||
.claude/settings.local.json
|
||||
|
||||
# Python build/test cache
|
||||
__pycache__/
|
||||
*.py[cod]
|
||||
|
||||
Executable
+183
@@ -0,0 +1,183 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Build a GitHub Wiki-compatible mirror from the Mintlify MDX sources."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import re
|
||||
import shutil
|
||||
from pathlib import Path, PurePosixPath
|
||||
from urllib.parse import urlsplit, urlunsplit
|
||||
|
||||
LANGUAGES = {"zh": "简体中文", "en": "English", "ja": "日本語"}
|
||||
DOC_EXTENSIONS = (".md", ".mdx", ".html")
|
||||
CALLOUTS = {"Info": "NOTE", "Note": "NOTE", "Tip": "TIP", "Warning": "WARNING"}
|
||||
MOVED_SECTIONS = {"platforms", "models", "pipelines", "knowledge", "mcp"}
|
||||
|
||||
|
||||
def page_name(path: PurePosixPath) -> str:
|
||||
return path.with_suffix("").as_posix().replace("/", "-")
|
||||
|
||||
|
||||
def extract_frontmatter(text: str) -> tuple[dict[str, str], str]:
|
||||
if not text.startswith("---\n"):
|
||||
return {}, text
|
||||
end = text.find("\n---\n", 4)
|
||||
if end == -1:
|
||||
return {}, text
|
||||
metadata: dict[str, str] = {}
|
||||
for line in text[4:end].splitlines():
|
||||
key, separator, value = line.partition(":")
|
||||
if separator:
|
||||
metadata[key.strip()] = value.strip().strip('"\'')
|
||||
return metadata, text[end + 5 :]
|
||||
|
||||
|
||||
def resolve_doc_target(source: PurePosixPath, target: str) -> str | None:
|
||||
parsed = urlsplit(target)
|
||||
if parsed.scheme or parsed.netloc or target.startswith("#"):
|
||||
return None
|
||||
raw_path = parsed.path
|
||||
if not raw_path:
|
||||
return None
|
||||
resolved = PurePosixPath(raw_path.lstrip("/")) if raw_path.startswith("/") else source.parent / raw_path
|
||||
parts: list[str] = []
|
||||
for part in resolved.parts:
|
||||
if part in ("", "."):
|
||||
continue
|
||||
if part == "..":
|
||||
if parts:
|
||||
parts.pop()
|
||||
else:
|
||||
parts.append(part)
|
||||
if not parts or parts[0] not in LANGUAGES:
|
||||
return None
|
||||
normalized = PurePosixPath(*parts)
|
||||
suffix = normalized.suffix.lower()
|
||||
if suffix in DOC_EXTENSIONS:
|
||||
normalized = normalized.with_suffix("")
|
||||
elif suffix:
|
||||
return None
|
||||
# Mintlify redirects these sections from their old deploy location. Apply the
|
||||
# same redirect in the flat Wiki mirror so historical source links still work.
|
||||
if len(normalized.parts) >= 3 and normalized.parts[1] == "deploy" and normalized.parts[2] in MOVED_SECTIONS:
|
||||
normalized = PurePosixPath(normalized.parts[0], "usage", *normalized.parts[2:])
|
||||
return urlunsplit(("", "", page_name(normalized), parsed.query, parsed.fragment))
|
||||
|
||||
|
||||
def rewrite_url(source: PurePosixPath, target: str, *, image: bool = False) -> str:
|
||||
parsed = urlsplit(target)
|
||||
if parsed.scheme or parsed.netloc or target.startswith(("#", "mailto:")):
|
||||
return target
|
||||
if image:
|
||||
path = parsed.path.lstrip("/") if parsed.path.startswith("/") else (source.parent / parsed.path).as_posix()
|
||||
return urlunsplit(("https", "raw.githubusercontent.com", f"/langbot-app/langbot-wiki/main/{path}", parsed.query, parsed.fragment))
|
||||
return resolve_doc_target(source, target) or target
|
||||
|
||||
|
||||
def convert_callouts(text: str) -> str:
|
||||
pattern = re.compile(
|
||||
r"<(Info|Note|Tip|Warning)(?:\s+title=(?:\"([^\"]*)\"|'([^']*)'))?\s*>(.*?)</\1>",
|
||||
re.DOTALL,
|
||||
)
|
||||
|
||||
def replace(match: re.Match[str]) -> str:
|
||||
component = match.group(1)
|
||||
title = match.group(2) or match.group(3) or ""
|
||||
body = match.group(4).strip()
|
||||
lines = [f"> [!{CALLOUTS[component]}]"]
|
||||
if title:
|
||||
lines.append(f"> **{title}**")
|
||||
lines.extend(">" if not line else f"> {line}" for line in body.splitlines())
|
||||
return "\n".join(lines)
|
||||
|
||||
previous = None
|
||||
while previous != text:
|
||||
previous = text
|
||||
text = pattern.sub(replace, text)
|
||||
return text
|
||||
|
||||
|
||||
def convert_mdx(source: PurePosixPath, text: str) -> tuple[str, str]:
|
||||
metadata, body = extract_frontmatter(text)
|
||||
title = metadata.get("title") or source.stem.replace("-", " ").title()
|
||||
body = convert_callouts(body)
|
||||
body = re.sub(
|
||||
r'<Accordion\s+title=(?:"([^"]*)"|\'([^\']*)\')\s*>',
|
||||
lambda m: f"\n<details>\n<summary><strong>{m.group(1) or m.group(2)}</strong></summary>\n",
|
||||
body,
|
||||
)
|
||||
body = body.replace("</Accordion>", "\n</details>")
|
||||
body = re.sub(r"</?AccordionGroup\s*>", "", body)
|
||||
body = re.sub(
|
||||
r"(!\[[^\]]*\]\()([^)\s]+)([^)]*\))",
|
||||
lambda m: m.group(1) + rewrite_url(source, m.group(2), image=True) + m.group(3),
|
||||
body,
|
||||
)
|
||||
body = re.sub(
|
||||
r"(?<!!)\[([^\]]+)\]\(([^)\s]+)([^)]*)\)",
|
||||
lambda m: f"[{m.group(1)}]({rewrite_url(source, m.group(2))}{m.group(3)})",
|
||||
body,
|
||||
)
|
||||
body = re.sub(
|
||||
r'(<img\b[^>]*?\bsrc=["\'])([^"\']+)(["\'])',
|
||||
lambda m: m.group(1) + rewrite_url(source, m.group(2), image=True) + m.group(3),
|
||||
body,
|
||||
flags=re.IGNORECASE,
|
||||
)
|
||||
body = body.replace("<br />", "<br>")
|
||||
body = re.sub(r"\n{3,}", "\n\n", body).strip()
|
||||
return title, f"# {title}\n\n{body}\n"
|
||||
|
||||
|
||||
def discover_documents(root: Path) -> list[Path]:
|
||||
return sorted(path for language in LANGUAGES for path in (root / language).rglob("*.mdx"))
|
||||
|
||||
|
||||
def build(root: Path, output: Path) -> int:
|
||||
documents = discover_documents(root)
|
||||
if not documents:
|
||||
raise SystemExit("No MDX documents found")
|
||||
if output.exists():
|
||||
shutil.rmtree(output)
|
||||
output.mkdir(parents=True)
|
||||
index: dict[str, list[tuple[str, str]]] = {language: [] for language in LANGUAGES}
|
||||
for path in documents:
|
||||
source = PurePosixPath(path.relative_to(root).as_posix())
|
||||
title, converted = convert_mdx(source, path.read_text(encoding="utf-8"))
|
||||
name = page_name(source)
|
||||
(output / f"{name}.md").write_text(converted, encoding="utf-8")
|
||||
index[source.parts[0]].append((name, title))
|
||||
home = [
|
||||
"# LangBot Documentation",
|
||||
"",
|
||||
"This GitHub Wiki is automatically synchronized from [langbot-app/langbot-wiki](https://github.com/langbot-app/langbot-wiki).",
|
||||
"",
|
||||
]
|
||||
sidebar = ["## LangBot Documentation", ""]
|
||||
for language, label in LANGUAGES.items():
|
||||
pages = index[language]
|
||||
home.extend((f"## {label}", "", f"[{pages[0][1]}]({pages[0][0]})", ""))
|
||||
sidebar.extend((f"### {label}", ""))
|
||||
sidebar.extend(f"- [{title}]({name})" for name, title in pages)
|
||||
sidebar.append("")
|
||||
(output / "Home.md").write_text("\n".join(home).rstrip() + "\n", encoding="utf-8")
|
||||
(output / "_Sidebar.md").write_text("\n".join(sidebar).rstrip() + "\n", encoding="utf-8")
|
||||
(output / "_Footer.md").write_text(
|
||||
"Automatically synchronized from [langbot-app/langbot-wiki](https://github.com/langbot-app/langbot-wiki).\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
return len(documents)
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--root", type=Path, default=Path(__file__).resolve().parents[1])
|
||||
parser.add_argument("--output", type=Path, default=Path("dist/github-wiki"))
|
||||
args = parser.parse_args()
|
||||
count = build(args.root.resolve(), args.output.resolve())
|
||||
print(f"Built {count} GitHub Wiki pages in {args.output}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,73 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib.util
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path, PurePosixPath
|
||||
|
||||
MODULE_PATH = Path(__file__).resolve().parents[1] / "scripts" / "build-github-wiki.py"
|
||||
SPEC = importlib.util.spec_from_file_location("build_github_wiki", MODULE_PATH)
|
||||
assert SPEC and SPEC.loader
|
||||
wiki = importlib.util.module_from_spec(SPEC)
|
||||
SPEC.loader.exec_module(wiki)
|
||||
|
||||
|
||||
class BuildGitHubWikiTest(unittest.TestCase):
|
||||
def test_converts_frontmatter_callout_links_and_images(self) -> None:
|
||||
source = PurePosixPath("zh/insight/guide.mdx")
|
||||
title, output = wiki.convert_mdx(
|
||||
source,
|
||||
'''---
|
||||
title: "开始使用"
|
||||
description: "ignored"
|
||||
---
|
||||
<Tip title="提示">
|
||||
阅读[模型文档](/zh/usage/models/readme)。
|
||||
</Tip>
|
||||
|
||||
|
||||
''',
|
||||
)
|
||||
self.assertEqual(title, "开始使用")
|
||||
self.assertTrue(output.startswith("# 开始使用\n"))
|
||||
self.assertNotIn("description:", output)
|
||||
self.assertIn("> [!TIP]", output)
|
||||
self.assertIn("> **提示**", output)
|
||||
self.assertIn("[模型文档](zh-usage-models-readme)", output)
|
||||
self.assertIn(
|
||||
"https://raw.githubusercontent.com/langbot-app/langbot-wiki/main/images/ui.png",
|
||||
output,
|
||||
)
|
||||
|
||||
def test_resolves_relative_document_links(self) -> None:
|
||||
source = PurePosixPath("en/usage/mcp/readme.mdx")
|
||||
self.assertEqual(
|
||||
wiki.resolve_doc_target(source, "../models/readme#tools"),
|
||||
"en-usage-models-readme#tools",
|
||||
)
|
||||
self.assertEqual(
|
||||
wiki.resolve_doc_target(source, "../../deploy/settings.html"),
|
||||
"en-deploy-settings",
|
||||
)
|
||||
self.assertEqual(
|
||||
wiki.resolve_doc_target(source, "/en/deploy/pipelines/readme#request-variables"),
|
||||
"en-usage-pipelines-readme#request-variables",
|
||||
)
|
||||
|
||||
def test_build_creates_flat_pages_and_navigation(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as temp:
|
||||
root = Path(temp)
|
||||
for language in wiki.LANGUAGES:
|
||||
path = root / language / "guide.mdx"
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text(f"---\ntitle: {language} Guide\n---\nContent\n", encoding="utf-8")
|
||||
output = root / "out"
|
||||
count = wiki.build(root, output)
|
||||
self.assertEqual(count, 3)
|
||||
self.assertTrue((output / "zh-guide.md").is_file())
|
||||
self.assertIn("[zh Guide](zh-guide)", (output / "_Sidebar.md").read_text())
|
||||
self.assertTrue((output / "Home.md").is_file())
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user