mirror of
https://github.com/langbot-app/LangBot.git
synced 2026-10-28 19:35:27 +08:00
301 lines
11 KiB
Python
301 lines
11 KiB
Python
"""Run the API documentation snippets with the installed SDK and a mock Host.
|
|
|
|
Usage: python scripts/verify-plugin-api-examples.py
|
|
This checks SDK calls and result shapes, not external platform delivery.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import ast
|
|
import inspect
|
|
import re
|
|
from pathlib import Path
|
|
from types import SimpleNamespace
|
|
from typing import Any
|
|
from unittest.mock import AsyncMock
|
|
|
|
from langbot_plugin.api.entities.builtin.runner.context import RunnerContext
|
|
from langbot_plugin.api.entities.builtin.runner.errors import (
|
|
AgentAPIError,
|
|
AgentAPIException,
|
|
)
|
|
from langbot_plugin.api.proxies.runner import RunnerAPIProxy
|
|
from langbot_plugin.api.proxies.langbot_api import LangBotAPIProxy
|
|
from langbot_plugin.api.proxies.event_context import EventContextProxy
|
|
from langbot_plugin.api.proxies.execute_context import ExecuteContextProxy
|
|
from langbot_plugin.api.entities.builtin.provider.session import Session
|
|
from langbot_plugin.api.proxies.invocation import bind_invocation
|
|
from langbot_plugin.api.entities.builtin.provider.message import Message
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
|
|
|
|
class Host:
|
|
def __init__(self):
|
|
self.calls = []
|
|
|
|
async def call_action(self, action, data, timeout=15):
|
|
name = action.value
|
|
self.calls.append((name, data))
|
|
responses = {
|
|
"invoke_llm": {
|
|
"message": {"role": "assistant", "content": "Hello!"},
|
|
"usage": {"total_tokens": 8},
|
|
},
|
|
"invoke_rerank": {"results": [{"index": 0, "relevance_score": 0.9}]},
|
|
"list_parsers": {
|
|
"parsers": [{"plugin_author": "demo", "plugin_name": "Parser"}]
|
|
},
|
|
"invoke_parser": {"text": "guide", "sections": [], "metadata": {}},
|
|
"vector_list": {
|
|
"items": [{"id": "chunk", "document": "guide", "metadata": {}}],
|
|
"total": 1,
|
|
},
|
|
"list_plugins_manifest": {"plugins": []},
|
|
"list_commands": {"commands": []},
|
|
"call_platform_api": {"result": {"id": "123456", "name": "Example"}},
|
|
"call_tool": {"result": {"id": "alice"}},
|
|
"reply_stream": {"result": {"ok": True, "mock": True}},
|
|
"history_page": {
|
|
"items": [],
|
|
"has_more": not data.get("before_cursor"),
|
|
"prev_cursor": "older",
|
|
"next_cursor": None,
|
|
},
|
|
"reply_message": {},
|
|
"set_query_var": {},
|
|
"state_set": {"success": True},
|
|
"state_get": {"value": "setup"},
|
|
}
|
|
if name not in responses:
|
|
raise AssertionError(f"Unexpected Host action: {name}")
|
|
return responses[name]
|
|
|
|
async def call_action_generator(self, action, data, timeout=15):
|
|
assert action.value == "invoke_llm_stream"
|
|
yield {"chunk": {"role": "assistant", "content": "Hello", "is_final": False}}
|
|
yield {"chunk": {"role": "assistant", "content": "!", "is_final": True}}
|
|
yield {"usage": {"total_tokens": 8}}
|
|
|
|
|
|
def context():
|
|
return RunnerContext.model_validate(
|
|
{
|
|
"run_id": "docs-run",
|
|
"trigger": {"type": "user_message"},
|
|
"event": {
|
|
"event_id": "docs-event",
|
|
"event_type": "message.received",
|
|
"source": "test",
|
|
},
|
|
"input": {"content": "Hello"},
|
|
"delivery": {"surface": "test"},
|
|
"runtime": {},
|
|
"resources": {
|
|
"models": [{"model_id": "model"}],
|
|
"tools": [
|
|
{"tool_name": "event_get_actor"},
|
|
{"tool_name": "event_reply"},
|
|
],
|
|
},
|
|
"context": {
|
|
"available_apis": {
|
|
"history_page": True,
|
|
"state": True,
|
|
"reply_stream": True,
|
|
}
|
|
},
|
|
}
|
|
)
|
|
|
|
|
|
def snippets(locale, page):
|
|
source = (ROOT / locale / f"plugin/dev/apis/{page}.mdx").read_text()
|
|
if page == "common":
|
|
headings = {
|
|
"zh": ("### 模型流式输出与用量", "## 其他资源 API"),
|
|
"en": ("### Model streaming and usage", "## Additional resource APIs"),
|
|
"ja": ("### モデルのストリーミングと使用量", "## その他のリソース API"),
|
|
}[locale]
|
|
start = source.index(headings[0])
|
|
end = source.index("\n### ", start + len(headings[0]))
|
|
source = source[start:end] + source[source.index(headings[1]) :]
|
|
return re.findall(r"^```python\n(.*?)^```", source, re.M | re.S)
|
|
|
|
|
|
def verify_context_reference():
|
|
source = (ROOT / "zh/plugin/dev/apis/agent-run.mdx").read_text()
|
|
query_methods = {
|
|
"reply",
|
|
"get_bot_uuid",
|
|
"set_query_var",
|
|
"get_query_var",
|
|
"get_query_vars",
|
|
"create_new_conversation",
|
|
"list_pipeline_knowledge_bases",
|
|
"retrieve_knowledge",
|
|
}
|
|
runner_methods = {
|
|
name
|
|
for name, value in RunnerContext.__dict__.items()
|
|
if callable(value) and not name.startswith("_")
|
|
}
|
|
runner_methods.discard("model_post_init")
|
|
|
|
for prefix in ("event_context", "context"):
|
|
missing = sorted(
|
|
name for name in query_methods if f"{prefix}.{name}(" not in source
|
|
)
|
|
assert not missing, f"{prefix} methods missing from context reference: {missing}"
|
|
|
|
for name in ("prevent_default", "prevent_postorder"):
|
|
assert f"event_context.{name}(" in source
|
|
|
|
missing = sorted(name for name in runner_methods if f"ctx.{name}(" not in source)
|
|
assert not missing, f"RunnerContext methods missing from context reference: {missing}"
|
|
assert "`platform_event`" in source
|
|
|
|
required_fields = {
|
|
"instance_uuid",
|
|
"workspace_uuid",
|
|
"placement_generation",
|
|
"query_id",
|
|
"query_uuid",
|
|
"eid",
|
|
"event_name",
|
|
"event",
|
|
"is_prevent_default",
|
|
"is_prevent_postorder",
|
|
"session",
|
|
"command_text",
|
|
"full_command_text",
|
|
"command",
|
|
"crt_command",
|
|
"params",
|
|
"crt_params",
|
|
"privilege",
|
|
*RunnerContext.model_fields,
|
|
}
|
|
missing = sorted(name for name in required_fields if f"`{name}`" not in source)
|
|
assert not missing, f"Context fields missing from context reference: {missing}"
|
|
assert "spec.permissions" not in source
|
|
assert "spec.capabilities" not in source
|
|
|
|
|
|
async def execute(block, namespace):
|
|
# Examples are function bodies; signature reference blocks may include stubs.
|
|
function = "async def example():\n" + "\n".join(
|
|
" " + line for line in block.splitlines()
|
|
)
|
|
tree = ast.parse(function)
|
|
for node in ast.walk(tree):
|
|
if not isinstance(node, ast.Call) or not isinstance(node.func, ast.Attribute):
|
|
continue
|
|
owner = ast.unparse(node.func.value)
|
|
cls = (
|
|
LangBotAPIProxy
|
|
if owner == "self.plugin"
|
|
else EventContextProxy
|
|
if owner == "event_context"
|
|
else ExecuteContextProxy
|
|
if owner == "context"
|
|
else RunnerContext
|
|
if owner == "ctx"
|
|
else RunnerAPIProxy
|
|
if owner == "api"
|
|
else None
|
|
)
|
|
if cls is not None:
|
|
signature = inspect.signature(getattr(cls, node.func.attr))
|
|
signature.bind(
|
|
None,
|
|
*([None] * len(node.args)),
|
|
**{kw.arg: None for kw in node.keywords},
|
|
)
|
|
exec(compile(tree, "plugin-api-doc-example", "exec"), namespace)
|
|
await namespace["example"]()
|
|
|
|
|
|
async def main():
|
|
verify_context_reference()
|
|
total = 0
|
|
for page in ("common", "agent-run", "platform"):
|
|
blocks = snippets("zh", page)
|
|
for locale in ("en", "ja"):
|
|
assert blocks == snippets(locale, page), f"{page}: {locale} code differs"
|
|
for block in blocks:
|
|
host = Host()
|
|
ctx = context()
|
|
api = RunnerAPIProxy(ctx, host)
|
|
ctx._api = api
|
|
ctx._results = asyncio.Queue()
|
|
session = Session(launcher_type="group", launcher_id="docs-group", sender_id="docs-user")
|
|
namespace = {
|
|
"event_context": EventContextProxy.model_construct(
|
|
query_id=42,
|
|
event=SimpleNamespace(sender_id="docs-user"),
|
|
plugin_runtime_handler=host,
|
|
),
|
|
"context": ExecuteContextProxy.model_construct(
|
|
query_id=43,
|
|
plugin_runtime_handler=host,
|
|
crt_params=["Hello!"],
|
|
session=session,
|
|
),
|
|
"session": session,
|
|
"Any": Any,
|
|
"Message": Message,
|
|
"api": api,
|
|
"ctx": ctx,
|
|
"self": SimpleNamespace(
|
|
plugin=LangBotAPIProxy(host), get_run_api=lambda _: api
|
|
),
|
|
"model_uuid": "model",
|
|
"rerank_model_uuid": "model",
|
|
"collection_id": "collection",
|
|
"storage_path": "files/guide.pdf",
|
|
"bot_uuid": "bot",
|
|
"print": lambda *args: None,
|
|
}
|
|
with bind_invocation(host, api):
|
|
await execute(block, namespace)
|
|
for name, data in host.calls:
|
|
if name in {"reply_message", "set_query_var"}:
|
|
expected_query = 42 if "event_context" in block else 43
|
|
assert data["query_id"] == expected_query
|
|
assert "run_id" not in data
|
|
if "reply.update" in block:
|
|
ops = [
|
|
data["operation"]
|
|
for name, data in host.calls
|
|
if name == "reply_stream"
|
|
]
|
|
assert ops == ["update", "update", "finish"], ops
|
|
if "older =" in block:
|
|
pages = [data for name, data in host.calls if name == "history_page"]
|
|
assert len(pages) == 2 and pages[1]["before_cursor"] == "older"
|
|
if "except PermissionDeniedError" in block:
|
|
# Exercise both documented error handlers with the real SDK proxy.
|
|
api._allowed_tool_names = frozenset()
|
|
await execute(block, namespace)
|
|
ctx.log.assert_awaited()
|
|
api._allowed_tool_names = frozenset({"event_get_actor"})
|
|
host.call_action = AsyncMock(
|
|
side_effect=AgentAPIException(
|
|
AgentAPIError(
|
|
code="host.action_error", message="Failed", retryable=False
|
|
)
|
|
)
|
|
)
|
|
await execute(block, namespace)
|
|
assert ctx.log.await_count == 2
|
|
total += 1
|
|
print(
|
|
f"{total} API examples passed; all three locales have identical code. Host delivery is mocked."
|
|
)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
asyncio.run(main())
|